make-mp-data 2.0.21 → 2.0.23

package/lib/orchestrators/user-loop.js

@@ -21,185 +21,216 @@ import { makeSCD } from "../generators/scd.js";
  * @returns {Promise<void>}
  */
 export async function userLoop(context) {
-    const { config, storage, defaults } = context;
-    const chance = u.getChance();
-    const concurrency = config?.concurrency || Math.min(os.cpus().length * 2, 16);
-    const USER_CONN = pLimit(concurrency);
-    
-    const {
-        verbose,
-        numUsers,
-        numEvents,
-        isAnonymous,
-        hasAvatar,
-        hasAnonIds,
-        hasSessionIds,
-        hasLocation,
-        funnels,
-        userProps,
-        scdProps,
-        numDays,
-        percentUsersBornInDataset = 5,
-    } = config;
-    
-    const { eventData, userProfilesData, scdTableData } = storage;
-    const avgEvPerUser = numEvents / numUsers;
-    const startTime = Date.now();
-
-    // Create batches for parallel processing
-    const batchSize = Math.max(1, Math.ceil(numUsers / concurrency));
-    const userPromises = [];
-    
-    for (let i = 0; i < numUsers; i++) {
-        const userPromise = USER_CONN(async () => {
-            context.incrementUserCount();
-            const eps = Math.floor(context.getEventCount() / ((Date.now() - startTime) / 1000));
-            
-            if (verbose) {
-                u.progress([
-                    ["users", context.getUserCount()], 
-                    ["events", context.getEventCount()], 
-                    ["eps", eps]
-                ]);
-            }
-            
-            const userId = chance.guid();
-            const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds });
-            const { distinct_id, created } = user;
-            const userIsBornInDataset = chance.bool({ likelihood: percentUsersBornInDataset });
-            let numEventsPreformed = 0;
-            
-            if (!userIsBornInDataset) delete user.created;
-            
-            // Calculate time adjustments
-            const daysShift = context.getDaysShift();
-            const adjustedCreated = userIsBornInDataset 
-                ? dayjs(created).subtract(daysShift, 'd') 
-                : dayjs.unix(global.FIXED_BEGIN);
-
-            if (hasLocation) {
-                const location = u.pickRandom(u.choose(defaults.locationsUsers));
-                for (const key in location) {
-                    user[key] = location[key];
-                }
-            }
-
-            // Profile creation
-            const profile = await makeUserProfile(context, userProps, user);
-
-            // SCD creation
-            // @ts-ignore
-            const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user' || !scd.type);
-            const scdTableKeys = Object.keys(scdUserTables);
-
-            const userSCD = {};
-            for (const [index, key] of scdTableKeys.entries()) {
-                const { max = 100 } = scdProps[key];
-                const mutations = chance.integer({ min: 1, max });
-                const changes = await makeSCD(context, scdProps[key], key, distinct_id, mutations, created);
-                userSCD[key] = changes;
-                
-                await config.hook(changes, "scd-pre", { 
-                    profile, 
-                    type: 'user', 
-                    scd: { [key]: scdProps[key] }, 
-                    config, 
-                    allSCDs: userSCD 
-                });
-            }
-
-            let numEventsThisUserWillPreform = Math.floor(chance.normal({
-                mean: avgEvPerUser,
-                dev: avgEvPerUser / u.integer(u.integer(2, 5), u.integer(2, 7))
-            }) * 0.714159265359);
-
-            // Power users and low-activity users logic
-            chance.bool({ likelihood: 20 }) ? numEventsThisUserWillPreform *= 5 : null;
-            chance.bool({ likelihood: 15 }) ? numEventsThisUserWillPreform *= 0.333 : null;
-            numEventsThisUserWillPreform = Math.round(numEventsThisUserWillPreform);
-
-            let userFirstEventTime;
-
-            const firstFunnels = funnels.filter((f) => f.isFirstFunnel).reduce(u.weighFunnels, []);
-            const usageFunnels = funnels.filter((f) => !f.isFirstFunnel).reduce(u.weighFunnels, []);
-
-            const secondsInDay = 86400;
-            const noise = () => chance.integer({ min: 0, max: secondsInDay });
-            let usersEvents = [];
-
-            if (firstFunnels.length && userIsBornInDataset) {
-                const firstFunnel = chance.pickone(firstFunnels, user);
-                const firstTime = adjustedCreated.subtract(noise(), 'seconds').unix();
-                const [data, userConverted] = await makeFunnel(context, firstFunnel, user, firstTime, profile, userSCD);
-                
-                const timeShift = context.getTimeShift();
-                userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
-                numEventsPreformed += data.length;
-                usersEvents = usersEvents.concat(data);
-                
-                if (!userConverted) {
-                    // if (verbose) {
-                    //     u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
-                    // }
-                    return;
-                }
-            } else {
-                userFirstEventTime = adjustedCreated.subtract(noise(), 'seconds').unix();
-            }
-
-            while (numEventsPreformed < numEventsThisUserWillPreform) {
-                if (usageFunnels.length) {
-                    const currentFunnel = chance.pickone(usageFunnels);
-                    const [data, userConverted] = await makeFunnel(context, currentFunnel, user, userFirstEventTime, profile, userSCD);
-                    numEventsPreformed += data.length;
-                    usersEvents = usersEvents.concat(data);
-                } else {
-                    const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(config.events), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true);
-                    numEventsPreformed++;
-                    usersEvents = usersEvents.concat(data);
-                }
-            }
-
-            // Hook for processing all user events
-            if (config.hook) {
-                const newEvents = await config.hook(usersEvents, "everything", { 
-                    profile, 
-                    scd: userSCD, 
-                    config, 
-                    userIsBornInDataset 
-                });
-                if (Array.isArray(newEvents)) usersEvents = newEvents;
-            }
-
-            // Store all user data
-            await userProfilesData.hookPush(profile);
-
-            if (Object.keys(userSCD).length) {
-                for (const [key, changesArray] of Object.entries(userSCD)) {
-                    for (const changes of changesArray) {
-                        try {
-                            const target = scdTableData.filter(arr => arr.scdKey === key).pop();
-                            await target.hookPush(changes, { profile, type: 'user' });
-                        }
-                        catch (e) {
-                            // This is probably a test
-                            const target = scdTableData[0];
-                            await target.hookPush(changes, { profile, type: 'user' });
-                        }
-                    }
-                }
-            }
-            
-            await eventData.hookPush(usersEvents, { profile });
-
-            if (verbose) {
-                // u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
-            }
-        });
-        
-        userPromises.push(userPromise);
-    }
-    
-    // Wait for all users to complete
-    await Promise.all(userPromises);
+	const { config, storage, defaults } = context;
+	const chance = u.getChance();
+	const concurrency = config?.concurrency || Math.min(os.cpus().length * 2, 16);
+	const USER_CONN = pLimit(concurrency);
+
+	const {
+		verbose,
+		numUsers,
+		numEvents,
+		isAnonymous,
+		hasAvatar,
+		hasAnonIds,
+		hasSessionIds,
+		hasLocation,
+		funnels,
+		userProps,
+		scdProps,
+		numDays,
+		percentUsersBornInDataset = 5,
+	} = config;
+
+	const { eventData, userProfilesData, scdTableData } = storage;
+	const avgEvPerUser = numEvents / numUsers;
+	const startTime = Date.now();
+
+	// Create batches for parallel processing
+	const batchSize = Math.max(1, Math.ceil(numUsers / concurrency));
+	const userPromises = [];
+
+	for (let i = 0; i < numUsers; i++) {
+		const userPromise = USER_CONN(async () => {
+			context.incrementUserCount();
+			const eps = Math.floor(context.getEventCount() / ((Date.now() - startTime) / 1000));
+
+			if (verbose) {
+				u.progress([
+					["users", context.getUserCount()],
+					["events", context.getEventCount()],
+					["eps", eps]
+				]);
+			}
+
+			const userId = chance.guid();
+			const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds });
+			const { distinct_id, created } = user;
+			const userIsBornInDataset = chance.bool({ likelihood: percentUsersBornInDataset });
+			let numEventsPreformed = 0;
+
+			if (!userIsBornInDataset) delete user.created;
+
+			// Calculate time adjustments
+			const daysShift = context.getDaysShift();
+			const adjustedCreated = userIsBornInDataset
+				? dayjs(created).subtract(daysShift, 'd')
+				: dayjs.unix(global.FIXED_BEGIN);
+
+			if (hasLocation) {
+				const location = u.pickRandom(u.choose(defaults.locationsUsers));
+				for (const key in location) {
+					user[key] = location[key];
+				}
+			}
+
+			// Profile creation
+			const profile = await makeUserProfile(context, userProps, user);
+
+			// Call user hook after profile creation
+			if (config.hook) {
+				await config.hook(profile, "user", { 
+					user, 
+					config,
+					userIsBornInDataset 
+				});
+			}
+
+			// SCD creation
+			// @ts-ignore
+			const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user' || !scd.type);
+			const scdTableKeys = Object.keys(scdUserTables);
+
+			const userSCD = {};
+			for (const [index, key] of scdTableKeys.entries()) {
+				const { max = 100 } = scdProps[key];
+				const mutations = chance.integer({ min: 1, max });
+				const changes = await makeSCD(context, scdProps[key], key, distinct_id, mutations, created);
+				userSCD[key] = changes;
+
+				await config.hook(changes, "scd-pre", {
+					profile,
+					type: 'user',
+					scd: { [key]: scdProps[key] },
+					config,
+					allSCDs: userSCD
+				});
+			}
+
+			let numEventsThisUserWillPreform = Math.floor(chance.normal({
+				mean: avgEvPerUser,
+				dev: avgEvPerUser / u.integer(u.integer(2, 5), u.integer(2, 7))
+			}) * 0.714159265359);
+
+			// Power users and low-activity users logic
+			chance.bool({ likelihood: 20 }) ? numEventsThisUserWillPreform *= 5 : null;
+			chance.bool({ likelihood: 15 }) ? numEventsThisUserWillPreform *= 0.333 : null;
+			numEventsThisUserWillPreform = Math.round(numEventsThisUserWillPreform);
+
+			let userFirstEventTime;
+
+			const firstFunnels = funnels.filter((f) => f.isFirstFunnel)
+				.filter((f) => !f.conditions || matchConditions(profile, f.conditions))
+				.reduce(weighFunnels, []);
+			const usageFunnels = funnels.filter((f) => !f.isFirstFunnel)
+				.filter((f) => !f.conditions || matchConditions(profile, f.conditions))
+				.reduce(weighFunnels, []);
+
+			const secondsInDay = 86400;
+			const noise = () => chance.integer({ min: 0, max: secondsInDay });
+			let usersEvents = [];
+
+			// PATH FOR USERS BORN IN DATASET AND PERFORMING FIRST FUNNEL
+			if (firstFunnels.length && userIsBornInDataset) {
+				const firstFunnel = chance.pickone(firstFunnels, user);
+				const firstTime = adjustedCreated.subtract(noise(), 'seconds').unix();
+				const [data, userConverted] = await makeFunnel(context, firstFunnel, user, firstTime, profile, userSCD);
+
+				const timeShift = context.getTimeShift();
+				userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
+				numEventsPreformed += data.length;
+				usersEvents = usersEvents.concat(data);
+
+				if (!userConverted) {
+					// if (verbose) {
+					//     u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
+					// }
+					return;
+				}
+			} else {
+				userFirstEventTime = adjustedCreated.subtract(noise(), 'seconds').unix();
+			}
+
+			// ALL SUBSEQUENT FUNNELS
+			while (numEventsPreformed < numEventsThisUserWillPreform) {
+				if (usageFunnels.length) {
+					const currentFunnel = chance.pickone(usageFunnels);
+					const [data, userConverted] = await makeFunnel(context, currentFunnel, user, userFirstEventTime, profile, userSCD);
+					numEventsPreformed += data.length;
+					usersEvents = usersEvents.concat(data);
+				} else {
+					const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(config.events), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true);
+					numEventsPreformed++;
+					usersEvents = usersEvents.concat(data);
+				}
+			}
+
+			// Hook for processing all user events
+			if (config.hook) {
+				const newEvents = await config.hook(usersEvents, "everything", {
+					profile,
+					scd: userSCD,
+					config,
+					userIsBornInDataset
+				});
+				if (Array.isArray(newEvents)) usersEvents = newEvents;
+			}
+
+			// Store all user data
+			await userProfilesData.hookPush(profile);
+
+			if (Object.keys(userSCD).length) {
+				for (const [key, changesArray] of Object.entries(userSCD)) {
+					for (const changes of changesArray) {
+						try {
+							const target = scdTableData.filter(arr => arr.scdKey === key).pop();
+							await target.hookPush(changes, { profile, type: 'user' });
+						}
+						catch (e) {
+							// This is probably a test
+							const target = scdTableData[0];
+							await target.hookPush(changes, { profile, type: 'user' });
+						}
+					}
+				}
+			}
+
+			await eventData.hookPush(usersEvents, { profile });
+
+			if (verbose) {
+				// u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
+			}
+		});
+
+		userPromises.push(userPromise);
+	}
+
+	// Wait for all users to complete
+	await Promise.all(userPromises);
+}
+
+
+export function weighFunnels(acc, funnel) {
+	const weight = funnel?.weight || 1;
+	for (let i = 0; i < weight; i++) {
+		acc.push(funnel);
+	}
+	return acc;
+}
+
+export function matchConditions(profile, conditions) {
+	for (const [key, value] of Object.entries(conditions)) {
+		if (profile[key] !== value) return false;
+	}
+	return true;
 }

package/lib/templates/abbreviated.d.ts

@@ -94,6 +94,9 @@ interface Funnel {
     
     /** Properties that will be attached to every event generated within this funnel journey. */
     props?: Record<string, ValueValid>;
+    
+    /** User property conditions that determine which users are eligible for this funnel. Only users whose properties match these conditions will be considered for this funnel. */
+    conditions?: Record<string, ValueValid>;
 }
 
 
@@ -153,6 +156,4 @@ interface LookupTableSchema {
     
     /** A dictionary of attributes (columns) for the table and their possible values. */
     attributes: Record<string, ValueValid>;
-}
-/**
- 
+}

package/lib/templates/instructions.txt

@@ -56,6 +56,7 @@ Core Requirements:
 		- Use groups if the prompt involves B2B, teams, companies, or organizations.
 		- Use SCD if user or group traits change over time (e.g., subscription tier).
 		- Use lookup tables if events reference external entities with their own attributes (e.g., product_id, video_id).
+		- Use funnel conditions when different user segments or cohorts should have different behavioral patterns (e.g., premium vs free users, students vs teachers, rider vs driver, doctor vs patient).
 
 	4. Available Functions: You have access to these built-in functions: date, weighNumRange, range, and the chance library.
 

package/lib/templates/{dungeon-template.js → scratch-dungeon-template.js}

@@ -1,9 +1,15 @@
+/**
+ * this file is NOT currently used
+ */
+
 import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
+dayjs.extend(utc);
 import "dotenv/config";
-import { weighNumRange, range, date, initChance, exhaust, choose, integer, decimal } from "../../lib/utils/utils.js";
+import { weighNumRange, range, date, initChance, exhaust, choose, integer, decimal, odds } from "../utils/utils.js";
+const { NODE_ENV = "unknown" } = process.env;
+
 
-dayjs.extend(utc);
 
 /**
  * Dungeon template for AI-generated configurations
@@ -29,7 +35,7 @@ export function createDungeonTemplate({
     const chance = u.initChance(seed);
     const numEvents = numUsers * 100;
 
-    /** @typedef  {import("../../types.d.ts").Dungeon} Config */
+    /** @typedef  {import("../../types.js").Dungeon} Config */
 
     /** @type {Config} */
     const config = {

package/lib/templates/verbose-schema.js

@@ -1,3 +1,9 @@
+/**
+ * @fileoverview this is a highly verbose schema for a dungeon that shows all the options available
+ * and how they might be implemented with extensive comments so an AI can understand it
+ * it is not meant to be used as a template, but rather as a reference for how to create a dungeon
+ * it is also used as a test for the AI to see if it can generate a dungeon with the same structure
+ */
 
 
 
@@ -6,8 +12,9 @@ const chance = new Chance();
 import dayjs from "dayjs";
 import utc from "dayjs/plugin/utc.js";
 dayjs.extend(utc);
-import { uid, comma } from "ak-tools";
-import { pickAWinner, weighNumRange, date, integer, weighChoices, range } from "../utils/utils.js";
+import "dotenv/config";
+import { weighNumRange, range, date, initChance, exhaust, choose, integer, decimal, odds } from "../../lib/utils/utils.js";
+const { NODE_ENV = "unknown" } = process.env;
 
 
 /** @type {import("../../types.js").Dungeon} */
@@ -119,8 +126,9 @@ const DUNGEON = {
 	 */
 	userProps: {
 		title: ["Mr.", "Ms.", "Mrs.", "Dr.", "Prof.", "Sir", "Madam", "Lord", "Lady", "Dame", "Baron", "Baroness", "Count", "Countess", "Viscount", "Viscountess", "Marquis", "Marchioness"],
+		role: ["basic", "basic", "basic", "premium", "admin"], // role property that can be used in funnel conditions
 		luckyNumber: weighNumRange(42, 420, .3),
-		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceros beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
+		spiritAnimal: ["duck", "dog", "otter", "penguin", "cat", "elephant", "lion", "cheetah", "giraffe", "zebra", "rhino", "hippo", "whale", "dolphin", "shark", "octopus", "squid", "jellyfish", "starfish", "seahorse", "crab", "lobster", "shrimp", "clam", "snail", "slug", "butterfly", "moth", "bee", "wasp", "ant", "beetle", "ladybug", "caterpillar", "centipede", "millipede", "scorpion", "spider", "tarantula", "tick", "mite", "mosquito", "fly", "dragonfly", "damselfly", "grasshopper", "cricket", "locust", "mantis", "cockroach", "termite", "praying mantis", "walking stick", "stick bug", "leaf insect", "lacewing", "aphid", "cicada", "thrips", "psyllid", "scale insect", "whitefly", "mealybug", "planthopper", "leafhopper", "treehopper", "flea", "louse", "bedbug", "flea beetle", "weevil", "longhorn beetle", "leaf beetle", "tiger beetle", "ground beetle", "lady beetle", "firefly", "click beetle", "rove beetle", "scarab beetle", "dung beetle", "stag beetle", "rhinoceus beetle", "hercules beetle", "goliath beetle", "jewel beetle", "tortoise beetle"]
 	},
 	/**
 	 * Funnels represent intentional user journeys (e.g., sign-up, checkout), 
@@ -130,10 +138,14 @@ const DUNGEON = {
 	 * unless otherwise specified in the prompt.
 	 * Funnels are the primary mechanism used to generate the example data, and it's critical sequences match events in the events array.
 	 * there are many different options for the funnels like: 
-	 * 	isFirstFunnel, conversionRate, isChurnFunnel, order, props, requireRepeats, timeToConvert, weight
+	 * 	isFirstFunnel, conversionRate, isChurnFunnel, order, props, requireRepeats, timeToConvert, weight, conditions
 	 * 
 	 * isFirstFunnel are funnels a user will only go through once (like a sign up)
 	 * non isFirstFunnel are funnels a user will go through multiple times (like a purchase)
+	 * 
+	 * conditions are used to filter which users are eligible for a specific funnel based on their user properties
+	 * this is useful when different user segments should have different behavioral patterns
+	 * for example: premium users might have access to advanced features, students vs teachers have different workflows
 	 *
 	 */
 	funnels: [
@@ -170,6 +182,21 @@ const DUNGEON = {
 			props: {
 				"browsing type": ["casual", "intentional", "exploratory"] // you can add properties to the funnel
 			}
+		},
+		{
+			name: "premium user workflow",
+			description: "advanced features only available to premium users",
+			sequence: ["page view", "view item", "save item", "checkout"],
+			isFirstFunnel: false,
+			conversionRate: 85,
+			timeToConvert: 3,
+			conditions: {
+				role: "premium" // only users with role "premium" are eligible for this funnel
+			},
+			order: "sequential",
+			props: {
+				"feature_tier": "premium"
+			}
 		}
 	],
 	/**