make-mp-data 2.0.19 → 2.0.22

package/lib/orchestrators/user-loop.js

@@ -21,185 +21,216 @@ import { makeSCD } from "../generators/scd.js";
  * @returns {Promise<void>}
  */
 export async function userLoop(context) {
-    const { config, storage, defaults } = context;
-    const chance = u.getChance();
-    const concurrency = config?.concurrency || Math.min(os.cpus().length * 2, 16);
-    const USER_CONN = pLimit(concurrency);
-    
-    const {
-        verbose,
-        numUsers,
-        numEvents,
-        isAnonymous,
-        hasAvatar,
-        hasAnonIds,
-        hasSessionIds,
-        hasLocation,
-        funnels,
-        userProps,
-        scdProps,
-        numDays,
-        percentUsersBornInDataset = 5,
-    } = config;
-    
-    const { eventData, userProfilesData, scdTableData } = storage;
-    const avgEvPerUser = numEvents / numUsers;
-    const startTime = Date.now();
-
-    // Create batches for parallel processing
-    const batchSize = Math.max(1, Math.ceil(numUsers / concurrency));
-    const userPromises = [];
-    
-    for (let i = 0; i < numUsers; i++) {
-        const userPromise = USER_CONN(async () => {
-            context.incrementUserCount();
-            const eps = Math.floor(context.getEventCount() / ((Date.now() - startTime) / 1000));
-            
-            if (verbose) {
-                u.progress([
-                    ["users", context.getUserCount()], 
-                    ["events", context.getEventCount()], 
-                    ["eps", eps]
-                ]);
-            }
-            
-            const userId = chance.guid();
-            const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds });
-            const { distinct_id, created } = user;
-            const userIsBornInDataset = chance.bool({ likelihood: percentUsersBornInDataset });
-            let numEventsPreformed = 0;
-            
-            if (!userIsBornInDataset) delete user.created;
-            
-            // Calculate time adjustments
-            const daysShift = context.getDaysShift();
-            const adjustedCreated = userIsBornInDataset 
-                ? dayjs(created).subtract(daysShift, 'd') 
-                : dayjs.unix(global.FIXED_BEGIN);
-
-            if (hasLocation) {
-                const location = u.pickRandom(u.choose(defaults.locationsUsers));
-                for (const key in location) {
-                    user[key] = location[key];
-                }
-            }
-
-            // Profile creation
-            const profile = await makeUserProfile(context, userProps, user);
-
-            // SCD creation
-            // @ts-ignore
-            const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user' || !scd.type);
-            const scdTableKeys = Object.keys(scdUserTables);
-
-            const userSCD = {};
-            for (const [index, key] of scdTableKeys.entries()) {
-                const { max = 100 } = scdProps[key];
-                const mutations = chance.integer({ min: 1, max });
-                const changes = await makeSCD(context, scdProps[key], key, distinct_id, mutations, created);
-                userSCD[key] = changes;
-                
-                await config.hook(changes, "scd-pre", { 
-                    profile, 
-                    type: 'user', 
-                    scd: { [key]: scdProps[key] }, 
-                    config, 
-                    allSCDs: userSCD 
-                });
-            }
-
-            let numEventsThisUserWillPreform = Math.floor(chance.normal({
-                mean: avgEvPerUser,
-                dev: avgEvPerUser / u.integer(u.integer(2, 5), u.integer(2, 7))
-            }) * 0.714159265359);
-
-            // Power users and low-activity users logic
-            chance.bool({ likelihood: 20 }) ? numEventsThisUserWillPreform *= 5 : null;
-            chance.bool({ likelihood: 15 }) ? numEventsThisUserWillPreform *= 0.333 : null;
-            numEventsThisUserWillPreform = Math.round(numEventsThisUserWillPreform);
-
-            let userFirstEventTime;
-
-            const firstFunnels = funnels.filter((f) => f.isFirstFunnel).reduce(u.weighFunnels, []);
-            const usageFunnels = funnels.filter((f) => !f.isFirstFunnel).reduce(u.weighFunnels, []);
-
-            const secondsInDay = 86400;
-            const noise = () => chance.integer({ min: 0, max: secondsInDay });
-            let usersEvents = [];
-
-            if (firstFunnels.length && userIsBornInDataset) {
-                const firstFunnel = chance.pickone(firstFunnels, user);
-                const firstTime = adjustedCreated.subtract(noise(), 'seconds').unix();
-                const [data, userConverted] = await makeFunnel(context, firstFunnel, user, firstTime, profile, userSCD);
-                
-                const timeShift = context.getTimeShift();
-                userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
-                numEventsPreformed += data.length;
-                usersEvents = usersEvents.concat(data);
-                
-                if (!userConverted) {
-                    // if (verbose) {
-                    //     u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
-                    // }
-                    return;
-                }
-            } else {
-                userFirstEventTime = adjustedCreated.subtract(noise(), 'seconds').unix();
-            }
-
-            while (numEventsPreformed < numEventsThisUserWillPreform) {
-                if (usageFunnels.length) {
-                    const currentFunnel = chance.pickone(usageFunnels);
-                    const [data, userConverted] = await makeFunnel(context, currentFunnel, user, userFirstEventTime, profile, userSCD);
-                    numEventsPreformed += data.length;
-                    usersEvents = usersEvents.concat(data);
-                } else {
-                    const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(config.events), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true);
-                    numEventsPreformed++;
-                    usersEvents = usersEvents.concat(data);
-                }
-            }
-
-            // Hook for processing all user events
-            if (config.hook) {
-                const newEvents = await config.hook(usersEvents, "everything", { 
-                    profile, 
-                    scd: userSCD, 
-                    config, 
-                    userIsBornInDataset 
-                });
-                if (Array.isArray(newEvents)) usersEvents = newEvents;
-            }
-
-            // Store all user data
-            await userProfilesData.hookPush(profile);
-
-            if (Object.keys(userSCD).length) {
-                for (const [key, changesArray] of Object.entries(userSCD)) {
-                    for (const changes of changesArray) {
-                        try {
-                            const target = scdTableData.filter(arr => arr.scdKey === key).pop();
-                            await target.hookPush(changes, { profile, type: 'user' });
-                        }
-                        catch (e) {
-                            // This is probably a test
-                            const target = scdTableData[0];
-                            await target.hookPush(changes, { profile, type: 'user' });
-                        }
-                    }
-                }
-            }
-            
-            await eventData.hookPush(usersEvents, { profile });
-
-            if (verbose) {
-                // u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
-            }
-        });
-        
-        userPromises.push(userPromise);
-    }
-    
-    // Wait for all users to complete
-    await Promise.all(userPromises);
+	const { config, storage, defaults } = context;
+	const chance = u.getChance();
+	const concurrency = config?.concurrency || Math.min(os.cpus().length * 2, 16);
+	const USER_CONN = pLimit(concurrency);
+
+	const {
+		verbose,
+		numUsers,
+		numEvents,
+		isAnonymous,
+		hasAvatar,
+		hasAnonIds,
+		hasSessionIds,
+		hasLocation,
+		funnels,
+		userProps,
+		scdProps,
+		numDays,
+		percentUsersBornInDataset = 5,
+	} = config;
+
+	const { eventData, userProfilesData, scdTableData } = storage;
+	const avgEvPerUser = numEvents / numUsers;
+	const startTime = Date.now();
+
+	// Create batches for parallel processing
+	const batchSize = Math.max(1, Math.ceil(numUsers / concurrency));
+	const userPromises = [];
+
+	for (let i = 0; i < numUsers; i++) {
+		const userPromise = USER_CONN(async () => {
+			context.incrementUserCount();
+			const eps = Math.floor(context.getEventCount() / ((Date.now() - startTime) / 1000));
+
+			if (verbose) {
+				u.progress([
+					["users", context.getUserCount()],
+					["events", context.getEventCount()],
+					["eps", eps]
+				]);
+			}
+
+			const userId = chance.guid();
+			const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds });
+			const { distinct_id, created } = user;
+			const userIsBornInDataset = chance.bool({ likelihood: percentUsersBornInDataset });
+			let numEventsPreformed = 0;
+
+			if (!userIsBornInDataset) delete user.created;
+
+			// Calculate time adjustments
+			const daysShift = context.getDaysShift();
+			const adjustedCreated = userIsBornInDataset
+				? dayjs(created).subtract(daysShift, 'd')
+				: dayjs.unix(global.FIXED_BEGIN);
+
+			if (hasLocation) {
+				const location = u.pickRandom(u.choose(defaults.locationsUsers));
+				for (const key in location) {
+					user[key] = location[key];
+				}
+			}
+
+			// Profile creation
+			const profile = await makeUserProfile(context, userProps, user);
+
+			// Call user hook after profile creation
+			if (config.hook) {
+				await config.hook(profile, "user", { 
+					user, 
+					config,
+					userIsBornInDataset 
+				});
+			}
+
+			// SCD creation
+			// @ts-ignore
+			const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user' || !scd.type);
+			const scdTableKeys = Object.keys(scdUserTables);
+
+			const userSCD = {};
+			for (const [index, key] of scdTableKeys.entries()) {
+				const { max = 100 } = scdProps[key];
+				const mutations = chance.integer({ min: 1, max });
+				const changes = await makeSCD(context, scdProps[key], key, distinct_id, mutations, created);
+				userSCD[key] = changes;
+
+				await config.hook(changes, "scd-pre", {
+					profile,
+					type: 'user',
+					scd: { [key]: scdProps[key] },
+					config,
+					allSCDs: userSCD
+				});
+			}
+
+			let numEventsThisUserWillPreform = Math.floor(chance.normal({
+				mean: avgEvPerUser,
+				dev: avgEvPerUser / u.integer(u.integer(2, 5), u.integer(2, 7))
+			}) * 0.714159265359);
+
+			// Power users and low-activity users logic
+			chance.bool({ likelihood: 20 }) ? numEventsThisUserWillPreform *= 5 : null;
+			chance.bool({ likelihood: 15 }) ? numEventsThisUserWillPreform *= 0.333 : null;
+			numEventsThisUserWillPreform = Math.round(numEventsThisUserWillPreform);
+
+			let userFirstEventTime;
+
+			const firstFunnels = funnels.filter((f) => f.isFirstFunnel)
+				.filter((f) => !f.conditions || matchConditions(profile, f.conditions))
+				.reduce(weighFunnels, []);
+			const usageFunnels = funnels.filter((f) => !f.isFirstFunnel)
+				.filter((f) => !f.conditions || matchConditions(profile, f.conditions))
+				.reduce(weighFunnels, []);
+
+			const secondsInDay = 86400;
+			const noise = () => chance.integer({ min: 0, max: secondsInDay });
+			let usersEvents = [];
+
+			// PATH FOR USERS BORN IN DATASET AND PERFORMING FIRST FUNNEL
+			if (firstFunnels.length && userIsBornInDataset) {
+				const firstFunnel = chance.pickone(firstFunnels, user);
+				const firstTime = adjustedCreated.subtract(noise(), 'seconds').unix();
+				const [data, userConverted] = await makeFunnel(context, firstFunnel, user, firstTime, profile, userSCD);
+
+				const timeShift = context.getTimeShift();
+				userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
+				numEventsPreformed += data.length;
+				usersEvents = usersEvents.concat(data);
+
+				if (!userConverted) {
+					// if (verbose) {
+					//     u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
+					// }
+					return;
+				}
+			} else {
+				userFirstEventTime = adjustedCreated.subtract(noise(), 'seconds').unix();
+			}
+
+			// ALL SUBSEQUENT FUNNELS
+			while (numEventsPreformed < numEventsThisUserWillPreform) {
+				if (usageFunnels.length) {
+					const currentFunnel = chance.pickone(usageFunnels);
+					const [data, userConverted] = await makeFunnel(context, currentFunnel, user, userFirstEventTime, profile, userSCD);
+					numEventsPreformed += data.length;
+					usersEvents = usersEvents.concat(data);
+				} else {
+					const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(config.events), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true);
+					numEventsPreformed++;
+					usersEvents = usersEvents.concat(data);
+				}
+			}
+
+			// Hook for processing all user events
+			if (config.hook) {
+				const newEvents = await config.hook(usersEvents, "everything", {
+					profile,
+					scd: userSCD,
+					config,
+					userIsBornInDataset
+				});
+				if (Array.isArray(newEvents)) usersEvents = newEvents;
+			}
+
+			// Store all user data
+			await userProfilesData.hookPush(profile);
+
+			if (Object.keys(userSCD).length) {
+				for (const [key, changesArray] of Object.entries(userSCD)) {
+					for (const changes of changesArray) {
+						try {
+							const target = scdTableData.filter(arr => arr.scdKey === key).pop();
+							await target.hookPush(changes, { profile, type: 'user' });
+						}
+						catch (e) {
+							// This is probably a test
+							const target = scdTableData[0];
+							await target.hookPush(changes, { profile, type: 'user' });
+						}
+					}
+				}
+			}
+
+			await eventData.hookPush(usersEvents, { profile });
+
+			if (verbose) {
+				// u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
+			}
+		});
+
+		userPromises.push(userPromise);
+	}
+
+	// Wait for all users to complete
+	await Promise.all(userPromises);
+}
+
+
+export function weighFunnels(acc, funnel) {
+	const weight = funnel?.weight || 1;
+	for (let i = 0; i < weight; i++) {
+		acc.push(funnel);
+	}
+	return acc;
+}
+
+export function matchConditions(profile, conditions) {
+	for (const [key, value] of Object.entries(conditions)) {
+		if (profile[key] !== value) return false;
+	}
+	return true;
 }

package/lib/orchestrators/worker-manager.js

@@ -3,6 +3,9 @@
  * Handles distributed processing across multiple cloud function workers
  */
 
+/** @typedef {import('../../types.js').Context} Context */
+/** @typedef {import('../../types.js').Dungeon} Dungeon */
+
 import pLimit from 'p-limit';
 import { GoogleAuth } from 'google-auth-library';
 import { timer, uid, sLog } from 'ak-tools';
@@ -138,7 +141,7 @@ export async function handleCloudFunctionEntry(req, res, mainFunction) {
     try {
         if (!script) throw new Error("no script");
 
-        /** @type {Config} */
+        /** @type {Dungeon} */
         const config = eval(script);
         
         if (isReplica) {
@@ -148,7 +151,7 @@ export async function handleCloudFunctionEntry(req, res, mainFunction) {
             params.seed = newSeed;
         }
 
-        /** @type {Config} */
+        /** @type {Dungeon} */
         const optionsYouCantChange = {
             verbose: false
         };

package/lib/templates/abbreviated.d.ts

@@ -0,0 +1,159 @@
+/**
+ * A "validValue" can be a primitive, an array of primitives, 
+ * or a function that returns a primitive or an array of primitives.
+ * This is the building block for all property values in the dungeon.
+ */
+type Primitives = string | number | boolean | Date;
+type ValueValid = Primitives | Primitives[] | (() => Primitives | Primitives[]);
+
+
+/**
+ * The main configuration object for the entire data generation spec, known as a "Dungeon".
+ * This is the high-level object you will be constructing.
+ */
+export interface Dungeon {
+    /** A list of all possible events that can occur in the simulation. */
+    events?: EventConfig[];
+    
+    /** A list of event sequences that represent user journeys (e.g., sign-up, purchase). */
+    funnels?: Funnel[];
+    
+    /** Properties that are attached to every event for all users. */
+    superProps?: Record<string, ValueValid>;
+    
+    /** Properties that define the characteristics of individual users. */
+    userProps?: Record<string, ValueValid>;
+    
+    /** Properties that change for users or groups over time (Slowly Changing Dimensions). */
+    scdProps?: Record<string, SCDProp>;
+    
+    /** Defines group entities, like companies or teams, and how many of each to create. */
+    groupKeys?: [string, number, string[]?][]; // [key, numGroups, optional_events_for_group]
+    
+    /** Properties that define the characteristics of the groups defined in groupKeys. */
+    groupProps?: Record<string, Record<string, ValueValid>>;
+
+    /** Events that are attributed to a group entity rather than an individual user. */
+    groupEvents?: GroupEventConfig[];
+    
+    /** Static data tables (e.g., product catalogs) that can be referenced in events. */
+    lookupTables?: LookupTableSchema[];
+
+}
+
+
+/**
+ * Defines a single event, its properties, and its likelihood of occurring.
+ */
+interface EventConfig {
+    /** The name of the event (e.g., "Page View", "Add to Cart"). */
+    event?: string;
+    
+    /** The relative frequency of this event compared to others. Higher numbers are more frequent. */
+    weight?: number;
+    
+    /** A dictionary of properties associated with this event. */
+    properties?: Record<string, ValueValid>;
+    
+    /** If true, this event will be the first event for a new user. */
+    isFirstEvent?: boolean;
+    
+    /** If true, this event signifies that a user has churned. */
+    isChurnEvent?: boolean;
+}
+
+
+/**
+ * Defines a sequence of events that represents a meaningful user journey or workflow.
+ */
+interface Funnel {
+    /** The name of the funnel (e.g., "Purchase Funnel"). */
+    name?: string;
+
+    /** A list of event names that make up the sequence of this funnel. */
+    sequence: string[];
+    
+    /** The likelihood that a user will attempt this funnel. */
+    weight?: number;
+    
+    /** If true, this funnel is part of the initial user experience (e.g., onboarding). */
+    isFirstFunnel?: boolean;
+    
+    /** The probability (0-100) that a user who starts the funnel will complete it. */
+    conversionRate?: number;
+    
+    /** The average time (in hours) it takes a user to complete the funnel. */
+    timeToConvert?: number;
+    
+    /** * Defines the ordering of events within the funnel for a user.
+     * "sequential": Events must happen in the exact order defined in `sequence`.
+     * "random": Events can happen in any order.
+     * "first-and-last-fixed": The first and last events are fixed, but the middle ones are random.
+     */
+    order?: "sequential" | "random" | "first-and-last-fixed" | "first-fixed" | "last-fixed" | "interrupted";
+    
+    /** Properties that will be attached to every event generated within this funnel journey. */
+    props?: Record<string, ValueValid>;
+    
+    /** User property conditions that determine which users are eligible for this funnel. Only users whose properties match these conditions will be considered for this funnel. */
+    conditions?: Record<string, ValueValid>;
+}
+
+
+/**
+ * Defines a "Slowly Changing Dimension" — a property of a user or group
+ * that changes periodically over time (e.g., subscription plan, user role).
+ */
+interface SCDProp {
+    /** The entity this property belongs to ('user' or a group key like 'company_id'). */
+    type?: string;
+    
+    /** How often the value of this property can change. */
+    frequency: "day" | "week" | "month" | "year";
+    
+    /** A list or function defining the possible values for this property. */
+    values: ValueValid;
+    
+    /** * 'fixed': Changes occur exactly on the frequency interval.
+     * 'fuzzy': Changes occur randomly around the frequency interval.
+     */
+    timing: "fixed" | "fuzzy";
+
+    /** The maximum number of times this property can change for a single entity. */
+    max?: number;
+}
+
+
+/**
+ * Defines an event that is attributed to a group and occurs on a regular schedule.
+ * (e.g., a monthly subscription charge for a company).
+ */
+interface GroupEventConfig extends EventConfig {
+    /** How often the event occurs (in days). */
+    frequency: number; 
+    
+    /** The group key this event is associated with (e.g., "company_id"). */
+    group_key: string; 
+    
+    /** If true, a random user within the group is also associated with the event. */
+    attribute_to_user: boolean; 
+    
+    /** The number of entities in this group. */
+    group_size: number; 
+}
+
+
+/**
+ * Defines the schema for a static lookup table, which can be used to enrich event data.
+ * For example, a "products" table could hold details about product IDs.
+ */
+interface LookupTableSchema {
+    /** The name of the key that will be used to join this table in an event (e.g., "product_id"). */
+    key: string;
+    
+    /** The number of unique rows to generate for this table. */
+    entries: number;
+    
+    /** A dictionary of attributes (columns) for the table and their possible values. */
+    attributes: Record<string, ValueValid>;
+}

package/lib/{data → templates}/defaults.js

@@ -4,8 +4,8 @@
  * @fileoverview Contains default values for campaigns, devices, locations, and domains
  */
 
-/** @typedef {import('../../types.d.ts').main.Dungeon} Config */
-/** @typedef {import('../../types.d.ts').main.ValueValid} ValueValid */
+/** @typedef {import('../../types.js').Dungeon} Config */
+/** @typedef {import('../../types.js').ValueValid} ValueValid */
 
 //? https://docs.mixpanel.com/docs/data-structure/property-reference#default-properties