make-mp-data 1.5.56 → 2.0.1

package/lib/generators/events.js

@@ -0,0 +1,242 @@
+/**
+ * Event generator module
+ * Creates individual Mixpanel events with realistic properties and timing
+ */
+
+/** @typedef {import('../../types').Dungeon} Config */
+/** @typedef {import('../../types').EventConfig} EventConfig */
+/** @typedef {import('../../types').ValueValid} ValueValid */
+/** @typedef {import('../../types').EventSchema} EventSchema */
+/** @typedef {import('../../types').Context} Context */
+
+import dayjs from "dayjs";
+import { md5 } from "ak-tools";
+import * as u from "../utils/utils.js";
+
+/**
+ * Creates a Mixpanel event with a flat shape
+ * @param {Context} context - Context object containing config, defaults, etc.
+ * @param {string} distinct_id - User identifier
+ * @param {number} earliestTime - Unix timestamp for earliest possible event time
+ * @param {Object} chosenEvent - Event configuration object
+ * @param {string[]} [anonymousIds] - Array of anonymous/device IDs
+ * @param {string[]} [sessionIds] - Array of session IDs
+ * @param {Object} [superProps] - Super properties to add to event
+ * @param {Array} [groupKeys] - Group key configurations
+ * @param {boolean} [isFirstEvent] - Whether this is the user's first event
+ * @param {boolean} [skipDefaults] - Whether to skip adding default properties
+ * @returns {Promise<Object>} Generated event object
+ */
+export async function makeEvent(
+    context, 
+    distinct_id, 
+    earliestTime, 
+    chosenEvent, 
+    anonymousIds = [], 
+    sessionIds = [], 
+    superProps = {}, 
+    groupKeys = [], 
+    isFirstEvent = false, 
+    skipDefaults = false
+) {
+    // Validate required parameters
+    if (!distinct_id) throw new Error("no distinct_id");
+    if (!earliestTime) throw new Error("no earliestTime");
+    if (!chosenEvent) throw new Error("no chosenEvent");
+
+    // Update context metrics
+    context.incrementOperations();
+    context.incrementEvents();
+
+    const { config, defaults } = context;
+    const chance = u.getChance();
+    
+    // Extract soup configuration for time distribution
+    const { mean = 0, deviation = 2, peaks = 5 } = config.soup || {};
+    
+    // Extract feature flags from config
+    const {
+        hasAndroidDevices = false,
+        hasBrowser = false,
+        hasCampaigns = false,
+        hasDesktopDevices = false,
+        hasIOSDevices = false,
+        hasLocation = false
+    } = config;
+
+    // Create base event template
+    const eventTemplate = {
+        event: chosenEvent.event,
+        source: "dm4",
+        time: "",
+        insert_id: "",
+    };
+
+    let defaultProps = {};
+    let devicePool = [];
+
+    // Add default properties based on configuration
+    if (hasLocation) {
+        defaultProps.location = u.shuffleArray(defaults.locationsEvents()).pop();
+    }
+    
+    if (hasBrowser) {
+        defaultProps.browser = u.choose(defaults.browsers());
+    }
+    
+    // Build device pool based on enabled device types
+    if (hasAndroidDevices) devicePool.push(defaults.androidDevices());
+    if (hasIOSDevices) devicePool.push(defaults.iOSDevices());
+    if (hasDesktopDevices) devicePool.push(defaults.desktopDevices());
+
+    // Add campaigns with attribution likelihood
+    if (hasCampaigns && chance.bool({ likelihood: 25 })) {
+        defaultProps.campaigns = u.shuffleArray(defaults.campaigns()).pop();
+    }
+    
+    // Select device from pool
+    const devices = devicePool.flat();
+    if (devices.length) {
+        defaultProps.device = u.shuffleArray(devices).pop();
+    }
+
+    // Set event time using TimeSoup for realistic distribution
+    if (earliestTime) {
+        if (isFirstEvent) {
+            eventTemplate.time = dayjs.unix(earliestTime).toISOString();
+        } else {
+            eventTemplate.time = u.TimeSoup(earliestTime, context.FIXED_NOW, peaks, deviation, mean);
+        }
+    }
+
+    // Add anonymous and session identifiers
+    if (anonymousIds.length) {
+        eventTemplate.device_id = chance.pickone(anonymousIds);
+    }
+    
+    if (sessionIds.length) {
+        eventTemplate.session_id = chance.pickone(sessionIds);
+    }
+
+    // Sometimes add user_id (for attribution modeling)
+    if (!isFirstEvent && chance.bool({ likelihood: 42 })) {
+        eventTemplate.user_id = distinct_id;
+    }
+
+    // Ensure we have either user_id or device_id
+    if (!eventTemplate.user_id && !eventTemplate.device_id) {
+        eventTemplate.user_id = distinct_id;
+    }
+
+    // Merge custom properties with super properties
+    const props = { ...chosenEvent.properties, ...superProps };
+
+    // Add custom properties from event configuration
+    for (const key in props) {
+        try {
+            eventTemplate[key] = u.choose(props[key]);
+        } catch (e) {
+            console.error(`error with ${key} in ${chosenEvent.event} event`, e);
+            // Continue processing other properties
+        }
+    }
+
+    // Add default properties if not skipped
+    if (!skipDefaults) {
+        addDefaultProperties(eventTemplate, defaultProps);
+    }
+
+    // Add group properties
+    addGroupProperties(eventTemplate, groupKeys);
+
+    // Generate unique insert_id
+    eventTemplate.insert_id = md5(JSON.stringify(eventTemplate));
+
+    // Apply time shift to move events to current timeline
+    if (earliestTime) {
+        const timeShift = dayjs().add(2, "day").diff(dayjs.unix(context.FIXED_NOW), "seconds");
+        const timeShifted = dayjs(eventTemplate.time).add(timeShift, "seconds").toISOString();
+        eventTemplate.time = timeShifted;
+    }
+
+    return eventTemplate;
+}
+
+/**
+ * Adds default properties to an event template
+ * Handles complex nested property structures
+ * @param {Object} eventTemplate - Event object to modify
+ * @param {Object} defaultProps - Default properties to add
+ */
+function addDefaultProperties(eventTemplate, defaultProps) {
+    for (const key in defaultProps) {
+        if (Array.isArray(defaultProps[key])) {
+            const choice = u.choose(defaultProps[key]);
+            
+            if (typeof choice === "string") {
+                if (!eventTemplate[key]) eventTemplate[key] = choice;
+            }
+            else if (Array.isArray(choice)) {
+                for (const subChoice of choice) {
+                    if (!eventTemplate[key]) eventTemplate[key] = subChoice;
+                }
+            }
+            else if (typeof choice === "object") {
+                addNestedObjectProperties(eventTemplate, choice);
+            }
+        }
+        else if (typeof defaultProps[key] === "object") {
+            addNestedObjectProperties(eventTemplate, defaultProps[key]);
+        }
+        else {
+            if (!eventTemplate[key]) eventTemplate[key] = defaultProps[key];
+        }
+    }
+}
+
+/**
+ * Adds nested object properties to event template
+ * @param {Object} eventTemplate - Event object to modify
+ * @param {Object} obj - Object with properties to add
+ */
+function addNestedObjectProperties(eventTemplate, obj) {
+    for (const subKey in obj) {
+        if (typeof obj[subKey] === "string") {
+            if (!eventTemplate[subKey]) eventTemplate[subKey] = obj[subKey];
+        }
+        else if (Array.isArray(obj[subKey])) {
+            const subChoice = u.choose(obj[subKey]);
+            if (!eventTemplate[subKey]) eventTemplate[subKey] = subChoice;
+        }
+        else if (typeof obj[subKey] === "object") {
+            for (const subSubKey in obj[subKey]) {
+                if (!eventTemplate[subSubKey]) {
+                    eventTemplate[subSubKey] = obj[subKey][subSubKey];
+                }
+            }
+        }
+    }
+}
+
+/**
+ * Adds group properties to event based on group key configuration
+ * @param {Object} eventTemplate - Event object to modify
+ * @param {Array} groupKeys - Array of group key configurations
+ */
+function addGroupProperties(eventTemplate, groupKeys) {
+    for (const groupPair of groupKeys) {
+        const groupKey = groupPair[0];
+        const groupCardinality = groupPair[1];
+        const groupEvents = groupPair[2] || [];
+
+        // Empty array for group events means all events get the group property
+        if (!groupEvents.length) {
+            eventTemplate[groupKey] = u.pick(u.weighNumRange(1, groupCardinality));
+        }
+        
+        // Only add group property if event is in the specified group events
+        if (groupEvents.includes(eventTemplate.event)) {
+            eventTemplate[groupKey] = u.pick(u.weighNumRange(1, groupCardinality));
+        }
+    }
+}

package/lib/generators/funnels.js

@@ -0,0 +1,330 @@
+/**
+ * Funnel generator module
+ * Creates conversion sequences with realistic timing and ordering
+ */
+
+/** @typedef {import('../../types').Context} Context */
+
+import dayjs from "dayjs";
+import { clone } from "ak-tools";
+import * as u from "../utils/utils.js";
+import { makeEvent } from "./events.js";
+
+/**
+ * Creates a funnel (sequence of events) for a user with conversion logic
+ * @param {Context} context - Context object containing config, defaults, etc.
+ * @param {Object} funnel - Funnel configuration
+ * @param {Object} user - User object with distinct_id, created, etc.
+ * @param {number} firstEventTime - Unix timestamp for first event
+ * @param {Object} profile - User profile object
+ * @param {Object} scd - Slowly changing dimensions object
+ * @returns {Promise<[Array, boolean]>} Tuple of [events, didConvert]
+ */
+export async function makeFunnel(context, funnel, user, firstEventTime, profile = {}, scd = {}) {
+    if (!funnel) throw new Error("no funnel");
+    if (!user) throw new Error("no user");
+
+    const { config } = context;
+    const chance = u.getChance();
+    const { hook = async (a) => a } = config;
+    
+    // Get session start events if configured
+    const sessionStartEvents = config.events?.filter(a => a.isSessionStartEvent) || [];
+
+    // Call pre-funnel hook
+    await hook(funnel, "funnel-pre", { user, profile, scd, funnel, config });
+    
+    // Extract funnel configuration
+    let {
+        sequence,
+        conversionRate = 50,
+        order = 'sequential',
+        timeToConvert = 1,
+        props = {},
+        requireRepeats = false,
+    } = funnel;
+    
+    const { distinct_id, created, anonymousIds = [], sessionIds = [] } = user;
+    const { superProps = {}, groupKeys = [] } = config;
+
+    // Choose properties for this funnel instance
+    const chosenFunnelProps = { ...props, ...superProps };
+    for (const key in props) {
+        try {
+            chosenFunnelProps[key] = u.choose(chosenFunnelProps[key]);
+        } catch (e) {
+            console.error(`error with ${key} in ${funnel.sequence.join(" > ")} funnel`, e);
+        }
+    }
+
+    // Build event specifications for funnel steps
+    const funnelPossibleEvents = buildFunnelEvents(context, sequence, chosenFunnelProps);
+    
+    // Handle repeat logic and conversion rate adjustment
+    const { processedEvents, adjustedConversionRate } = processEventRepeats(
+        funnelPossibleEvents, 
+        requireRepeats, 
+        conversionRate, 
+        chance
+    );
+
+    // Determine if user converts and how many steps they'll take
+    const { doesUserConvert, numStepsUserWillTake } = determineConversion(
+        adjustedConversionRate, 
+        sequence.length, 
+        chance
+    );
+
+    // Get steps user will actually take
+    const funnelStepsUserWillTake = processedEvents.slice(0, numStepsUserWillTake);
+
+    // Apply ordering strategy
+    const funnelActualOrder = applyOrderingStrategy(
+        funnelStepsUserWillTake, 
+        order, 
+        config, 
+        sequence
+    );
+
+    // Add timing offsets to events
+    const funnelEventsWithTiming = addTimingOffsets(
+        funnelActualOrder, 
+        timeToConvert, 
+        numStepsUserWillTake
+    );
+
+    // Add session start event if configured
+    if (sessionStartEvents.length) {
+        const sessionStartEvent = chance.pickone(sessionStartEvents);
+        sessionStartEvent.relativeTimeMs = -15000; // 15 seconds before funnel
+        funnelEventsWithTiming.push(sessionStartEvent);
+    }
+
+    // Generate actual events with timing
+    const finalEvents = await generateFunnelEvents(
+        context,
+        funnelEventsWithTiming,
+        distinct_id,
+        firstEventTime || dayjs(created).unix(),
+        anonymousIds,
+        sessionIds,
+        groupKeys
+    );
+
+    // Call post-funnel hook
+    await hook(finalEvents, "funnel-post", { user, profile, scd, funnel, config });
+
+    return [finalEvents, doesUserConvert];
+}
+
+/**
+ * Builds event specifications for funnel steps
+ * @param {Context} context - Context object
+ * @param {Array} sequence - Array of event names
+ * @param {Object} chosenFunnelProps - Properties to apply to all events
+ * @returns {Array} Array of event specifications
+ */
+function buildFunnelEvents(context, sequence, chosenFunnelProps) {
+    const { config } = context;
+    
+    return sequence.map((eventName) => {
+        const foundEvent = config.events?.find((e) => e.event === eventName);
+        const eventSpec = clone(foundEvent) || { event: eventName, properties: {} };
+        
+        // Process event properties
+        for (const key in eventSpec.properties) {
+            try {
+                eventSpec.properties[key] = u.choose(eventSpec.properties[key]);
+            } catch (e) {
+                console.error(`error with ${key} in ${eventSpec.event} event`, e);
+            }
+        }
+        
+        // Clean up funnel-specific properties
+        delete eventSpec.isFirstEvent;
+        delete eventSpec.weight;
+        
+        // Merge funnel properties
+        eventSpec.properties = { ...eventSpec.properties, ...chosenFunnelProps };
+        
+        return eventSpec;
+    });
+}
+
+/**
+ * Processes event repeats and adjusts conversion rate
+ * @param {Array} events - Array of event specifications
+ * @param {boolean} requireRepeats - Whether repeats are required
+ * @param {number} conversionRate - Base conversion rate
+ * @param {Object} chance - Chance.js instance
+ * @returns {Object} Object with processedEvents and adjustedConversionRate
+ */
+function processEventRepeats(events, requireRepeats, conversionRate, chance) {
+    let adjustedConversionRate = conversionRate;
+    
+    const processedEvents = events.reduce((acc, step) => {
+        if (!requireRepeats) {
+            if (acc.find(e => e.event === step.event)) {
+                if (chance.bool({ likelihood: 50 })) {
+                    adjustedConversionRate = Math.floor(adjustedConversionRate * 1.35); // Increase conversion rate
+                    acc.push(step);
+                } else {
+                    adjustedConversionRate = Math.floor(adjustedConversionRate * 0.70); // Reduce conversion rate
+                    return acc; // Skip the step
+                }
+            } else {
+                acc.push(step);
+            }
+        } else {
+            acc.push(step);
+        }
+        return acc;
+    }, []);
+
+    // Clamp conversion rate
+    if (adjustedConversionRate > 100) adjustedConversionRate = 100;
+    if (adjustedConversionRate < 0) adjustedConversionRate = 0;
+
+    return { processedEvents, adjustedConversionRate };
+}
+
+/**
+ * Determines if user converts and how many steps they'll take
+ * @param {number} conversionRate - Adjusted conversion rate
+ * @param {number} totalSteps - Total number of steps in funnel
+ * @param {Object} chance - Chance.js instance
+ * @returns {Object} Object with doesUserConvert and numStepsUserWillTake
+ */
+function determineConversion(conversionRate, totalSteps, chance) {
+    const doesUserConvert = chance.bool({ likelihood: conversionRate });
+    const numStepsUserWillTake = doesUserConvert ? 
+        totalSteps : 
+        u.integer(1, totalSteps - 1);
+    
+    return { doesUserConvert, numStepsUserWillTake };
+}
+
+/**
+ * Applies ordering strategy to funnel steps
+ * @param {Array} steps - Funnel steps to order
+ * @param {string} order - Ordering strategy
+ * @param {Object} config - Configuration object
+ * @param {Array} sequence - Original sequence for interrupted mode
+ * @returns {Array} Ordered funnel steps
+ */
+function applyOrderingStrategy(steps, order, config, sequence) {
+    switch (order) {
+        case "sequential":
+            return steps;
+        case "random":
+            return u.shuffleArray(steps);
+        case "first-fixed":
+            return u.shuffleExceptFirst(steps);
+        case "last-fixed":
+            return u.shuffleExceptLast(steps);
+        case "first-and-last-fixed":
+            return u.fixFirstAndLast(steps);
+        case "middle-fixed":
+            return u.shuffleOutside(steps);
+        case "interrupted":
+            const potentialSubstitutes = config.events
+                ?.filter(e => !e.isFirstEvent)
+                ?.filter(e => !sequence.includes(e.event)) || [];
+            return u.interruptArray(steps, potentialSubstitutes);
+        default:
+            return steps;
+    }
+}
+
+/**
+ * Adds timing offsets to funnel events
+ * @param {Array} events - Events to add timing to
+ * @param {number} timeToConvert - Total time to convert (in hours)
+ * @param {number} numSteps - Number of steps in funnel
+ * @returns {Array} Events with timing information
+ */
+function addTimingOffsets(events, timeToConvert, numSteps) {
+    const msInHour = 60000 * 60;
+    let lastTimeJump = 0;
+    
+    return events.map((event, index) => {
+        if (index === 0) {
+            event.relativeTimeMs = 0;
+            return event;
+        }
+
+        // Calculate base increment for each step
+        const baseIncrement = (timeToConvert * msInHour) / numSteps;
+
+        // Add random fluctuation
+        const fluctuation = u.integer(
+            -baseIncrement / u.integer(3, 5), 
+            baseIncrement / u.integer(3, 5)
+        );
+
+        // Ensure increasing timestamps
+        const previousTime = lastTimeJump;
+        const currentTime = previousTime + baseIncrement + fluctuation;
+        const chosenTime = Math.max(currentTime, previousTime + 1);
+        
+        lastTimeJump = chosenTime;
+        event.relativeTimeMs = chosenTime;
+        
+        return event;
+    });
+}
+
+/**
+ * Generates actual events with proper timing
+ * @param {Context} context - Context object
+ * @param {Array} eventsWithTiming - Events with timing information
+ * @param {string} distinct_id - User ID
+ * @param {number} earliestTime - Base timestamp
+ * @param {Array} anonymousIds - Anonymous IDs
+ * @param {Array} sessionIds - Session IDs
+ * @param {Array} groupKeys - Group keys
+ * @returns {Promise<Array>} Generated events
+ */
+async function generateFunnelEvents(
+    context, 
+    eventsWithTiming, 
+    distinct_id, 
+    earliestTime, 
+    anonymousIds, 
+    sessionIds, 
+    groupKeys
+) {
+    let funnelStartTime;
+    
+    const finalEvents = await Promise.all(eventsWithTiming.map(async (event, index) => {
+        const newEvent = await makeEvent(
+            context, 
+            distinct_id, 
+            earliestTime, 
+            event, 
+            anonymousIds, 
+            sessionIds, 
+            {}, 
+            groupKeys
+        );
+        
+        if (index === 0) {
+            funnelStartTime = dayjs(newEvent.time);
+            delete newEvent.relativeTimeMs;
+            return newEvent;
+        }
+        
+        try {
+            newEvent.time = dayjs(funnelStartTime)
+                .add(event.relativeTimeMs, "milliseconds")
+                .toISOString();
+            delete newEvent.relativeTimeMs;
+            return newEvent;
+        } catch (e) {
+            console.error("Error setting funnel event time:", e);
+            return newEvent;
+        }
+    }));
+
+    return finalEvents;
+}