@ak--47/dungeon-master 1.0.0

package/lib/generators/funnels.js

@@ -0,0 +1,407 @@
+/**
+ * Funnel generator module
+ * Creates conversion sequences with realistic timing and ordering
+ */
+
+/** @typedef {import('../../types').Context} Context */
+
+import dayjs from "dayjs";
+import * as u from "../utils/utils.js";
+import { makeEvent } from "./events.js";
+import { dataLogger as logger } from "../utils/logger.js";
+
+/**
+ * Creates a funnel (sequence of events) for a user with conversion logic
+ * @param {Context} context - Context object containing config, defaults, etc.
+ * @param {Object} funnel - Funnel configuration
+ * @param {Object} user - User object with distinct_id, created, etc.
+ * @param {number} firstEventTime - Unix timestamp for first event
+ * @param {Object} profile - User profile object
+ * @param {Object} scd - Slowly changing dimensions object
+ * @returns {Promise<[Array, boolean]>} Tuple of [events, didConvert]
+ */
+export async function makeFunnel(context, funnel, user, firstEventTime, profile = {}, scd = {}) {
+	if (!funnel) throw new Error("no funnel");
+	if (!user) throw new Error("no user");
+
+	const { config } = context;
+	const chance = u.getChance();
+	const { hook = async (a) => a } = config;
+
+	// Get session start events if configured
+	const sessionStartEvents = config.events?.filter(a => a.isSessionStartEvent) || [];
+
+	// Clone funnel to avoid mutating the original object
+	funnel = { ...funnel };
+
+	// Experiment handling: if funnel.experiment === true, create 3 variants
+	let experimentVariant = null;
+	let experimentName = null;
+
+	if (funnel.experiment) {
+		experimentName = funnel.name + ` Experiment` || "Unnamed Funnel";
+
+		// Evenly distribute across 3 variants (33.33% each) using seeded chance
+		const randomValue = chance.floating({ min: 0, max: 1 });
+		if (randomValue < 0.333) {
+			// Variant A: WORSE conversion, slower
+			funnel.conversionRate = Math.max(1, Math.floor(funnel.conversionRate * 0.7));
+			funnel.timeToConvert = Math.max(0.1, funnel.timeToConvert * 1.5);
+			experimentVariant = "A";
+		} else if (randomValue < 0.666) {
+			// Variant B: BETTER conversion, faster
+			funnel.conversionRate = Math.min(100, Math.ceil(funnel.conversionRate * 1.3));
+			funnel.timeToConvert = Math.max(0.1, funnel.timeToConvert * 0.7);
+			experimentVariant = "B";
+		} else {
+			// Variant C: CONTROL - original values (no changes)
+			experimentVariant = "C";
+		}
+
+		// Mark that this funnel has experiment metadata (used later)
+		funnel._experimentName = experimentName;
+		funnel._experimentVariant = experimentVariant;
+
+		// Insert $experiment_started at beginning of sequence (clone array to avoid mutation)
+		funnel.sequence = ["$experiment_started", ...funnel.sequence];
+	}
+
+	// Call pre-funnel hook
+	await hook(funnel, "funnel-pre", { user, profile, scd, funnel, config, firstEventTime });
+
+	// Extract funnel configuration
+	let {
+		sequence,
+		conversionRate = 50,
+		order = 'sequential',
+		timeToConvert = 1,
+		props = {},
+		requireRepeats = false,
+		_experimentName: expName,
+		_experimentVariant: expVariant,
+		bindPropsIndex = 0
+	} = funnel;
+
+	const { distinct_id, created, anonymousIds = [], sessionIds = [] } = user;
+	const { superProps = {}, groupKeys = [] } = config;
+
+	// Choose properties for this funnel instance
+	const chosenFunnelProps = { ...props, ...superProps };
+	for (const key in props) {
+		try {
+			chosenFunnelProps[key] = u.choose(chosenFunnelProps[key]);
+		} catch (e) {
+			logger.error({ err: e, key, funnel: funnel.sequence.join(" > ") }, `Error processing property ${key} in funnel`);
+		}
+	}
+
+	// Build event specifications for funnel steps
+	const funnelPossibleEvents = buildFunnelEvents(context, sequence, chosenFunnelProps, bindPropsIndex, expName, expVariant);
+
+	// Handle repeat logic and conversion rate adjustment
+	const { processedEvents, adjustedConversionRate } = processEventRepeats(
+		funnelPossibleEvents,
+		requireRepeats,
+		conversionRate,
+		chance
+	);
+
+	// Determine if user converts and how many steps they'll take
+	const { doesUserConvert, numStepsUserWillTake } = determineConversion(
+		adjustedConversionRate,
+		sequence.length,
+		chance
+	);
+
+	// Get steps user will actually take
+	const funnelStepsUserWillTake = processedEvents.slice(0, numStepsUserWillTake);
+
+	// Apply ordering strategy
+	const funnelActualOrder = applyOrderingStrategy(
+		funnelStepsUserWillTake,
+		order,
+		config,
+		sequence
+	);
+
+	// Add timing offsets to events
+	const funnelEventsWithTiming = addTimingOffsets(
+		funnelActualOrder,
+		timeToConvert,
+		numStepsUserWillTake
+	);
+
+	// Add session start event if configured (clone to avoid mutating shared config)
+	if (sessionStartEvents.length) {
+		const sessionStartEvent = { ...chance.pickone(sessionStartEvents), relativeTimeMs: -15000 };
+		funnelEventsWithTiming.push(sessionStartEvent);
+	}
+
+	// Generate actual events with timing
+	const finalEvents = await generateFunnelEvents(
+		context,
+		funnelEventsWithTiming,
+		distinct_id,
+		firstEventTime || dayjs(created).unix(),
+		anonymousIds,
+		sessionIds,
+		groupKeys
+	);
+
+	// Call post-funnel hook
+	await hook(finalEvents, "funnel-post", { user, profile, scd, funnel, config });
+
+	return [finalEvents, doesUserConvert];
+}
+
+/**
+ * Builds event specifications for funnel steps
+ * @param {Context} context - Context object
+ * @param {Array} sequence - Array of event names
+ * @param {Object} chosenFunnelProps - Properties to apply to all events
+ * @param {number} bindPropsIndex - Index at which to bind properties (if applicable)
+ * @param {string} [experimentName] - Name of experiment (if experiment is enabled)
+ * @param {string} [experimentVariant] - Variant name (A, B, or C)
+ * @returns {Array} Array of event specifications
+ */
+function buildFunnelEvents(context, sequence, chosenFunnelProps, bindPropsIndex, experimentName, experimentVariant) {
+	const { config } = context;
+
+	return sequence.map((eventName, currentIndex) => {
+		// Handle $experiment_started event specially
+		if (eventName === "$experiment_started" && experimentName && experimentVariant) {
+			return {
+				event: "$experiment_started",
+				properties: {
+					"Experiment name": experimentName,
+					"Variant name": experimentVariant
+				}
+			};
+		}
+
+		const foundEvent = config.events?.find((e) => e.event === eventName);
+
+		// PERFORMANCE: Shallow copy instead of deepClone for better performance
+		// We only need to copy the top-level structure since we're rebuilding properties anyway
+		const eventSpec = foundEvent ? {
+			event: foundEvent.event,
+			properties: { ...foundEvent.properties }
+		} : { event: eventName, properties: {} };
+
+		// Process event properties
+		for (const key in eventSpec.properties) {
+			try {
+				eventSpec.properties[key] = u.choose(eventSpec.properties[key]);
+			} catch (e) {
+				logger.error({ err: e, key, event: eventSpec.event }, `Error processing property ${key} in ${eventSpec.event} event`);
+			}
+		}
+
+		// Merge funnel properties (no need to delete properties since we're creating a new object)
+		eventSpec.properties = { ...eventSpec.properties, ...chosenFunnelProps };
+
+
+		if (bindPropsIndex && currentIndex < bindPropsIndex) {
+			// Remove funnel properties that were added but should not be bound yet
+			for (const key in chosenFunnelProps) {
+				delete eventSpec.properties[key];
+			}
+		}
+
+		return eventSpec;
+	});
+}
+
+/**
+ * Processes event repeats and adjusts conversion rate
+ * @param {Array} events - Array of event specifications
+ * @param {boolean} requireRepeats - Whether repeats are required
+ * @param {number} conversionRate - Base conversion rate
+ * @param {Object} chance - Chance.js instance
+ * @returns {Object} Object with processedEvents and adjustedConversionRate
+ */
+function processEventRepeats(events, requireRepeats, conversionRate, chance) {
+	let adjustedConversionRate = conversionRate;
+
+	const processedEvents = events.reduce((acc, step) => {
+		if (!requireRepeats) {
+			if (acc.find(e => e.event === step.event)) {
+				if (chance.bool({ likelihood: 50 })) {
+					adjustedConversionRate = Math.floor(adjustedConversionRate * 1.35); // Increase conversion rate
+					acc.push(step);
+				} else {
+					adjustedConversionRate = Math.floor(adjustedConversionRate * 0.70); // Reduce conversion rate
+					return acc; // Skip the step
+				}
+			} else {
+				acc.push(step);
+			}
+		} else {
+			acc.push(step);
+		}
+		return acc;
+	}, []);
+
+	// Clamp conversion rate
+	if (adjustedConversionRate > 100) adjustedConversionRate = 100;
+	if (adjustedConversionRate < 0) adjustedConversionRate = 0;
+
+	return { processedEvents, adjustedConversionRate };
+}
+
+/**
+ * Determines if user converts and how many steps they'll take
+ * @param {number} conversionRate - Adjusted conversion rate
+ * @param {number} totalSteps - Total number of steps in funnel
+ * @param {Object} chance - Chance.js instance
+ * @returns {Object} Object with doesUserConvert and numStepsUserWillTake
+ */
+function determineConversion(conversionRate, totalSteps, chance) {
+	const doesUserConvert = chance.bool({ likelihood: conversionRate });
+	const numStepsUserWillTake = doesUserConvert ?
+		totalSteps :
+		u.integer(1, totalSteps - 1);
+
+	return { doesUserConvert, numStepsUserWillTake };
+}
+
+/**
+ * Applies ordering strategy to funnel steps
+ * @param {Array} steps - Funnel steps to order
+ * @param {string} order - Ordering strategy
+ * @param {Object} config - Configuration object
+ * @param {Array} sequence - Original sequence for interrupted mode
+ * @returns {Array} Ordered funnel steps
+ */
+function applyOrderingStrategy(steps, order, config, sequence) {
+	switch (order) {
+		case "sequential":
+			return steps;
+		case "random":
+			return u.shuffleArray(steps);
+		case "first-fixed":
+			return u.shuffleExceptFirst(steps);
+		case "last-fixed":
+			return u.shuffleExceptLast(steps);
+		case "first-and-last-fixed":
+			return u.fixFirstAndLast(steps);
+		case "middle-fixed":
+			return u.shuffleOutside(steps);
+		case "interrupted":
+			const potentialSubstitutes = config.events
+				?.filter(e => !e.isFirstEvent)
+				?.filter(e => !sequence.includes(e.event)) || [];
+			return u.interruptArray(steps, potentialSubstitutes);
+		default:
+			return steps;
+	}
+}
+
+/**
+ * Adds timing offsets to funnel events
+ * @param {Array} events - Events to add timing to
+ * @param {number} timeToConvert - Total time to convert (in hours)
+ * @param {number} numSteps - Number of steps in funnel
+ * @returns {Array} Events with timing information
+ */
+function addTimingOffsets(events, timeToConvert, numSteps) {
+	const msInHour = 60000 * 60;
+	let lastTimeJump = 0;
+
+	return events.map((event, index) => {
+		if (index === 0) {
+			event.relativeTimeMs = 0;
+			return event;
+		}
+
+		// Calculate base increment for each step
+		const baseIncrement = (timeToConvert * msInHour) / numSteps;
+
+		// Add random fluctuation
+		const fluctuation = u.integer(
+			-baseIncrement / u.integer(3, 5),
+			baseIncrement / u.integer(3, 5)
+		);
+
+		// Ensure increasing timestamps
+		const previousTime = lastTimeJump;
+		const currentTime = previousTime + baseIncrement + fluctuation;
+		const chosenTime = Math.max(currentTime, previousTime + 1);
+
+		lastTimeJump = chosenTime;
+		event.relativeTimeMs = chosenTime;
+
+		return event;
+	});
+}
+
+/**
+ * Generates actual events with proper timing
+ * @param {Context} context - Context object
+ * @param {Array} eventsWithTiming - Events with timing information
+ * @param {string} distinct_id - User ID
+ * @param {number} earliestTime - Base timestamp
+ * @param {Array} anonymousIds - Anonymous IDs
+ * @param {Array} sessionIds - Session IDs
+ * @param {Array} groupKeys - Group keys
+ * @returns {Promise<Array>} Generated events
+ */
+async function generateFunnelEvents(
+	context,
+	eventsWithTiming,
+	distinct_id,
+	earliestTime,
+	anonymousIds,
+	sessionIds,
+	groupKeys
+) {
+	let funnelStartTime;
+
+	const finalEvents = await Promise.all(eventsWithTiming.map(async (event, index) => {
+		const newEvent = await makeEvent(
+			context,
+			distinct_id,
+			earliestTime,
+			event,
+			anonymousIds,
+			sessionIds,
+			{},
+			groupKeys,
+			false  // Let all funnel events use TimeSoup for proper time distribution
+		);
+
+		if (index === 0) {
+			const parsedTime = dayjs(newEvent.time);
+			// Validate the first event's time - if invalid, use TimeSoup-generated time as-is
+			funnelStartTime = parsedTime.isValid() ? parsedTime : null;
+			delete newEvent.relativeTimeMs;
+			return newEvent;
+		}
+
+		// If funnelStartTime is invalid, just use the TimeSoup-generated time from makeEvent
+		if (!funnelStartTime || !funnelStartTime.isValid()) {
+			delete newEvent.relativeTimeMs;
+			return newEvent;
+		}
+
+		try {
+			let computedTime = dayjs(funnelStartTime).add(event.relativeTimeMs, "milliseconds");
+			// Drop events that would land in the future
+			if (context.MAX_TIME && computedTime.unix() > context.MAX_TIME) {
+				newEvent._drop = true;
+			}
+			if (computedTime.isValid()) {
+				newEvent.time = computedTime.toISOString();
+			}
+			// If invalid, keep the TimeSoup-generated time from makeEvent
+			delete newEvent.relativeTimeMs;
+			return newEvent;
+		} catch (e) {
+			// Graceful fallback: keep the TimeSoup-generated time from makeEvent
+			delete newEvent.relativeTimeMs;
+			return newEvent;
+		}
+	}));
+
+	return finalEvents;
+}

package/lib/generators/mirror.js

@@ -0,0 +1,167 @@
+/**
+ * Mirror dataset generator module
+ * Creates mirror datasets in a future state with different transformation strategies
+ */
+
+/** @typedef {import('../../types').Context} Context */
+
+import dayjs from "dayjs";
+import * as u from "../utils/utils.js";
+
+/**
+ * Takes event data and creates mirror datasets in a future state
+ * depending on the mirror strategy configuration
+ * @param {Context} context - Context object containing config, defaults, etc.
+ * @returns {Promise<void>}
+ */
+export async function makeMirror(context) {
+    const { config, storage } = context;
+    const { mirrorProps } = config;
+    const { eventData, mirrorEventData } = storage;
+    
+    if (!mirrorProps || Object.keys(mirrorProps).length === 0) {
+        return; // No mirror properties configured
+    }
+    
+    const now = dayjs();
+
+    for (const oldEvent of eventData) {
+        let newEvent = null;
+        const eventTime = dayjs(oldEvent.time);
+        const delta = now.diff(eventTime, "day");
+
+        for (const mirrorProp in mirrorProps) {
+            const prop = mirrorProps[mirrorProp];
+            const { 
+                daysUnfilled = 7, 
+                events = "*", 
+                strategy = "create", 
+                values = [] 
+            } = prop;
+            
+            // Check if this event should be processed
+            if (shouldProcessEvent(oldEvent.event, events)) {
+                // Clone event only when needed
+                if (!newEvent) {
+                    newEvent = u.deepClone(oldEvent);
+                }
+
+                // Apply the specified strategy
+                applyMirrorStrategy(
+                    strategy, 
+                    newEvent, 
+                    oldEvent, 
+                    mirrorProp, 
+                    values, 
+                    delta, 
+                    daysUnfilled
+                );
+            }
+        }
+
+        // Push the processed event (or original if no changes)
+        const mirrorDataPoint = newEvent || oldEvent;
+        await mirrorEventData.hookPush(mirrorDataPoint);
+    }
+}
+
+/**
+ * Determines if an event should be processed based on event filter
+ * @param {string} eventName - Name of the event to check
+ * @param {string|Array} eventFilter - Event filter ("*" for all, or array of event names)
+ * @returns {boolean} True if event should be processed
+ */
+function shouldProcessEvent(eventName, eventFilter) {
+    if (eventFilter === "*") {
+        return true;
+    }
+    
+    if (Array.isArray(eventFilter)) {
+        return eventFilter.includes(eventName);
+    }
+    
+    return false;
+}
+
+/**
+ * Applies the specified mirror strategy to an event
+ * @param {string} strategy - Mirror strategy to apply
+ * @param {Object} newEvent - Event object to modify
+ * @param {Object} oldEvent - Original event object
+ * @param {string} propName - Property name to modify
+ * @param {Array} values - Possible values for the property
+ * @param {number} delta - Days between event time and now
+ * @param {number} daysUnfilled - Days threshold for fill strategy
+ */
+function applyMirrorStrategy(strategy, newEvent, oldEvent, propName, values, delta, daysUnfilled) {
+    switch (strategy) {
+        case "create":
+            // Always add the property with a random value
+            newEvent[propName] = u.choose(values);
+            break;
+            
+        case "delete":
+            // Remove the property from the event
+            delete newEvent[propName];
+            break;
+            
+        case "fill":
+            // Fill missing properties if enough time has passed
+            if (delta >= daysUnfilled) {
+                oldEvent[propName] = u.choose(values);
+            }
+            newEvent[propName] = u.choose(values);
+            break;
+            
+        case "update":
+            // Update only if property doesn't exist
+            if (!oldEvent[propName]) {
+                newEvent[propName] = u.choose(values);
+            } else {
+                newEvent[propName] = oldEvent[propName];
+            }
+            break;
+            
+        default:
+            throw new Error(`Unknown mirror strategy: ${strategy}`);
+    }
+}
+
+/**
+ * Validates mirror properties configuration
+ * @param {Object} mirrorProps - Mirror properties configuration to validate
+ * @returns {boolean} True if valid, throws error if invalid
+ */
+export function validateMirrorProps(mirrorProps) {
+    if (!mirrorProps || typeof mirrorProps !== 'object') {
+        return true; // Mirror props are optional
+    }
+    
+    const validStrategies = ['create', 'delete', 'fill', 'update'];
+    
+    for (const [propName, config] of Object.entries(mirrorProps)) {
+        if (!config || typeof config !== 'object') {
+            throw new Error(`Mirror property '${propName}' must have a configuration object`);
+        }
+        
+        const { strategy = 'create', values = [], events = '*', daysUnfilled = 7 } = config;
+        
+        if (!validStrategies.includes(strategy)) {
+            throw new Error(`Invalid mirror strategy '${strategy}' for property '${propName}'. Must be one of: ${validStrategies.join(', ')}`);
+        }
+        
+        if (strategy !== 'delete' && (!values || !Array.isArray(values) || values.length === 0)) {
+            throw new Error(`Mirror property '${propName}' with strategy '${strategy}' must have non-empty values array`);
+        }
+        
+        if (events !== '*' && (!Array.isArray(events) || events.length === 0)) {
+            throw new Error(`Mirror property '${propName}' events filter must be "*" or non-empty array`);
+        }
+        
+        if (typeof daysUnfilled !== 'number' || daysUnfilled < 0) {
+            throw new Error(`Mirror property '${propName}' daysUnfilled must be a non-negative number`);
+        }
+    }
+    
+    return true;
+}