make-mp-data 1.5.55 → 2.0.0

package/lib/generators/mirror.js

@@ -0,0 +1,168 @@
+/**
+ * Mirror dataset generator module
+ * Creates mirror datasets in a future state with different transformation strategies
+ */
+
+/** @typedef {import('../../types').Context} Context */
+
+import dayjs from "dayjs";
+import { clone } from "ak-tools";
+import * as u from "../utils/utils.js";
+
+/**
+ * Takes event data and creates mirror datasets in a future state
+ * depending on the mirror strategy configuration
+ * @param {Context} context - Context object containing config, defaults, etc.
+ * @returns {Promise<void>}
+ */
+export async function makeMirror(context) {
+    const { config, storage } = context;
+    const { mirrorProps } = config;
+    const { eventData, mirrorEventData } = storage;
+    
+    if (!mirrorProps || Object.keys(mirrorProps).length === 0) {
+        return; // No mirror properties configured
+    }
+    
+    const now = dayjs();
+
+    for (const oldEvent of eventData) {
+        let newEvent = null;
+        const eventTime = dayjs(oldEvent.time);
+        const delta = now.diff(eventTime, "day");
+
+        for (const mirrorProp in mirrorProps) {
+            const prop = mirrorProps[mirrorProp];
+            const { 
+                daysUnfilled = 7, 
+                events = "*", 
+                strategy = "create", 
+                values = [] 
+            } = prop;
+            
+            // Check if this event should be processed
+            if (shouldProcessEvent(oldEvent.event, events)) {
+                // Clone event only when needed
+                if (!newEvent) {
+                    newEvent = clone(oldEvent);
+                }
+
+                // Apply the specified strategy
+                applyMirrorStrategy(
+                    strategy, 
+                    newEvent, 
+                    oldEvent, 
+                    mirrorProp, 
+                    values, 
+                    delta, 
+                    daysUnfilled
+                );
+            }
+        }
+
+        // Push the processed event (or original if no changes)
+        const mirrorDataPoint = newEvent || oldEvent;
+        await mirrorEventData.hookPush(mirrorDataPoint);
+    }
+}
+
+/**
+ * Determines if an event should be processed based on event filter
+ * @param {string} eventName - Name of the event to check
+ * @param {string|Array} eventFilter - Event filter ("*" for all, or array of event names)
+ * @returns {boolean} True if event should be processed
+ */
+function shouldProcessEvent(eventName, eventFilter) {
+    if (eventFilter === "*") {
+        return true;
+    }
+    
+    if (Array.isArray(eventFilter)) {
+        return eventFilter.includes(eventName);
+    }
+    
+    return false;
+}
+
+/**
+ * Applies the specified mirror strategy to an event
+ * @param {string} strategy - Mirror strategy to apply
+ * @param {Object} newEvent - Event object to modify
+ * @param {Object} oldEvent - Original event object
+ * @param {string} propName - Property name to modify
+ * @param {Array} values - Possible values for the property
+ * @param {number} delta - Days between event time and now
+ * @param {number} daysUnfilled - Days threshold for fill strategy
+ */
+function applyMirrorStrategy(strategy, newEvent, oldEvent, propName, values, delta, daysUnfilled) {
+    switch (strategy) {
+        case "create":
+            // Always add the property with a random value
+            newEvent[propName] = u.choose(values);
+            break;
+            
+        case "delete":
+            // Remove the property from the event
+            delete newEvent[propName];
+            break;
+            
+        case "fill":
+            // Fill missing properties if enough time has passed
+            if (delta >= daysUnfilled) {
+                oldEvent[propName] = u.choose(values);
+            }
+            newEvent[propName] = u.choose(values);
+            break;
+            
+        case "update":
+            // Update only if property doesn't exist
+            if (!oldEvent[propName]) {
+                newEvent[propName] = u.choose(values);
+            } else {
+                newEvent[propName] = oldEvent[propName];
+            }
+            break;
+            
+        default:
+            throw new Error(`Unknown mirror strategy: ${strategy}`);
+    }
+}
+
+/**
+ * Validates mirror properties configuration
+ * @param {Object} mirrorProps - Mirror properties configuration to validate
+ * @returns {boolean} True if valid, throws error if invalid
+ */
+export function validateMirrorProps(mirrorProps) {
+    if (!mirrorProps || typeof mirrorProps !== 'object') {
+        return true; // Mirror props are optional
+    }
+    
+    const validStrategies = ['create', 'delete', 'fill', 'update'];
+    
+    for (const [propName, config] of Object.entries(mirrorProps)) {
+        if (!config || typeof config !== 'object') {
+            throw new Error(`Mirror property '${propName}' must have a configuration object`);
+        }
+        
+        const { strategy = 'create', values = [], events = '*', daysUnfilled = 7 } = config;
+        
+        if (!validStrategies.includes(strategy)) {
+            throw new Error(`Invalid mirror strategy '${strategy}' for property '${propName}'. Must be one of: ${validStrategies.join(', ')}`);
+        }
+        
+        if (strategy !== 'delete' && (!values || !Array.isArray(values) || values.length === 0)) {
+            throw new Error(`Mirror property '${propName}' with strategy '${strategy}' must have non-empty values array`);
+        }
+        
+        if (events !== '*' && (!Array.isArray(events) || events.length === 0)) {
+            throw new Error(`Mirror property '${propName}' events filter must be "*" or non-empty array`);
+        }
+        
+        if (typeof daysUnfilled !== 'number' || daysUnfilled < 0) {
+            throw new Error(`Mirror property '${propName}' daysUnfilled must be a non-negative number`);
+        }
+    }
+    
+    return true;
+}

package/lib/generators/profiles.js

@@ -0,0 +1,93 @@
+/**
+ * Profile generator module
+ * Creates user and group profiles with realistic properties
+ */
+
+/** @typedef {import('../../types').Context} Context */
+
+import * as u from "../utils/utils.js";
+
+/**
+ * Creates a user or group profile by choosing from available property values
+ * @param {Context} context - Context object containing config, defaults, etc.
+ * @param {Object} props - Properties to include in the profile
+ * @param {Object} defaults - Default values to merge with props
+ * @returns {Promise<Object>} Generated profile object
+ */
+export async function makeProfile(context, props = {}, defaults = {}) {
+    // Update operation counter
+    context.incrementOperations();
+    
+    // Keys that should not be processed with the choose function
+    const keysToNotChoose = ["anonymousIds", "sessionIds"];
+
+    // Start with defaults
+    const profile = { ...defaults };
+
+    // Process default values first
+    for (const key in profile) {
+        if (keysToNotChoose.includes(key)) continue;
+        
+        try {
+            profile[key] = u.choose(profile[key]);
+        } catch (e) {
+            console.error(`Error processing default property ${key}:`, e);
+            // Keep original value on error
+        }
+    }
+
+    // Process provided props (these override defaults)
+    for (const key in props) {
+        try {
+            profile[key] = u.choose(props[key]);
+        } catch (e) {
+            console.error(`Error processing property ${key}:`, e);
+            // Keep original value on error
+        }
+    }
+
+    return profile;
+}
+
+/**
+ * Creates a user profile with typical user properties
+ * @param {Context} context - Context object
+ * @param {Object} userProps - User-specific properties
+ * @param {Object} baseProfile - Base profile to extend
+ * @returns {Promise<Object>} Generated user profile
+ */
+export async function makeUserProfile(context, userProps = {}, baseProfile = {}) {
+    const { config } = context;
+    
+    // Combine user props with any configured user properties
+    const combinedProps = {
+        ...config.userProps,
+        ...userProps
+    };
+    
+    return makeProfile(context, combinedProps, baseProfile);
+}
+
+/**
+ * Creates a group profile with group-specific properties
+ * @param {Context} context - Context object
+ * @param {string} groupKey - Group identifier
+ * @param {Object} groupProps - Group-specific properties
+ * @param {Object} baseProfile - Base profile to extend
+ * @returns {Promise<Object>} Generated group profile
+ */
+export async function makeGroupProfile(context, groupKey, groupProps = {}, baseProfile = {}) {
+    const { config } = context;
+    
+    // Get group properties from config for this specific group
+    const configGroupProps = config.groupProps?.[groupKey] || {};
+    
+    // Combine with provided props
+    const combinedProps = {
+        ...configGroupProps,
+        ...groupProps,
+        groupKey // Always include the group key
+    };
+    
+    return makeProfile(context, combinedProps, baseProfile);
+}

package/lib/generators/scd.js

@@ -0,0 +1,102 @@
+/**
+ * Slowly Changing Dimensions (SCD) generator module
+ * Creates time-series data showing how properties change over time
+ */
+
+/** @typedef {import('../../types').Context} Context */
+
+import dayjs from "dayjs";
+import * as u from "../utils/utils.js";
+import { makeProfile } from "./profiles.js";
+
+/**
+ * Creates SCD (Slowly Changing Dimensions) entries for a given property
+ * @param {Context} context - Context object containing config, defaults, etc.
+ * @param {Array|Object} scdProp - SCD property configuration or simple array of values
+ * @param {string} scdKey - Key name for the SCD property
+ * @param {string} distinct_id - User/entity identifier
+ * @param {number} mutations - Number of mutations to create
+ * @param {string | number} created - Creation timestamp for the entity
+ * @returns {Promise<Array>} Array of SCD entries
+ */
+export async function makeSCD(context, scdProp, scdKey, distinct_id, mutations, created) {
+    // Convert simple array to full configuration object
+    if (Array.isArray(scdProp)) {
+        scdProp = { 
+            values: scdProp, 
+            frequency: 'week', 
+            max: 10, 
+            timing: 'fuzzy', 
+            type: 'user' 
+        };
+    }
+	if (typeof created === 'number') created = dayjs.unix(created).toISOString();
+    
+    const { frequency, max, timing, values, type = "user" } = scdProp;
+    
+    // Return empty array if no values provided
+    if (JSON.stringify(values) === "{}" || JSON.stringify(values) === "[]") {
+        return [];
+    }
+    
+    const scdEntries = [];
+    let lastInserted = dayjs(created);
+    const deltaDays = dayjs().diff(lastInserted, "day");
+    const uuidKeyName = type === 'user' ? 'distinct_id' : type;
+
+    for (let i = 0; i < mutations; i++) {
+        // Stop if we've reached the current time
+        if (lastInserted.isAfter(dayjs())) break;
+        
+        // Create profile with the SCD property
+        const scd = await makeProfile(context, { [scdKey]: values }, { [uuidKeyName]: distinct_id });
+
+        // Create SCD entry with all required properties
+        const scdEntry = {
+            ...scd,
+            [uuidKeyName]: scd.distinct_id || distinct_id,
+            startTime: null,
+            insertTime: null
+        };
+
+        // Set start time based on timing strategy
+        if (timing === 'fixed') {
+            switch (frequency) {
+                case "day":
+                    scdEntry.startTime = lastInserted.add(1, "day").startOf('day').toISOString();
+                    break;
+                case "week":
+                    scdEntry.startTime = lastInserted.add(1, "week").startOf('week').toISOString();
+                    break;
+                case "month":
+                    scdEntry.startTime = lastInserted.add(1, "month").startOf('month').toISOString();
+                    break;
+            }
+        }
+
+        if (timing === 'fuzzy') {
+            scdEntry.startTime = lastInserted.toISOString();
+        }
+
+        // Set insert time (slightly after start time)
+        const insertTime = lastInserted.add(u.integer(1, 9000), "seconds");
+        scdEntry.insertTime = insertTime.toISOString();
+
+        // Only add entry if all required properties are set
+        if (scdEntry.hasOwnProperty('insertTime') && scdEntry.hasOwnProperty('startTime')) {
+            scdEntries.push(scdEntry);
+        }
+
+        // Advance time for next entry
+        lastInserted = lastInserted
+            .add(u.integer(0, deltaDays), "day")
+            .subtract(u.integer(1, 9000), "seconds");
+    }
+
+    // De-duplicate on startTime
+    const deduped = scdEntries.filter((entry, index, self) =>
+        index === self.findIndex((t) => t.startTime === entry.startTime)
+    );
+    
+    return deduped;
+}

package/lib/orchestrators/mixpanel-sender.js

@@ -0,0 +1,222 @@
+/**
+ * Mixpanel Sender Orchestrator module
+ * Handles sending all data types to Mixpanel
+ */
+
+/** @typedef {import('../../types').Context} Context */
+
+import dayjs from "dayjs";
+import path from "path";
+import { clone, comma, ls, rm } from "ak-tools";
+import mp from "mixpanel-import";
+
+/**
+ * Sends the data to Mixpanel
+ * @param {Context} context - Context object containing config, storage, etc.
+ * @returns {Promise<Object>} Import results for all data types
+ */
+export async function sendToMixpanel(context) {
+    const { config, storage } = context;
+    const {
+        adSpendData,
+        eventData,
+        groupProfilesData,
+        lookupTableData,
+        mirrorEventData,
+        scdTableData,
+        userProfilesData,
+        groupEventData
+    } = storage;
+    
+    const { 
+        token, 
+        region, 
+        writeToDisk = true,
+        format,
+        serviceAccount,
+        projectId,
+        serviceSecret
+    } = config;
+    
+    const importResults = { events: {}, users: {}, groups: [] };
+    const isBATCH_MODE = context.isBatchMode();
+    const isCLI = context.isCLI();
+    const NODE_ENV = process.env.NODE_ENV || "unknown";
+
+    /** @type {import('mixpanel-import').Creds} */
+    const creds = { token };
+    const mpImportFormat = format === "json" ? "jsonl" : "csv";
+    
+    /** @type {import('mixpanel-import').Options} */
+    const commonOpts = {
+        region,
+        fixData: true,
+        verbose: false,
+        forceStream: true,
+        strict: true,
+        epochEnd: dayjs().unix(),
+        dryRun: false,
+        abridged: false,
+        fixJson: true,
+        showProgress: NODE_ENV === "dev" ? true : false,
+        streamFormat: mpImportFormat
+    };
+
+    if (isCLI) commonOpts.showProgress = true;
+
+    // Import events
+    if (eventData || isBATCH_MODE) {
+        log(`importing events to mixpanel...\n`);
+        let eventDataToImport = clone(eventData);
+        if (isBATCH_MODE) {
+            const writeDir = eventData.getWriteDir();
+            const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+            eventDataToImport = files.filter(f => f.includes('-EVENTS-'));
+        }
+        const imported = await mp(creds, eventDataToImport, {
+            recordType: "event",
+            ...commonOpts,
+        });
+        log(`\tsent ${comma(imported.success)} events\n`);
+        importResults.events = imported;
+    }
+
+    // Import user profiles
+    if (userProfilesData || isBATCH_MODE) {
+        log(`importing user profiles to mixpanel...\n`);
+        let userProfilesToImport = clone(userProfilesData);
+        if (isBATCH_MODE) {
+            const writeDir = userProfilesData.getWriteDir();
+            const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+            userProfilesToImport = files.filter(f => f.includes('-USERS-'));
+        }
+        const imported = await mp(creds, userProfilesToImport, {
+            recordType: "user",
+            ...commonOpts,
+        });
+        log(`\tsent ${comma(imported.success)} user profiles\n`);
+        importResults.users = imported;
+    }
+
+    // Import ad spend data  
+    if (groupEventData || isBATCH_MODE) {
+        log(`importing ad spend data to mixpanel...\n`);
+        let adSpendDataToImport = clone(adSpendData);
+        if (isBATCH_MODE) {
+            const writeDir = adSpendData.getWriteDir();
+            const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+            adSpendDataToImport = files.filter(f => f.includes('-AD-SPEND-'));
+        }
+        const imported = await mp(creds, adSpendDataToImport, {
+            recordType: "event",
+            ...commonOpts,
+        });
+        log(`\tsent ${comma(imported.success)} ad spend events\n`);
+        importResults.adSpend = imported;
+    }
+
+    // Import group profiles
+    if (groupProfilesData || isBATCH_MODE) {
+        for (const groupEntity of groupProfilesData) {
+            const groupKey = groupEntity?.groupKey;
+            log(`importing ${groupKey} profiles to mixpanel...\n`);
+            let groupProfilesToImport = clone(groupEntity);
+            if (isBATCH_MODE) {
+                const writeDir = groupEntity.getWriteDir();
+                const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+                groupProfilesToImport = files.filter(f => f.includes(`-GROUPS-${groupKey}`));
+            }
+            const imported = await mp({ token, groupKey }, groupProfilesToImport, {
+                recordType: "group",
+                ...commonOpts,
+            });
+            log(`\tsent ${comma(imported.success)} ${groupKey} profiles\n`);
+            importResults.groups.push(imported);
+        }
+    }
+
+    // Import group events
+    if (groupEventData || isBATCH_MODE) {
+        log(`importing group events to mixpanel...\n`);
+        let groupEventDataToImport = clone(groupEventData);
+        if (isBATCH_MODE) {
+            const writeDir = groupEventData.getWriteDir();
+            const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+            groupEventDataToImport = files.filter(f => f.includes('-GROUP-EVENTS-'));
+        }
+        const imported = await mp(creds, groupEventDataToImport, {
+            recordType: "event",
+            ...commonOpts,
+            strict: false
+        });
+        log(`\tsent ${comma(imported.success)} group events\n`);
+        importResults.groupEvents = imported;
+    }
+
+    // Import SCD data (requires service account)
+    if (serviceAccount && projectId && serviceSecret) {
+        if (scdTableData || isBATCH_MODE) {
+            log(`importing SCD data to mixpanel...\n`);
+            for (const scdEntity of scdTableData) {
+                const scdKey = scdEntity?.scdKey;
+                log(`importing ${scdKey} SCD data to mixpanel...\n`);
+                let scdDataToImport = clone(scdEntity);
+                if (isBATCH_MODE) {
+                    const writeDir = scdEntity.getWriteDir();
+                    const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+                    scdDataToImport = files.filter(f => f.includes(`-SCD-${scdKey}`));
+                }
+
+                /** @type {import('mixpanel-import').Options} */
+                const options = {
+                    recordType: "scd",
+                    scdKey,
+                    scdType: scdEntity.dataType,
+                    scdLabel: `${scdKey}-scd`,
+                    ...commonOpts,
+                };
+
+                if (scdEntity.entityType !== "user") options.groupKey = scdEntity.entityType;
+                
+                const imported = await mp(
+                    {
+                        token,
+                        acct: serviceAccount,
+                        pass: serviceSecret,
+                        project: projectId
+                    },
+                    scdDataToImport,
+                    options
+                );
+                log(`\tsent ${comma(imported.success)} ${scdKey} SCD data\n`);
+                importResults[`${scdKey}_scd`] = imported;
+            }
+        }
+    }
+
+    // Clean up batch files if needed
+    if (!writeToDisk && isBATCH_MODE) {
+        const writeDir = eventData?.getWriteDir() || userProfilesData?.getWriteDir();
+        const listDir = await ls(writeDir.split(path.basename(writeDir)).join(""));
+        const files = listDir.filter(f => 
+            f.includes('-EVENTS-') || 
+            f.includes('-USERS-') || 
+            f.includes('-AD-SPEND-') || 
+            f.includes('-GROUPS-') || 
+            f.includes('-GROUP-EVENTS-')
+        );
+        for (const file of files) {
+            await rm(file);
+        }
+    }
+    
+    return importResults;
+}
+
+/**
+ * Simple logging function
+ * @param {string} message - Message to log
+ */
+function log(message) {
+    console.log(message);
+}