make-mp-data 1.5.55 → 2.0.0

package/{components → lib/cli}/cli.js

@@ -1,5 +1,9 @@
-const yargs = require('yargs');
-const { version } = require('../package.json');
+import yargs from 'yargs';
+import fs from 'fs';
+
+/** @typedef {import('../../types').Dungeon} Config */
+
+const { version } = JSON.parse(fs.readFileSync('package.json', 'utf8'));
 
 const hero = String.raw`
                         
@@ -47,7 +51,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'string'
 		})
 		.option("format", {
-			demandOption: false,		
+			demandOption: false,
 			alias: 'f',
 			describe: 'csv or json',
 			type: 'string'
@@ -104,6 +108,14 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
+		.options("simple", {
+			demandOption: false,
+			default: false,
+			describe: 'use simple data model (basic events and users)',
+			alias: 'simp',
+			type: 'boolean',
+			coerce: boolCoerce
+		})
 		.option("writeToDisk", {
 			demandOption: false,
 			default: true,
@@ -141,7 +153,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			describe: 'create a PNG chart from data',
 			type: 'boolean',
 			coerce: boolCoerce
-		})		
+		})
 		.option("hasAdSpend", {
 			alias: 'ads',
 			demandOption: false,
@@ -198,11 +210,14 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
-		
+
 		.help()
 		.wrap(null)
 		.argv;
 
+	//cli is always verbose mode:
+	args.verbose = true;
+
 	return args;
 
 }
@@ -218,4 +233,4 @@ function boolCoerce(value, foo) {
 }
 
 
-module.exports = cliParams;
+export default cliParams;

package/lib/cloud-function.js

@@ -0,0 +1,20 @@
+/**
+ * Cloud Function Entry Point
+ * Provides a clean interface for Google Cloud Functions deployment
+ */
+
+/** @typedef {import('../types').Dungeon} Config */
+
+import functions from '@google-cloud/functions-framework';
+import { handleCloudFunctionEntry } from './orchestrators/worker-manager.js';
+import main from '../index.js';
+
+/**
+ * Cloud Function HTTP entry point
+ * Handles distributed data generation across multiple workers
+ */
+functions.http('entry', async (req, res) => {
+    await handleCloudFunctionEntry(req, res, main);
+});
+
+export { handleCloudFunctionEntry, main };

package/lib/core/config-validator.js

@@ -0,0 +1,248 @@
+/**
+ * Configuration validation and enrichment module
+ * Extracted from index.js validateDungeonConfig function
+ */
+
+import dayjs from "dayjs";
+import { makeName, clone } from "ak-tools";
+import * as u from "../utils/utils.js";
+
+/**
+ * Infers funnels from the provided events
+ * @param {Array} events - Array of event configurations
+ * @returns {Array} Array of inferred funnel configurations
+ */
+function inferFunnels(events) {
+    const createdFunnels = [];
+    const firstEvents = events.filter((e) => e.isFirstEvent).map((e) => e.event);
+    const usageEvents = events.filter((e) => !e.isFirstEvent).map((e) => e.event);
+    const numFunnelsToCreate = Math.ceil(usageEvents.length);
+    
+    /** @type {import('../../types.js').Funnel} */
+    const funnelTemplate = {
+        sequence: [],
+        conversionRate: 50,
+        order: 'sequential',
+        requireRepeats: false,
+        props: {},
+        timeToConvert: 1,
+        isFirstFunnel: false,
+        weight: 1
+    };
+    
+    // Create funnels for first events
+    if (firstEvents.length) {
+        for (const event of firstEvents) {
+            createdFunnels.push({ 
+                ...clone(funnelTemplate), 
+                sequence: [event], 
+                isFirstFunnel: true, 
+                conversionRate: 100 
+            });
+        }
+    }
+
+    // At least one funnel with all usage events
+    createdFunnels.push({ ...clone(funnelTemplate), sequence: usageEvents });
+
+    // Create random funnels for the rest
+    for (let i = 1; i < numFunnelsToCreate; i++) {
+        /** @type {import('../../types.js').Funnel} */
+        const funnel = { ...clone(funnelTemplate) };
+        funnel.conversionRate = u.integer(25, 75);
+        funnel.timeToConvert = u.integer(1, 10);
+        funnel.weight = u.integer(1, 10);
+        const sequence = u.shuffleArray(usageEvents).slice(0, u.integer(2, usageEvents.length));
+        funnel.sequence = sequence;
+        funnel.order = 'random';
+        createdFunnels.push(funnel);
+    }
+
+    return createdFunnels;
+}
+
+/**
+ * Validates and enriches a dungeon configuration object
+ * @param {Object} config - Raw configuration object
+ * @returns {Object} Validated and enriched configuration
+ */
+export function validateDungeonConfig(config) {
+    const chance = u.getChance();
+    
+    // Extract configuration with defaults
+    let {
+        seed,
+        numEvents = 100_000,
+        numUsers = 1000,
+        numDays = 30,
+        epochStart = 0,
+        epochEnd = dayjs().unix(),
+        events = [{ event: "foo" }, { event: "bar" }, { event: "baz" }],
+        superProps = { luckyNumber: [2, 2, 4, 4, 42, 42, 42, 2, 2, 4, 4, 42, 42, 42, 420] },
+        funnels = [],
+        userProps = {
+            spiritAnimal: chance.animal.bind(chance),
+        },
+        scdProps = {},
+        mirrorProps = {},
+        groupKeys = [],
+        groupProps = {},
+        lookupTables = [],
+        hasAnonIds = false,
+        hasSessionIds = false,
+        format = "csv",
+        token = null,
+        region = "US",
+        writeToDisk = false,
+        verbose = false,
+        makeChart = false,
+        soup = {},
+        hook = (record) => record,
+        hasAdSpend = false,
+        hasCampaigns = false,
+        hasLocation = false,
+        hasAvatar = false,
+        isAnonymous = false,
+        hasBrowser = false,
+        hasAndroidDevices = false,
+        hasDesktopDevices = false,
+        hasIOSDevices = false,
+        alsoInferFunnels = false,
+        name = "",
+        batchSize = 500_000,
+        concurrency = 500
+    } = config;
+
+    // Ensure defaults for deep objects
+    if (!config.superProps) config.superProps = superProps;
+    if (!config.userProps || Object.keys(config?.userProps || {})) config.userProps = userProps;
+
+    // Setting up "TIME"
+    if (epochStart && !numDays) numDays = dayjs.unix(epochEnd).diff(dayjs.unix(epochStart), "day");
+    if (!epochStart && numDays) epochStart = dayjs.unix(epochEnd).subtract(numDays, "day").unix();
+    if (epochStart && numDays) { } // noop
+    if (!epochStart && !numDays) {
+        throw new Error("Either epochStart or numDays must be provided");
+    }
+
+    // Generate simulation name
+    config.simulationName = name || makeName();
+    config.name = config.simulationName;
+
+    // Validate events
+    if (!events || !events.length) events = [{ event: "foo" }, { event: "bar" }, { event: "baz" }];
+    
+    // Convert string events to objects
+    if (typeof events[0] === "string") {
+        events = events.map(e => ({ event: e }));
+    }
+
+    // Handle funnel inference
+    if (alsoInferFunnels) {
+        const inferredFunnels = inferFunnels(events);
+        funnels = [...funnels, ...inferredFunnels];
+    }
+
+    // Create funnel for events not in other funnels
+    const eventContainedInFunnels = Array.from(funnels.reduce((acc, f) => {
+        const events = f.sequence;
+        events.forEach(event => acc.add(event));
+        return acc;
+    }, new Set()));
+
+    const eventsNotInFunnels = events
+        .filter(e => !e.isFirstEvent)
+        .filter(e => !eventContainedInFunnels.includes(e.event))
+        .map(e => e.event);
+        
+    if (eventsNotInFunnels.length) {
+        const sequence = u.shuffleArray(eventsNotInFunnels.flatMap(event => {
+            let evWeight;
+            // First check the config
+            if (config.events) {
+                evWeight = config.events.find(e => e.event === event)?.weight || 1;
+            }
+            // Fallback on default
+            else {
+                evWeight = 1;
+            }
+            return Array(evWeight).fill(event);
+        }));
+
+        funnels.push({
+            sequence,
+            conversionRate: 50,
+            order: 'random',
+            timeToConvert: 24 * 14,
+            requireRepeats: false,
+        });
+    }
+
+    // Event validation 
+    const validatedEvents = u.validateEventConfig(events);
+    
+    // Build final config object
+    const validatedConfig = {
+        ...config,
+        concurrency,
+        funnels,
+        batchSize,
+        seed,
+        numEvents,
+        numUsers,
+        numDays,
+        epochStart,
+        epochEnd,
+        events: validatedEvents,
+        superProps,
+        userProps,
+        scdProps,
+        mirrorProps,
+        groupKeys,
+        groupProps,
+        lookupTables,
+        hasAnonIds,
+        hasSessionIds,
+        format,
+        token,
+        region,
+        writeToDisk,
+        verbose,
+        makeChart,
+        soup,
+        hook,
+        hasAdSpend,
+        hasCampaigns,
+        hasLocation,
+        hasAvatar,
+        isAnonymous,
+        hasBrowser,
+        hasAndroidDevices,
+        hasDesktopDevices,
+        hasIOSDevices,
+        simulationName: config.simulationName,
+        name: config.name
+    };
+
+    return validatedConfig;
+}
+
+/**
+ * Validates configuration for required fields
+ * @param {Object} config - Configuration to validate
+ * @throws {Error} If required fields are missing
+ */
+export function validateRequiredConfig(config) {
+    if (!config) {
+        throw new Error("Configuration is required");
+    }
+    
+    if (typeof config !== 'object') {
+        throw new Error("Configuration must be an object");
+    }
+    
+    // Could add more specific validation here
+    return true;
+}
+
+export { inferFunnels };

package/lib/core/context.js

@@ -0,0 +1,180 @@
+/**
+ * Context module - replaces global variables with a context object
+ * Provides centralized state management and dependency injection
+ */
+
+/** @typedef {import('../../types').Dungeon} Dungeon */
+/** @typedef {import('../../types').Storage} Storage */
+/** @typedef {import('../../types').Context} Context */
+/** @typedef {import('../../types').RuntimeState} RuntimeState */
+/** @typedef {import('../../types').Defaults} Defaults */
+
+import dayjs from "dayjs";
+import { campaigns, devices, locations } from '../data/defaults.js';
+import * as u from '../utils/utils.js';
+
+/**
+ * Creates a defaults factory function that computes weighted defaults
+ * @param {Dungeon} config - Configuration object
+ * @param {Array} campaignData - Campaign data array
+ * @returns {Defaults} Defaults object with factory functions
+ */
+function createDefaults(config, campaignData) {
+    const { singleCountry } = config;
+    
+    // Pre-compute weighted arrays based on configuration
+    const locationsUsers = singleCountry ? 
+        locations.filter(l => l.country === singleCountry) : 
+        locations;
+    
+    const locationsEvents = singleCountry ? 
+        locations.filter(l => l.country === singleCountry) : 
+        locations;
+
+    return {
+        locationsUsers: () => u.weighArray(locationsUsers),
+        locationsEvents: () => u.weighArray(locationsEvents),
+        iOSDevices: () => u.weighArray(devices.iosDevices),
+        androidDevices: () => u.weighArray(devices.androidDevices),
+        desktopDevices: () => u.weighArray(devices.desktopDevices),
+        browsers: () => u.weighArray(devices.browsers),
+        campaigns: () => u.weighArray(campaignData)
+    };
+}
+
+/**
+ * Creates a runtime state object for tracking execution state
+ * @returns {RuntimeState} Runtime state with counters and flags
+ */
+function createRuntimeState() {
+    return {
+        operations: 0,
+        eventCount: 0,
+        userCount: 0,
+        isBatchMode: false,
+        verbose: false,
+        isCLI: false
+    };
+}
+
+/**
+ * Context factory that creates a complete context object for data generation
+ * @param {Dungeon} config - Validated configuration object
+ * @param {Storage|null} storage - Storage containers (optional, can be set later)
+ * @returns {Context} Context object containing all state and dependencies
+ */
+export function createContext(config, storage = null) {
+    // Import campaign data (could be made configurable)
+    const campaignData = campaigns;
+    
+    // Create computed defaults based on config
+    const defaults = createDefaults(config, campaignData);
+    
+    // Create runtime state
+    const runtime = createRuntimeState();
+    
+    // Set runtime flags from config
+    runtime.verbose = config.verbose || false;
+    runtime.isBatchMode = config.batchSize && config.batchSize < config.numEvents;
+    runtime.isCLI = process.argv[1].endsWith('index.js') || process.argv[1].endsWith('cli.js');
+    
+    const context = {
+        config,
+        storage,
+        defaults,
+        campaigns: campaignData,
+        runtime,
+        
+        // Helper methods for updating state
+        incrementOperations() {
+            runtime.operations++;
+        },
+        
+        incrementEvents() {
+            runtime.eventCount++;
+        },
+        
+        incrementUsers() {
+            runtime.userCount++;
+        },
+        
+        setStorage(storageObj) {
+            this.storage = storageObj;
+        },
+        
+        // Getter methods for runtime state
+        getOperations() {
+            return runtime.operations;
+        },
+        
+        getEventCount() {
+            return runtime.eventCount;
+        },
+        
+        getUserCount() {
+            return runtime.userCount;
+        },
+        
+        incrementUserCount() {
+            runtime.userCount++;
+        },
+        
+        incrementEventCount() {
+            runtime.eventCount++;
+        },
+        
+        isBatchMode() {
+            return runtime.isBatchMode;
+        },
+        
+        isCLI() {
+            return runtime.isCLI;
+        },
+        
+        // Time helper methods
+        getTimeShift() {
+            const actualNow = dayjs().add(2, "day");
+            return actualNow.diff(dayjs.unix(global.FIXED_NOW), "seconds");
+        },
+        
+        getDaysShift() {
+            const actualNow = dayjs().add(2, "day");
+            return actualNow.diff(dayjs.unix(global.FIXED_NOW), "days");
+        },
+        
+        // Time constants (previously globals)
+        FIXED_NOW: global.FIXED_NOW,
+        FIXED_BEGIN: global.FIXED_BEGIN
+    };
+    
+    return context;
+}
+
+/**
+ * Updates an existing context with new storage containers
+ * @param {Context} context - Existing context object
+ * @param {Storage} storage - New storage containers
+ * @returns {Context} Updated context object
+ */
+export function updateContextWithStorage(context, storage) {
+    context.storage = storage;
+    return context;
+}
+
+/**
+ * Validates that a context object has all required properties
+ * @param {Context} context - Context to validate
+ * @throws {Error} If context is missing required properties
+ */
+export function validateContext(context) {
+    const required = ['config', 'defaults', 'campaigns', 'runtime'];
+    const missing = required.filter(prop => !context[prop]);
+    
+    if (missing.length > 0) {
+        throw new Error(`Context is missing required properties: ${missing.join(', ')}`);
+    }
+    
+    if (!context.config.numUsers || !context.config.numEvents) {
+        throw new Error('Context config must have numUsers and numEvents');
+    }
+}