make-mp-data 1.5.56 → 2.0.1

package/lib/core/storage.js

@@ -0,0 +1,268 @@
+/**
+ * Storage module providing HookArray functionality for data transformation and batching
+ * Extracted from index.js to eliminate global dependencies
+ */
+
+/** @typedef {import('../../types').Context} Context */
+
+import { existsSync } from "fs";
+import pLimit from 'p-limit';
+import os from "os";
+import path from "path";
+import * as u from "../utils/utils.js";
+
+/**
+ * Creates a hooked array that transforms data on push and handles batching/disk writes
+ * @param {Array} arr - Base array to enhance
+ * @param {Object} opts - Configuration options
+ * @param {Function} opts.hook - Transform function applied to each item
+ * @param {string} opts.type - Type identifier for the hook function
+ * @param {string} opts.filepath - Base filename for disk writes
+ * @param {string} opts.format - Output format ('csv' or 'json')
+ * @param {number} opts.concurrency - Max concurrent file operations
+ * @param {Context} opts.context - Context object with config, batchSize, etc.
+ * @returns {Promise<Array>} Enhanced array with hookPush and flush methods
+ */
+export async function createHookArray(arr = [], opts = {}) {
+	const {
+		hook = a => a,
+		type = "",
+		filepath = "./defaultFile",
+		format = "csv",
+		concurrency = 1,
+		context = {},
+		...rest
+	} = opts;
+
+	const FILE_CONN = pLimit(concurrency);
+	const { config = {}, runtime = {} } = context;
+	const BATCH_SIZE = config.batchSize || 1_000_000;
+	const NODE_ENV = process.env.NODE_ENV || "unknown";
+
+	let batch = 0;
+	let writeDir;
+	let isBatchMode = runtime.isBatchMode || false;
+
+	// Determine write directory
+	const dataFolder = path.resolve("./data");
+	if (existsSync(dataFolder)) writeDir = dataFolder;
+	else writeDir = path.resolve("./");
+
+	if (NODE_ENV?.toLowerCase()?.startsWith("prod")) {
+		writeDir = path.resolve(os.tmpdir());
+	}
+
+	if (typeof config.writeToDisk === "string" && config.writeToDisk.startsWith('gs://')) {
+		writeDir = config.writeToDisk;
+	}
+
+	function getWritePath() {
+		if (isBatchMode) {
+			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}-part-${batch.toString()}.${format}`;
+			return path.join(writeDir, `${filepath}-part-${batch.toString()}.${format}`);
+		}
+		else {
+			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}.${format}`;
+			return path.join(writeDir, `${filepath}.${format}`);
+		}
+	}
+
+	function getWriteDir() {
+		return path.join(writeDir, `${filepath}.${format}`);
+	}
+
+	async function transformThenPush(item, meta) {
+		if (item === null || item === undefined) return false;
+		if (typeof item === 'object' && Object.keys(item).length === 0) return false;
+
+		const allMetaData = { ...rest, ...meta };
+
+		if (Array.isArray(item)) {
+			for (const i of item) {
+				try {
+					const enriched = await hook(i, type, allMetaData);
+					if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
+					else arr.push(enriched);
+				} catch (e) {
+					console.error(`\n\nyour hook had an error\n\n`, e);
+					arr.push(i);
+				}
+			}
+		} else {
+			try {
+				const enriched = await hook(item, type, allMetaData);
+				if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
+				else arr.push(enriched);
+			} catch (e) {
+				console.error(`\n\nyour hook had an error\n\n`, e);
+				arr.push(item);
+			}
+		}
+
+		if (arr.length > BATCH_SIZE) {
+			isBatchMode = true;
+			runtime.isBatchMode = true; // Update runtime state
+			batch++;
+			const writePath = getWritePath();
+			const writeResult = await FILE_CONN(() => writeToDisk(arr, { writePath }));
+			return writeResult;
+		} else {
+			return Promise.resolve(false);
+		}
+	}
+
+	async function writeToDisk(data, options) {
+		const { writePath } = options;
+		let writeResult;
+
+		if (config.verbose) {
+			console.log(`\n\n\twriting ${writePath}\n\n`);
+		}
+
+		switch (format) {
+			case "csv":
+				writeResult = await u.streamCSV(writePath, data);
+				break;
+			case "json":
+				writeResult = await u.streamJSON(writePath, data);
+				break;
+			default:
+				throw new Error(`format ${format} is not supported`);
+		}
+
+		if (isBatchMode) data.length = 0;
+		return writeResult;
+	}
+
+	async function flush() {
+		if (arr.length > 0) {
+			batch++;
+			const writePath = getWritePath();
+			await FILE_CONN(() => writeToDisk(arr, { writePath }));
+			if (isBatchMode) arr.length = 0; // free up memory for batch mode
+		}
+	}
+
+	// Enhance the array with our methods
+	const enrichedArray = arr;
+	enrichedArray.hookPush = transformThenPush;
+	enrichedArray.flush = flush;
+	enrichedArray.getWriteDir = getWriteDir;
+	enrichedArray.getWritePath = getWritePath;
+
+	// Add additional properties from rest
+	for (const key in rest) {
+		enrichedArray[key.toString()] = rest[key];
+	}
+
+	return enrichedArray;
+}
+
+/**
+ * Storage manager class for initializing and managing all storage containers
+ */
+export class StorageManager {
+	constructor(context) {
+		this.context = context;
+	}
+
+	/**
+	 * Initialize all storage containers for the data generation process
+	 * @returns {import('../../types').Storage} Storage containers object
+	 */
+	async initializeContainers() {
+		const { config } = this.context;
+
+		const storage = {
+			eventData: await createHookArray([], {
+				hook: config.hook,
+				type: "event",
+				filepath: `${config.simulationName || 'events'}-EVENTS`,
+				format: config.format || "csv",
+				concurrency: config.concurrency || 1,
+				context: this.context
+			}),
+
+			userProfilesData: await createHookArray([], {
+				hook: config.hook,
+				type: "user",
+				filepath: `${config.simulationName || 'users'}-USERS`,
+				format: config.format || "csv",
+				concurrency: config.concurrency || 1,
+				context: this.context
+			}),
+
+			adSpendData: await createHookArray([], {
+				hook: config.hook,
+				type: "adspend",
+				filepath: `${config.simulationName || 'adspend'}-ADSPEND`,
+				format: config.format || "csv",
+				concurrency: config.concurrency || 1,
+				context: this.context
+			}),
+
+			scdTableData: [],
+			groupProfilesData: [],
+			lookupTableData: [],
+
+			mirrorEventData: await createHookArray([], {
+				hook: config.hook,
+				type: "mirror",
+				filepath: `${config.simulationName || 'mirror'}-MIRROR`,
+				format: config.format || "csv",
+				concurrency: config.concurrency || 1,
+				context: this.context
+			})
+		};
+
+		// Initialize SCD tables if configured
+		if (config.scdProps && Object.keys(config.scdProps).length > 0) {
+			for (const scdKey of Object.keys(config.scdProps)) {
+				const scdArray = await createHookArray([], {
+					hook: config.hook,
+					type: "scd",
+					filepath: `${config.simulationName || 'scd'}-${scdKey}-SCD`,
+					format: config.format || "csv",
+					concurrency: config.concurrency || 1,
+					context: this.context
+				});
+				scdArray.scdKey = scdKey;
+				storage.scdTableData.push(scdArray);
+			}
+		}
+
+		// Initialize group profile tables if configured
+		if (config.groupKeys && config.groupKeys.length > 0) {
+			for (const [groupKey] of config.groupKeys) {
+				const groupArray = await createHookArray([], {
+					hook: config.hook,
+					type: "group",
+					filepath: `${config.simulationName || 'groups'}-${groupKey}-GROUPS`,
+					format: config.format || "csv",
+					concurrency: config.concurrency || 1,
+					context: this.context
+				});
+				groupArray.groupKey = groupKey;
+				storage.groupProfilesData.push(groupArray);
+			}
+		}
+
+		// Initialize lookup tables if configured
+		if (config.lookupTables && config.lookupTables.length > 0) {
+			for (const lookupConfig of config.lookupTables) {
+				const lookupArray = await createHookArray([], {
+					hook: config.hook,
+					type: "lookup",
+					filepath: `${config.simulationName || 'lookup'}-${lookupConfig.key}-LOOKUP`,
+					format: config.format || "csv",
+					concurrency: config.concurrency || 1,
+					context: this.context
+				});
+				lookupArray.lookupKey = lookupConfig.key;
+				storage.lookupTableData.push(lookupArray);
+			}
+		}
+
+		return storage;
+	}
+}

package/{components → lib/data}/defaults.js

@@ -1,8 +1,17 @@
 /* cSpell:disable */
+/**
+ * Default data sets for generating realistic test data
+ * @fileoverview Contains default values for campaigns, devices, locations, and domains
+ */
+
+/** @typedef {import('../../types.d.ts').main.Dungeon} Config */
+/** @typedef {import('../../types.d.ts').main.ValueValid} ValueValid */
+
 //? https://docs.mixpanel.com/docs/data-structure/property-reference#default-properties
 
 const domainSuffix = ["com", "com", "com", "com", "net", "org", "net", "org", "io", "co", "co.uk", "us", "biz", "info", "gov", "edu"];
 const domainPrefix = ["gmail", "gmail", "gmail", "gmail", "gmail", "gmail", "yahoo", "yahoo", "icloud", "icloud", "icloud", "icloud", "hotmail", "hotmail", "gmail", "gmail", "gmail", "gmail", "gmail", "gmail", "yahoo", "yahoo", "icloud", "icloud", "icloud", "hotmail", "hotmail", "outlook", "aol", "outlook", "aol", "protonmail", "zoho", "gmx", "yandex", "mail", "inbox", "fastmail", "tutanota", "mailfence", "disroot", "riseup", "posteo", "runbox", "kolabnow", "mailbox", "scryptmail", "ctemplar", "countermail", "hushmail", "startmail", "privatemail"];
+
 const campaigns = [
 	{
 		utm_campaign: ["$organic"],
@@ -25,7 +34,6 @@ const campaigns = [
 		utm_content: ["sc_control_group", "sc_variant_A", "sc_variant_B", "sc_variant_C", "sc_variant_D"],
 		utm_term: ["sc_jan_feb", "sc_mar_apr", "sc_may_jun", "sc_jul_aug", "sc_sep_oct", "sc_nov_dec"]
 	},
-	
 	{
 		utm_source: ["linkedin"],
 		utm_campaign: ["li_free_trial", "li_discount_US", "li_fall_sale", "li_holiday_special", "li_lookalike_audience"],
@@ -315,7 +323,6 @@ const androidDevices = [
 	}
 ];
 
-
 const desktopDevices = [
 	{
 		model: 'iMac 24-inch (M1, 2021)',
@@ -446,7 +453,6 @@ const desktopDevices = [
 	}
 ];
 
-
 const locations = [
 	{
 		"country": "United States",
@@ -1242,19 +1248,16 @@ const locations = [
 	}
 ];
 
-
-
-
-
-module.exports = {
+export {
 	campaigns,
-	devices: {
-		browsers,
-		androidDevices,
-		iosDevices,
-		desktopDevices
-	},
 	locations,
 	domainSuffix,
 	domainPrefix
 };
+
+export const devices = {
+	browsers,
+	androidDevices,
+	iosDevices,
+	desktopDevices
+};

package/lib/generators/adspend.js

@@ -0,0 +1,133 @@
+/**
+ * Ad Spend generator module
+ * Creates realistic advertising spend events with UTM parameters and metrics
+ */
+
+/** @typedef {import('../../types').Context} Context */
+
+import dayjs from "dayjs";
+import { md5 } from "ak-tools";
+import * as u from "../utils/utils.js";
+
+/**
+ * Creates ad spend events for a given day and campaign configurations
+ * @param {Context} context - Context object containing config, defaults, etc.
+ * @param {string} day - ISO date string for the ad spend day
+ * @param {Array} campaigns - Array of campaign configurations (optional, uses context.campaigns if not provided)
+ * @returns {Promise<Array>} Array of ad spend event objects
+ */
+export async function makeAdSpend(context, day, campaigns = null) {
+    // Update operation counter
+    context.incrementOperations();
+    
+    // Use campaigns from context if not provided
+    const campaignConfigs = campaigns || context.campaigns;
+    
+    if (!campaignConfigs || campaignConfigs.length === 0) {
+        return [];
+    }
+    
+    const chance = u.getChance();
+    const adSpendEvents = [];
+    
+    for (const network of campaignConfigs) {
+        const networkCampaigns = network.utm_campaign;
+        
+        for (const campaign of networkCampaigns) {
+            // Skip organic campaigns
+            if (campaign === "$organic") continue;
+
+            // Generate realistic ad spend metrics
+            const adSpendEvent = createAdSpendEvent(network, campaign, day, chance);
+            adSpendEvents.push(adSpendEvent);
+        }
+    }
+
+    return adSpendEvents;
+}
+
+/**
+ * Creates a single ad spend event with realistic metrics
+ * @param {Object} network - Network configuration object
+ * @param {string} campaign - Campaign name
+ * @param {string} day - ISO date string
+ * @param {Object} chance - Chance.js instance
+ * @returns {Object} Ad spend event object
+ */
+function createAdSpendEvent(network, campaign, day, chance) {
+    // Generate realistic cost
+    const cost = chance.floating({ min: 10, max: 250, fixed: 2 });
+
+    // Generate realistic CPC and CTR
+    const avgCPC = chance.floating({ min: 0.33, max: 2.00, fixed: 4 });
+    const avgCTR = chance.floating({ min: 0.05, max: 0.25, fixed: 4 });
+
+    // Calculate derived metrics
+    const clicks = Math.floor(cost / avgCPC);
+    const impressions = Math.floor(clicks / avgCTR);
+    const views = Math.floor(impressions * avgCTR);
+
+    // Generate UTM parameters
+    const utm_medium = u.choose(u.pickAWinner(network.utm_medium)());
+    const utm_content = u.choose(u.pickAWinner(network.utm_content)());
+    const utm_term = u.choose(u.pickAWinner(network.utm_term)());
+
+    // Create unique identifiers
+    const id = network.utm_source[0] + '-' + campaign;
+    const uid = md5(id);
+
+    return {
+        event: "$ad_spend",
+        time: day,
+        source: 'dm4',
+        utm_campaign: campaign,
+        campaign_id: id,
+        insert_id: uid,
+        network: network.utm_source[0].toUpperCase(),
+        distinct_id: network.utm_source[0].toUpperCase(),
+        utm_source: network.utm_source[0],
+        utm_medium,
+        utm_content,
+        utm_term,
+        clicks,
+        views,
+        impressions,
+        cost,
+        date: dayjs(day).format("YYYY-MM-DD"),
+    };
+}
+
+/**
+ * Validates campaign configuration
+ * @param {Array} campaigns - Campaign configurations to validate
+ * @returns {boolean} True if valid, throws error if invalid
+ */
+export function validateCampaigns(campaigns) {
+    if (!Array.isArray(campaigns)) {
+        throw new Error("Campaigns must be an array");
+    }
+    
+    for (const network of campaigns) {
+        if (!network.utm_source || !Array.isArray(network.utm_source)) {
+            throw new Error("Each campaign network must have utm_source array");
+        }
+        
+        if (!network.utm_campaign || !Array.isArray(network.utm_campaign)) {
+            throw new Error("Each campaign network must have utm_campaign array");
+        }
+        
+        if (!network.utm_medium || !Array.isArray(network.utm_medium)) {
+            throw new Error("Each campaign network must have utm_medium array");
+        }
+        
+        if (!network.utm_content || !Array.isArray(network.utm_content)) {
+            throw new Error("Each campaign network must have utm_content array");
+        }
+        
+        if (!network.utm_term || !Array.isArray(network.utm_term)) {
+            throw new Error("Each campaign network must have utm_term array");
+        }
+    }
+    
+    return true;
+}