make-mp-data 1.4.4 → 1.5.0

package/{core → src}/utils.js

@@ -15,8 +15,8 @@ const { domainSuffix, domainPrefix } = require('./defaults');
 /** @typedef {import('../types').Config} Config */
 /** @typedef {import('../types').EventConfig} EventConfig */
 /** @typedef {import('../types').ValueValid} ValueValid */
-/** @typedef {import('../types').EnrichedArray} EnrichArray */
-/** @typedef {import('../types').EnrichArrayOptions} EnrichArrayOptions */
+/** @typedef {import('../types').HookedArray} hookArray */
+/** @typedef {import('../types').hookArrayOptions} hookArrayOptions */
 /** @typedef {import('../types').Person} Person */
 /** @typedef {import('../types').Funnel} Funnel */
 
@@ -34,14 +34,15 @@ RNG
 /**
  * the random number generator initialization function
  * @param  {string} seed
+ * @returns {Chance}
  */
 function initChance(seed) {
 	if (process.env.SEED) seed = process.env.SEED;  // Override seed with environment variable if available
 	if (!chanceInitialized) {
 		globalChance = new Chance(seed);
-		if (global.MP_SIMULATION_CONFIG) global.MP_SIMULATION_CONFIG.chance = globalChance;
 		chanceInitialized = true;
 	}
+	return globalChance;
 }
 
 /**
@@ -50,12 +51,11 @@ function initChance(seed) {
  */
 function getChance() {
 	if (!chanceInitialized) {
-		const seed = process.env.SEED || global.MP_SIMULATION_CONFIG?.seed;
+		const seed = process.env.SEED || "";
 		if (!seed) {
-			return new Chance();
+			return new Chance(); // this is a new RNG and therefore not deterministic
 		}
-		initChance(seed);
-		return globalChance;
+		return initChance(seed);
 	}
 	return globalChance;
 }
@@ -242,64 +242,6 @@ function integer(min = 1, max = 100) {
 };
 
 
-/**
- * Creates a function that generates a weighted list of items
- * with a higher likelihood of picking a specified index and clear second and third place indices.
- * 
- * @param {Array} items - The list of items to pick from.
- * @param {number} [mostChosenIndex] - The index of the item to be most favored.
- * @returns {function} - A function that returns a weighted list of items.
- */
-function pickAWinner(items, mostChosenIndex) {
-	const chance = getChance();
-
-	// Ensure mostChosenIndex is within the bounds of the items array
-	if (!items) return () => { return ""; };
-	if (!items.length) return () => { return ""; };
-	if (!mostChosenIndex) mostChosenIndex = chance.integer({ min: 0, max: items.length - 1 });
-	if (mostChosenIndex >= items.length) mostChosenIndex = items.length - 1;
-
-	// Calculate second and third most chosen indices
-	const secondMostChosenIndex = (mostChosenIndex + 1) % items.length;
-	const thirdMostChosenIndex = (mostChosenIndex + 2) % items.length;
-
-	// Return a function that generates a weighted list
-	return function () {
-		const weighted = [];
-		for (let i = 0; i < 10; i++) {
-			const rand = chance.d10(); // Random number between 1 and 10
-
-			// 35% chance to favor the most chosen index
-			if (chance.bool({ likelihood: 35 })) {
-				// 50% chance to slightly alter the index
-				if (chance.bool({ likelihood: 50 })) {
-					weighted.push(items[mostChosenIndex]);
-				} else {
-					const addOrSubtract = chance.bool({ likelihood: 50 }) ? -rand : rand;
-					let newIndex = mostChosenIndex + addOrSubtract;
-
-					// Ensure newIndex is within bounds
-					if (newIndex < 0) newIndex = 0;
-					if (newIndex >= items.length) newIndex = items.length - 1;
-					weighted.push(items[newIndex]);
-				}
-			}
-			// 25% chance to favor the second most chosen index
-			else if (chance.bool({ likelihood: 25 })) {
-				weighted.push(items[secondMostChosenIndex]);
-			}
-			// 15% chance to favor the third most chosen index
-			else if (chance.bool({ likelihood: 15 })) {
-				weighted.push(items[thirdMostChosenIndex]);
-			}
-			// Otherwise, pick a random item from the list
-			else {
-				weighted.push(chance.pickone(items));
-			}
-		}
-		return weighted;
-	};
-}
 
 
 
@@ -359,61 +301,16 @@ function mapToRange(value, mean, sd) {
  * @param  {number} step=1
  */
 function range(a, b, step = 1) {
+	const arr = [];
 	step = !step ? 1 : step;
 	b = b / step;
 	for (var i = a; i <= b; i++) {
-		this.push(i * step);
+		arr.push(i * step);
 	}
-	return this;
+	return arr;
 };
 
 
-/**
- * create funnels out of random events
- * @param {EventConfig[]} events
- */
-function inferFunnels(events) {
-	const createdFunnels = [];
-	const firstEvents = events.filter((e) => e.isFirstEvent).map((e) => e.event);
-	const usageEvents = events.filter((e) => !e.isFirstEvent).map((e) => e.event);
-	const numFunnelsToCreate = Math.ceil(usageEvents.length);
-	/** @type {Funnel} */
-	const funnelTemplate = {
-		sequence: [],
-		conversionRate: 50,
-		order: 'sequential',
-		requireRepeats: false,
-		props: {},
-		timeToConvert: 1,
-		isFirstFunnel: false,
-		weight: 1
-	};
-	if (firstEvents.length) {
-		for (const event of firstEvents) {
-			createdFunnels.push({ ...clone(funnelTemplate), sequence: [event], isFirstFunnel: true, conversionRate: 100 });
-		}
-	}
-
-	//at least one funnel with all usage events
-	createdFunnels.push({ ...clone(funnelTemplate), sequence: usageEvents });
-
-	//for the rest, make random funnels
-	followUpFunnels: for (let i = 1; i < numFunnelsToCreate; i++) {
-		/** @type {Funnel} */
-		const funnel = { ...clone(funnelTemplate) };
-		funnel.conversionRate = integer(25, 75);
-		funnel.timeToConvert = integer(1, 10);
-		funnel.weight = integer(1, 10);
-		const sequence = shuffleArray(usageEvents).slice(0, integer(2, usageEvents.length));
-		funnel.sequence = sequence;
-		funnel.order = 'random';
-		createdFunnels.push(funnel);
-	}
-
-	return createdFunnels;
-
-}
-
 
 /*
 ----
@@ -491,7 +388,7 @@ function weighFunnels(acc, funnel) {
  * @param  {number} skew=1
  * @param  {number} size=100
  */
-function weightedRange(min, max, skew = 1, size = 50) {
+function weighNumRange(min, max, skew = 1, size = 50) {
 	if (size > 2000) size = 2000;
 	const mean = (max + min) / 2;
 	const sd = (max - min) / 4;
@@ -508,13 +405,16 @@ function weightedRange(min, max, skew = 1, size = 50) {
 	return array;
 }
 
+/**
+ * arbitrarily weigh an array of values to create repeats
+ * @param  {Array<any>} arr
+ */
 function weighArray(arr) {
-
 	// Calculate the upper bound based on the size of the array with added noise
 	const maxCopies = arr.length + integer(1, arr.length);
 
 	// Create an empty array to store the weighted elements
-	let weightedArray = [];
+	const weightedArray = [];
 
 	// Iterate over the input array and copy each element a random number of times
 	arr.forEach(element => {
@@ -527,6 +427,106 @@ function weighArray(arr) {
 	return weightedArray;
 }
 
+/**
+ * Creates a function that generates a weighted array of values.
+ * 
+ * @overload
+ * @param {Array<{value: string, weight: number}>} items - An array of weighted objects or an array of strings.
+ * @returns {function(): Array<string>} A function that returns a weighted array of values when called.
+ * 
+ * @overload
+ * @param {Array<string>} items - An array of strings.
+ * @returns {function(): Array<string>} A function that returns a weighted array with automatically assigned random weights to each string.
+ */
+
+function weighChoices(items) {
+	let weightedItems;
+
+	// If items are strings, assign unique random weights
+	if (items.every(item => typeof item === 'string')) {
+		const weights = shuffleArray(range(1, items.length));
+		weightedItems = items.map((item, index) => ({
+			value: item,
+			weight: weights[index]
+		}));
+	} else {
+		weightedItems = items;
+	}
+
+	return function generateWeightedArray() {
+		const weightedArray = [];
+
+		// Add each value to the array the number of times specified by its weight
+		weightedItems.forEach(({ value, weight }) => {
+			if (!weight) weight = 1;
+			for (let i = 0; i < weight; i++) {
+				weightedArray.push(value);
+			}
+		});
+
+		return weightedArray;
+	};
+}
+
+/**
+ * Creates a function that generates a weighted list of items
+ * with a higher likelihood of picking a specified index and clear second and third place indices.
+ * 
+ * @param {Array} items - The list of items to pick from.
+ * @param {number} [mostChosenIndex] - The index of the item to be most favored.
+ * @returns {function} - A function that returns a weighted list of items.
+ */
+function pickAWinner(items, mostChosenIndex) {
+	const chance = getChance();
+
+	// Ensure mostChosenIndex is within the bounds of the items array
+	if (!items) return () => { return ""; };
+	if (!items.length) return () => { return ""; };
+	if (!mostChosenIndex) mostChosenIndex = chance.integer({ min: 0, max: items.length - 1 });
+	if (mostChosenIndex >= items.length) mostChosenIndex = items.length - 1;
+
+	// Calculate second and third most chosen indices
+	const secondMostChosenIndex = (mostChosenIndex + 1) % items.length;
+	const thirdMostChosenIndex = (mostChosenIndex + 2) % items.length;
+
+	// Return a function that generates a weighted list
+	return function () {
+		const weighted = [];
+		for (let i = 0; i < 10; i++) {
+			const rand = chance.d10(); // Random number between 1 and 10
+
+			// 35% chance to favor the most chosen index
+			if (chance.bool({ likelihood: 35 })) {
+				// 50% chance to slightly alter the index
+				if (chance.bool({ likelihood: 50 })) {
+					weighted.push(items[mostChosenIndex]);
+				} else {
+					const addOrSubtract = chance.bool({ likelihood: 50 }) ? -rand : rand;
+					let newIndex = mostChosenIndex + addOrSubtract;
+
+					// Ensure newIndex is within bounds
+					if (newIndex < 0) newIndex = 0;
+					if (newIndex >= items.length) newIndex = items.length - 1;
+					weighted.push(items[newIndex]);
+				}
+			}
+			// 25% chance to favor the second most chosen index
+			else if (chance.bool({ likelihood: 25 })) {
+				weighted.push(items[secondMostChosenIndex]);
+			}
+			// 15% chance to favor the third most chosen index
+			else if (chance.bool({ likelihood: 15 })) {
+				weighted.push(items[thirdMostChosenIndex]);
+			}
+			// Otherwise, pick a random item from the list
+			else {
+				weighted.push(chance.pickone(items));
+			}
+		}
+		return weighted;
+	};
+}
+
 /*
 ----
 SHUFFLERS
@@ -627,7 +627,7 @@ function validateEventConfig(events) {
 	return cleanEventConfig;
 }
 
-function validateTime(chosenTime, earliestTime, latestTime) {
+function validTime(chosenTime, earliestTime, latestTime) {
 	if (!earliestTime) earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
 	if (!latestTime) latestTime = global.NOW;
 
@@ -644,6 +644,17 @@ function validateTime(chosenTime, earliestTime, latestTime) {
 	return false;
 }
 
+function validEvent(row) {
+	if (!row) return false;
+	if (!row.event) return false;
+	if (!row.time) return false;
+	if (!row.device_id && !row.user_id) return false;
+	if (!row.insert_id) return false;
+	if (!row.source) return false;
+	if (typeof row.time !== 'string') return false;
+	return true;
+}
+
 
 /*
 ----
@@ -651,76 +662,18 @@ META
 ----
 */
 
-/** 
- * our meta programming function which lets you mutate items as they are pushed into an array
- * @param  {any[]} arr
- * @param  {EnrichArrayOptions} opts
- * @returns {EnrichArray}}
- */
-function enrichArray(arr = [], opts = {}) {
-	const { hook = a => a, type = "", ...rest } = opts;
-
-	function transformThenPush(item) {
-		if (item === null) return false;
-		if (item === undefined) return false;
-		if (typeof item === 'object') {
-			if (Object.keys(item).length === 0) return false;
-		}
 
-		//hook is passed an array 
-		if (Array.isArray(item)) {
-			for (const i of item) {
-				try {
-					const enriched = hook(i, type, rest);
-					if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
-					else arr.push(enriched);
-
-				}
-				catch (e) {
-					console.error(`\n\nyour hook had an error\n\n`, e);
-					arr.push(i);
-					return false;
-				}
-
-			}
-			return true;
-		}
-
-		//hook is passed a single item
-		else {
-			try {
-				const enriched = hook(item, type, rest);
-				if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
-				else arr.push(enriched);
-				return true;
-			}
-			catch (e) {
-				console.error(`\n\nyour hook had an error\n\n`, e);
-				arr.push(item);
-				return false;
-			}
-		}
-
-	}
-
-	/** @type {EnrichArray} */
-	// @ts-ignore
-	const enrichedArray = arr;
-
-
-	enrichedArray.hookPush = transformThenPush;
-
-
-	return enrichedArray;
-};
 
+/**
+ * @param  {Config} config
+ */
 function buildFileNames(config) {
 	const { format = "csv", groupKeys = [], lookupTables = [] } = config;
 	let extension = "";
 	extension = format === "csv" ? "csv" : "json";
 	// const current = dayjs.utc().format("MM-DD-HH");
-	const simName = config.simulationName;
-	let writeDir = "./";
+	let simName = config.simulationName;
+	let writeDir = typeof config.writeToDisk === 'string' ? config.writeToDisk : "./";
 	if (config.writeToDisk) {
 		const dataFolder = path.resolve("./data");
 		if (existsSync(dataFolder)) writeDir = dataFolder;
@@ -732,13 +685,17 @@ function buildFileNames(config) {
 	const writePaths = {
 		eventFiles: [path.join(writeDir, `${simName}-EVENTS.${extension}`)],
 		userFiles: [path.join(writeDir, `${simName}-USERS.${extension}`)],
-		adSpendFiles: [path.join(writeDir, `${simName}-AD-SPEND.${extension}`)],
+		adSpendFiles: [],
 		scdFiles: [],
 		mirrorFiles: [],
 		groupFiles: [],
 		lookupFiles: [],
 		folder: writeDir,
 	};
+	//add ad spend files
+	if (config?.hasAdSpend) {
+		writePaths.adSpendFiles.push(path.join(writeDir, `${simName}-AD-SPEND.${extension}`));
+	}
 
 	//add SCD files
 	const scdKeys = Object.keys(config?.scdProps || {});
@@ -781,7 +738,6 @@ function buildFileNames(config) {
  * @param  {[string, number][]} arrayOfArrays
  */
 function progress(arrayOfArrays) {
-	// @ts-ignore
 	readline.cursorTo(process.stdout, 0);
 	let message = "";
 	for (const status of arrayOfArrays) {
@@ -819,8 +775,9 @@ CORE
 */
 
 //the function which generates $distinct_id + $anonymous_ids, $session_ids, and created, skewing towards the present
-function generateUser(user_id, numDays, amplitude = 1, frequency = 1, skew = 1) {
+function generateUser(user_id, opts, amplitude = 1, frequency = 1, skew = 1) {
 	const chance = getChance();
+	const { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds } = opts;
 	// Uniformly distributed `u`, then skew applied
 	let u = Math.pow(chance.random(), skew);
 
@@ -832,16 +789,18 @@ function generateUser(user_id, numDays, amplitude = 1, frequency = 1, skew = 1)
 
 	// Clamp values to ensure they are within the desired range
 	daysAgoBorn = Math.min(daysAgoBorn, numDays);
+	const props = person(user_id, daysAgoBorn, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds);
 
 	const user = {
 		distinct_id: user_id,
-		...person(numDays),
+		...props,
 	};
 
 
 	return user;
 }
 
+let soupHits = 0;
 /**
  * build sign waves basically
  * @param  {number} [earliestTime]
@@ -867,9 +826,12 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 	let isValidTime = false;
 	do {
 		iterations++;
+		soupHits++;
 		offset = chance.normal({ mean: mean, dev: chunkSize / deviation });
-		isValidTime = validateTime(chunkMid + offset, earliestTime, latestTime);
-		if (iterations > 10000) throw new Error("Too many iterations");
+		isValidTime = validTime(chunkMid + offset, earliestTime, latestTime);
+		if (iterations > 25000) {
+			throw `${iterations} iterations... exceeded`;
+		}
 	} while (chunkMid + offset < chunkStart || chunkMid + offset > chunkEnd);
 
 	try {
@@ -890,15 +852,17 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
  * @param {string} userId
  * @param  {number} bornDaysAgo=30
  * @param {boolean} isAnonymous
+ * @param {boolean} hasAvatar
+ * @param {boolean} hasAnonIds
+ * @param {boolean} hasSessionIds
  * @return {Person}
  */
-function person(userId, bornDaysAgo = 30, isAnonymous = false) {
+function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false, hasAnonIds = false, hasSessionIds = false) {
 	const chance = getChance();
 	//names and photos
 	const l = chance.letter.bind(chance);
 	let gender = chance.pickone(['male', 'female']);
 	if (!gender) gender = "female";
-	// @ts-ignore
 	let first = chance.first({ gender });
 	let last = chance.last();
 	let name = `${first} ${last}`;
@@ -926,21 +890,23 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false) {
 		user.name = "Anonymous User";
 		user.email = l() + l() + `*`.repeat(integer(3, 6)) + l() + `@` + l() + `*`.repeat(integer(3, 6)) + l() + `.` + choose(domainSuffix);
 		delete user.avatar;
-
 	}
 
+	if (!hasAvatar) delete user.avatar;
+
 	//anon Ids
-	if (global.MP_SIMULATION_CONFIG?.anonIds) {
+	if (hasAnonIds) {
 		const clusterSize = integer(2, 10);
 		for (let i = 0; i < clusterSize; i++) {
 			const anonId = uid(42);
 			user.anonymousIds.push(anonId);
 		}
-
 	}
 
+	if (!hasAnonIds) delete user.anonymousIds;
+
 	//session Ids
-	if (global.MP_SIMULATION_CONFIG?.sessionIds) {
+	if (hasSessionIds) {
 		const sessionSize = integer(5, 30);
 		for (let i = 0; i < sessionSize; i++) {
 			const sessionId = [uid(5), uid(5), uid(5), uid(5)].join("-");
@@ -948,6 +914,8 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false) {
 		}
 	}
 
+	if (!hasSessionIds) delete user.sessionIds;
+
 	return user;
 };
 
@@ -1022,11 +990,14 @@ module.exports = {
 
 	initChance,
 	getChance,
-	validateTime,
+
+	validTime,
+	validEvent,
+
 	boxMullerRandom,
 	applySkew,
 	mapToRange,
-	weightedRange,
+	weighNumRange,
 	progress,
 	range,
 	openFinder,
@@ -1044,11 +1015,10 @@ module.exports = {
 	shuffleOutside,
 	interruptArray,
 	generateUser,
-	enrichArray,
 	optimizedBoxMuller,
 	buildFileNames,
 	streamJSON,
 	streamCSV,
-	inferFunnels,
-	datesBetween
+	datesBetween,
+	weighChoices
 };

package/tests/benchmark/concurrency.mjs

@@ -0,0 +1,52 @@
+/*
+----
+TO DOs
+----
+*/
+
+//!feature: fixedTimeFunnel? if set this funnel will occur for all users at the same time ['cards charged', 'charge complete']
+//!feature: churn ... is churnFunnel, possible to return, etc
+//!feature: send SCD data to mixpanel (blocked on dev)
+//!feature: send and map lookup tables to mixpanel (also blocked on dev)
+//!bug: using --mc flag reverts to --complex for some reason
+
+
+import main from "../../index.js";
+import simple from '../../schemas/simple.js';
+
+/** @typedef {import('../../types').Config} Config */
+
+/** @type {Config} */
+const noWrites = {
+	...simple,
+	numUsers: 10_000,
+	numEvents: 250_000,
+	writeToDisk: false,
+};
+
+/** @type {Config} */
+const yesWrites = {
+	...noWrites,
+	writeToDisk: true
+};
+
+console.log('concurrency benchmarking');
+
+const concurrency = [1, 2, 3, 4, 5];
+
+const results = [];
+for (const concurrent of concurrency) {
+	console.log(`concurrency: ${concurrent}`);
+	// @ts-ignore
+	const test = await main({ ...noWrites, concurrency: concurrent });
+	results.push({ human: test.time.human, concurrency: concurrent });
+	console.log(`\t\tdone: ${test.time.human}\n\n`);
+}
+
+const display = results.map((r) => {
+	return `concurrency: ${r.concurrency} | duration: ${r.human}`;
+});
+
+console.log(display.join('\n\n'));
+
+debugger;