make-mp-data 1.3.4 → 1.4.1

package/utils.js

@@ -1,17 +1,73 @@
 const fs = require('fs');
-const Papa = require('papaparse');
 const Chance = require('chance');
-const chance = new Chance();
 const readline = require('readline');
 const { comma, uid } = require('ak-tools');
 const { spawn } = require('child_process');
 const dayjs = require('dayjs');
 const utc = require('dayjs/plugin/utc');
-
+const path = require('path');
+const { mkdir } = require('ak-tools');
 dayjs.extend(utc);
+require('dotenv').config();
+
+/** @typedef {import('./types').Config} Config */
+/** @typedef {import('./types').EventConfig} EventConfig */
+/** @typedef {import('./types').ValueValid} ValueValid */
+/** @typedef {import('./types').EnrichedArray} EnrichArray */
+/** @typedef {import('./types').EnrichArrayOptions} EnrichArrayOptions */
+/** @typedef {import('./types').Person} Person */
+/** @typedef {import('./types').Funnel} Funnel */
 
+let globalChance;
+let chanceInitialized = false;
+
+/*
+----
+RNG
+----
+*/
+
+/**
+ * the random number generator initialization function
+ * @param  {string} seed
+ */
+function initChance(seed) {
+	if (process.env.SEED) seed = process.env.SEED;  // Override seed with environment variable if available
+	if (!chanceInitialized) {
+		globalChance = new Chance(seed);
+		if (global.MP_SIMULATION_CONFIG) global.MP_SIMULATION_CONFIG.chance = globalChance;
+		chanceInitialized = true;
+	}
+}
 
+/**
+ * the random number generator getter function
+ * @returns {Chance}
+ */
+function getChance() {
+	if (!chanceInitialized) {
+		const seed = process.env.SEED || global.MP_SIMULATION_CONFIG?.seed;
+		if (!seed) {
+			return new Chance();
+		}
+		initChance(seed);
+		return globalChance;
+	}
+	return globalChance;
+}
+
+/*
+----
+PICKERS
+----
+*/
+
+/**
+ * choose a value from an array or a function
+ * @param  {ValueValid} items
+ */
 function pick(items) {
+	const chance = getChance();
 	if (!Array.isArray(items)) {
 		if (typeof items === 'function') {
 			const selection = items();
@@ -28,36 +84,41 @@ function pick(items) {
 	return chance.pickone(items);
 };
 
+/**
+ * returns a random date in the past or future
+ * @param  {number} inTheLast=30
+ * @param  {boolean} isPast=true
+ * @param  {string} format='YYYY-MM-DD'
+ */
 function date(inTheLast = 30, isPast = true, format = 'YYYY-MM-DD') {
-	const now = dayjs.utc();
-	// dates must be in the the last 10 years
+	const chance = getChance();
+	const now = global.NOW ? dayjs.unix(global.NOW) : dayjs();
 	if (Math.abs(inTheLast) > 365 * 10) inTheLast = chance.integer({ min: 1, max: 180 });
 	return function () {
-		try {
-			const when = chance.integer({ min: 0, max: Math.abs(inTheLast) });
-			let then;
-			if (isPast) {
-				then = now.subtract(when, 'day')
-					.subtract(integer(0, 23), 'hour')
-					.subtract(integer(0, 59), 'minute')
-					.subtract(integer(0, 59), 'second');
-			}
-			if (!isPast) {
-				then = now.add(when, 'day')
-					.add(integer(0, 23), 'hour')
-					.add(integer(0, 59), 'minute')
-					.add(integer(0, 59), 'second');
-			}
-			if (format) return then?.format(format);
-			if (!format) return then?.toISOString();
-		}
-		catch (e) {
-			if (format) return now?.format(format);
-			if (!format) return now?.toISOString();
+		const when = chance.integer({ min: 0, max: Math.abs(inTheLast) });
+		let then;
+		if (isPast) {
+			then = now.subtract(when, 'day')
+				.subtract(integer(0, 23), 'hour')
+				.subtract(integer(0, 59), 'minute')
+				.subtract(integer(0, 59), 'second');
+		} else {
+			then = now.add(when, 'day')
+				.add(integer(0, 23), 'hour')
+				.add(integer(0, 59), 'minute')
+				.add(integer(0, 59), 'second');
 		}
+
+		return format ? then.format(format) : then.toISOString();
 	};
-};
+}
 
+/**
+ * returns pairs of random date in the past or future
+ * @param  {number} inTheLast=30
+ * @param  {number} numPairs=5
+ * @param  {string} format='YYYY-MM-DD'
+ */
 function dates(inTheLast = 30, numPairs = 5, format = 'YYYY-MM-DD') {
 	const pairs = [];
 	for (let i = 0; i < numPairs; i++) {
@@ -66,11 +127,17 @@ function dates(inTheLast = 30, numPairs = 5, format = 'YYYY-MM-DD') {
 	return pairs;
 };
 
-function day(start, end) {
+/**
+ * returns a random date
+ * @param  {any} start
+ * @param  {any} end=global.NOW
+ */
+function day(start, end = global.NOW) {
+	const chance = getChance();
 	const format = 'YYYY-MM-DD';
 	return function (min, max) {
 		start = dayjs(start);
-		end = dayjs(end);
+		end = dayjs.unix(global.NOW);
 		const diff = end.diff(start, 'day');
 		const delta = chance.integer({ min: min, max: diff });
 		const day = start.add(delta, 'day');
@@ -83,7 +150,12 @@ function day(start, end) {
 
 };
 
+/**
+ * similar to pick
+ * @param  {ValueValid} value
+ */
 function choose(value) {
+	const chance = getChance();
 	try {
 		// Keep resolving the value if it's a function
 		while (typeof value === 'function') {
@@ -111,14 +183,24 @@ function choose(value) {
 		return '';
 	}
 }
+
+/**
+ * keeps picking from an array until the array is exhausted
+ * @param  {Array} arr
+ */
 function exhaust(arr) {
 	return function () {
 		return arr.shift();
 	};
 };
 
-
+/**
+ * returns a random integer between min and max
+ * @param  {number} min=1
+ * @param  {number} max=100
+ */
 function integer(min = 1, max = 100) {
+	const chance = getChance();
 	if (min === max) {
 		return min;
 	}
@@ -141,15 +223,69 @@ function integer(min = 1, max = 100) {
 };
 
 
-// Box-Muller transform to generate standard normally distributed values
+function pickAWinner(items, mostChosenIndex) {
+	const chance = getChance();
+	if (mostChosenIndex > items.length) mostChosenIndex = items.length;
+	return function () {
+		const weighted = [];
+		for (let i = 0; i < 10; i++) {
+			if (chance.bool({ likelihood: integer(10, 35) })) {
+				if (chance.bool({ likelihood: 50 })) {
+					weighted.push(items[mostChosenIndex]);
+				}
+				else {
+					const rand = chance.d10();
+					const addOrSubtract = chance.bool({ likelihood: 50 }) ? -rand : rand;
+					let newIndex = mostChosenIndex + addOrSubtract;
+					if (newIndex < 0) newIndex = 0;
+					if (newIndex > items.length) newIndex = items.length;
+					weighted.push(items[newIndex]);
+				}
+			}
+			else {
+				weighted.push(chance.pickone(items));
+			}
+		}
+		return weighted;
+
+	};
+}
+
+
+/*
+----
+GENERATORS
+----
+*/
+
+/**
+ * returns a random float between 0 and 1
+ * a substitute for Math.random
+ */
 function boxMullerRandom() {
+	const chance = getChance();
 	let u = 0, v = 0;
-	while (u === 0) u = Math.random();
-	while (v === 0) v = Math.random();
+	while (u === 0) u = chance.floating({ min: 0, max: 1, fixed: 13 });
+	while (v === 0) v = chance.floating({ min: 0, max: 1, fixed: 13 });
 	return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
 };
 
-// Apply skewness to the value
+function optimizedBoxMuller() {
+	const chance = getChance();
+	const u = Math.max(Math.min(chance.normal({ mean: .5, dev: .25 }), 1), 0);
+	const v = Math.max(Math.min(chance.normal({ mean: .5, dev: .25 }), 1), 0);
+	return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
+
+}
+
+/**
+ * applies a skew to a value;
+ * Skew=0.5: When the skew is 0.5, the distribution becomes more compressed, with values clustering closer to the mean.
+ * Skew=1: With a skew of 1, the distribution remains unchanged, as this is equivalent to applying no skew.
+ * Skew=2: When the skew is 2, the distribution spreads out, with values extending further from the mean.
+ * @param  {number} value
+ * @param  {number} skew
+ */
 function applySkew(value, skew) {
 	if (skew === 1) return value;
 	// Adjust the value based on skew
@@ -162,34 +298,103 @@ function mapToRange(value, mean, sd) {
 	return Math.round(value * sd + mean);
 };
 
-function unOptimizedWeightedRange(min, max, size = 100, skew = 1) {
-	const mean = (max + min) / 2;
-	const sd = (max - min) / 4;
-	let array = [];
+/**
+ * generate a range of numbers
+ * @param  {number} a
+ * @param  {number} b
+ * @param  {number} step=1
+ */
+function range(a, b, step = 1) {
+	step = !step ? 1 : step;
+	b = b / step;
+	for (var i = a; i <= b; i++) {
+		this.push(i * step);
+	}
+	return this;
+};
 
-	for (let i = 0; i < size; i++) {
-		let normalValue = boxMullerRandom();
-		let skewedValue = applySkew(normalValue, skew);
-		let mappedValue = mapToRange(skewedValue, mean, sd);
 
-		// Ensure the mapped value is within our min-max range
-		if (mappedValue >= min && mappedValue <= max) {
-			array.push(mappedValue);
-		} else {
-			i--; // If out of range, redo this iteration
-		}
-	}
+/*
+----
+STREAMERS
+----
+*/
+
+function streamJSON(path, data) {
+	return new Promise((resolve, reject) => {
+		const writeStream = fs.createWriteStream(path, { encoding: 'utf8' });
+		data.forEach(item => {
+			writeStream.write(JSON.stringify(item) + '\n');
+		});
+		writeStream.end();
+		writeStream.on('finish', () => {
+			resolve(path);
+		});
+		writeStream.on('error', reject);
+	});
+}
+
+function streamCSV(path, data) {
+	return new Promise((resolve, reject) => {
+		const writeStream = fs.createWriteStream(path, { encoding: 'utf8' });
+		// Extract all unique keys from the data array
+		const columns = getUniqueKeys(data);  // Assuming getUniqueKeys properly retrieves all keys
+
+		// Stream the header
+		writeStream.write(columns.join(',') + '\n');
+
+		// Stream each data row
+		data.forEach(item => {
+			for (const key in item) {
+				// Ensure all nested objects are properly stringified
+				if (typeof item[key] === "object") item[key] = JSON.stringify(item[key]);
+			}
+			const row = columns.map(col => item[col] ? `"${item[col].toString().replace(/"/g, '""')}"` : "").join(',');
+			writeStream.write(row + '\n');
+		});
+
+		writeStream.end();
+		writeStream.on('finish', () => {
+			resolve(path);
+		});
+		writeStream.on('error', reject);
+	});
+}
 
-	return array;
-};
 
-// optimized weighted range
-function weightedRange(min, max, size = 100, skew = 1) {
+/*
+----
+WEIGHERS
+----
+*/
+
+function weighFunnels(acc, funnel) {
+	const weight = funnel?.weight || 1;
+	for (let i = 0; i < weight; i++) {
+		acc.push(funnel);
+	}
+	return acc;
+}
+
+/**
+ * a utility function to generate a range of numbers within a given skew
+ * Skew = 0.5: The values are more concentrated towards the extremes (both ends of the range) with a noticeable dip in the middle. The distribution appears more "U" shaped. Larger sizes result in smoother distributions but maintain the overall shape.
+ * Skew = 1: This represents the default normal distribution without skew. The values are normally distributed around the mean. Larger sizes create a clearer bell-shaped curve.
+ * Skew = 2: The values are more concentrated towards the mean, with a steeper drop-off towards the extremes. The distribution appears more peaked, resembling a "sharper" bell curve. Larger sizes enhance the clarity of this peaked distribution.
+ * Size represents the size of the pool to choose from; Larger sizes result in smoother distributions but maintain the overall shape.
+ * @param  {number} min
+ * @param  {number} max
+ * @param  {number} skew=1
+ * @param  {number} size=100
+ */
+function weightedRange(min, max, skew = 1, size = 50) {
+	if (size > 2000) size = 2000;
 	const mean = (max + min) / 2;
 	const sd = (max - min) / 4;
 	const array = [];
 	while (array.length < size) {
-		const normalValue = boxMullerRandom();
+		// const normalValue = boxMullerRandom();
+		const normalValue = optimizedBoxMuller();
 		const skewedValue = applySkew(normalValue, skew);
 		const mappedValue = mapToRange(skewedValue, mean, sd);
 		if (mappedValue >= min && mappedValue <= max) {
@@ -199,24 +404,277 @@ function weightedRange(min, max, size = 100, skew = 1) {
 	return array;
 }
 
+function weighArray(arr) {
 
-function progress(thing, p) {
+	// Calculate the upper bound based on the size of the array with added noise
+	const maxCopies = arr.length + integer(1, arr.length);
+
+	// Create an empty array to store the weighted elements
+	let weightedArray = [];
+
+	// Iterate over the input array and copy each element a random number of times
+	arr.forEach(element => {
+		let copies = integer(1, maxCopies);
+		for (let i = 0; i < copies; i++) {
+			weightedArray.push(element);
+		}
+	});
+
+	return weightedArray;
+}
+
+/*
+----
+SHUFFLERS
+----
+*/
+
+// Function to shuffle array
+function shuffleArray(array) {
+	const chance = getChance();
+	for (let i = array.length - 1; i > 0; i--) {
+		const j = chance.integer({ min: 0, max: i });
+		[array[i], array[j]] = [array[j], array[i]];
+	}
+	return array;
+}
+
+function shuffleExceptFirst(array) {
+	if (array.length <= 1) return array;
+	const restShuffled = shuffleArray(array.slice(1));
+	return [array[0], ...restShuffled];
+}
+
+function shuffleExceptLast(array) {
+	if (array.length <= 1) return array;
+	const restShuffled = shuffleArray(array.slice(0, -1));
+	return [...restShuffled, array[array.length - 1]];
+}
+
+function fixFirstAndLast(array) {
+	if (array.length <= 2) return array;
+	const middleShuffled = shuffleArray(array.slice(1, -1));
+	return [array[0], ...middleShuffled, array[array.length - 1]];
+}
+
+function shuffleMiddle(array) {
+	if (array.length <= 2) return array;
+	const middleShuffled = shuffleArray(array.slice(1, -1));
+	return [array[0], ...middleShuffled, array[array.length - 1]];
+}
+
+function shuffleOutside(array) {
+	if (array.length <= 2) return array;
+	const middleFixed = array.slice(1, -1);
+	const outsideShuffled = shuffleArray([array[0], array[array.length - 1]]);
+	return [outsideShuffled[0], ...middleFixed, outsideShuffled[1]];
+}
+
+/**
+ * @param  {EventConfig[]} funnel
+ * @param  {EventConfig[]} possibles
+ */
+function interruptArray(funnel, possibles, percent = 50) {
+	if (!Array.isArray(funnel)) return funnel;
+	if (!Array.isArray(possibles)) return funnel;
+	if (!funnel.length) return funnel;
+	if (!possibles.length) return funnel;
+	const ignorePositions = [0, funnel.length - 1];
+	const chance = getChance();
+	loopSteps: for (const [index, event] of funnel.entries()) {
+		if (ignorePositions.includes(index)) continue loopSteps;
+		if (chance.bool({ likelihood: percent })) {
+			funnel[index] = chance.pickone(possibles);
+		}
+	}
+
+	return funnel;
+}
+
+/*
+----
+VALIDATORS
+----
+*/
+
+
+/**
+ * @param  {EventConfig[] | string[]} events
+ */
+function validateEventConfig(events) {
+	if (!Array.isArray(events)) throw new Error("events must be an array");
+	const cleanEventConfig = [];
+	for (const event of events) {
+		if (typeof event === "string") {
+			/** @type {EventConfig} */
+			const eventTemplate = {
+				event,
+				isFirstEvent: false,
+				properties: {},
+				weight: integer(1, 5)
+			};
+			cleanEventConfig.push(eventTemplate);
+		}
+		if (typeof event === "object") {
+			cleanEventConfig.push(event);
+		}
+	}
+	return cleanEventConfig;
+}
+
+function validateTime(chosenTime, earliestTime, latestTime) {
+	if (!earliestTime) earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
+	if (!latestTime) latestTime = global.NOW;
+
+	if (typeof chosenTime === 'number') {
+		if (chosenTime > 0) {
+			if (chosenTime > earliestTime) {
+				if (chosenTime < latestTime) {
+					return true;
+				}
+
+			}
+		}
+	}
+	return false;
+}
+
+
+/*
+----
+META
+----
+*/
+
+/** 
+ * our meta programming function which lets you mutate items as they are pushed into an array
+ * @param  {any[]} arr
+ * @param  {EnrichArrayOptions} opts
+ * @returns {EnrichArray}}
+ */
+function enrichArray(arr = [], opts = {}) {
+	const { hook = a => a, type = "", ...rest } = opts;
+
+	function transformThenPush(item) {
+		if (item === null) return false;
+		if (item === undefined) return false;
+		if (typeof item === 'object') {
+			if (Object.keys(item).length === 0) return false;
+		}
+
+		//hook is passed an array 
+		if (Array.isArray(item)) {
+			for (const i of item) {
+				try {
+					const enriched = hook(i, type, rest);
+					if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
+					else arr.push(enriched);
+
+				}
+				catch (e) {
+					console.error(`\n\nyour hook had an error\n\n`, e);
+					arr.push(i);
+					return false;
+				}
+
+			}
+			return true;
+		}
+
+		//hook is passed a single item
+		else {
+			try {
+				const enriched = hook(item, type, rest);
+				if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
+				else arr.push(enriched);
+				return true;
+			}
+			catch (e) {
+				console.error(`\n\nyour hook had an error\n\n`, e);
+				arr.push(item);
+				return false;
+			}
+		}
+
+	}
+
+	/** @type {EnrichArray} */
 	// @ts-ignore
-	readline.cursorTo(process.stdout, 0);
-	process.stdout.write(`${thing} processed ... ${comma(p)}`);
+	const enrichedArray = arr;
+
+
+	enrichedArray.hookPush = transformThenPush;
+
+
+	return enrichedArray;
 };
 
-function range(a, b, step = 1) {
-	step = !step ? 1 : step;
-	b = b / step;
-	for (var i = a; i <= b; i++) {
-		this.push(i * step);
+function buildFileNames(config) {
+	const { format = "csv", groupKeys = [], lookupTables = [] } = config;
+	let extension = "";
+	extension = format === "csv" ? "csv" : "json";
+	// const current = dayjs.utc().format("MM-DD-HH");
+	const simName = config.simulationName;
+	let writeDir = "./";
+	if (config.writeToDisk) writeDir = mkdir("./data");
+	if (typeof writeDir !== "string") throw new Error("writeDir must be a string");
+	if (typeof simName !== "string") throw new Error("simName must be a string");
+
+	const writePaths = {
+		eventFiles: [path.join(writeDir, `${simName}-EVENTS.${extension}`)],
+		userFiles: [path.join(writeDir, `${simName}-USERS.${extension}`)],
+		scdFiles: [],
+		mirrorFiles: [],
+		groupFiles: [],
+		lookupFiles: [],
+		folder: writeDir,
+	};
+
+	//add SCD files
+	const scdKeys = Object.keys(config?.scdProps || {});
+	for (const key of scdKeys) {
+		writePaths.scdFiles.push(
+			path.join(writeDir, `${simName}-${key}-SCD.${extension}`)
+		);
 	}
-	return this;
+
+	//add group files
+	for (const groupPair of groupKeys) {
+		const groupKey = groupPair[0];
+
+		writePaths.groupFiles.push(
+			path.join(writeDir, `${simName}-${groupKey}-GROUP.${extension}`)
+		);
+	}
+
+	//add lookup files
+	for (const lookupTable of lookupTables) {
+		const { key } = lookupTable;
+		writePaths.lookupFiles.push(
+			//lookups are always CSVs
+			path.join(writeDir, `${simName}-${key}-LOOKUP.csv`)
+		);
+	}
+
+	//add mirror files
+	const mirrorProps = config?.mirrorProps || {};
+	if (Object.keys(mirrorProps).length) {
+		writePaths.mirrorFiles.push(
+			path.join(writeDir, `${simName}-MIRROR.${extension}`)
+		);
+	}
+
+	return writePaths;
+}
+
+
+function progress(thing, p) {
+	// @ts-ignore
+	readline.cursorTo(process.stdout, 0);
+	process.stdout.write(`${thing} processed ... ${comma(p)}`);
 };
 
 
-//helper to open the finder
 function openFinder(path, callback) {
 	path = path || '/';
 	let p = spawn('open', [path]);
@@ -234,56 +692,112 @@ function getUniqueKeys(data) {
 	return Array.from(keysSet);
 };
 
-//
+
+
+/*
+----
+CORE
+----
+*/
+
+//the function which generates $distinct_id + $anonymous_ids, $session_ids, and created, skewing towards the present
+function generateUser(user_id, numDays, amplitude = 1, frequency = 1, skew = 1) {
+	const chance = getChance();
+	// Uniformly distributed `u`, then skew applied
+	let u = Math.pow(chance.random(), skew);
+
+	// Sine function for a smoother curve
+	const sineValue = (Math.sin(u * Math.PI * frequency - Math.PI / 2) * amplitude + 1) / 2;
+
+	// Scale the sineValue to the range of days
+	let daysAgoBorn = Math.round(sineValue * (numDays - 1)) + 1;
+
+	// Clamp values to ensure they are within the desired range
+	daysAgoBorn = Math.min(daysAgoBorn, numDays);
+
+	const user = {
+		distinct_id: user_id,
+		...person(numDays),
+	};
+
+
+	return user;
+}
+
 /**
- * makes a random-sized array of emojis
- * @param  {number} max=10
- * @param  {boolean} array=false
+ * build sign waves basically
+ * @param  {number} [earliestTime]
+ * @param  {number} [latestTime]
+ * @param  {number} [peaks=5]
  */
-function generateEmoji(max = 10, array = false) {
-	return function () {
-		const emojis = ['😀', '😂', '😍', '😎', '😜', '😇', '😡', '😱', '😭', '😴', '🤢', '🤠', '🤡', '👽', '👻', '💩', '👺', '👹', '👾', '🤖', '🤑', '🤗', '🤓', '🤔', '🤐', '😀', '😂', '😍', '😎', '😜', '😇', '😡', '😱', '😭', '😴', '🤢', '🤠', '🤡', '👽', '👻', '💩', '👺', '👹', '👾', '🤖', '🤑', '🤗', '🤓', '🤔', '🤐', '😈', '👿', '👦', '👧', '👨', '👩', '👴', '👵', '👶', '🧒', '👮', '👷', '💂', '🕵', '👩‍⚕️', '👨‍⚕️', '👩‍🌾', '👨‍🌾', '👩‍🍳', '👨‍🍳', '👩‍🎓', '👨‍🎓', '👩‍🎤', '👨‍🎤', '👩‍🏫', '👨‍🏫', '👩‍🏭', '👨‍🏭', '👩‍💻', '👨‍💻', '👩‍💼', '👨‍💼', '👩‍🔧', '👨‍🔧', '👩‍🔬', '👨‍🔬', '👩‍🎨', '👨‍🎨', '👩‍🚒', '👨‍🚒', '👩‍✈️', '👨‍✈️', '👩‍🚀', '👨‍🚀', '👩‍⚖️', '👨‍⚖️', '🤶', '🎅', '👸', '🤴', '👰', '🤵', '👼', '🤰', '🙇', '💁', '🙅', '🙆', '🙋', '🤦', '🤷', '🙎', '🙍', '💇', '💆', '🕴', '💃', '🕺', '🚶', '🏃', '🤲', '👐', '🙌', '👏', '🤝', '👍', '👎', '👊', '✊', '🤛', '🤜', '🤞', '✌️', '🤟', '🤘', '👌', '👈', '👉', '👆', '👇', '☝️', '✋', '🤚', '🖐', '🖖', '👋', '🤙', '💪', '🖕', '✍️', '🤳', '💅', '👂', '👃', '👣', '👀', '👁', '🧠', '👅', '👄', '💋', '👓', '🕶', '👔', '👕', '👖', '🧣', '🧤', '🧥', '🧦', '👗', '👘', '👙', '👚', '👛', '👜', '👝', '🛍', '🎒', '👞', '👟', '👠', '👡', '👢', '👑', '👒', '🎩', '🎓', '🧢', '⛑', '📿', '💄', '💍', '💎', '🔇', '🔈', '🔉', '🔊', '📢', '📣', '📯', '🔔', '🔕', '🎼', '🎵', '🎶', '🎙', '🎚', '🎛', '🎤', '🎧', '📻', '🎷', '🎸', '🎹', '🎺', '🎻', '🥁', '📱', '📲', '💻', '🖥', '🖨', '🖱', '🖲', '🕹', '🗜', '💽', '💾', '💿', '📀', '📼', '📷', '📸', '📹', '🎥', '📽', '🎞', '📞', '☎️', '📟', '📠', '📺', '📻', '🎙', '📡', '🔍', '🔎', '🔬', '🔭', '📡', '💡', '🔦', '🏮', '📔', '📕', '📖', '📗', '📘', '📙', '📚', '📓', '📒', '📃', '📜', '📄', '📰', '🗞', '📑', '🔖', '🏷', '💰', '💴', '💵', '💶', '💷', '💸', '💳', '🧾', '💹', '💱', '💲', '✉️', '📧', '📨', '📩', '📤', '📥', '📦', '📫', '📪', '📬', '📭', '📮', '🗳', '✏️', '✒️', '🖋', '🖊', '🖌', '🖍', '📝', '💼', '📁', '📂', '🗂', '📅', '📆', '🗒', '🗓', '📇', '📈', '📉', '📊', '📋', '📌', '📍', '📎', '🖇', '📏', '📐', '✂️', '🗃', '🗄', '🗑', '🔒', '🔓', '🔏', '🔐', '🔑', '🗝', '🔨', '⛏', '⚒', '🛠', '🗡', '⚔️', '🔫', '🏹', '🛡', '🔧', '🔩', '⚙️', '🗜', '⚖️', '🔗', '⛓', '🧰', '🧲', '⚗️', '🧪', '🧫', '🧬', '🔬', '🔭', '📡', '💉', '💊', '🛏', '🛋', '🚪', '🚽', '🚿', '🛁', '🧴', '🧷', '🧹', '🧺', '🧻', '🧼', '🧽', '🧯', '🚬', '⚰️', '⚱️', '🗿', '🏺', '🧱', '🎈', '🎏', '🎀', '🎁', '🎊', '🎉', '🎎', '🏮', '🎐', '🧧', '✉️', '📩', '📨', '📧'];
-		let num = integer(1, max);
-		let arr = [];
-		for (let i = 0; i < num; i++) {
-			arr.push(chance.pickone(emojis));
-		}
-		if (array) return arr;
-		if (!array) return arr.join(', ');
-		return "🤷";
-	};
-};
+function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0) {
+	if (!earliestTime) earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
+	if (!latestTime) latestTime = global.NOW;
+	const chance = getChance();
+	const totalRange = latestTime - earliestTime;
+	const chunkSize = totalRange / peaks;
+
+	// Select a random chunk based on the number of peaks
+	const peakIndex = integer(0, peaks - 1);
+	const chunkStart = earliestTime + peakIndex * chunkSize;
+	const chunkEnd = chunkStart + chunkSize;
+	const chunkMid = (chunkStart + chunkEnd) / 2;
+
+	// Generate a single timestamp within this chunk using a normal distribution centered at chunkMid
+	let offset;
+	let iterations = 0;
+	let isValidTime = false;
+	do {
+		iterations++;
+		offset = chance.normal({ mean: mean, dev: chunkSize / deviation });
+		isValidTime = validateTime(chunkMid + offset, earliestTime, latestTime);
+		if (iterations > 10000) throw new Error("Too many iterations");
+	} while (chunkMid + offset < chunkStart || chunkMid + offset > chunkEnd);
+
+	try {
+		return dayjs.unix(chunkMid + offset).toISOString();
+	}
+
+	catch (e) {
+		//escape hatch
+		// console.log('BAD TIME', e?.message);
+		return dayjs.unix(integer(earliestTime, latestTime)).toISOString();
+	}
+}
+
+
 
-/** @typedef {import('./types').Person} Person */
 
 /**
  * @param  {number} bornDaysAgo=30
  * @return {Person}
  */
 function person(bornDaysAgo = 30) {
+	const chance = getChance();
 	//names and photos
 	let gender = chance.pickone(['male', 'female']);
 	if (!gender) gender = "female";
 	// @ts-ignore
 	const first = chance.first({ gender });
 	const last = chance.last();
-	const $name = `${first} ${last}`;
-	const $email = `${first[0]}.${last}@${chance.domain()}.com`;
+	const name = `${first} ${last}`;
+	const email = `${first[0]}.${last}@${chance.domain()}.com`;
 	const avatarPrefix = `https://randomuser.me/api/portraits`;
 	const randomAvatarNumber = chance.integer({
 		min: 1,
 		max: 99
 	});
 	const avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
-	const $avatar = avatarPrefix + avPath;
-	const $created = date(bornDaysAgo, true)();
+	const avatar = avatarPrefix + avPath;
+	const created = dayjs.unix(global.NOW).subtract(bornDaysAgo, 'day').format('YYYY-MM-DD');
+	// const created = date(bornDaysAgo, true)();
 
 	/** @type {Person} */
 	const user = {
-		$name,
-		$email,
-		$avatar,
-		$created,
+		name,
+		email,
+		avatar,
+		created,
 		anonymousIds: [],
 		sessionIds: []
 	};
@@ -311,76 +825,59 @@ function person(bornDaysAgo = 30) {
 };
 
 
-function weighList(items, mostChosenIndex) {
-	if (mostChosenIndex > items.length) mostChosenIndex = items.length;
-	return function () {
-		const weighted = [];
-		for (let i = 0; i < 10; i++) {
-			if (chance.bool({ likelihood: integer(10, 35) })) {
-				if (chance.bool({ likelihood: 50 })) {
-					weighted.push(items[mostChosenIndex]);
-				}
-				else {
-					const rand = chance.d10();
-					const addOrSubtract = chance.bool({ likelihood: 50 }) ? -rand : rand;
-					let newIndex = mostChosenIndex + addOrSubtract;
-					if (newIndex < 0) newIndex = 0;
-					if (newIndex > items.length) newIndex = items.length;
-					weighted.push(items[newIndex]);
-				}
-			}
-			else {
-				weighted.push(chance.pickone(items));
-			}
-		}
-		return weighted;
-
-	};
-}
 
 
+//UNUSED
 
+function fixFunkyTime(earliestTime, latestTime) {
+	if (!earliestTime) earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
+	// if (typeof earliestTime !== "number") {
+	// 	if (parseInt(earliestTime) > 0) earliestTime = parseInt(earliestTime);
+	// 	if (dayjs(earliestTime).isValid()) earliestTime = dayjs(earliestTime).unix();
+	// }
+	if (typeof earliestTime !== "number") earliestTime = dayjs.unix(earliestTime).unix();
+	if (typeof latestTime !== "number") latestTime = global.NOW;
+	if (typeof latestTime === "number" && latestTime > global.NOW) latestTime = global.NOW;
+	if (earliestTime > latestTime) {
+		const tempEarlyTime = earliestTime;
+		const tempLateTime = latestTime;
+		earliestTime = tempLateTime;
+		latestTime = tempEarlyTime;
+	}
+	if (earliestTime === latestTime) {
+		earliestTime = dayjs.unix(earliestTime)
+			.subtract(integer(1, 14), "day")
+			.subtract(integer(1, 23), "hour")
+			.subtract(integer(1, 59), "minute")
+			.subtract(integer(1, 59), "second")
+			.unix();
+	}
+	return [earliestTime, latestTime];
 
-function streamJSON(path, data) {
-	return new Promise((resolve, reject) => {
-		const writeStream = fs.createWriteStream(path, { encoding: 'utf8' });
-		data.forEach(item => {
-			writeStream.write(JSON.stringify(item) + '\n');
-		});
-		writeStream.end();
-		writeStream.on('finish', () => {
-			resolve(path);
-		});
-		writeStream.on('error', reject);
-	});
 }
 
-function streamCSV(path, data) {
-	return new Promise((resolve, reject) => {
-		const writeStream = fs.createWriteStream(path, { encoding: 'utf8' });
-		// Extract all unique keys from the data array
-		const columns = getUniqueKeys(data);  // Assuming getUniqueKeys properly retrieves all keys
 
-		// Stream the header
-		writeStream.write(columns.join(',') + '\n');
 
-		// Stream each data row
-		data.forEach(item => {
-			for (const key in item) {
-				// Ensure all nested objects are properly stringified
-				if (typeof item[key] === "object") item[key] = JSON.stringify(item[key]);
-			}
-			const row = columns.map(col => item[col] ? `"${item[col].toString().replace(/"/g, '""')}"` : "").join(',');
-			writeStream.write(row + '\n');
-		});
+/**
+ * makes a random-sized array of emojis
+ * @param  {number} max=10
+ * @param  {boolean} array=false
+ */
+function generateEmoji(max = 10, array = false) {
+	const chance = getChance();
+	return function () {
+		const emojis = ['😀', '😂', '😍', '😎', '😜', '😇', '😡', '😱', '😭', '😴', '🤢', '🤠', '🤡', '👽', '👻', '💩', '👺', '👹', '👾', '🤖', '🤑', '🤗', '🤓', '🤔', '🤐', '😀', '😂', '😍', '😎', '😜', '😇', '😡', '😱', '😭', '😴', '🤢', '🤠', '🤡', '👽', '👻', '💩', '👺', '👹', '👾', '🤖', '🤑', '🤗', '🤓', '🤔', '🤐', '😈', '👿', '👦', '👧', '👨', '👩', '👴', '👵', '👶', '🧒', '👮', '👷', '💂', '🕵', '👩‍⚕️', '👨‍⚕️', '👩‍🌾', '👨‍🌾', '👩‍🍳', '👨‍🍳', '👩‍🎓', '👨‍🎓', '👩‍🎤', '👨‍🎤', '👩‍🏫', '👨‍🏫', '👩‍🏭', '👨‍🏭', '👩‍💻', '👨‍💻', '👩‍💼', '👨‍💼', '👩‍🔧', '👨‍🔧', '👩‍🔬', '👨‍🔬', '👩‍🎨', '👨‍🎨', '👩‍🚒', '👨‍🚒', '👩‍✈️', '👨‍✈️', '👩‍🚀', '👨‍🚀', '👩‍⚖️', '👨‍⚖️', '🤶', '🎅', '👸', '🤴', '👰', '🤵', '👼', '🤰', '🙇', '💁', '🙅', '🙆', '🙋', '🤦', '🤷', '🙎', '🙍', '💇', '💆', '🕴', '💃', '🕺', '🚶', '🏃', '🤲', '👐', '🙌', '👏', '🤝', '👍', '👎', '👊', '✊', '🤛', '🤜', '🤞', '✌️', '🤟', '🤘', '👌', '👈', '👉', '👆', '👇', '☝️', '✋', '🤚', '🖐', '🖖', '👋', '🤙', '💪', '🖕', '✍️', '🤳', '💅', '👂', '👃', '👣', '👀', '👁', '🧠', '👅', '👄', '💋', '👓', '🕶', '👔', '👕', '👖', '🧣', '🧤', '🧥', '🧦', '👗', '👘', '👙', '👚', '👛', '👜', '👝', '🛍', '🎒', '👞', '👟', '👠', '👡', '👢', '👑', '👒', '🎩', '🎓', '🧢', '⛑', '📿', '💄', '💍', '💎', '🔇', '🔈', '🔉', '🔊', '📢', '📣', '📯', '🔔', '🔕', '🎼', '🎵', '🎶', '🎙', '🎚', '🎛', '🎤', '🎧', '📻', '🎷', '🎸', '🎹', '🎺', '🎻', '🥁', '📱', '📲', '💻', '🖥', '🖨', '🖱', '🖲', '🕹', '🗜', '💽', '💾', '💿', '📀', '📼', '📷', '📸', '📹', '🎥', '📽', '🎞', '📞', '☎️', '📟', '📠', '📺', '📻', '🎙', '📡', '🔍', '🔎', '🔬', '🔭', '📡', '💡', '🔦', '🏮', '📔', '📕', '📖', '📗', '📘', '📙', '📚', '📓', '📒', '📃', '📜', '📄', '📰', '🗞', '📑', '🔖', '🏷', '💰', '💴', '💵', '💶', '💷', '💸', '💳', '🧾', '💹', '💱', '💲', '✉️', '📧', '📨', '📩', '📤', '📥', '📦', '📫', '📪', '📬', '📭', '📮', '🗳', '✏️', '✒️', '🖋', '🖊', '🖌', '🖍', '📝', '💼', '📁', '📂', '🗂', '📅', '📆', '🗒', '🗓', '📇', '📈', '📉', '📊', '📋', '📌', '📍', '📎', '🖇', '📏', '📐', '✂️', '🗃', '🗄', '🗑', '🔒', '🔓', '🔏', '🔐', '🔑', '🗝', '🔨', '⛏', '⚒', '🛠', '🗡', '⚔️', '🔫', '🏹', '🛡', '🔧', '🔩', '⚙️', '🗜', '⚖️', '🔗', '⛓', '🧰', '🧲', '⚗️', '🧪', '🧫', '🧬', '🔬', '🔭', '📡', '💉', '💊', '🛏', '🛋', '🚪', '🚽', '🚿', '🛁', '🧴', '🧷', '🧹', '🧺', '🧻', '🧼', '🧽', '🧯', '🚬', '⚰️', '⚱️', '🗿', '🏺', '🧱', '🎈', '🎏', '🎀', '🎁', '🎊', '🎉', '🎎', '🏮', '🎐', '🧧', '✉️', '📩', '📨', '📧'];
+		let num = integer(1, max);
+		let arr = [];
+		for (let i = 0; i < num; i++) {
+			arr.push(chance.pickone(emojis));
+		}
+		if (array) return arr;
+		if (!array) return arr.join(', ');
+		return "🤷";
+	};
+};
 
-		writeStream.end();
-		writeStream.on('finish', () => {
-			resolve(path);
-		});
-		writeStream.on('error', reject);
-	});
-}
 
 
 module.exports = {
@@ -391,7 +888,14 @@ module.exports = {
 	choose,
 	exhaust,
 	integer,
+	TimeSoup,
 
+	generateEmoji,
+
+
+	initChance,
+	getChance,
+	validateTime,
 	boxMullerRandom,
 	applySkew,
 	mapToRange,
@@ -400,11 +904,22 @@ module.exports = {
 	range,
 	openFinder,
 	getUniqueKeys,
-	generateEmoji,
 	person,
-	weighList,
-
-
+	pickAWinner,
+	weighArray,
+	weighFunnels,
+	validateEventConfig,
+	shuffleArray,
+	shuffleExceptFirst,
+	shuffleExceptLast,
+	fixFirstAndLast,
+	shuffleMiddle,
+	shuffleOutside,
+	interruptArray,
+	generateUser,
+	enrichArray,
+	optimizedBoxMuller,
+	buildFileNames,
 	streamJSON,
 	streamCSV
 };