make-mp-data 1.4.0 → 1.4.2

package/utils.js

@@ -1,7 +1,7 @@
 const fs = require('fs');
 const Chance = require('chance');
 const readline = require('readline');
-const { comma, uid } = require('ak-tools');
+const { comma, uid, clone } = require('ak-tools');
 const { spawn } = require('child_process');
 const dayjs = require('dayjs');
 const utc = require('dayjs/plugin/utc');
@@ -11,10 +11,12 @@ dayjs.extend(utc);
 require('dotenv').config();
 
 /** @typedef {import('./types').Config} Config */
+/** @typedef {import('./types').EventConfig} EventConfig */
 /** @typedef {import('./types').ValueValid} ValueValid */
 /** @typedef {import('./types').EnrichedArray} EnrichArray */
 /** @typedef {import('./types').EnrichArrayOptions} EnrichArrayOptions */
 /** @typedef {import('./types').Person} Person */
+/** @typedef {import('./types').Funnel} Funnel */
 
 let globalChance;
 let chanceInitialized = false;
@@ -223,6 +225,7 @@ function integer(min = 1, max = 100) {
 
 function pickAWinner(items, mostChosenIndex) {
 	const chance = getChance();
+	if (!mostChosenIndex) mostChosenIndex = integer(0, items.length);
 	if (mostChosenIndex > items.length) mostChosenIndex = items.length;
 	return function () {
 		const weighted = [];
@@ -250,6 +253,48 @@ function pickAWinner(items, mostChosenIndex) {
 }
 
 
+function inferFunnels(events) {
+	const createdFunnels = [];
+	const firstEvents = events.filter((e) => e.isFirstEvent).map((e) => e.event);
+	const usageEvents = events.filter((e) => !e.isFirstEvent).map((e) => e.event);
+	const numFunnelsToCreate = Math.ceil(usageEvents.length);
+	/** @type {Funnel} */
+	const funnelTemplate = {
+		sequence: [],
+		conversionRate: 50,
+		order: 'sequential',
+		requireRepeats: false,
+		props: {},
+		timeToConvert: 1,
+		isFirstFunnel: false,
+		weight: 1
+	};
+	if (firstEvents.length) {
+		for (const event of firstEvents) {
+			createdFunnels.push({ ...clone(funnelTemplate), sequence: [event], isFirstFunnel: true, conversionRate: 100 });
+		}
+	}
+
+	//at least one funnel with all usage events
+	createdFunnels.push({ ...clone(funnelTemplate), sequence: usageEvents });
+
+	//for the rest, make random funnels
+	followUpFunnels: for (let i = 1; i < numFunnelsToCreate; i++) {
+		/** @type {Funnel} */
+		const funnel = { ...clone(funnelTemplate) };
+		funnel.conversionRate = integer(25, 75);
+		funnel.timeToConvert = integer(1, 10);
+		funnel.weight = integer(1, 10);
+		const sequence = shuffleArray(usageEvents).slice(0, integer(2, usageEvents.length));
+		funnel.sequence = sequence;
+		funnel.order = 'random';
+		createdFunnels.push(funnel);
+	}
+
+	return createdFunnels;
+
+}
+
 /*
 ----
 GENERATORS
@@ -268,6 +313,17 @@ function boxMullerRandom() {
 	return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
 };
 
+function optimizedBoxMuller() {
+	const chance = getChance();
+	const u = Math.max(Math.min(chance.normal({ mean: .5, dev: .25 }), 1), 0);
+	const v = Math.max(Math.min(chance.normal({ mean: .5, dev: .25 }), 1), 0);
+	const result = Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
+	//ensure we didn't get infinity
+	if (result === Infinity || result === -Infinity) return chance.floating({ min: 0, max: 1 });
+	return result;
+
+}
+
 /**
  * applies a skew to a value;
  * Skew=0.5: When the skew is 0.5, the distribution becomes more compressed, with values clustering closer to the mean.
@@ -377,12 +433,14 @@ function weighFunnels(acc, funnel) {
  * @param  {number} skew=1
  * @param  {number} size=100
  */
-function weightedRange(min, max, skew = 1, size = 100) {
+function weightedRange(min, max, skew = 1, size = 50) {
+	if (size > 2000) size = 2000;
 	const mean = (max + min) / 2;
 	const sd = (max - min) / 4;
 	const array = [];
 	while (array.length < size) {
-		const normalValue = boxMullerRandom();
+		// const normalValue = boxMullerRandom();
+		const normalValue = optimizedBoxMuller();
 		const skewedValue = applySkew(normalValue, skew);
 		const mappedValue = mapToRange(skewedValue, mean, sd);
 		if (mappedValue >= min && mappedValue <= max) {
@@ -458,6 +516,76 @@ function shuffleOutside(array) {
 	return [outsideShuffled[0], ...middleFixed, outsideShuffled[1]];
 }
 
+/**
+ * @param  {EventConfig[]} funnel
+ * @param  {EventConfig[]} possibles
+ */
+function interruptArray(funnel, possibles, percent = 50) {
+	if (!Array.isArray(funnel)) return funnel;
+	if (!Array.isArray(possibles)) return funnel;
+	if (!funnel.length) return funnel;
+	if (!possibles.length) return funnel;
+	const ignorePositions = [0, funnel.length - 1];
+	const chance = getChance();
+	loopSteps: for (const [index, event] of funnel.entries()) {
+		if (ignorePositions.includes(index)) continue loopSteps;
+		if (chance.bool({ likelihood: percent })) {
+			funnel[index] = chance.pickone(possibles);
+		}
+	}
+
+	return funnel;
+}
+
+/*
+----
+VALIDATORS
+----
+*/
+
+
+/**
+ * @param  {EventConfig[] | string[]} events
+ */
+function validateEventConfig(events) {
+	if (!Array.isArray(events)) throw new Error("events must be an array");
+	const cleanEventConfig = [];
+	for (const event of events) {
+		if (typeof event === "string") {
+			/** @type {EventConfig} */
+			const eventTemplate = {
+				event,
+				isFirstEvent: false,
+				properties: {},
+				weight: integer(1, 5)
+			};
+			cleanEventConfig.push(eventTemplate);
+		}
+		if (typeof event === "object") {
+			cleanEventConfig.push(event);
+		}
+	}
+	return cleanEventConfig;
+}
+
+function validateTime(chosenTime, earliestTime, latestTime) {
+	if (!earliestTime) earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
+	if (!latestTime) latestTime = global.NOW;
+
+	if (typeof chosenTime === 'number') {
+		if (chosenTime > 0) {
+			if (chosenTime > earliestTime) {
+				if (chosenTime < latestTime) {
+					return true;
+				}
+
+			}
+		}
+	}
+	return false;
+}
+
+
 /*
 ----
 META
@@ -474,33 +602,43 @@ function enrichArray(arr = [], opts = {}) {
 	const { hook = a => a, type = "", ...rest } = opts;
 
 	function transformThenPush(item) {
-		if (item === null) return 0;
-		if (item === undefined) return 0;
+		if (item === null) return false;
+		if (item === undefined) return false;
 		if (typeof item === 'object') {
-			if (Object.keys(item).length === 0) return 0;
+			if (Object.keys(item).length === 0) return false;
 		}
+
+		//hook is passed an array 
 		if (Array.isArray(item)) {
 			for (const i of item) {
 				try {
 					const enriched = hook(i, type, rest);
-					arr.push(enriched);
+					if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
+					else arr.push(enriched);
+
 				}
 				catch (e) {
 					console.error(`\n\nyour hook had an error\n\n`, e);
 					arr.push(i);
+					return false;
 				}
 
 			}
-			return -1;
+			return true;
 		}
+
+		//hook is passed a single item
 		else {
 			try {
 				const enriched = hook(item, type, rest);
-				return arr.push(enriched);
+				if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
+				else arr.push(enriched);
+				return true;
 			}
 			catch (e) {
 				console.error(`\n\nyour hook had an error\n\n`, e);
-				return arr.push(item);
+				arr.push(item);
+				return false;
 			}
 		}
 
@@ -674,50 +812,39 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 }
 
 
-function validateTime(chosenTime, earliestTime, latestTime) {
-	if (!earliestTime) earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
-	if (!latestTime) latestTime = global.NOW;
-
-	if (typeof chosenTime === 'number') {
-		if (chosenTime > 0) {
-			if (chosenTime > earliestTime) {
-				if (chosenTime < latestTime) {
-					return true;
-				}
-
-			}
-		}
-	}
-	return false;
-}
 
 
 /**
+ * @param {string} userId
  * @param  {number} bornDaysAgo=30
+ * @param {boolean} isAnonymous
  * @return {Person}
  */
-function person(bornDaysAgo = 30) {
+function person(userId, bornDaysAgo = 30, isAnonymous = false) {
 	const chance = getChance();
 	//names and photos
+	const l = chance.letter;
 	let gender = chance.pickone(['male', 'female']);
 	if (!gender) gender = "female";
 	// @ts-ignore
-	const first = chance.first({ gender });
-	const last = chance.last();
-	const name = `${first} ${last}`;
-	const email = `${first[0]}.${last}@${chance.domain()}.com`;
-	const avatarPrefix = `https://randomuser.me/api/portraits`;
-	const randomAvatarNumber = chance.integer({
+	let first = chance.first({ gender });
+	let last = chance.last();
+	let name = `${first} ${last}`;
+	let email = `${first[0]}.${last}@${chance.domain()}.com`;
+	let avatarPrefix = `https://randomuser.me/api/portraits`;
+	let randomAvatarNumber = chance.integer({
 		min: 1,
 		max: 99
 	});
-	const avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
-	const avatar = avatarPrefix + avPath;
-	const created = dayjs.unix(global.NOW).subtract(bornDaysAgo, 'day').format('YYYY-MM-DD');
+	let avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
+	let avatar = avatarPrefix + avPath;
+	let created = dayjs.unix(global.NOW).subtract(bornDaysAgo, 'day').format('YYYY-MM-DD');
 	// const created = date(bornDaysAgo, true)();
 
+
 	/** @type {Person} */
 	const user = {
+		distinct_id: userId,
 		name,
 		email,
 		avatar,
@@ -726,6 +853,13 @@ function person(bornDaysAgo = 30) {
 		sessionIds: []
 	};
 
+	if (isAnonymous) {
+		user.name = "Anonymous User";
+		user.email = `${l()}${l()}****${l()}${l()}@${l()}**${l()}*.com`;		
+		delete user.avatar;		
+
+	}
+
 	//anon Ids
 	if (global.MP_SIMULATION_CONFIG?.anonIds) {
 		const clusterSize = integer(2, 10);
@@ -819,7 +953,7 @@ module.exports = {
 
 	initChance,
 	getChance,
-
+	validateTime,
 	boxMullerRandom,
 	applySkew,
 	mapToRange,
@@ -832,18 +966,19 @@ module.exports = {
 	pickAWinner,
 	weighArray,
 	weighFunnels,
-
+	validateEventConfig,
 	shuffleArray,
 	shuffleExceptFirst,
 	shuffleExceptLast,
 	fixFirstAndLast,
 	shuffleMiddle,
 	shuffleOutside,
-
+	interruptArray,
 	generateUser,
 	enrichArray,
-
+	optimizedBoxMuller,
 	buildFileNames,
 	streamJSON,
-	streamCSV
+	streamCSV,
+	inferFunnels
 };