make-mp-data 1.4.0 → 1.4.1

package/schemas/simple.js

@@ -37,7 +37,7 @@ const config = {
 			event: "checkout",
 			weight: 2,
 			properties: {
-				amount: weightedRange(5, 500, .25, 1000),
+				amount: weightedRange(5, 500, .25),
 				currency: ["USD", "CAD", "EUR", "BTC", "ETH", "JPY"],
 				coupon: ["none", "none", "none", "none", "10%OFF", "20%OFF", "10%OFF", "20%OFF", "30%OFF", "40%OFF", "50%OFF"],
 				numItems: weightedRange(1, 10),
@@ -48,7 +48,7 @@ const config = {
 			event: "add to cart",
 			weight: 4,
 			properties: {
-				amount: weightedRange(5, 500, .25, 1000),
+				amount: weightedRange(5, 500, .25),
 				rating: weightedRange(1, 5),
 				reviews: weightedRange(0, 35),
 				isFeaturedItem: [true, false, false],
@@ -71,7 +71,7 @@ const config = {
 			properties: {
 				videoCategory: pickAWinner(videoCategories, integer(0, 9)),
 				isFeaturedItem: [true, false, false],
-				watchTimeSec: weightedRange(10, 600, .25, 1000),
+				watchTimeSec: weightedRange(10, 600, .25),
 				quality: ["2160p", "1440p", "1080p", "720p", "480p", "360p", "240p"],
 				format: ["mp4", "avi", "mov", "mpg"],
 				uploader_id: chance.guid.bind(chance)
@@ -133,7 +133,7 @@ const config = {
 		profit: { events: ["checkout"], values: [4, 2, 42, 420] },
 		watchTimeSec: {
 			events: ["watch video"],
-			values: weightedRange(50, 1200, 6, 247)
+			values: weightedRange(50, 1200, 6)
 		}
 
 	},

package/scratch.mjs

@@ -1,17 +1,19 @@
 import main from "./index.js";
 import amir from './customers/amir.js';
 import simple from './schemas/simple.js';
+import funnels from './schemas/funnels.js';
+import foobar from './schemas/foobar.js';
+import complex from './schemas/complex.js';
+import deepNest from './schemas/deepNest.js';
 import execSync from 'child_process';
 
 
 /** @type {main.Config} */
 const spec = {	
-	...amir,
-	numUsers: 1000,
-	numEvents: 100000,
+	...funnels,	
 	writeToDisk: false,
 	verbose: true,
-	makeChart: true,	
+	makeChart: false,	
 };
 
 

package/tests/unit.test.js

@@ -39,7 +39,11 @@ const { applySkew,
 	buildFileNames,
 	TimeSoup,
 	getChance,
-	initChance
+	initChance,
+	validateEventConfig,
+	validateTime,
+	interruptArray,
+	optimizedBoxMuller
 } = require('../utils');
 
 
@@ -59,7 +63,7 @@ describe('timesoup', () => {
 });
 
 
-describe('naming things', () => {
+describe('names', () => {
 
 	test('default config', () => {
 		const config = { simulationName: 'testSim' };
@@ -182,7 +186,7 @@ describe('naming things', () => {
 });
 
 
-describe('determined random', () => {
+describe('determinism', () => {
 	test('initializes RNG with seed from environment variable', () => {
 		process.env.SEED = 'test-seed';
 		// @ts-ignore
@@ -206,8 +210,8 @@ describe('determined random', () => {
 });
 
 
-describe('generateUser', () => {
-	test('creates a user with valid fields', () => {
+describe('generation', () => {
+	test('users: can make', () => {
 		const numDays = 30;
 		const user = generateUser('uuid-123', numDays);
 		expect(user).toHaveProperty('distinct_id');
@@ -219,7 +223,7 @@ describe('generateUser', () => {
 		expect(user).toHaveProperty('sessionIds');
 	});
 
-	test('creates a user with a created date within the specified range', () => {
+	test('user: in time range', () => {
 		const numDays = 30;
 		const user = generateUser('uuid-123', numDays);
 		const createdDate = dayjs(user.created, 'YYYY-MM-DD');
@@ -228,9 +232,86 @@ describe('generateUser', () => {
 	});
 });
 
+describe('validation', () => {
 
+	beforeAll(() => {
+		global.NOW = 1672531200; // fixed point in time for testing
+	});
+
+	test('events: throws non array', () => {
+		// @ts-ignore
+		expect(() => validateEventConfig("not an array")).toThrow("events must be an array");
+	});
+
+	test('events: strings', () => {
+		const events = ["event1", "event2"];
+		const result = validateEventConfig(events);
+
+		expect(result).toEqual([
+			{ event: "event1", isFirstEvent: false, properties: {}, weight: expect.any(Number) },
+			{ event: "event2", isFirstEvent: false, properties: {}, weight: expect.any(Number) },
+		]);
+
+		result.forEach(event => {
+			expect(event.weight).toBeGreaterThanOrEqual(1);
+			expect(event.weight).toBeLessThanOrEqual(5);
+		});
+	});
+
+	test('events: objects', () => {
+		const events = [{ event: "event1", properties: { a: 1 } }, { event: "event2", properties: { b: 2 } }];
+		const result = validateEventConfig(events);
 
-describe('enrich array', () => {
+		expect(result).toEqual(events);
+	});
+
+	test('events: mix', () => {
+		const events = ["event1", { event: "event2", properties: { b: 2 } }];
+		// @ts-ignore
+		const result = validateEventConfig(events);
+
+		expect(result).toEqual([
+			{ event: "event1", isFirstEvent: false, properties: {}, weight: expect.any(Number) },
+			{ event: "event2", properties: { b: 2 } }
+		]);
+
+		expect(result[0].weight).toBeGreaterThanOrEqual(1);
+		expect(result[0].weight).toBeLessThanOrEqual(5);
+	});
+
+	test('dates: between', () => {
+		const chosenTime = global.NOW - (60 * 60 * 24 * 15); // 15 days ago
+		const earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
+		const latestTime = global.NOW;
+		expect(validateTime(chosenTime, earliestTime, latestTime)).toBe(true);
+	});
+
+	test('dates: outside earliest', () => {
+		const chosenTime = global.NOW - (60 * 60 * 24 * 31); // 31 days ago
+		const earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
+		const latestTime = global.NOW;
+		expect(validateTime(chosenTime, earliestTime, latestTime)).toBe(false);
+	});
+
+	test('dates: outside latest', () => {
+		const chosenTime = -1;
+		const earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
+		const latestTime = global.NOW;
+		expect(validateTime(chosenTime, earliestTime, latestTime)).toBe(false);
+	});
+
+	test('dates: inference in', () => {
+		const chosenTime = global.NOW - (60 * 60 * 24 * 15); // 15 days ago
+		expect(validateTime(chosenTime)).toBe(true);
+	});
+
+	test('dates: inference out', () => {
+		const chosenTime = global.NOW - (60 * 60 * 24 * 31); // 31 days ago
+		expect(validateTime(chosenTime)).toBe(false);
+	});
+});
+
+describe('enrichment', () => {
 	test('hook works', () => {
 		const arr = [];
 		const hook = (item) => item * 2;
@@ -238,7 +319,7 @@ describe('enrich array', () => {
 		enrichedArray.hookPush(1);
 		enrichedArray.hookPush(2);
 		expect(enrichedArray.includes(2)).toBeTruthy();
-		expect(enrichedArray.includes(4)).toBeTruthy();		
+		expect(enrichedArray.includes(4)).toBeTruthy();
 	});
 
 	test('filter empties', () => {
@@ -247,7 +328,7 @@ describe('enrich array', () => {
 		const enrichedArray = enrichArray(arr, { hook });
 		enrichedArray.hookPush(null);
 		enrichedArray.hookPush(undefined);
-		enrichedArray.hookPush({});		
+		enrichedArray.hookPush({});
 		enrichedArray.hookPush({ a: 1 });
 		enrichedArray.hookPush([1, 2]);
 		expect(enrichedArray).toHaveLength(3);
@@ -256,12 +337,14 @@ describe('enrich array', () => {
 		expect(enrichedArray.includes('[object Object]')).toBeTruthy();
 		expect(enrichedArray.includes('1')).toBeTruthy();
 		expect(enrichedArray.includes('2')).toBeTruthy();
-		
+
 	});
+
+
 });
 
 
-describe('utils', () => {
+describe('utilities', () => {
 
 	test('pick: works', () => {
 		const array = [1, 2, 3];
@@ -343,13 +426,13 @@ describe('utils', () => {
 	test('weightedRange:  within range', () => {
 		const values = weightedRange(5, 15);
 		expect(values.every(v => v >= 5 && v <= 15)).toBe(true);
-		expect(values.length).toBe(100);
+		expect(values.length).toBe(50);
 	});
 
 	test('applySkew: skews', () => {
 		const value = boxMullerRandom();
 		const skewedValue = applySkew(value, .25);
-		expect(Math.abs(skewedValue)).toBeGreaterThanOrEqual(Math.abs(value));
+		expect(Math.abs(skewedValue)).toBeLessThanOrEqual(Math.abs(value));
 	});
 
 	test('mapToRange: works', () => {
@@ -537,5 +620,17 @@ describe('utils', () => {
 		expect(stdDev).toBeCloseTo(1, 1);
 	});
 
+	test('optimized box normal distribution', () => {
+		const values = [];
+		for (let i = 0; i < 10000; i++) {
+			values.push(optimizedBoxMuller());
+		}
+		const mean = values.reduce((sum, val) => sum + val, 0) / values.length;
+		const variance = values.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / values.length;
+		const stdDev = Math.sqrt(variance);
+		expect(mean).toBeCloseTo(0, 1);
+		expect(stdDev).toBeCloseTo(1, 1);
+	});
+
 
 });

package/types.d.ts

@@ -60,7 +60,7 @@ declare namespace main {
   }
 
   export interface EnrichedArray<T> extends Array<T> {
-    hookPush: (item: T) => number;
+    hookPush: (item: T) => boolean;
   }
 
   export interface EventConfig {
@@ -85,13 +85,20 @@ declare namespace main {
     sequence: string[];
     weight?: number;
     isFirstFunnel?: boolean;
+	/**
+	 * If true, the funnel will require the user to repeat the sequence of events in order to convert
+	 * If false, the user does not need to repeat the sequence of events in order to convert
+	 * ^ when false, users who repeat the repetitive steps are more likely to convert
+	 */
+	requireRepeats?: boolean;
     order?:
       | "sequential"
       | "first-fixed"
       | "last-fixed"
       | "random"
       | "first-and-last-fixed"
-      | "middle-fixed";
+      | "middle-fixed"
+	  | "interrupted";
     conversionRate?: number;
     timeToConvert?: number;
     props?: Record<string, ValueValid>;

package/utils.js

@@ -11,10 +11,12 @@ dayjs.extend(utc);
 require('dotenv').config();
 
 /** @typedef {import('./types').Config} Config */
+/** @typedef {import('./types').EventConfig} EventConfig */
 /** @typedef {import('./types').ValueValid} ValueValid */
 /** @typedef {import('./types').EnrichedArray} EnrichArray */
 /** @typedef {import('./types').EnrichArrayOptions} EnrichArrayOptions */
 /** @typedef {import('./types').Person} Person */
+/** @typedef {import('./types').Funnel} Funnel */
 
 let globalChance;
 let chanceInitialized = false;
@@ -268,6 +270,14 @@ function boxMullerRandom() {
 	return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
 };
 
+function optimizedBoxMuller() {
+	const chance = getChance();
+	const u = Math.max(Math.min(chance.normal({ mean: .5, dev: .25 }), 1), 0);
+	const v = Math.max(Math.min(chance.normal({ mean: .5, dev: .25 }), 1), 0);
+	return Math.sqrt(-2.0 * Math.log(u)) * Math.cos(2.0 * Math.PI * v);
+
+}
+
 /**
  * applies a skew to a value;
  * Skew=0.5: When the skew is 0.5, the distribution becomes more compressed, with values clustering closer to the mean.
@@ -377,12 +387,14 @@ function weighFunnels(acc, funnel) {
  * @param  {number} skew=1
  * @param  {number} size=100
  */
-function weightedRange(min, max, skew = 1, size = 100) {
+function weightedRange(min, max, skew = 1, size = 50) {
+	if (size > 2000) size = 2000;
 	const mean = (max + min) / 2;
 	const sd = (max - min) / 4;
 	const array = [];
 	while (array.length < size) {
-		const normalValue = boxMullerRandom();
+		// const normalValue = boxMullerRandom();
+		const normalValue = optimizedBoxMuller();
 		const skewedValue = applySkew(normalValue, skew);
 		const mappedValue = mapToRange(skewedValue, mean, sd);
 		if (mappedValue >= min && mappedValue <= max) {
@@ -458,6 +470,76 @@ function shuffleOutside(array) {
 	return [outsideShuffled[0], ...middleFixed, outsideShuffled[1]];
 }
 
+/**
+ * @param  {EventConfig[]} funnel
+ * @param  {EventConfig[]} possibles
+ */
+function interruptArray(funnel, possibles, percent = 50) {
+	if (!Array.isArray(funnel)) return funnel;
+	if (!Array.isArray(possibles)) return funnel;
+	if (!funnel.length) return funnel;
+	if (!possibles.length) return funnel;
+	const ignorePositions = [0, funnel.length - 1];
+	const chance = getChance();
+	loopSteps: for (const [index, event] of funnel.entries()) {
+		if (ignorePositions.includes(index)) continue loopSteps;
+		if (chance.bool({ likelihood: percent })) {
+			funnel[index] = chance.pickone(possibles);
+		}
+	}
+
+	return funnel;
+}
+
+/*
+----
+VALIDATORS
+----
+*/
+
+
+/**
+ * @param  {EventConfig[] | string[]} events
+ */
+function validateEventConfig(events) {
+	if (!Array.isArray(events)) throw new Error("events must be an array");
+	const cleanEventConfig = [];
+	for (const event of events) {
+		if (typeof event === "string") {
+			/** @type {EventConfig} */
+			const eventTemplate = {
+				event,
+				isFirstEvent: false,
+				properties: {},
+				weight: integer(1, 5)
+			};
+			cleanEventConfig.push(eventTemplate);
+		}
+		if (typeof event === "object") {
+			cleanEventConfig.push(event);
+		}
+	}
+	return cleanEventConfig;
+}
+
+function validateTime(chosenTime, earliestTime, latestTime) {
+	if (!earliestTime) earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
+	if (!latestTime) latestTime = global.NOW;
+
+	if (typeof chosenTime === 'number') {
+		if (chosenTime > 0) {
+			if (chosenTime > earliestTime) {
+				if (chosenTime < latestTime) {
+					return true;
+				}
+
+			}
+		}
+	}
+	return false;
+}
+
+
 /*
 ----
 META
@@ -474,33 +556,43 @@ function enrichArray(arr = [], opts = {}) {
 	const { hook = a => a, type = "", ...rest } = opts;
 
 	function transformThenPush(item) {
-		if (item === null) return 0;
-		if (item === undefined) return 0;
+		if (item === null) return false;
+		if (item === undefined) return false;
 		if (typeof item === 'object') {
-			if (Object.keys(item).length === 0) return 0;
+			if (Object.keys(item).length === 0) return false;
 		}
+
+		//hook is passed an array 
 		if (Array.isArray(item)) {
 			for (const i of item) {
 				try {
 					const enriched = hook(i, type, rest);
-					arr.push(enriched);
+					if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
+					else arr.push(enriched);
+
 				}
 				catch (e) {
 					console.error(`\n\nyour hook had an error\n\n`, e);
 					arr.push(i);
+					return false;
 				}
 
 			}
-			return -1;
+			return true;
 		}
+
+		//hook is passed a single item
 		else {
 			try {
 				const enriched = hook(item, type, rest);
-				return arr.push(enriched);
+				if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
+				else arr.push(enriched);
+				return true;
 			}
 			catch (e) {
 				console.error(`\n\nyour hook had an error\n\n`, e);
-				return arr.push(item);
+				arr.push(item);
+				return false;
 			}
 		}
 
@@ -674,22 +766,6 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 }
 
 
-function validateTime(chosenTime, earliestTime, latestTime) {
-	if (!earliestTime) earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
-	if (!latestTime) latestTime = global.NOW;
-
-	if (typeof chosenTime === 'number') {
-		if (chosenTime > 0) {
-			if (chosenTime > earliestTime) {
-				if (chosenTime < latestTime) {
-					return true;
-				}
-
-			}
-		}
-	}
-	return false;
-}
 
 
 /**
@@ -819,7 +895,7 @@ module.exports = {
 
 	initChance,
 	getChance,
-
+	validateTime,
 	boxMullerRandom,
 	applySkew,
 	mapToRange,
@@ -832,17 +908,17 @@ module.exports = {
 	pickAWinner,
 	weighArray,
 	weighFunnels,
-
+	validateEventConfig,
 	shuffleArray,
 	shuffleExceptFirst,
 	shuffleExceptLast,
 	fixFirstAndLast,
 	shuffleMiddle,
 	shuffleOutside,
-
+	interruptArray,
 	generateUser,
 	enrichArray,
-
+	optimizedBoxMuller,
 	buildFileNames,
 	streamJSON,
 	streamCSV