make-mp-data 1.3.4 → 1.4.1

package/tests/unit.test.js

@@ -1,19 +1,58 @@
 const generate = require('../index.js');
 const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
+const fs = require('fs');
 const u = require('ak-tools');
 dayjs.extend(utc);
-const { timeSoup } = generate;
 require('dotenv').config();
 
-
-
-describe('timeSoup', () => {
-	test('always positive dates', () => {
+const { applySkew,
+	boxMullerRandom,
+	choose,
+	date,
+	dates,
+	day,
+	exhaust,
+	generateEmoji,
+	getUniqueKeys,
+	integer,
+	mapToRange,
+	person,
+	pick,
+	range,
+	pickAWinner,
+	weightedRange,
+	enrichArray,
+	fixFirstAndLast,
+	generateUser,
+	openFinder,
+	progress,
+	shuffleArray,
+	shuffleExceptFirst,
+	shuffleExceptLast,
+	shuffleMiddle,
+	shuffleOutside,
+	streamCSV,
+	streamJSON,
+	weighArray,
+	weighFunnels,
+	buildFileNames,
+	TimeSoup,
+	getChance,
+	initChance,
+	validateEventConfig,
+	validateTime,
+	interruptArray,
+	optimizedBoxMuller
+} = require('../utils');
+
+
+describe('timesoup', () => {
+	test('always valid times', () => {
 		const dates = [];
-		for (let i = 0; i < 20000; i++) {
-			const earliest = dayjs().subtract(u.rand(2, 360), 'D');
-			dates.push(timeSoup());
+		for (let i = 0; i < 10000; i++) {
+			const earliest = dayjs().subtract(u.rand(5, 50), 'D');
+			dates.push(TimeSoup());
 		}
 		const tooOld = dates.filter(d => dayjs(d).isBefore(dayjs.unix(0)));
 		const badYear = dates.filter(d => !d.startsWith('202'));
@@ -24,10 +63,288 @@ describe('timeSoup', () => {
 });
 
 
+describe('names', () => {
+
+	test('default config', () => {
+		const config = { simulationName: 'testSim' };
+		const result = buildFileNames(config);
+		expect(result.eventFiles).toEqual(['testSim-EVENTS.csv']);
+		expect(result.userFiles).toEqual(['testSim-USERS.csv']);
+		expect(result.scdFiles).toEqual([]);
+		expect(result.groupFiles).toEqual([]);
+		expect(result.lookupFiles).toEqual([]);
+		expect(result.mirrorFiles).toEqual([]);
+		expect(result.folder).toEqual('./');
+	});
+
+	test('json format', () => {
+		const config = { simulationName: 'testSim', format: 'json' };
+		const result = buildFileNames(config);
+		expect(result.eventFiles).toEqual(['testSim-EVENTS.json']);
+		expect(result.userFiles).toEqual(['testSim-USERS.json']);
+	});
+
+	test('with scdProps', () => {
+		const config = {
+			simulationName: 'testSim',
+			scdProps: { prop1: {}, prop2: {} }
+		};
+		const result = buildFileNames(config);
+		expect(result.scdFiles).toEqual([
+			'testSim-prop1-SCD.csv',
+			'testSim-prop2-SCD.csv'
+		]);
+	});
+
+	test('with groupKeys', () => {
+		const config = {
+			simulationName: 'testSim',
+			groupKeys: [['group1'], ['group2']]
+		};
+		const result = buildFileNames(config);
+		expect(result.groupFiles).toEqual([
+			'testSim-group1-GROUP.csv',
+			'testSim-group2-GROUP.csv'
+		]);
+	});
+
+	test('with lookupTables', () => {
+		const config = {
+			simulationName: 'testSim',
+			lookupTables: [{ key: 'lookup1' }, { key: 'lookup2' }]
+		};
+		const result = buildFileNames(config);
+		expect(result.lookupFiles).toEqual([
+			'testSim-lookup1-LOOKUP.csv',
+			'testSim-lookup2-LOOKUP.csv'
+		]);
+	});
+
+	test('with mirrorProps', () => {
+		const config = {
+			simulationName: 'testSim',
+			mirrorProps: { prop1: {} }
+		};
+		const result = buildFileNames(config);
+		expect(result.mirrorFiles).toEqual(['testSim-MIRROR.csv']);
+	});
+
+	test('writeToDisk', async () => {
+		const config = { simulationName: 'testSim', writeToDisk: true };
+		const result = await buildFileNames(config);
+		expect(result.folder).toBeDefined();
+
+	});
+
+
+	test('invalid simName', () => {
+		const config = { simulationName: 123 };
+		expect(() => buildFileNames(config)).toThrow('simName must be a string');
+	});
+
+
+	test('streamJSON: writes to file', async () => {
+		const path = 'test.json';
+		const data = [{ a: 1, b: 2 }, { a: 3, b: 4 }];
+		await streamJSON(path, data);
+		const content = fs.readFileSync(path, 'utf8');
+		const lines = content.trim().split('\n').map(line => JSON.parse(line));
+		expect(lines).toEqual(data);
+		fs.unlinkSync(path);
+	});
+
+	test('streamCSV: writes to file', async () => {
+		const path = 'test.csv';
+		const data = [{ a: 1, b: 2 }, { a: 3, b: 4 }];
+		await streamCSV(path, data);
+		const content = fs.readFileSync(path, 'utf8');
+		const lines = content.trim().split('\n');
+		expect(lines.length).toBe(3); // Including header
+		fs.unlinkSync(path);
+	});
+
+
+	test('generateUser: works', () => {
+		const uuid = { guid: jest.fn().mockReturnValue('uuid-123') };
+		const numDays = 30;
+		const user = generateUser(numDays);
+		expect(user).toHaveProperty('distinct_id');
+		expect(user).toHaveProperty('name');
+		expect(user).toHaveProperty('email');
+		expect(user).toHaveProperty('avatar');
+	});
+
+	test('enrichArray: works', () => {
+		const arr = [];
+		const enrichedArray = enrichArray(arr);
+		enrichedArray.hookPush(1);
+		enrichedArray.hookPush(2);
+		const match = JSON.stringify(enrichedArray) === JSON.stringify([1, 2]);
+		expect(match).toEqual(true);
+	});
+
+});
+
+
+describe('determinism', () => {
+	test('initializes RNG with seed from environment variable', () => {
+		process.env.SEED = 'test-seed';
+		// @ts-ignore
+		initChance();
+		const chance = getChance();
+		expect(chance).toBeDefined();
+		expect(chance.random()).toBeGreaterThanOrEqual(0);
+		expect(chance.random()).toBeLessThanOrEqual(1);
+
+	});
+
+	test('initializes RNG only once', () => {
+		const seed = 'initial-seed';
+		initChance(seed);
+		const chance1 = getChance();
+		initChance('new-seed');
+		const chance2 = getChance();
+		expect(chance1).toBe(chance2);
+
+	});
+});
+
+
+describe('generation', () => {
+	test('users: can make', () => {
+		const numDays = 30;
+		const user = generateUser('uuid-123', numDays);
+		expect(user).toHaveProperty('distinct_id');
+		expect(user).toHaveProperty('name');
+		expect(user).toHaveProperty('email');
+		expect(user).toHaveProperty('avatar');
+		expect(user).toHaveProperty('created');
+		expect(user).toHaveProperty('anonymousIds');
+		expect(user).toHaveProperty('sessionIds');
+	});
+
+	test('user: in time range', () => {
+		const numDays = 30;
+		const user = generateUser('uuid-123', numDays);
+		const createdDate = dayjs(user.created, 'YYYY-MM-DD');
+		expect(createdDate.isValid()).toBeTruthy();
+		expect(createdDate.isBefore(dayjs.unix(global.NOW))).toBeTruthy();
+	});
+});
+
+describe('validation', () => {
+
+	beforeAll(() => {
+		global.NOW = 1672531200; // fixed point in time for testing
+	});
+
+	test('events: throws non array', () => {
+		// @ts-ignore
+		expect(() => validateEventConfig("not an array")).toThrow("events must be an array");
+	});
+
+	test('events: strings', () => {
+		const events = ["event1", "event2"];
+		const result = validateEventConfig(events);
+
+		expect(result).toEqual([
+			{ event: "event1", isFirstEvent: false, properties: {}, weight: expect.any(Number) },
+			{ event: "event2", isFirstEvent: false, properties: {}, weight: expect.any(Number) },
+		]);
+
+		result.forEach(event => {
+			expect(event.weight).toBeGreaterThanOrEqual(1);
+			expect(event.weight).toBeLessThanOrEqual(5);
+		});
+	});
+
+	test('events: objects', () => {
+		const events = [{ event: "event1", properties: { a: 1 } }, { event: "event2", properties: { b: 2 } }];
+		const result = validateEventConfig(events);
+
+		expect(result).toEqual(events);
+	});
+
+	test('events: mix', () => {
+		const events = ["event1", { event: "event2", properties: { b: 2 } }];
+		// @ts-ignore
+		const result = validateEventConfig(events);
+
+		expect(result).toEqual([
+			{ event: "event1", isFirstEvent: false, properties: {}, weight: expect.any(Number) },
+			{ event: "event2", properties: { b: 2 } }
+		]);
+
+		expect(result[0].weight).toBeGreaterThanOrEqual(1);
+		expect(result[0].weight).toBeLessThanOrEqual(5);
+	});
+
+	test('dates: between', () => {
+		const chosenTime = global.NOW - (60 * 60 * 24 * 15); // 15 days ago
+		const earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
+		const latestTime = global.NOW;
+		expect(validateTime(chosenTime, earliestTime, latestTime)).toBe(true);
+	});
+
+	test('dates: outside earliest', () => {
+		const chosenTime = global.NOW - (60 * 60 * 24 * 31); // 31 days ago
+		const earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
+		const latestTime = global.NOW;
+		expect(validateTime(chosenTime, earliestTime, latestTime)).toBe(false);
+	});
+
+	test('dates: outside latest', () => {
+		const chosenTime = -1;
+		const earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
+		const latestTime = global.NOW;
+		expect(validateTime(chosenTime, earliestTime, latestTime)).toBe(false);
+	});
+
+	test('dates: inference in', () => {
+		const chosenTime = global.NOW - (60 * 60 * 24 * 15); // 15 days ago
+		expect(validateTime(chosenTime)).toBe(true);
+	});
+
+	test('dates: inference out', () => {
+		const chosenTime = global.NOW - (60 * 60 * 24 * 31); // 31 days ago
+		expect(validateTime(chosenTime)).toBe(false);
+	});
+});
+
+describe('enrichment', () => {
+	test('hook works', () => {
+		const arr = [];
+		const hook = (item) => item * 2;
+		const enrichedArray = enrichArray(arr, { hook });
+		enrichedArray.hookPush(1);
+		enrichedArray.hookPush(2);
+		expect(enrichedArray.includes(2)).toBeTruthy();
+		expect(enrichedArray.includes(4)).toBeTruthy();
+	});
+
+	test('filter empties', () => {
+		const arr = [];
+		const hook = (item) => item ? item.toString() : item;
+		const enrichedArray = enrichArray(arr, { hook });
+		enrichedArray.hookPush(null);
+		enrichedArray.hookPush(undefined);
+		enrichedArray.hookPush({});
+		enrichedArray.hookPush({ a: 1 });
+		enrichedArray.hookPush([1, 2]);
+		expect(enrichedArray).toHaveLength(3);
+		expect(enrichedArray.includes('null')).toBeFalsy();
+		expect(enrichedArray.includes('undefined')).toBeFalsy();
+		expect(enrichedArray.includes('[object Object]')).toBeTruthy();
+		expect(enrichedArray.includes('1')).toBeTruthy();
+		expect(enrichedArray.includes('2')).toBeTruthy();
+
+	});
+
+
+});
 
-const { applySkew, boxMullerRandom, choose, date, dates, day, exhaust, generateEmoji, getUniqueKeys, integer, mapToRange, person, pick, range, weighList, weightedRange } = require('../utils');
 
-describe('utils', () => {
+describe('utilities', () => {
 
 	test('pick: works', () => {
 		const array = [1, 2, 3];
@@ -58,9 +375,9 @@ describe('utils', () => {
 
 	test('person: fields', () => {
 		const generatedPerson = person();
-		expect(generatedPerson).toHaveProperty('$name');
-		expect(generatedPerson).toHaveProperty('$email');
-		expect(generatedPerson).toHaveProperty('$avatar');
+		expect(generatedPerson).toHaveProperty('name');
+		expect(generatedPerson).toHaveProperty('email');
+		expect(generatedPerson).toHaveProperty('avatar');
 	});
 
 
@@ -73,7 +390,7 @@ describe('utils', () => {
 	test('date: future', () => {
 		const futureDate = date(10, false, 'YYYY-MM-DD')();
 		expect(dayjs(futureDate, 'YYYY-MM-DD').isValid()).toBeTruthy();
-		expect(dayjs(futureDate).isAfter(dayjs())).toBeTruthy();
+		expect(dayjs(futureDate).isAfter(dayjs.unix(global.NOW))).toBeTruthy();
 	});
 
 	test('dates: pairs', () => {
@@ -107,15 +424,15 @@ describe('utils', () => {
 	});
 
 	test('weightedRange:  within range', () => {
-		const values = weightedRange(5, 15, 100);
+		const values = weightedRange(5, 15);
 		expect(values.every(v => v >= 5 && v <= 15)).toBe(true);
-		expect(values.length).toBe(100);
+		expect(values.length).toBe(50);
 	});
 
 	test('applySkew: skews', () => {
 		const value = boxMullerRandom();
 		const skewedValue = applySkew(value, .25);
-		expect(Math.abs(skewedValue)).toBeGreaterThanOrEqual(Math.abs(value));
+		expect(Math.abs(skewedValue)).toBeLessThanOrEqual(Math.abs(value));
 	});
 
 	test('mapToRange: works', () => {
@@ -146,6 +463,19 @@ describe('utils', () => {
 	});
 
 
+	test('times', () => {
+		const dates = [];
+		for (let i = 0; i < 10000; i++) {
+			const earliest = dayjs().subtract(u.rand(5, 50), 'D');
+			dates.push(TimeSoup());
+		}
+		const tooOld = dates.filter(d => dayjs(d).isBefore(dayjs.unix(0)));
+		const badYear = dates.filter(d => !d.startsWith('202'));
+		expect(dates.every(d => dayjs(d).isAfter(dayjs.unix(0)))).toBe(true);
+		expect(dates.every(d => d.startsWith('202'))).toBe(true);
+
+	});
+
 	test('date', () => {
 		const result = date();
 		expect(dayjs(result()).isValid()).toBe(true);
@@ -206,4 +536,101 @@ describe('utils', () => {
 	});
 
 
+	test('weighArray: works', () => {
+		const arr = ['a', 'b', 'c'];
+		const weightedArr = weighArray(arr);
+		expect(weightedArr.length).toBeGreaterThanOrEqual(arr.length);
+	});
+
+	test('weighFunnels: works', () => {
+		const acc = [];
+		const funnel = { weight: 3 };
+		const result = weighFunnels(acc, funnel);
+		expect(result.length).toBe(3);
+	});
+
+	test('progress: outputs correctly', () => {
+		// @ts-ignore
+		const mockStdoutWrite = jest.spyOn(process.stdout, 'write').mockImplementation(() => { });
+		progress('test', 50);
+		expect(mockStdoutWrite).toHaveBeenCalled();
+		mockStdoutWrite.mockRestore();
+	});
+
+	test('range: works', () => {
+		const result = [];
+		range.call(result, 1, 5);
+		expect(result).toEqual([1, 2, 3, 4, 5]);
+	});
+
+
+
+	test('shuffleArray: works', () => {
+		const arr = [1, 2, 3, 4, 5];
+		const shuffled = shuffleArray([...arr]);
+		expect(shuffled).not.toEqual(arr);
+		expect(shuffled.sort()).toEqual(arr.sort());
+	});
+
+	test('shuffleExceptFirst: works', () => {
+		const arr = [1, 2, 3, 4, 5];
+		const shuffled = shuffleExceptFirst([...arr]);
+		expect(shuffled[0]).toBe(arr[0]);
+		expect(shuffled.slice(1).sort()).toEqual(arr.slice(1).sort());
+	});
+
+	test('shuffleExceptLast: works', () => {
+		const arr = [1, 2, 3, 4, 5];
+		const shuffled = shuffleExceptLast([...arr]);
+		expect(shuffled[shuffled.length - 1]).toBe(arr[arr.length - 1]);
+		expect(shuffled.slice(0, -1).sort()).toEqual(arr.slice(0, -1).sort());
+	});
+
+	test('fixFirstAndLast: works', () => {
+		const arr = [1, 2, 3, 4, 5];
+		const shuffled = fixFirstAndLast([...arr]);
+		expect(shuffled[0]).toBe(arr[0]);
+		expect(shuffled[shuffled.length - 1]).toBe(arr[arr.length - 1]);
+		expect(shuffled.slice(1, -1).sort()).toEqual(arr.slice(1, -1).sort());
+	});
+
+	test('shuffleMiddle: works', () => {
+		const arr = [1, 2, 3, 4, 5];
+		const shuffled = shuffleMiddle([...arr]);
+		expect(shuffled[0]).toBe(arr[0]);
+		expect(shuffled[shuffled.length - 1]).toBe(arr[arr.length - 1]);
+		expect(shuffled.slice(1, -1).sort()).toEqual(arr.slice(1, -1).sort());
+	});
+
+	test('shuffleOutside: works', () => {
+		const arr = [1, 2, 3, 4, 5];
+		const shuffled = shuffleOutside([...arr]);
+		expect(shuffled.slice(1, -1)).toEqual(arr.slice(1, -1));
+	});
+
+	test('box normal distribution', () => {
+		const values = [];
+		for (let i = 0; i < 10000; i++) {
+			values.push(boxMullerRandom());
+		}
+		const mean = values.reduce((sum, val) => sum + val, 0) / values.length;
+		const variance = values.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / values.length;
+		const stdDev = Math.sqrt(variance);
+		expect(mean).toBeCloseTo(0, 1);
+		expect(stdDev).toBeCloseTo(1, 1);
+	});
+
+	test('optimized box normal distribution', () => {
+		const values = [];
+		for (let i = 0; i < 10000; i++) {
+			values.push(optimizedBoxMuller());
+		}
+		const mean = values.reduce((sum, val) => sum + val, 0) / values.length;
+		const variance = values.reduce((sum, val) => sum + Math.pow(val - mean, 2), 0) / values.length;
+		const stdDev = Math.sqrt(variance);
+		expect(mean).toBeCloseTo(0, 1);
+		expect(stdDev).toBeCloseTo(1, 1);
+	});
+
+
 });

package/tsconfig.json

@@ -11,7 +11,7 @@
   },
   "include": [
     "**/*.js"
-  ],
+, "soupTemplates.mjs", "testSoup.mjs", "testCases.mjs"  ],
   "exclude": [
     "node_modules"
   ]

package/types.d.ts

@@ -9,16 +9,20 @@ declare namespace main {
     token?: string;
     seed?: string;
     numDays?: number;
+    epochStart?: number;
+    epochEnd?: number;
     numEvents?: number;
     numUsers?: number;
     format?: "csv" | "json";
     region?: "US" | "EU";
-    events?: EventConfig[];
+    chance?: any;
+    events?: EventConfig[]; //can also be a array of strings
     superProps?: Record<string, ValueValid>;
+    funnels?: Funnel[];
     userProps?: Record<string, ValueValid>;
     scdProps?: Record<string, ValueValid>;
     mirrorProps?: Record<string, MirrorProps>;
-    groupKeys?: [string, number][] | [string, number, string[]][];
+    groupKeys?: [string, number][] | [string, number, string[]][]; // [key, numGroups, [events]]
     groupProps?: Record<string, Record<string, ValueValid>>;
     lookupTables?: LookupTable[];
     writeToDisk?: boolean;
@@ -26,19 +30,37 @@ declare namespace main {
     verbose?: boolean;
     anonIds?: boolean;
     sessionIds?: boolean;
+    makeChart?: boolean | string;
+    soup?: soup;
     hook?: Hook<any>;
   }
 
-  export type Hook<T> = (record: any, type: string, meta: any) => T;
+  type soup = {
+    deviation?: number;
+    peaks?: number;
+    mean?: number;
+  };
+
+  type hookTypes =
+    | "event"
+    | "user"
+    | "group"
+    | "lookup"
+    | "scd"
+    | "mirror"
+    | "funnel-pre"
+    | "funnel-post"
+    | "";
+  export type Hook<T> = (record: any, type: hookTypes, meta: any) => T;
 
   export interface EnrichArrayOptions<T> {
     hook?: Hook<T>;
-    type?: string;
+    type?: hookTypes;
     [key: string]: any;
   }
 
   export interface EnrichedArray<T> extends Array<T> {
-    hPush: (item: T) => number;
+    hookPush: (item: T) => boolean;
   }
 
   export interface EventConfig {
@@ -46,6 +68,40 @@ declare namespace main {
     weight?: number;
     properties?: Record<string, ValueValid>;
     isFirstEvent?: boolean;
+    relativeTimeMs?: number;
+  }
+
+  export interface EventSpec {
+	event: string;
+	time: string;
+	insert_id: string;
+	device_id?: string;
+	session_id?: string;
+	user_id?: string;	
+	[key: string]: ValueValid;
+  }
+
+  export interface Funnel {
+    sequence: string[];
+    weight?: number;
+    isFirstFunnel?: boolean;
+	/**
+	 * If true, the funnel will require the user to repeat the sequence of events in order to convert
+	 * If false, the user does not need to repeat the sequence of events in order to convert
+	 * ^ when false, users who repeat the repetitive steps are more likely to convert
+	 */
+	requireRepeats?: boolean;
+    order?:
+      | "sequential"
+      | "first-fixed"
+      | "last-fixed"
+      | "random"
+      | "first-and-last-fixed"
+      | "middle-fixed"
+	  | "interrupted";
+    conversionRate?: number;
+    timeToConvert?: number;
+    props?: Record<string, ValueValid>;
   }
 
   export interface MirrorProps {
@@ -59,7 +115,7 @@ declare namespace main {
     attributes: Record<string, ValueValid>;
   }
 
-  export interface SCDTable {
+  export interface SCDTableRow {
     distinct_id: string;
     insertTime: string;
     startTime: string;
@@ -72,17 +128,17 @@ declare namespace main {
     scdTableData: any[];
     groupProfilesData: GroupProfilesData[];
     lookupTableData: LookupTableData[];
-    import?: ImportResults;
+    importResults?: ImportResults;
     files?: string[];
   };
 
   export interface EventData {
     event: string;
-    $source: string;
+    source: string;
     time: string;
-    $device_id?: string;
-    $session_id?: string;
-    $user_id?: string;
+    device_id?: string;
+    session_id?: string;
+    user_id?: string;
     [key: string]: any;
   }
 
@@ -107,12 +163,22 @@ declare namespace main {
     bytes: number;
   }
   export interface Person {
-    $name: string;
-    $email: string;
-    $avatar: string;
-    $created: string | undefined;
+    name: string;
+    email: string;
+    avatar: string;
+    created: string | undefined;
     anonymousIds: string[];
     sessionIds: string[];
+    distinct_id?: string;
+  }
+
+  export interface UserProfile {
+    name?: string;
+    email?: string;
+    avatar?: string;
+    created: string | undefined;
+    distinct_id: string;
+    [key: string]: ValueValid;
   }
 }