make-mp-data 1.4.5 → 1.5.1

package/tests/unit.test.js

@@ -1,4 +1,4 @@
-const generate = require('../core/index.js');
+const generate = require('../index.js');
 const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 const fs = require('fs');
@@ -9,7 +9,7 @@ require('dotenv').config();
 /** @typedef {import('../types').Config} Config */
 /** @typedef {import('../types').EventConfig} EventConfig */
 /** @typedef {import('../types').ValueValid} ValueValid */
-/** @typedef {import('../types').EnrichedArray} hookArray */
+/** @typedef {import('../types').HookedArray} hookArray */
 /** @typedef {import('../types').hookArrayOptions} hookArrayOptions */
 /** @typedef {import('../types').Person} Person */
 /** @typedef {import('../types').Funnel} Funnel */
@@ -32,7 +32,7 @@ const {
 	range,
 	pickAWinner,
 	weighNumRange,
-	hookArray,
+
 	fixFirstAndLast,
 	generateUser,
 	openFinder,
@@ -51,13 +51,17 @@ const {
 	getChance,
 	initChance,
 	validateEventConfig,
-	validateTime,
+	validTime,
 	interruptArray,
 	optimizedBoxMuller,
-	inferFunnels,
+
 	datesBetween,
 	weighChoices
-} = require('../core/utils.js');
+} = require('../components/utils.js');
+
+const main = require('../index.js');
+//todo: test for funnel inference
+const { hookArray, inferFunnels } = main.meta;
 
 
 describe('timesoup', () => {
@@ -264,22 +268,22 @@ describe('generation', () => {
 	test('user: works', () => {
 		const uuid = { guid: jest.fn().mockReturnValue('uuid-123') };
 		const numDays = 30;
-		const user = generateUser(numDays);
+		const user = generateUser('123', { numDays });
 		expect(user).toHaveProperty('distinct_id');
 		expect(user).toHaveProperty('name');
 		expect(user).toHaveProperty('email');
-		expect(user).toHaveProperty('avatar');
+		expect(user).not.toHaveProperty('avatar');
 		expect(user).toHaveProperty('created');
-		expect(user).toHaveProperty('anonymousIds');
-		expect(user).toHaveProperty('sessionIds');
+		expect(user).not.toHaveProperty('anonymousIds');
+		expect(user).not.toHaveProperty('sessionIds');
 	});
 
 	test('user: in time range', () => {
 		const numDays = 30;
-		const user = generateUser('uuid-123', numDays);
+		const user = generateUser('uuid-123', { numDays });
 		const createdDate = dayjs(user.created, 'YYYY-MM-DD');
 		expect(createdDate.isValid()).toBeTruthy();
-		expect(createdDate.isBefore(dayjs.unix(global.NOW))).toBeTruthy();
+		expect(createdDate.isBefore(dayjs())).toBeTruthy();
 	});
 
 
@@ -290,10 +294,10 @@ describe('generation', () => {
 		expect(user.distinct_id).toBe('uuid-123');
 		expect(user).toHaveProperty('name');
 		expect(user).toHaveProperty('email');
-		expect(user).toHaveProperty('avatar');
+		expect(user).not.toHaveProperty('avatar');
 		expect(user).toHaveProperty('created');
-		expect(user).toHaveProperty('anonymousIds');
-		expect(user).toHaveProperty('sessionIds');
+		expect(user).not.toHaveProperty('anonymousIds');
+		expect(user).not.toHaveProperty('sessionIds');
 	});
 
 	test('person: anon', () => {
@@ -301,13 +305,13 @@ describe('generation', () => {
 		const user = person('uuid-123', numDays, true);
 		expect(user).toHaveProperty('distinct_id');
 		expect(user).toHaveProperty('name');
-		expect(user.name).toBe('Anonymous User')
+		expect(user.name).toBe('Anonymous User');
 		expect(user).toHaveProperty('email');
 		expect(user.email.includes('*')).toBeTruthy();
 		expect(user).not.toHaveProperty('avatar');
 		expect(user).toHaveProperty('created');
-		expect(user).toHaveProperty('anonymousIds');
-		expect(user).toHaveProperty('sessionIds');
+		expect(user).not.toHaveProperty('anonymousIds');
+		expect(user).not.toHaveProperty('sessionIds');
 	});
 
 
@@ -389,7 +393,8 @@ describe('generation', () => {
 describe('validation', () => {
 
 	beforeAll(() => {
-		global.NOW = 1672531200; // fixed point in time for testing
+		global.FIXED_NOW = 1672531200; // fixed point in time for testing
+		global.FIXED_BEGIN = global.FIXED_NOW - (60 * 60 * 24 * 30); // 30 days ago
 	});
 
 	test('events: non arrays', () => {
@@ -434,67 +439,67 @@ describe('validation', () => {
 	});
 
 	test('time: between', () => {
-		const chosenTime = global.NOW - (60 * 60 * 24 * 15); // 15 days ago
-		const earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
-		const latestTime = global.NOW;
-		expect(validateTime(chosenTime, earliestTime, latestTime)).toBe(true);
+		const chosenTime = global.FIXED_NOW - (60 * 60 * 24 * 15); // 15 days ago
+		const earliestTime = global.FIXED_NOW - (60 * 60 * 24 * 30); // 30 days ago
+		const latestTime = global.FIXED_NOW;
+		expect(validTime(chosenTime, earliestTime, latestTime)).toBe(true);
 	});
 
 	test('time: outside earliest', () => {
-		const chosenTime = global.NOW - (60 * 60 * 24 * 31); // 31 days ago
-		const earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
-		const latestTime = global.NOW;
-		expect(validateTime(chosenTime, earliestTime, latestTime)).toBe(false);
+		const chosenTime = global.FIXED_NOW - (60 * 60 * 24 * 31); // 31 days ago
+		const earliestTime = global.FIXED_NOW - (60 * 60 * 24 * 30); // 30 days ago
+		const latestTime = global.FIXED_NOW;
+		expect(validTime(chosenTime, earliestTime, latestTime)).toBe(false);
 	});
 
 	test('time: outside latest', () => {
 		const chosenTime = -1;
-		const earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
-		const latestTime = global.NOW;
-		expect(validateTime(chosenTime, earliestTime, latestTime)).toBe(false);
+		const earliestTime = global.FIXED_NOW - (60 * 60 * 24 * 30); // 30 days ago
+		const latestTime = global.FIXED_NOW;
+		expect(validTime(chosenTime, earliestTime, latestTime)).toBe(false);
 	});
 
 	test('time: inference in', () => {
-		const chosenTime = global.NOW - (60 * 60 * 24 * 15); // 15 days ago
-		expect(validateTime(chosenTime)).toBe(true);
+		const chosenTime = global.FIXED_NOW - (60 * 60 * 24 * 15); // 15 days ago
+		expect(validTime(chosenTime)).toBe(true);
 	});
 
 	test('time: inference out', () => {
-		const chosenTime = global.NOW - (60 * 60 * 24 * 31); // 31 days ago
-		expect(validateTime(chosenTime)).toBe(false);
+		const chosenTime = global.FIXED_NOW - (60 * 60 * 24 * 31); // 31 days ago
+		expect(validTime(chosenTime)).toBe(false);
 	});
 });
 
 describe('enrichment', () => {
 
-	test('hooks: noop', () => {
+	test('hooks: noop', async () => {
 		const arr = [];
-		const enrichedArray = hookArray(arr);
-		enrichedArray.hookPush(1);
-		enrichedArray.hookPush(2);
+		const enrichedArray = await hookArray(arr);
+		await enrichedArray.hookPush(1);
+		await enrichedArray.hookPush(2);
 		const match = JSON.stringify(enrichedArray) === JSON.stringify([1, 2]);
 		expect(match).toEqual(true);
 	});
 
-	test('hook: double', () => {
+	test('hook: double', async () => {
 		const arr = [];
 		const hook = (item) => item * 2;
-		const enrichedArray = hookArray(arr, { hook });
-		enrichedArray.hookPush(1);
-		enrichedArray.hookPush(2);
+		const enrichedArray = await hookArray(arr, { hook });
+		await enrichedArray.hookPush(1);
+		await enrichedArray.hookPush(2);
 		expect(enrichedArray.includes(2)).toBeTruthy();
 		expect(enrichedArray.includes(4)).toBeTruthy();
 	});
 
-	test('hooks: filter', () => {
+	test('hooks: filter', async () => {
 		const arr = [];
 		const hook = (item) => item ? item.toString() : item;
-		const enrichedArray = hookArray(arr, { hook });
-		enrichedArray.hookPush(null);
-		enrichedArray.hookPush(undefined);
-		enrichedArray.hookPush({});
-		enrichedArray.hookPush({ a: 1 });
-		enrichedArray.hookPush([1, 2]);
+		const enrichedArray = await hookArray(arr, { hook });
+		await enrichedArray.hookPush(null);
+		await enrichedArray.hookPush(undefined);
+		await enrichedArray.hookPush({});
+		await enrichedArray.hookPush({ a: 1 });
+		await enrichedArray.hookPush([1, 2]);
 		expect(enrichedArray).toHaveLength(3);
 		expect(enrichedArray.includes('null')).toBeFalsy();
 		expect(enrichedArray.includes('undefined')).toBeFalsy();
@@ -543,7 +548,7 @@ describe('utilities', () => {
 	test('date: future', () => {
 		const futureDate = date(10, false, 'YYYY-MM-DD')();
 		expect(dayjs(futureDate, 'YYYY-MM-DD').isValid()).toBeTruthy();
-		expect(dayjs(futureDate).isAfter(dayjs.unix(global.NOW))).toBeTruthy();
+		expect(dayjs(futureDate).isAfter(dayjs.unix(global.FIXED_NOW))).toBeTruthy();
 	});
 
 	test('dates: pairs', () => {
@@ -664,7 +669,7 @@ describe('utilities', () => {
 	});
 
 	test('range: works', () => {
-		const result = range(1,5);
+		const result = range(1, 5);
 		expect(result).toEqual([1, 2, 3, 4, 5]);
 	});
 

package/tsconfig.json

@@ -11,7 +11,7 @@
   },
   "include": [
     "**/*.js"
-, "soupTemplates.mjs", "tests/testSoup.mjs", "tests/testCases.mjs"  ],
+, "soupTemplates.mjs", "tests/testSoup.mjs", "tests/testCases.mjs", "tests/benchmark/concurrency.mjs", "dungeons/pos.mjs", "scripts/new-project.mjs"  ],
   "exclude": [
     "node_modules"
   ]

package/types.d.ts

@@ -13,6 +13,7 @@ declare namespace main {
    * main config object for the entire data generation
    */
   export interface Config {
+    // constants
     token?: string;
     seed?: string;
     numDays?: number;
@@ -20,9 +21,18 @@ declare namespace main {
     epochEnd?: number;
     numEvents?: number;
     numUsers?: number;
+    format?: "csv" | "json" | string;
+    region?: "US" | "EU";
+	concurrency?: number;
+	batchSize?: number;
+
+    // ids
+    simulationName?: string;
+    name?: string;
 
     //switches
     isAnonymous?: boolean;
+    hasAvatar?: boolean;
     hasLocation?: boolean;
     hasCampaigns?: boolean;
     hasAdSpend?: boolean;
@@ -30,10 +40,15 @@ declare namespace main {
     hasAndroidDevices?: boolean;
     hasDesktopDevices?: boolean;
     hasBrowser?: boolean;
+    writeToDisk?: boolean | string;
+    verbose?: boolean;
+    hasAnonIds?: boolean;
+    hasSessionIds?: boolean;
+	alsoInferFunnels?: boolean;
+    makeChart?: boolean | string;
 
-    format?: "csv" | "json";
-    region?: "US" | "EU";
-    events?: EventConfig[]; //can also be a array of strings
+    //models
+    events?: EventConfig[]; //| string[]; //can also be a array of strings
     superProps?: Record<string, ValueValid>;
     funnels?: Funnel[];
     userProps?: Record<string, ValueValid>;
@@ -42,14 +57,11 @@ declare namespace main {
     groupKeys?: [string, number][] | [string, number, string[]][]; // [key, numGroups, [events]]
     groupProps?: Record<string, Record<string, ValueValid>>;
     lookupTables?: LookupTableSchema[];
-    writeToDisk?: boolean;
-    simulationName?: string;
-    verbose?: boolean;
-    anonIds?: boolean;
-    sessionIds?: boolean;
-    makeChart?: boolean | string;
     soup?: soup;
     hook?: Hook<any>;
+
+    //allow anything to be on the config
+    [key: string]: any;
   }
 
   /**
@@ -85,14 +97,42 @@ declare namespace main {
   export interface hookArrayOptions<T> {
     hook?: Hook<T>;
     type?: hookTypes;
+    filename?: string;
+    format?: "csv" | "json" | string;
+	concurrency?: number;
     [key: string]: any;
   }
 
   /**
    * an enriched array is an array that has a hookPush method that can be used to transform-then-push items into the array
    */
-  export interface EnrichedArray<T> extends Array<T> {
-    hookPush: (item: T) => boolean;
+  export interface HookedArray<T> extends Array<T> {
+    hookPush: (item: T | T[]) => any;
+    flush: () => void;
+    getWriteDir: () => string;
+	getWritePath: () => string;
+    [key: string]: any;
+  }
+
+  export type AllData =
+    | HookedArray<EventSchema>
+    | HookedArray<UserProfile>
+    | HookedArray<GroupProfileSchema>
+    | HookedArray<LookupTableSchema>
+    | HookedArray<SCDSchema>
+    | any[];
+
+  /**
+   * the storage object is a key-value store that holds arrays of data
+   */
+  export interface Storage {
+    eventData?: HookedArray<EventSchema>;
+    mirrorEventData?: HookedArray<EventSchema>;
+    userProfilesData?: HookedArray<UserProfile>;
+    adSpendData?: HookedArray<EventSchema>;
+    groupProfilesData?: HookedArray<GroupProfileSchema>[];
+    lookupTableData?: HookedArray<LookupTableSchema>[];
+    scdTableData?: HookedArray<SCDSchema>[];
   }
 
   /**
@@ -107,6 +147,20 @@ declare namespace main {
     relativeTimeMs?: number;
   }
 
+  /**
+   * the generated event data
+   */
+  export interface EventSchema {
+    event: string;
+    time: string;
+    source: string;
+    insert_id: string;
+    device_id?: string;
+    session_id?: string;
+    user_id?: string;
+    [key: string]: ValueValid;
+  }
+
   /**
    * how we define funnels and their properties
    */
@@ -133,13 +187,15 @@ declare namespace main {
      * how the events in the funnel are ordered for each user
      */
     order?:
+      | string
       | "sequential"
       | "first-fixed"
       | "last-fixed"
       | "random" //totally shuffled
       | "first-and-last-fixed"
       | "middle-fixed"
-      | "interrupted"; //todo: explain this
+      | "interrupted";
+
     /**
      * todo: implement this
      * if set, the funnel might be the last thing the user does
@@ -178,29 +234,10 @@ declare namespace main {
      */
     strategy?: "create" | "update" | "fill" | "delete" | "";
     values?: ValueValid[];
-  }
-
-  /**
-   * the generated event data
-   */
-  export interface EventSchema {
-    event: string;
-    time: string;
-    insert_id: string;
-    device_id?: string;
-    session_id?: string;
-    user_id?: string;
-    [key: string]: ValueValid;
-  }
-
-  export interface EventData {
-    event: string;
-    source: string;
-    time: string;
-    device_id?: string;
-    session_id?: string;
-    user_id?: string;
-    [key: string]: any;
+    /**
+     * optional: for 'fill' mode, daysUnfilled will dictate where the cutoff is in the unfilled data
+     */
+    daysUnfilled?: number;
   }
 
   export interface UserProfile {
@@ -214,8 +251,8 @@ declare namespace main {
 
   export interface Person {
     name: string;
-    email: string;
-    avatar: string;
+    email?: string;
+    avatar?: string;
     created: string | undefined;
     anonymousIds: string[];
     sessionIds: string[];
@@ -262,14 +299,21 @@ declare namespace main {
    * the end result of the data generation
    */
   export type Result = {
-    eventData: EventData[];
+    eventData: EventSchema[];
+    mirrorEventData: EventSchema[];
     userProfilesData: any[];
     scdTableData: any[];
-    adSpendData: EventData[];
+    adSpendData: EventSchema[];
     groupProfilesData: GroupProfileSchema[];
     lookupTableData: LookupTableData[];
     importResults?: ImportResults;
     files?: string[];
+    time?: {
+      start: number;
+      end: number;
+      delta: number;
+      human: string;
+    };
   };
 }