make-mp-data 1.5.0 → 1.5.2

package/tests/int.test.js

@@ -6,7 +6,7 @@ dayjs.extend(utc);
 require('dotenv').config();
 const path = require('path');
 
-/** @typedef {import('../types.js').Config} Config */
+/** @typedef {import('../types.js').Dungeon} Config */
 /** @typedef {import('../types.js').EventConfig} EventConfig */
 /** @typedef {import("../types.js").EventSchema} EventSchema */
 /** @typedef {import('../types.js').ValueValid} ValueValid */
@@ -23,7 +23,7 @@ const { generators, orchestrators, meta } = MAIN;
 const { makeAdSpend, makeEvent, makeFunnel, makeProfile, makeSCD, makeMirror } = generators;
 const { sendToMixpanel, userLoop, validateDungeonConfig } = orchestrators;
 const { hookArray, inferFunnels } = meta;
-const { validEvent } = require('../src/utils.js');
+const { validEvent } = require('../components/utils.js');
 
 
 // Mock the global variables
@@ -31,7 +31,7 @@ let CAMPAIGNS;
 let DEFAULTS;
 let STORAGE;
 let CONFIG;
-const { campaigns, devices, locations } = require('../src/defaults.js');
+const { campaigns, devices, locations } = require('../components/defaults.js');
 
 beforeEach(async () => {
 	// Reset global variables before each test
@@ -72,7 +72,8 @@ beforeEach(async () => {
 	global.DEFAULTS = DEFAULTS;
 	global.STORAGE = STORAGE;
 	global.CONFIG = CONFIG;
-	global.NOW = dayjs().unix(); // Mock global NOW
+	const FIXED_NOW = dayjs('2024-02-02').unix();
+	global.FIXED_NOW = FIXED_NOW;
 
 });
 
@@ -137,7 +138,7 @@ describe('generators', () => {
 				prop3: ["value5"]
 			},
 		};
-		const result = await makeEvent("known_id", dayjs().subtract(1, 'd').unix(), eventConfig, ["anon_id"], ["session_id"]);
+		const result = await makeEvent("known_id", dayjs.unix(global.FIXED_NOW).subtract(30, 'd').unix(), eventConfig, ["anon_id"], ["session_id"]);
 		expect(result).toHaveProperty('event', 'test_event');
 		expect(result).toHaveProperty('device_id', 'anon_id');
 		// expect(result).toHaveProperty('user_id', 'known_id'); // Known ID not always on the event
@@ -154,7 +155,7 @@ describe('generators', () => {
 
 	test('makeEvent: opt params', async () => {
 		const eventConfig = { event: "test_event", properties: {} };
-		const result = await makeEvent("known_id", dayjs().subtract(1, 'd').unix(), eventConfig);
+		const result = await makeEvent("known_id",dayjs.unix(global.FIXED_NOW).subtract(30, 'd').unix(), eventConfig);
 		expect(result).toHaveProperty('event', 'test_event');
 		expect(result).toHaveProperty('user_id', 'known_id');
 		expect(result).toHaveProperty('source', 'dm4');
@@ -170,7 +171,7 @@ describe('generators', () => {
 				prop2: ["value3", "value4"]
 			},
 		};
-		const result = await makeEvent("known_id", dayjs().subtract(1, 'd').unix(), eventConfig, ["anon_id"], ["session_id"]);
+		const result = await makeEvent("known_id",dayjs.unix(global.FIXED_NOW).subtract(30, 'd').unix(), eventConfig, ["anon_id"], ["session_id"]);
 		expect(result.prop1 === "value1" || result.prop1 === "value2").toBeTruthy();
 		expect(result.prop2 === "value3" || result.prop2 === "value4").toBeTruthy();
 	});
@@ -189,7 +190,7 @@ describe('generators', () => {
 		/** @type {Record<string, SCDSchema[]>} */
 		const scd = { "scd_example": [{ distinct_id: "user1", insertTime: dayjs().toISOString(), startTime: dayjs().toISOString() }] };
 
-		const [result, converted] = await makeFunnel(funnelConfig, user, dayjs().unix(), profile, scd, {});
+		const [result, converted] = await makeFunnel(funnelConfig, user,dayjs.unix(global.FIXED_NOW).subtract(30, 'd').unix(), profile, scd, {});
 		expect(result.length).toBe(2);
 		expect(converted).toBe(true);
 		expect(result.every(e => validEvent(e))).toBeTruthy();
@@ -205,7 +206,7 @@ describe('generators', () => {
 		const profile = { created: dayjs().toISOString(), distinct_id: "user1" };
 		const scd = { "scd_example": [{ distinct_id: "user1", insertTime: dayjs().toISOString(), startTime: dayjs().toISOString() }] };
 
-		const [result, converted] = await makeFunnel(funnelConfig, user, dayjs().unix(), profile, scd, {});
+		const [result, converted] = await makeFunnel(funnelConfig, user, dayjs.unix(global.FIXED_NOW).subtract(30, 'd').unix(), profile, scd, {});
 		expect(result.length).toBeGreaterThanOrEqual(1);
 		expect(result.length).toBeLessThanOrEqual(3);
 		expect(result.every(e => validEvent(e))).toBeTruthy();
@@ -221,7 +222,7 @@ describe('generators', () => {
 		const profile = { created: dayjs().toISOString(), distinct_id: "user1" };
 		const scd = { "scd_example": [{ distinct_id: "user1", insertTime: dayjs().toISOString(), startTime: dayjs().toISOString() }] };
 
-		const [result, converted] = await makeFunnel(funnelConfig, user, dayjs().unix(), profile, scd, {});
+		const [result, converted] = await makeFunnel(funnelConfig, user, dayjs.unix(global.FIXED_NOW).subtract(30, 'd').unix(), profile, scd, {});
 		expect(result.length).toBe(3);
 		expect(converted).toBe(true);
 		expect(result.every(e => validEvent(e))).toBeTruthy();
@@ -490,12 +491,12 @@ describe('orchestrators', () => {
 		};
 		const result = await sendToMixpanel(CONFIG, STORAGE);
 		expect(result.events.success).toBe(0);
-		expect(result.users.success).toBe(0)
+		expect(result.users.success).toBe(0);
 		expect(result.groups).toHaveLength(0);
 	});
 
 
-	test('userLoop: works (no funnels)', async () => {
+	test('userLoop: works (no funnels; no inference)', async () => {
 		/** @type {Config} */
 		const config = {
 			numUsers: 2,
@@ -508,6 +509,7 @@ describe('orchestrators', () => {
 			hasAnonIds: false,
 			hasSessionIds: false,
 			hasLocation: false,
+			alsoInferFunnels: false,
 			events: [{ event: "foo" }, { event: "bar" }, { event: "baz" }]
 		};
 		await userLoop(config, STORAGE);

package/tests/jest.config.js

@@ -7,6 +7,13 @@ const jestConfig = {
 	verbose: isDebugMode,
 	watch: false,	
 	projects: [
+		{
+			"displayName": "cli",
+			"testMatch": [
+				"<rootDir>/tests/cli.test.js"
+			],
+			maxWorkers: 1
+		},
 		{
 			"displayName": "e2e",
 			"testMatch": [
@@ -14,6 +21,7 @@ const jestConfig = {
 			],		
 			// @ts-ignore
 			maxWorkers: 1
+
 			
 		},
 		{

package/tests/testCases.mjs

@@ -226,4 +226,4 @@ const TEST_CASES = [
 
 ];
 
-module.exports = TEST_CASES;
+export default TEST_CASES;

package/tests/testSoup.mjs

@@ -1,8 +1,9 @@
-import { generateLineChart } from '../src/chart.js';
-import { TimeSoup } from '../src/utils.js';
+import { generateLineChart } from '../components/chart.js';
+import { TimeSoup } from '../components/utils.js';
 import dayjs from 'dayjs';
 import { progress } from 'ak-tools';
 import TEST_CASES from './testCases.mjs';
+
 import execSync from 'child_process';
 
 async function genViz(soup) {
@@ -12,7 +13,7 @@ async function genViz(soup) {
 	const end = dayjs().unix();
 	console.log(`\n\nTEST CASE: ${name}\n\n`);
 	for (let i = 0; i < 100_000; i++) {
-		progress('processing', i);
+		progress([['processing', i]]);
 		const time = TimeSoup(start, end, peaks, deviation, mean);
 		data.push({ time });
 	}

package/tests/unit.test.js

@@ -6,7 +6,7 @@ const u = require('ak-tools');
 dayjs.extend(utc);
 require('dotenv').config();
 
-/** @typedef {import('../types').Config} Config */
+/** @typedef {import('../types').Dungeon} Config */
 /** @typedef {import('../types').EventConfig} EventConfig */
 /** @typedef {import('../types').ValueValid} ValueValid */
 /** @typedef {import('../types').HookedArray} hookArray */
@@ -57,7 +57,7 @@ const {
 
 	datesBetween,
 	weighChoices
-} = require('../src/utils.js');
+} = require('../components/utils.js');
 
 const main = require('../index.js');
 //todo: test for funnel inference
@@ -283,7 +283,7 @@ describe('generation', () => {
 		const user = generateUser('uuid-123', { numDays });
 		const createdDate = dayjs(user.created, 'YYYY-MM-DD');
 		expect(createdDate.isValid()).toBeTruthy();
-		expect(createdDate.isBefore(dayjs.unix(global.NOW))).toBeTruthy();
+		expect(createdDate.isBefore(dayjs())).toBeTruthy();
 	});
 
 
@@ -393,7 +393,8 @@ describe('generation', () => {
 describe('validation', () => {
 
 	beforeAll(() => {
-		global.NOW = 1672531200; // fixed point in time for testing
+		global.FIXED_NOW = 1672531200; // fixed point in time for testing
+		global.FIXED_BEGIN = global.FIXED_NOW - (60 * 60 * 24 * 30); // 30 days ago
 	});
 
 	test('events: non arrays', () => {
@@ -438,33 +439,33 @@ describe('validation', () => {
 	});
 
 	test('time: between', () => {
-		const chosenTime = global.NOW - (60 * 60 * 24 * 15); // 15 days ago
-		const earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
-		const latestTime = global.NOW;
+		const chosenTime = global.FIXED_NOW - (60 * 60 * 24 * 15); // 15 days ago
+		const earliestTime = global.FIXED_NOW - (60 * 60 * 24 * 30); // 30 days ago
+		const latestTime = global.FIXED_NOW;
 		expect(validTime(chosenTime, earliestTime, latestTime)).toBe(true);
 	});
 
 	test('time: outside earliest', () => {
-		const chosenTime = global.NOW - (60 * 60 * 24 * 31); // 31 days ago
-		const earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
-		const latestTime = global.NOW;
+		const chosenTime = global.FIXED_NOW - (60 * 60 * 24 * 31); // 31 days ago
+		const earliestTime = global.FIXED_NOW - (60 * 60 * 24 * 30); // 30 days ago
+		const latestTime = global.FIXED_NOW;
 		expect(validTime(chosenTime, earliestTime, latestTime)).toBe(false);
 	});
 
 	test('time: outside latest', () => {
 		const chosenTime = -1;
-		const earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
-		const latestTime = global.NOW;
+		const earliestTime = global.FIXED_NOW - (60 * 60 * 24 * 30); // 30 days ago
+		const latestTime = global.FIXED_NOW;
 		expect(validTime(chosenTime, earliestTime, latestTime)).toBe(false);
 	});
 
 	test('time: inference in', () => {
-		const chosenTime = global.NOW - (60 * 60 * 24 * 15); // 15 days ago
+		const chosenTime = global.FIXED_NOW - (60 * 60 * 24 * 15); // 15 days ago
 		expect(validTime(chosenTime)).toBe(true);
 	});
 
 	test('time: inference out', () => {
-		const chosenTime = global.NOW - (60 * 60 * 24 * 31); // 31 days ago
+		const chosenTime = global.FIXED_NOW - (60 * 60 * 24 * 31); // 31 days ago
 		expect(validTime(chosenTime)).toBe(false);
 	});
 });
@@ -547,7 +548,7 @@ describe('utilities', () => {
 	test('date: future', () => {
 		const futureDate = date(10, false, 'YYYY-MM-DD')();
 		expect(dayjs(futureDate, 'YYYY-MM-DD').isValid()).toBeTruthy();
-		expect(dayjs(futureDate).isAfter(dayjs.unix(global.NOW))).toBeTruthy();
+		expect(dayjs(futureDate).isAfter(dayjs.unix(global.FIXED_NOW))).toBeTruthy();
 	});
 
 	test('dates: pairs', () => {

package/tsconfig.json

@@ -11,7 +11,7 @@
   },
   "include": [
     "**/*.js"
-, "soupTemplates.mjs", "tests/testSoup.mjs", "tests/testCases.mjs", "tests/benchmark/concurrency.mjs"  ],
+, "soupTemplates.mjs", "tests/testSoup.mjs", "tests/testCases.mjs", "tests/benchmark/concurrency.mjs", "dungeons/customers/pos.mjs", "scripts/new-project.mjs"  ],
   "exclude": [
     "node_modules"
   ]

package/types.d.ts

@@ -12,7 +12,7 @@ declare namespace main {
   /**
    * main config object for the entire data generation
    */
-  export interface Config {
+  export interface Dungeon {
     // constants
     token?: string;
     seed?: string;
@@ -23,9 +23,13 @@ declare namespace main {
     numUsers?: number;
     format?: "csv" | "json" | string;
     region?: "US" | "EU";
-	concurrency?: number;
-	batchSize?: number;
+    concurrency?: number;
+    batchSize?: number;
 
+	serviceAccount?: string;
+	serviceSecret?: string;
+	projectId?: string;
+	
     // ids
     simulationName?: string;
     name?: string;
@@ -44,6 +48,7 @@ declare namespace main {
     verbose?: boolean;
     hasAnonIds?: boolean;
     hasSessionIds?: boolean;
+    alsoInferFunnels?: boolean;
     makeChart?: boolean | string;
 
     //models
@@ -51,18 +56,34 @@ declare namespace main {
     superProps?: Record<string, ValueValid>;
     funnels?: Funnel[];
     userProps?: Record<string, ValueValid>;
-    scdProps?: Record<string, ValueValid>;
+    scdProps?: Record<string, SCDProp>;
     mirrorProps?: Record<string, MirrorProps>;
     groupKeys?: [string, number][] | [string, number, string[]][]; // [key, numGroups, [events]]
     groupProps?: Record<string, Record<string, ValueValid>>;
+    groupEvents?: GroupEventConfig[];
     lookupTables?: LookupTableSchema[];
     soup?: soup;
     hook?: Hook<any>;
 
     //allow anything to be on the config
     [key: string]: any;
+
+	//probabilities
+	percentUsersBornInDataset?: number;
   }
 
+  export type complexSCDProp = {
+    type: string;
+    frequency: "day" | "week" | "month" | "year";
+    values: ValueValid;
+    timing: "fixed" | "fuzzy";
+    max?: number;
+  };
+
+  export type SimpleSCDProp = string[];
+
+  export type SCDProp = complexSCDProp | SimpleSCDProp;
+
   /**
    * the soup is a set of parameters that determine the distribution of events over time
    */
@@ -81,11 +102,14 @@ declare namespace main {
     | "group"
     | "lookup"
     | "scd"
+    | "scd-pre"
     | "mirror"
     | "funnel-pre"
     | "funnel-post"
     | "ad-spend"
     | "churn"
+    | "group-event"
+    | "everything"
     | "";
 
   /**
@@ -98,7 +122,7 @@ declare namespace main {
     type?: hookTypes;
     filename?: string;
     format?: "csv" | "json" | string;
-	concurrency?: number;
+    concurrency?: number;
     [key: string]: any;
   }
 
@@ -106,10 +130,10 @@ declare namespace main {
    * an enriched array is an array that has a hookPush method that can be used to transform-then-push items into the array
    */
   export interface HookedArray<T> extends Array<T> {
-    hookPush: (item: T | T[]) => any;
+    hookPush: (item: T | T[], ...meta) => any;
     flush: () => void;
     getWriteDir: () => string;
-	getWritePath: () => string;
+    getWritePath: () => string;
     [key: string]: any;
   }
 
@@ -132,6 +156,7 @@ declare namespace main {
     groupProfilesData?: HookedArray<GroupProfileSchema>[];
     lookupTableData?: HookedArray<LookupTableSchema>[];
     scdTableData?: HookedArray<SCDSchema>[];
+    groupEventData?: HookedArray<EventSchema>;
   }
 
   /**
@@ -146,6 +171,13 @@ declare namespace main {
     relativeTimeMs?: number;
   }
 
+  export interface GroupEventConfig extends EventConfig {
+    frequency: number; //how often the event occurs (in days)
+    group_key: string; //the key that the group is based on
+    attribute_to_user: boolean; //if true, the event also goes to a user
+    group_size: number; //the number of users in the group
+  }
+
   /**
    * the generated event data
    */
@@ -323,6 +355,6 @@ declare namespace main {
  * const gen = require('make-mp-data')
  * const dta = gen({writeToDisk: false})
  */
-declare function main(config: main.Config): Promise<main.Result>;
+declare function main(config: main.Dungeon): Promise<main.Result>;
 
 export = main;

package/scripts/deps.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-npm i mixpanel-import@latest --save
-npm i ak-tools@latest --save