make-mp-data 1.5.55 → 2.0.0

package/scripts/dana.mjs

@@ -0,0 +1,137 @@
+#!/usr/bin/env node
+import { fork } from 'child_process';
+import path from 'path';
+import dotenv from "dotenv";
+dotenv.config();
+const { NODE_ENV = "unknown" } = process.env;
+
+import main from "../index.js";
+import simple from '../dungeons/simple.js';
+import complex from '../dungeons/complex.js';
+import gaming from '../dungeons/customers/gaming.js';
+import big from '../dungeons/big.js';
+
+import Chance from 'chance';
+const chance = new Chance();
+const twentyFivePercent = () => chance.bool({ likelihood: 25 });
+
+// --- shared options ---
+let divisor = 1;
+if (NODE_ENV === "dev") divisor = 1000;
+
+const commonOptions = {
+	numEvents: 10_000_000 / divisor,
+	numUsers: 10_000_000 / 100 / divisor,
+	batchSize: 200_000,
+	hasAdSpend: false,
+	hasAnonIds: true,
+	hasSessionIds: true,
+	format: "json",
+	hasDesktopDevices: true,
+	hasIOSDevices: true,
+	hasAndroidDevices: true,
+	hasBrowser: true,
+	hasCampaigns: true,
+	verbose: true,
+	makeChart: false,
+	writeToDisk: true
+};
+
+
+
+
+const allSpecs = {
+	"simple-spec": { ...simple, ...commonOptions, name: "simple-spec" },
+	"complex-spec": { ...complex, ...commonOptions, name: "complex-spec" },
+	"meta-spec": { ...big, ...commonOptions, name: "meta-spec" },
+	"fun-spec": { ...gaming, ...commonOptions, name: "fun-spec" },
+	"bad-spec": {
+		...big, ...commonOptions, name: "bad-data-spec",
+		hook: function (record, type, meta) {
+			if (type === "event") {
+				if (twentyFivePercent()) delete record.event;
+				if (twentyFivePercent()) delete record.user_id;
+				if (twentyFivePercent()) delete record.session_id;
+				if (twentyFivePercent()) delete record.device_id;
+				if (twentyFivePercent()) delete record.time;
+				if (twentyFivePercent()) delete record.insert_id;
+			}
+			return record;
+		}
+	}
+};
+
+for (const [name, spec] of Object.entries(allSpecs)) {
+	delete spec.groupEvents;
+	delete spec.scdProps;
+	delete spec.lookupTables;
+	delete spec.groupKeys;
+	delete spec.groupProps;
+	delete spec.mirrorProps;
+	delete spec.token;
+	delete spec.secret;
+}
+
+const simulations = Object.entries(allSpecs).map(([name, spec]) => ({ name, spec }));
+
+// 1. If this process is a CHILD, just run the spec!
+if (process.env.__SIM_CHILD) {
+	const name = process.env.__SIM_NAME;
+	const spec = allSpecs[name];
+	(async () => {
+		const result = await main(spec);
+		// Send result to parent
+		if (process.send) process.send(result);
+		else console.log(JSON.stringify(result));
+		process.exit(0);
+	})();
+} else {
+	// 2. PARENT: fork a child for each simulation
+	const { promisify } = await import('util');
+	const limit = 5; // parallelism limit
+	let running = 0, idx = 0, results = [];
+
+	function runOne(sim) {
+		return new Promise((resolve, reject) => {
+			const child = fork(process.argv[1], [], {
+				env: { ...process.env, __SIM_CHILD: "1", __SIM_NAME: sim.name },
+				stdio: ['inherit', 'inherit', 'inherit', 'ipc']
+			});
+			child.on('message', (result) => resolve({ name: sim.name, result }));
+			child.on('error', reject);
+			child.on('exit', (code) => {
+				if (code !== 0) reject(new Error(`Child ${sim.name} exited code ${code}`));
+			});
+		});
+	}
+
+	const queue = [...simulations];
+	const resultsList = [];
+	async function runNext() {
+		while (running < limit && queue.length > 0) {
+			const sim = queue.shift();
+			running++;
+			runOne(sim)
+				.then(res => resultsList.push(res))
+				.catch(e => console.error("Sim error:", e))
+				.finally(() => {
+					running--;
+					runNext();
+				});
+		}
+	}
+	runNext();
+
+	// Wait until all are done
+	const wait = () => new Promise(resolve => {
+		const check = () => {
+			if (resultsList.length === simulations.length) resolve(resultsList);
+			else setTimeout(check, 100);
+		};
+		check();
+	});
+	(await wait()).forEach(res => {
+		console.log(`Finished simulation: ${res.name}`);
+		// optionally: console.log(res.result);
+	});
+}

package/scripts/new-dungeon.sh

@@ -8,13 +8,14 @@ random_file_name=$(mktemp ./dungeons/my-file-XXXXXXXX.js)
 
 # Initial text to write to the file
 initial_text='
+import dayjs from "dayjs";
+import utc from "dayjs/plugin/utc.js";
+import "dotenv/config";
+import * as u from "../lib/utils/utils.js";
+import * as v from "ak-tools";
+
 const SEED = "my-seed";
-const dayjs = require("dayjs");
-const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
-require("dotenv").config();
-const u = require("../components/utils");
-const v = require("ak-tools");
 const chance = u.initChance(SEED);
 const num_users = 25_000
 const days = 100
@@ -88,7 +89,7 @@ const config = {
 	}
 };
 
-module.exports = config;'
+export default config;'
 
 # Write the initial text to the new file
 echo "$initial_text" > "$random_file_name"

package/scripts/update-deps.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
 npm i mixpanel-import@latest --save
 npm i ak-tools@latest --save
-npm i ak-fetch@latest --save
+npm i ak-fetch@latest --save
+npm i ak-gemini@latest --save

package/tests/cli.test.js

@@ -1,27 +1,30 @@
-const generate = require('../index.js');
-require('dotenv').config();
-const { execSync } = require("child_process");
-const u = require('ak-tools');
-const Papa = require('papaparse');
-
-const simple = require('../dungeons/simple.js');
-const complex = require('../dungeons/complex.js');
-const anon = require('../dungeons/anon.js');
-const funnels = require('../dungeons/funnels.js');
-const foobar = require('../dungeons/foobar.js');
-const mirror = require('../dungeons/mirror.js');
-const adspend = require('../dungeons/adspend.js');
-const scd = require('../dungeons/scd.js');
+//@ts-nocheck
+import generate from '../index.js';
+import 'dotenv/config';
+import { execSync } from "child_process";
+import * as u from 'ak-tools';
+import Papa from 'papaparse';
+
+import simple from '../dungeons/simple.js';
+import complex from '../dungeons/complex.js';
+import anon from '../dungeons/anon.js';
+import funnels from '../dungeons/funnels.js';
+import foobar from '../dungeons/foobar.js';
+import mirror from '../dungeons/mirror.js';
+import adspend from '../dungeons/adspend.js';
+import scd from '../dungeons/scd.js';
 
 const timeout = 600000;
 const testToken = process.env.TEST_TOKEN || "hello token!";
 
-describe('cli', () => {
+// Use sequential execution to prevent CLI tests from interfering with each other
+// since they use execSync and create/modify files in the same directories
+describe.sequential('cli', () => {
 
 	test('sanity check', async () => {
 		console.log('SANITY TEST');
-		const run = execSync(`node ./index.js`);
-		const ending = `enjoy your data! :)`;
+		const run = execSync(`node ./index.js --numEvents 100 --numUsers 10`);
+		const ending = `completed successfully!`;
 		expect(run.toString().trim().endsWith(ending)).toBe(true);
 		const files = (await u.ls('./data')).filter(a => a.includes('.csv'));
 		expect(files.length).toBe(2);
@@ -33,8 +36,8 @@ describe('cli', () => {
 		const userProfilesData = (await u.load(users[0])).trim();
 		const parsedEvents = Papa.parse(eventData, { header: true }).data;
 		const parsedUsers = Papa.parse(userProfilesData, { header: true }).data;
-		expect(parsedEvents.length).toBeGreaterThan(42000);
-		expect(parsedUsers.length).toBeGreaterThan(420);
+		expect(parsedEvents.length).toBeGreaterThan(10);
+		expect(parsedUsers.length).toBeGreaterThan(5);
 		expect(parsedUsers.every(u => u.distinct_id)).toBe(true);
 		expect(parsedEvents.every(e => e.event)).toBe(true);
 		expect(parsedEvents.every(e => e.time)).toBe(true);
@@ -50,8 +53,8 @@ describe('cli', () => {
 
 	test('no args', async () => {
 		console.log('BARE CLI TEST');
-		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100`);
-		expect(run.toString().trim().includes('enjoy your data! :)')).toBe(true);
+		const run = execSync(`node ./index.js --numEvents 100 --numUsers 10`);
+		expect(run.toString().trim().includes('completed successfully!')).toBe(true);
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
 		expect(csvs.length).toBe(2);
 
@@ -59,16 +62,16 @@ describe('cli', () => {
 
 	test('--complex', async () => {
 		console.log('COMPLEX CLI TEST');
-		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --complex`, { stdio: "ignore" });
+		const run = execSync(`node ./index.js --numEvents 100 --numUsers 10 --seed "deal with it" --complex`, { stdio: "ignore" });
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.json'));
-		expect(csvs.length).toBe(8);
+		expect(csvs.length).toBeGreaterThan(7);
 
 	}, timeout);
 
 	test('--simple', async () => {
 		console.log('SIMPLE CLI TEST');
-		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --simple`);
-		expect(run.toString().trim().includes('enjoy your data! :)')).toBe(true);
+		const run = execSync(`node ./index.js --numEvents 100 --numUsers 10 --seed "deal with it" --simple`);
+		expect(run.toString().trim().includes('completed successfully!')).toBe(true);
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
 		expect(csvs.length).toBe(2);
 

package/tests/e2e.test.js

@@ -3,29 +3,30 @@
 /* eslint-disable no-undef */
 /* eslint-disable no-debugger */
 /* eslint-disable no-unused-vars */
-const generate = require('../index.js');
-require('dotenv').config();
-const { execSync } = require("child_process");
-const u = require('ak-tools');
-const Papa = require('papaparse');
-
-const simple = require('../dungeons/simple.js');
-const complex = require('../dungeons/complex.js');
-const anon = require('../dungeons/anon.js');
-const funnels = require('../dungeons/funnels.js');
-const foobar = require('../dungeons/foobar.js');
-const mirror = require('../dungeons/mirror.js');
-const adspend = require('../dungeons/adspend.js');
-const scd = require('../dungeons/scd.js');
-
-const timeout = 600000;
+import generate from '../index.js';
+import 'dotenv/config';
+import { execSync } from "child_process";
+import * as u from 'ak-tools';
+import Papa from 'papaparse';
+
+import simple from '../dungeons/simple.js';
+import complex from '../dungeons/complex.js';
+import anon from '../dungeons/anon.js';
+import funnels from '../dungeons/funnels.js';
+import foobar from '../dungeons/foobar.js';
+import mirror from '../dungeons/mirror.js';
+import adspend from '../dungeons/adspend.js';
+import scd from '../dungeons/scd.js';
+
+// 1 minute timeout
+const timeout = 60000;
 const testToken = process.env.TEST_TOKEN || "hello token!";
 
 describe('module', () => {
 
 	test('works as module (no config)', async () => {
 		console.log('MODULE TEST');
-		const results = await generate({ verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const results = await generate({ verbose: false, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
 		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
 		expect(groupProfilesData.length).toBe(0);
@@ -37,7 +38,7 @@ describe('module', () => {
 
 	test('works as module (simple)', async () => {
 		console.log('MODULE TEST: SIMPLE');
-		const results = await generate({ ...simple, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const results = await generate({ ...simple, verbose: false, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
 		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
 		expect(groupProfilesData.length).toBe(0);
@@ -49,7 +50,7 @@ describe('module', () => {
 
 	test('works as module (complex)', async () => {
 		console.log('MODULE TEST: COMPLEX');
-		const results = await generate({ ...complex, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const results = await generate({ ...complex, verbose: false, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
 		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
 		expect(groupProfilesData[0]?.length).toBe(500);
@@ -62,7 +63,7 @@ describe('module', () => {
 
 	test('works as module (funnels)', async () => {
 		console.log('MODULE TEST: FUNNELS');
-		const results = await generate({ ...funnels, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const results = await generate({ ...funnels, verbose: false, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
 		const { eventData, groupProfilesData, scdTableData, userProfilesData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
 		expect(groupProfilesData.length).toBe(3);
@@ -75,7 +76,7 @@ describe('module', () => {
 
 	test('works as module (mirror)', async () => {
 		console.log('MODULE TEST: MIRROR');
-		const results = await generate({ ...mirror, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const results = await generate({ ...mirror, verbose: false, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
 		const { eventData, userProfilesData, mirrorEventData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
 		expect(mirrorEventData.length).toBeGreaterThan(980);
@@ -87,7 +88,7 @@ describe('module', () => {
 
 	test('works as module (foobar)', async () => {
 		console.log('MODULE TEST: FOOBAR');
-		const results = await generate({ ...foobar, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const results = await generate({ ...foobar, verbose: false, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
 		const { eventData, userProfilesData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
 		expect(userProfilesData.length).toBe(100);
@@ -96,7 +97,7 @@ describe('module', () => {
 
 	test('works as module (adspend)', async () => {
 		console.log('MODULE TEST: ADSPEND');
-		const results = await generate({ ...adspend, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const results = await generate({ ...adspend, verbose: false, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
 		const { eventData, adSpendData, userProfilesData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
 		expect(userProfilesData.length).toBe(100);
@@ -107,17 +108,18 @@ describe('module', () => {
 
 	test('works as module (scd)', async () => {
 		console.log('MODULE TEST: scd');
-		scd;
-		const results = await generate({
+		// const scdSchema = (await import('../dungeons/scd.js')).default;
+		const config = {
 			...scd,
 			token: testToken,
 			serviceAccount: process.env.SERVICE_ACCOUNT,
 			projectId: process.env.PROJECT_ID,
 			serviceSecret: process.env.SERVICE_SECRET,
-			verbose: true, writeToDisk: false, numEvents: 100, numUsers: 10, seed: "deal with it"
-		});
-		const { importResults} = results;
-		const {MRR_scd, NPS_scd, plan_scd, role_scd} = importResults;
+			verbose: false, writeToDisk: false, numEvents: 100, numUsers: 10, seed: "deal with it"
+		};
+		const results = await generate(config);
+		const { importResults } = results;
+		const { MRR_scd, NPS_scd, plan_scd, role_scd } = importResults;
 		expect(MRR_scd.success).toBeGreaterThan(10);
 		expect(NPS_scd.success).toBeGreaterThan(10);
 		expect(plan_scd.success).toBeGreaterThan(10);
@@ -126,7 +128,7 @@ describe('module', () => {
 		expect(NPS_scd.failed).toBe(0);
 		expect(plan_scd.failed).toBe(0);
 		expect(role_scd.failed).toBe(0);
-	
+
 
 	}, timeout);
 
@@ -289,7 +291,7 @@ describe('options + tweaks', () => {
 
 	test('sends data to mixpanel', async () => {
 		console.log('NETWORK TEST');
-		const results = await generate({ verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it", token: testToken });
+		const results = await generate({ verbose: false, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it", token: testToken });
 		const { events, users, groups } = results.importResults;
 		expect(events.success).toBeGreaterThan(980);
 		expect(users.success).toBe(100);
@@ -298,7 +300,7 @@ describe('options + tweaks', () => {
 
 	test('every record is valid', async () => {
 		console.log('VALIDATION TEST');
-		const results = await generate({ verbose: true, writeToDisk: false, numEvents: 1000, numUsers: 100 });
+		const results = await generate({ verbose: false, writeToDisk: false, numEvents: 1000, numUsers: 100 });
 		const { eventData, userProfilesData } = results;
 		const areEventsValid = eventData.every(validateEvent);
 		const areUsersValid = userProfilesData.every(validateUser);
@@ -312,7 +314,7 @@ describe('options + tweaks', () => {
 
 	test('every date is valid', async () => {
 		console.log('DATE TEST');
-		const results = await generate({ ...simple, writeToDisk: false, verbose: true, numEvents: 1000, numUsers: 100 });
+		const results = await generate({ ...simple, writeToDisk: false, verbose: false, numEvents: 1000, numUsers: 100 });
 		const { eventData } = results;
 		const invalidDates = eventData.filter(e => !validTime(e.time));
 		expect(eventData.every(e => validTime(e.time))).toBe(true);
@@ -321,7 +323,7 @@ describe('options + tweaks', () => {
 
 	test('anonymous users', async () => {
 		console.log('ANON TEST');
-		const results = await generate({ ...anon, writeToDisk: false, verbose: true, numEvents: 1000, numUsers: 100 });
+		const results = await generate({ ...anon, writeToDisk: false, verbose: false, numEvents: 1000, numUsers: 100 });
 		const { userProfilesData } = results;
 		expect(userProfilesData.every(u => u.name === 'Anonymous User')).toBe(true);
 
@@ -329,7 +331,7 @@ describe('options + tweaks', () => {
 
 	test('no avatars (default)', async () => {
 		console.log('AVATAR TEST');
-		const results = await generate({ ...simple, writeToDisk: false, verbose: true, numEvents: 1000, numUsers: 100 });
+		const results = await generate({ ...simple, writeToDisk: false, verbose: false, numEvents: 1000, numUsers: 100 });
 		const { userProfilesData } = results;
 		expect(userProfilesData.every(u => !u.avatar)).toBe(true);
 
@@ -337,7 +339,7 @@ describe('options + tweaks', () => {
 
 	test('yes avatars', async () => {
 		console.log('AVATAR TEST');
-		const results = await generate({ ...simple, writeToDisk: false, verbose: true, numEvents: 1000, numUsers: 100, hasAvatar: true });
+		const results = await generate({ ...simple, writeToDisk: false, verbose: false, numEvents: 1000, numUsers: 100, hasAvatar: true });
 		const { userProfilesData } = results;
 		expect(userProfilesData.every(u => u.avatar)).toBe(true);