make-mp-data 1.4.4 → 1.5.0

package/tests/e2e.test.js

@@ -3,18 +3,22 @@
 /* eslint-disable no-undef */
 /* eslint-disable no-debugger */
 /* eslint-disable no-unused-vars */
-const generate = require('../core/index.js');
+const generate = require('../index.js');
 require('dotenv').config();
 const { execSync } = require("child_process");
 const u = require('ak-tools');
+const Papa = require('papaparse');
 
 const simple = require('../schemas/simple.js');
 const complex = require('../schemas/complex.js');
-const deep = require('../schemas/deepNest.js');
 const anon = require('../schemas/anon.js');
+const funnels = require('../schemas/funnels.js');
+const foobar = require('../schemas/foobar.js');
+const mirror = require('../schemas/mirror.js');
+const adspend = require('../schemas/adspend.js');
 
-const timeout = 60000;
-const testToken = process.env.TEST_TOKEN;
+const timeout = 600000;
+const testToken = process.env.TEST_TOKEN || "hello token!";
 
 describe('module', () => {
 
@@ -47,26 +51,62 @@ describe('module', () => {
 		const results = await generate({ ...complex, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
 		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
-		expect(groupProfilesData[0]?.data?.length).toBe(500);
+		expect(groupProfilesData[0]?.length).toBe(500);
 		expect(lookupTableData.length).toBe(2);
-		expect(lookupTableData[0].data.length).toBe(1000);
+		expect(lookupTableData[0].length).toBe(1000);
 		expect(scdTableData.length).toBe(5);
 		expect(userProfilesData.length).toBe(100);
 
 	}, timeout);
 
-	test('works as module (deep nest)', async () => {
-		console.log('MODULE TEST: DEEP NEST');
-		const results = await generate({ ...deep, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
-		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
+	test('works as module (funnels)', async () => {
+		console.log('MODULE TEST: FUNNELS');
+		const results = await generate({ ...funnels, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const { eventData, groupProfilesData, scdTableData, userProfilesData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
-		expect(groupProfilesData.length).toBe(0);
-		expect(lookupTableData.length).toBe(0);
-		expect(scdTableData.length).toBe(0);
+		expect(groupProfilesData.length).toBe(3);
+		expect(groupProfilesData[0]?.length).toBe(5000);
+		expect(groupProfilesData[1]?.length).toBe(500);
+		expect(groupProfilesData[2]?.length).toBe(50);
+		expect(scdTableData.length).toBe(2);
+		expect(scdTableData[0]?.length).toBeGreaterThan(200);
+		expect(scdTableData[1]?.length).toBeGreaterThan(200);
+		expect(userProfilesData.length).toBe(100);
+
+	}, timeout);
+
+	test('works as module (mirror)', async () => {
+		console.log('MODULE TEST: MIRROR');
+		const results = await generate({ ...mirror, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const { eventData, userProfilesData, mirrorEventData } = results;
+		expect(eventData.length).toBeGreaterThan(980);
+		expect(mirrorEventData.length).toBeGreaterThan(980);
+		expect(mirrorEventData.every(e => e.newlyCreated)).toBe(true);
+		expect(eventData.every(e => e.newlyCreated)).toBe(false);
 		expect(userProfilesData.length).toBe(100);
 
 	}, timeout);
 
+	test('works as module (foobar)', async () => {
+		console.log('MODULE TEST: FOOBAR');
+		const results = await generate({ ...foobar, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const { eventData, userProfilesData } = results;
+		expect(eventData.length).toBeGreaterThan(980);
+		expect(userProfilesData.length).toBe(100);
+
+	}, timeout);
+
+	test('works as module (adspend)', async () => {
+		console.log('MODULE TEST: ADSPEND');
+		const results = await generate({ ...adspend, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const { eventData, adSpendData, userProfilesData } = results;
+		expect(eventData.length).toBeGreaterThan(980);
+		expect(userProfilesData.length).toBe(100);
+		expect(adSpendData.length).toBe(14600);
+
+	}, timeout);
+
+
 	test('fails with invalid configuration', async () => {
 		try {
 			await generate({ numUsers: -10 });
@@ -78,83 +118,210 @@ describe('module', () => {
 
 	test('works with no params', async () => {
 		const { eventData, userProfilesData, groupProfilesData, files, importResults, lookupTableData, mirrorEventData, scdTableData } = await generate({ writeToDisk: false });
-		debugger;
+		expect(eventData.length).toBeGreaterThan(100000);
+		expect(userProfilesData.length).toBe(1000);
+		expect(groupProfilesData.length).toBe(0);
+		expect(importResults).toBe(undefined);
+		expect(scdTableData.length).toBe(0);
+		expect(lookupTableData.length).toBe(0);
+		expect(mirrorEventData.length).toBe(0);
 	}, timeout);
 
 
 
 
+});
+
+describe('batching', () => {
+	test('batch writes', async () => {
+		const results = await generate({ ...foobar, batchSize: 1000, writeToDisk: true, numEvents: 10_000, numUsers: 5000, seed: "deal" });
+		const { eventData, userProfilesData } = results;
+		const files = (await u.ls('./data')).filter(a => a.endsWith('.json'));
+		const eventFiles = files.filter(a => a.includes('EVENTS'));
+		const userFiles = files.filter(a => a.includes('USERS'));
+		const evWriteDir = eventData.getWriteDir();
+		const usWriteDir = userProfilesData.getWriteDir();
+		const evWritePath = eventData.getWritePath();
+		const usWritePath = userProfilesData.getWritePath();
+
+		const expectedEvWriteDir = `-EVENTS.json`;
+		const expectedUsWriteDir = `-USERS.json`;
+		const expectedWritePath = `-part-`;
+
+		expect(eventFiles.length).toBe(23);
+		expect(userFiles.length).toBe(5);
+
+		expect(eventFiles.filter(a => a.includes('part')).length).toBe(23);
+		expect(userFiles.filter(a => a.includes('part')).length).toBe(5);
+		expect(evWriteDir.endsWith(expectedEvWriteDir)).toBe(true);
+		expect(usWriteDir.endsWith(expectedUsWriteDir)).toBe(true);
+		expect(evWritePath.includes(expectedWritePath)).toBe(true);
+		expect(usWritePath.includes(expectedWritePath)).toBe(true);
+		
+	}, timeout);
+
+
+	test('dont batch', async () => {
+		const results = await generate({ ...foobar, writeToDisk: true, numEvents: 5000, numUsers: 1000, seed: "deal" });
+		const { eventData, userProfilesData } = results;
+		const files = await u.ls('./data');
+		const eventFiles = files.filter(a => a.includes('EVENTS'));
+		const userFiles = files.filter(a => a.includes('USERS'));
+		expect(eventFiles.length).toBe(1);
+		expect(userFiles.length).toBe(1);
+		expect(eventFiles.filter(a => a.includes('part')).length).toBe(0);
+		const evWriteDir = eventData.getWriteDir();
+		const usWriteDir = userProfilesData.getWriteDir();
+		const expectedEvWriteDir = `-EVENTS.json`;
+		const expectedUsWriteDir = `-USERS.json`;
+		expect(evWriteDir.endsWith(expectedEvWriteDir)).toBe(true);
+		expect(usWriteDir.endsWith(expectedUsWriteDir)).toBe(true);
+
+		const evWritePath = eventData.getWritePath();
+		const usWritePath = userProfilesData.getWritePath();
+		expect(evWritePath.endsWith(expectedEvWriteDir)).toBe(true);
+		expect(usWritePath.endsWith(expectedUsWriteDir)).toBe(true);
+		
+	}, timeout);
+	
+	test('send to mp: batches', async () => {
+		const results = await generate({ ...foobar, numDays: 90, hasAdSpend: true, token: testToken, batchSize: 4500, writeToDisk: true, numEvents: 10_000, numUsers: 5000, seed: "deal" });
+		const { importResults } = results;
+		const { adSpend, events, groups, users } = importResults;
+		expect(adSpend.success).toBeGreaterThan(0);
+		expect(events.success).toBeGreaterThan(0);
+		expect(users.success).toBeGreaterThan(0);
+		expect(groups[0].success).toBeGreaterThan(0);
+		expect(groups[1].success).toBeGreaterThan(0);
+		expect(adSpend.success).toBe(adSpend.total);
+		expect(events.success).toBe(events.total);
+	//	expect(users.success).toBe(users.total);
+		expect(groups.length).toBe(2);
+		expect(groups[0].success).toBe(groups[0].total);
+		expect(groups[1].success).toBe(groups[1].total);
+		expect(adSpend.failed).toBe(0);
+		expect(events.failed).toBe(0);
+		expect(users.failed).toBe(0);
+		expect(groups[0].failed).toBe(0);
+		expect(groups[1].failed).toBe(0);
+
+		
+	}, timeout);
+
+	test('send to mp: no batch', async () => {
+		const results = await generate({ ...foobar, numDays: 90, hasAdSpend: true, token: testToken, writeToDisk: true, numEvents: 5000, numUsers: 1000, seed: "deal" });
+		const { importResults } = results;
+		const { adSpend, events, groups, users } = importResults;
+		expect(adSpend.success).toBeGreaterThan(0);
+		expect(events.success).toBeGreaterThan(0);
+		expect(users.success).toBeGreaterThan(0);
+		expect(groups[0].success).toBeGreaterThan(0);
+		expect(groups[1].success).toBeGreaterThan(0);
+		expect(adSpend.success).toBe(adSpend.total);
+		expect(events.success).toBe(events.total);
+		expect(users.success).toBe(users.total);
+		expect(groups.length).toBe(2);
+		expect(groups[0].success).toBe(groups[0].total);
+		expect(groups[1].success).toBe(groups[1].total);
+		expect(adSpend.failed).toBe(0);
+		expect(events.failed).toBe(0);
+		expect(users.failed).toBe(0);
+		expect(groups[0].failed).toBe(0);
+		expect(groups[1].failed).toBe(0);
+		
+	}, timeout);
 });
 
 describe('cli', () => {
-	test('works as CLI (no args)', async () => {
-		console.log('COMPLEX CLI TEST');
-		const run = execSync(`node ./core/index.js --numEvents 1000 --numUsers 100`, { stdio: 'ignore' });
-		// expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+
+	test('sanity check', async () => {
+		console.log('SANITY TEST');
+		const run = execSync(`node ./index.js`);
+		const ending = `enjoy your data! :)`;
+		expect(run.toString().trim().endsWith(ending)).toBe(true);
+		const files = (await u.ls('./data')).filter(a => a.includes('.csv'));
+		expect(files.length).toBe(2);
+		const users = files.filter(a => a.includes('USERS'));
+		const events = files.filter(a => a.includes('EVENTS'));
+		expect(users.length).toBe(1);
+		expect(events.length).toBe(1);
+		const eventData = (await u.load(events[0])).trim();
+		const userProfilesData = (await u.load(users[0])).trim();
+		const parsedEvents = Papa.parse(eventData, { header: true }).data;
+		const parsedUsers = Papa.parse(userProfilesData, { header: true }).data;
+		expect(parsedEvents.length).toBeGreaterThan(42000);
+		expect(parsedUsers.length).toBeGreaterThan(420);
+		expect(parsedUsers.every(u => u.distinct_id)).toBe(true);
+		expect(parsedEvents.every(e => e.event)).toBe(true);
+		expect(parsedEvents.every(e => e.time)).toBe(true);
+		expect(parsedEvents.every(e => e.insert_id)).toBe(true);
+		expect(parsedEvents.every(e => e.device_id || e.user_id)).toBe(true);
+		expect(parsedUsers.every(u => u.name)).toBe(true);
+		expect(parsedUsers.every(u => u.email)).toBe(true);
+		expect(parsedUsers.every(u => u.created)).toBe(true);
+		expect(parsedUsers.every(u => u.avatar)).toBe(false);
+		expect(parsedEvents.every(e => validateEvent(e))).toBe(true);
+		expect(parsedUsers.every(u => validateUser(u))).toBe(true);
+	}, timeout);
+
+	test('no args', async () => {
+		console.log('BARE CLI TEST');
+		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100`);
+		expect(run.toString().trim().includes('enjoy your data! :)')).toBe(true);
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(4);
-		clearData();
+		expect(csvs.length).toBe(2);
+		
 	}, timeout);
 
-	test('works as CLI (complex)', async () => {
+	test('--complex', async () => {
 		console.log('COMPLEX CLI TEST');
-		const run = execSync(`node ./core/index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --complex`, { stdio: 'ignore' });
-		// expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --complex`, { stdio: "ignore" });
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
 		expect(csvs.length).toBe(13);
-		clearData();
+		
 	}, timeout);
 
-	test('works as CLI (simple)', async () => {
-		console.log('simple CLI TEST');
-		const run = execSync(`node ./core/index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --simple`);
-		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+	test('--simple', async () => {
+		console.log('SIMPLE CLI TEST');
+		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --simple`);
+		expect(run.toString().trim().includes('enjoy your data! :)')).toBe(true);
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(4);
-		clearData();
+		expect(csvs.length).toBe(2);
+	
 	}, timeout);
 
-	test('works as CLI (custom)', async () => {
-		console.log('custom CLI TEST');
-		const run = execSync(`node ./core/index.js ./schemas/deepNest.js`);
-		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
-		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(3);
-		clearData();
-	}, timeout);
 
 });
 
-
 describe('options + tweaks', () => {
 	test('creates sessionIds', async () => {
-		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, sessionIds: true });
+		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, hasSessionIds: true });
 		const { eventData } = results;
 		const sessionIds = eventData.map(a => a.session_id).filter(a => a);
 		expect(sessionIds.length).toBe(eventData.length);
 	}, timeout);
 
-	test('no sessionIds', async () => {
-		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, sessionIds: false });
+	test('no hasSessionIds', async () => {
+		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, hasSessionIds: false });
 		const { eventData } = results;
-		const sessionIds = eventData.map(a => a.session_id).filter(a => a);
-		expect(sessionIds.length).toBe(0);
+		const noSessionIds = eventData.map(a => a.session_id).filter(a => a);
+		expect(noSessionIds.length).toBe(0);
 	}, timeout);
 
 	test('creates anonymousIds', async () => {
-		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, anonIds: true });
+		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, hasAnonIds: true });
 		const { eventData } = results;
-		const anonIds = eventData.map(a => a.device_id).filter(a => a);
+		const anonymousEvents = eventData.map(a => a.device_id).filter(a => a);
 		const userIds = eventData.map(a => a.user_id).filter(a => a);
-		expect(anonIds.length).toBe(eventData.length);
-		expect(userIds.length).toBeLessThan(anonIds.length);
+		expect(anonymousEvents.length).toBe(eventData.length);
+		expect(userIds.length).toBeLessThan(anonymousEvents.length);
 	}, timeout);
 
 	test('no anonymousIds', async () => {
-		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, anonIds: false });
+		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, hasAnonIds: false });
 		const { eventData } = results;
-		const anonIds = eventData.map(a => a.device_id).filter(a => a);
-		expect(anonIds.length).toBe(0);
+		const unanonymousEvents = eventData.map(a => a.device_id).filter(a => a);
+		expect(unanonymousEvents.length).toBe(0);
 	}, timeout);
 
 	test('sends data to mixpanel', async () => {
@@ -168,7 +335,7 @@ describe('options + tweaks', () => {
 
 	test('every record is valid', async () => {
 		console.log('VALIDATION TEST');
-		const results = await generate({ verbose: true, writeToDisk: false, numEvents: 10000, numUsers: 500 });
+		const results = await generate({ verbose: true, writeToDisk: false, numEvents: 1000, numUsers: 100 });
 		const { eventData, userProfilesData } = results;
 		const areEventsValid = eventData.every(validateEvent);
 		const areUsersValid = userProfilesData.every(validateUser);
@@ -182,24 +349,44 @@ describe('options + tweaks', () => {
 
 	test('every date is valid', async () => {
 		console.log('DATE TEST');
-		const results = await generate({ ...simple, writeToDisk: false, verbose: true });
+		const results = await generate({ ...simple, writeToDisk: false, verbose: true, numEvents: 1000, numUsers: 100 });
 		const { eventData } = results;
-		const invalidDates = eventData.filter(e => !validateTime(e.time));
-		expect(eventData.every(e => validateTime(e.time))).toBe(true);
+		const invalidDates = eventData.filter(e => !validTime(e.time));
+		expect(eventData.every(e => validTime(e.time))).toBe(true);
 
 	}, timeout);
 
 	test('anonymous users', async () => {
 		console.log('ANON TEST');
-		const results = await generate({ ...anon, writeToDisk: false, verbose: true });
+		const results = await generate({ ...anon, writeToDisk: false, verbose: true, numEvents: 1000, numUsers: 100 });
 		const { userProfilesData } = results;
 		expect(userProfilesData.every(u => u.name === 'Anonymous User')).toBe(true);
 
 	}, timeout);
 
+	test('no avatars (default)', async () => {
+		console.log('AVATAR TEST');
+		const results = await generate({ ...simple, writeToDisk: false, verbose: true, numEvents: 1000, numUsers: 100 });
+		const { userProfilesData } = results;
+		expect(userProfilesData.every(u => !u.avatar)).toBe(true);
+
+	}, timeout);
+
+	test('yes avatars', async () => {
+		console.log('AVATAR TEST');
+		const results = await generate({ ...simple, writeToDisk: false, verbose: true, numEvents: 1000, numUsers: 100, hasAvatar: true });
+		const { userProfilesData } = results;
+		expect(userProfilesData.every(u => u.avatar)).toBe(true);
+
+	}, timeout);
+
+});
+
+beforeEach(() => {
+	clearData();
 });
 
-afterAll(() => {
+afterEach(() => {
 	clearData();
 });
 
@@ -234,7 +421,7 @@ function validateUser(user) {
 }
 
 
-function validateTime(str) {
+function validTime(str) {
 	if (!str) return false;
 	if (str.startsWith('-')) return false;
 	if (!str.startsWith('20')) return false;