make-mp-data 1.1.19 → 1.2.1

package/tests/e2e.test.js

@@ -8,66 +8,94 @@ require('dotenv').config();
 const { execSync } = require("child_process");
 const u = require('ak-tools');
 
-const simple = require('../examples/simple');
+const simple = require('../models/simple.js');
+const complex = require('../models/complex.js');
+const deep = require('../models/deepNest.js');
 
 const timeout = 60000;
 const testToken = process.env.TEST_TOKEN;
 
+describe('module', () => {
 
-describe('e2e', () => {
-
-	test('works as module', async () => {
+	test('works as module (no config)', async () => {
 		console.log('MODULE TEST');
-		const results = await generate({ verbose: false, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const results = await generate({ verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
 		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
 		expect(groupProfilesData.length).toBe(0);
 		expect(lookupTableData.length).toBe(0);
 		expect(scdTableData.length).toBeGreaterThan(200);
+		expect(userProfilesData.length).toBe(100);		
+
+	}, timeout);
+
+	test('works as module (simple)', async () => {
+		console.log('MODULE TEST: SIMPLE');
+		const results = await generate({ ...simple, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
+		expect(eventData.length).toBeGreaterThan(980);
+		expect(groupProfilesData.length).toBe(0);
+		expect(lookupTableData.length).toBe(0);
+		expect(scdTableData.length).toBe(0);
 		expect(userProfilesData.length).toBe(100);
 
 	}, timeout);
 
-	test('works as CLI', async () => {
-		console.log('CLI TEST');
-		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --verbose false`);
-		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
-		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(5);
+	test('works as module (complex)', async () => {
+		console.log('MODULE TEST: COMPLEX');
+		const results = await generate({ ...complex, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
+		expect(eventData.length).toBeGreaterThan(980);
+		expect(groupProfilesData[0]?.data?.length).toBe(350);
+		expect(lookupTableData.length).toBe(1);
+		expect(lookupTableData[0].data.length).toBe(1000);
+		expect(scdTableData.length).toBeGreaterThan(200);
+		expect(userProfilesData.length).toBe(100);
+
 	}, timeout);
 
-	test('sends data to mixpanel', async () => {
-		console.log('NETWORK TEST');
-		const results = await generate({verbose: false, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it", token: testToken });
-		const { events, users, groups } = results.import;
-		expect(events.success).toBeGreaterThan(980);
-		expect(users.success).toBe(100);
-		expect(groups.length).toBe(0);
+	test('works as module (deep nest)', async () => {
+		console.log('MODULE TEST: DEEP NEST');
+		const results = await generate({ ...deep, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
+		expect(eventData.length).toBeGreaterThan(980);
+		expect(groupProfilesData.length).toBe(0);
+		expect(lookupTableData.length).toBe(0);
+		expect(scdTableData.length).toBeGreaterThan(200);
+		expect(userProfilesData.length).toBe(100);
+
 	}, timeout);
 
-	test('every record is valid', async () => {
-		console.log('VALIDATION TEST');
-		const results = await generate({verbose: false, writeToDisk: false, numEvents: 10000, numUsers: 500 });
-		const { eventData, userProfilesData } = results;
-		const areEventsValid = eventData.every(validateEvent);
-		const areUsersValid = userProfilesData.every(validateUser);
 
-		const invalidEvents = eventData.filter(e => !validateEvent(e));
-		const invalidUsers = userProfilesData.filter(u => !validateUser(u));
+});
 
-		expect(areEventsValid).toBe(true);
-		expect(areUsersValid).toBe(true);
+describe('cli', () => {
+	test('works as CLI (complex)', async () => {
+		console.log('COMPLEX CLI TEST');
+		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --complex`);
+		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
+		expect(csvs.length).toBe(5);
+		clearData();
 	}, timeout);
 
-	test('every date is valid', async () => {
-		console.log('DATE TEST');
-		const results = await generate({ ...simple, writeToDisk: false, verbose: false });
-		const { eventData } = results;
-		const invalidDates = eventData.filter(e => !validateTime(e.time));
-		expect(eventData.every(e => validateTime(e.time))).toBe(true);
+	test('works as CLI (simple)', async () => {
+		console.log('simple CLI TEST');
+		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it"`);
+		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
+		expect(csvs.length).toBe(2);
+		clearData();
+	}, timeout);
 
-		
-	}, timeout)
+	test('works as CLI (custom)', async () => {
+		console.log('custom CLI TEST');
+		const run = execSync(`node ./index.js ./models/deepNest.js`);
+		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
+		expect(csvs.length).toBe(3);
+		clearData();
+	}, timeout);
 
 });
 
@@ -102,16 +130,47 @@ describe('options + tweaks', () => {
 		expect(anonIds.length).toBe(0);
 	}, timeout);
 
+	test('sends data to mixpanel', async () => {
+		console.log('NETWORK TEST');
+		const results = await generate({ verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it", token: testToken });
+		const { events, users, groups } = results.import;
+		expect(events.success).toBeGreaterThan(980);
+		expect(users.success).toBe(100);
+		expect(groups.length).toBe(0);
+	}, timeout);
 
-});
+	test('every record is valid', async () => {
+		console.log('VALIDATION TEST');
+		const results = await generate({ verbose: true, writeToDisk: false, numEvents: 10000, numUsers: 500 });
+		const { eventData, userProfilesData } = results;
+		const areEventsValid = eventData.every(validateEvent);
+		const areUsersValid = userProfilesData.every(validateUser);
 
+		const invalidEvents = eventData.filter(e => !validateEvent(e));
+		const invalidUsers = userProfilesData.filter(u => !validateUser(u));
 
+		expect(areEventsValid).toBe(true);
+		expect(areUsersValid).toBe(true);
+	}, timeout);
 
-afterEach(() => {
+	test('every date is valid', async () => {
+		console.log('DATE TEST');
+		const results = await generate({ ...simple, writeToDisk: false, verbose: true });
+		const { eventData } = results;
+		const invalidDates = eventData.filter(e => !validateTime(e.time));
+		expect(eventData.every(e => validateTime(e.time))).toBe(true);
+
+	}, timeout);
 
 });
 
 afterAll(() => {
+	clearData();
+});
+
+//helpers
+
+function clearData() {
 	try {
 		console.log('clearing...');
 		execSync(`npm run prune`);
@@ -120,9 +179,7 @@ afterAll(() => {
 	catch (err) {
 		console.log('error clearing files');
 	}
-});
-
-//helpers
+}
 
 function validateEvent(event) {
 	if (!event.event) return false;

package/tests/unit.test.js

@@ -7,6 +7,7 @@ const { timeSoup } = generate;
 require('dotenv').config();
 
 
+
 describe('timeSoup', () => {
 	test('always positive dates', () => {
 		const dates = [];
@@ -21,3 +22,157 @@ describe('timeSoup', () => {
 
 	});
 });
+
+
+
+const { applySkew, boxMullerRandom, choose, date, dates, day, exhaust, generateEmoji, getUniqueKeys, integer, makeHashTags, makeProducts, mapToRange, person, pick, range, weighList, weightedRange } = require('../utils');
+
+describe('utils', () => {
+
+	test('pick: works', () => {
+		const array = [1, 2, 3];
+		const item = pick(array);
+		expect(array).toContain(item);
+	});
+
+	test('pick: null', () => {
+		expect(pick(123)).toBe(123);
+	});
+
+
+
+	test('integer: diff', () => {
+		const min = 5;
+		const max = 10;
+		const result = integer(min, max);
+		expect(result).toBeGreaterThanOrEqual(min);
+		expect(result).toBeLessThanOrEqual(max);
+	});
+
+	test('integer: same', () => {
+		expect(integer(7, 7)).toBe(7);
+	});
+
+
+
+	test('hashtags', () => {
+		const hashtags = makeHashTags();
+		expect(hashtags).toBeInstanceOf(Array);
+		expect(hashtags).not.toHaveLength(0);
+		hashtags.forEach(tag => {
+			expect(tag).toMatch(/^#/);
+		});
+	});
+
+
+
+	test('person: fields', () => {
+		const generatedPerson = person();
+		expect(generatedPerson).toHaveProperty('$name');
+		expect(generatedPerson).toHaveProperty('$email');
+		expect(generatedPerson).toHaveProperty('$avatar');
+		expect(generatedPerson).toHaveProperty('anonymousIds');
+		expect(generatedPerson.anonymousIds).toBeInstanceOf(Array);
+	});
+
+
+	test('date: past date', () => {
+		const pastDate = date(10, true, 'YYYY-MM-DD')();
+		expect(dayjs(pastDate, 'YYYY-MM-DD').isValid()).toBeTruthy();
+		expect(dayjs(pastDate).isBefore(dayjs())).toBeTruthy();
+	});
+
+	test('date: future date', () => {
+		const futureDate = date(10, false, 'YYYY-MM-DD')();
+		expect(dayjs(futureDate, 'YYYY-MM-DD').isValid()).toBeTruthy();
+		expect(dayjs(futureDate).isAfter(dayjs())).toBeTruthy();
+	});
+
+	test('dates: returns pairs of dates', () => {
+		const datePairs = dates(10, 3, 'YYYY-MM-DD');
+		expect(datePairs).toBeInstanceOf(Array);
+		expect(datePairs).toHaveLength(3);
+		datePairs.forEach(pair => {
+			expect(pair).toHaveLength(2);
+		});
+	});
+
+	test('choose: choose from array', () => {
+		const options = ['apple', 'banana', 'cherry'];
+		const choice = choose(options);
+		expect(options).toContain(choice);
+	});
+
+	test('choose: execute function', () => {
+		const result = choose(() => 'test');
+		expect(result).toBe('test');
+	});
+
+	test('exhaust: exhaust array elements', () => {
+		const arr = [1, 2, 3];
+		const exhaustFn = exhaust([...arr]);
+		expect(exhaustFn()).toBe(1);
+		expect(exhaustFn()).toBe(2);
+		expect(exhaustFn()).toBe(3);
+		expect(exhaustFn()).toBeUndefined();
+	});
+
+	test('generateEmoji: returns string of emojis', () => {
+		const emojis = generateEmoji(5)();
+		expect(typeof emojis).toBe('string');
+		expect(emojis.split(', ').length).toBeLessThanOrEqual(5);
+	});
+
+	test('getUniqueKeys: find unique keys', () => {
+		const objects = [{ a: 1, b: 2 }, { a: 3, c: 4 }, { a: 5, b: 6 }];
+		const uniqueKeys = getUniqueKeys(objects);
+		expect(uniqueKeys).toEqual(expect.arrayContaining(['a', 'b', 'c']));
+	});
+
+
+	test('date: generates a valid date', () => {
+		const result = date();
+		expect(dayjs(result()).isValid()).toBe(true);
+	});
+
+	test('dates: generates an array of date pairs', () => {
+		const result = dates();
+		expect(result).toBeInstanceOf(Array);
+		expect(result.length).toBe(5); // Assuming default numPairs is 5
+		result.forEach(pair => {
+			expect(pair).toBeInstanceOf(Array);
+			expect(pair.length).toBe(2);
+			expect(dayjs(pair[0]()).isValid()).toBe(true);
+			expect(dayjs(pair[1]()).isValid()).toBe(true);
+		});
+	});
+
+	test('day: generates a day within range', () => {
+		const start = '2020-01-01';
+		const end = '2020-01-30';
+		const result = day(start, end);
+		const dayResult = result(0, 9);
+		expect(dayjs(dayResult.day).isAfter(dayjs(dayResult.start))).toBe(true);
+		expect(dayjs(dayResult.day).isBefore(dayjs(dayResult.end))).toBe(true);
+	});
+
+	test('exhaust: sequentially removes items from array', () => {
+		const arr = [1, 2, 3];
+		const next = exhaust(arr);
+		expect(next()).toBe(1);
+		expect(next()).toBe(2);
+		expect(next()).toBe(3);
+		expect(next()).toBe(undefined); // or whatever your implementation does after array is exhausted
+	});
+
+	test('generateEmoji: generates correct format and length', () => {
+		const result = generateEmoji();
+		const emojis = result();
+		expect(typeof emojis).toBe('string');
+		const emojiArray = emojis.split(', ');
+		expect(emojiArray.length).toBeLessThanOrEqual(10); // Assuming max default is 10
+		
+	});
+
+
+});

package/timesoup.js

@@ -0,0 +1,92 @@
+const Chance = require("chance");
+const chance = new Chance();
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+const { integer } = require('./utils');
+const NOW = dayjs().unix();
+
+const PEAK_DAYS = [
+	dayjs().subtract(2, "day").unix(),
+	dayjs().subtract(3, "day").unix(),
+	dayjs().subtract(5, "day").unix(),
+	dayjs().subtract(7, "day").unix(),
+	dayjs().subtract(11, "day").unix(),
+	dayjs().subtract(13, "day").unix(),
+	dayjs().subtract(17, "day").unix(),
+	dayjs().subtract(19, "day").unix(),
+	dayjs().subtract(23, "day").unix(),
+	dayjs().subtract(29, "day").unix(),
+];
+
+
+/**
+ * essentially, a timestamp generator with a twist
+ * @param {number} earliestTime - The earliest timestamp in Unix format.
+ * @param {number} latestTime - The latest timestamp in Unix format.
+ * @param {Array} peakDays - Array of Unix timestamps representing the start of peak days.
+ * @returns {number} - The generated event timestamp in Unix format.
+ */
+function AKsTimeSoup(earliestTime, latestTime = NOW, peakDays = PEAK_DAYS) {
+	let chosenTime;
+	let eventTime;
+	let validTime = false;
+
+	if (typeof earliestTime !== "number") {
+		if (parseInt(earliestTime) > 0) earliestTime = parseInt(earliestTime);
+		if (dayjs(earliestTime).isValid()) earliestTime = dayjs(earliestTime).unix();
+	}
+
+	while (!validTime) {
+
+		// Define business hours
+		const peakStartHour = 4; // 4 AM
+		const peakEndHour = 23; // 11 PM
+		const likelihoodOfPeakDay = chance.integer({ min: integer(5, 42), max: integer(43, 69) }); // Randomize likelihood with CHAOS!~~
+
+		// Select a day, with a preference for peak days
+		let selectedDay;
+		if (chance.bool({ likelihood: likelihoodOfPeakDay })) { // Randomized likelihood to pick a peak day
+			selectedDay = peakDays.length > 0 ? chance.pickone(peakDays) : integer(earliestTime, latestTime);
+		} else {
+			// Introduce minor peaks by allowing some events to still occur during business hours
+			selectedDay = chance.bool({ likelihood: integer(1, 42) })
+				? chance.pickone(peakDays)
+				: integer(earliestTime, latestTime);
+		}
+
+		// Normalize selectedDay to the start of the day
+		selectedDay = dayjs.unix(selectedDay).startOf('day').unix();
+
+		// Generate a random time within business hours with a higher concentration in the middle of the period
+		const businessStart = dayjs.unix(selectedDay).hour(peakStartHour).minute(0).second(0).unix();
+		const businessEnd = dayjs.unix(selectedDay).hour(peakEndHour).minute(0).second(0).unix();
+
+		if (selectedDay === peakDays[0]) {
+			// Use a skewed distribution for peak days
+			eventTime = chance.normal({ mean: (businessEnd + businessStart) / integer(1, 4), dev: (businessEnd - businessStart) / integer(2, 8) });
+		} else {
+			// For non-peak days, use a uniform distribution to add noise
+			eventTime = integer(integer(businessStart, businessEnd), integer(businessStart, businessEnd));
+		}
+
+		// usually, ensure the event time is within business hours
+		if (chance.bool({ likelihood: 42 })) eventTime = Math.min(Math.max(eventTime, businessStart), businessEnd);
+
+		if (eventTime > 0) validTime = true;
+		const parsedTime = dayjs.unix(eventTime).toISOString();
+		if (!parsedTime.startsWith('20')) validTime = false;
+
+	}
+	chosenTime = dayjs.unix(eventTime).toISOString();
+
+	//should never get here
+	if (eventTime < 0) debugger;
+	if (!chosenTime.startsWith('20')) debugger;
+
+
+	return chosenTime;
+}
+
+
+module.exports = AKsTimeSoup;