make-mp-data 1.1.18 → 1.2.0

package/.vscode/settings.json

@@ -1,5 +1,6 @@
 {
 	"cSpell.words": [
+		"timesoup",
 		"unparse"
 	],
 	"jest.runMode": "on-demand",

package/cli.js

@@ -35,7 +35,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'string'
 		})
 		.option("seed", {
-			demandOption: false,			
+			demandOption: false,
 			alias: 's',
 			describe: 'randomness seed; used to create distinct_ids',
 			type: 'string'
@@ -72,6 +72,24 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			describe: 'either US or EU',
 			type: 'string'
 		})
+		.options("complex", {
+			demandOption: false,
+			default: false,
+			describe: 'use complex data model (model all entities)',
+			alias: 'c',
+			type: 'boolean',
+			coerce: (value) => {
+				if (typeof value === 'boolean') return value;
+				if (value === 'true') {
+					return true;
+				}
+				if (value === 'false') {
+					return false;
+				}
+
+				return true;
+			}
+		})
 		.option("writeToDisk", {
 			demandOption: false,
 			default: true,

package/index.js

@@ -14,50 +14,19 @@ const Chance = require("chance");
 const chance = new Chance();
 const { touch, comma, bytesHuman, mkdir } = require("ak-tools");
 const Papa = require("papaparse");
-const {
-	weightedRange,
-	pick,
-	day,
-	integer,
-	makeProducts,
-	date,
-	progress,
-	person,
-	choose,
-	range,
-	exhaust,
-	openFinder,
-	applySkew,
-	boxMullerRandom,
-	getUniqueKeys
-} = require("./utils.js");
+const u = require("./utils.js");
+const AKsTimeSoup = require("./timesoup.js");
 const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const cliParams = require("./cli.js");
 const { makeName, md5 } = require('ak-tools');
-
-Array.prototype.pickOne = pick;
 const NOW = dayjs().unix();
 let VERBOSE = false;
 
 /** @typedef {import('./types.d.ts').Config} Config */
 /** @typedef {import('./types.d.ts').EventConfig} EventConfig */
 
-
-const PEAK_DAYS = [
-	dayjs().subtract(2, "day").unix(),
-	dayjs().subtract(3, "day").unix(),
-	dayjs().subtract(5, "day").unix(),
-	dayjs().subtract(7, "day").unix(),
-	dayjs().subtract(11, "day").unix(),
-	dayjs().subtract(13, "day").unix(),
-	dayjs().subtract(17, "day").unix(),
-	dayjs().subtract(19, "day").unix(),
-	dayjs().subtract(23, "day").unix(),
-	dayjs().subtract(29, "day").unix(),
-];
-
 /**
  * generates fake mixpanel data
  * @param  {Config} config
@@ -74,7 +43,7 @@ async function main(config) {
 			favoriteColor: ["red", "green", "blue", "yellow"],
 			spiritAnimal: chance.animal,
 		},
-		scdProps = { NPS: weightedRange(0, 10, 150, 1.6) },
+		scdProps = { NPS: u.weightedRange(0, 10, 150, 1.6) },
 		groupKeys = [],
 		groupProps = {},
 		lookupTables = [],
@@ -88,7 +57,7 @@ async function main(config) {
 	} = config;
 	VERBOSE = verbose;
 	config.simulationName = makeName();
-	global.config = config;
+	global.MP_SIMULATION_CONFIG = config;
 	const uuidChance = new Chance(seed);
 	log(`------------------SETUP------------------`);
 	log(`\nyour data simulation will heretofore be known as: \n\n\t${config.simulationName.toUpperCase()}...\n`);
@@ -99,18 +68,18 @@ async function main(config) {
 	//the function which generates $distinct_id + $anonymous_ids, $session_ids, and $created, skewing towards the present
 	function generateUser() {
 		const distinct_id = uuidChance.guid();
-		let z = boxMullerRandom();
+		let z = u.boxMullerRandom();
 		const skew = chance.normal({ mean: 10, dev: 3 });
-		z = applySkew(z, skew);
+		z = u.applySkew(z, skew);
 
 		// Scale and shift the normally distributed value to fit the range of days
-		const maxZ = integer(2, 4);
+		const maxZ = u.integer(2, 4);
 		const scaledZ = (z / maxZ + 1) / 2;
 		const daysAgoBorn = Math.round(scaledZ * (numDays - 1)) + 1;
 
 		return {
 			distinct_id,
-			...person(daysAgoBorn),
+			...u.person(daysAgoBorn),
 		};
 	}
 
@@ -136,16 +105,16 @@ async function main(config) {
 	const avgEvPerUser = Math.floor(numEvents / numUsers);
 
 	//user loop
-	log(`---------------SIMULATION----------------`, `\n\n`);
+	log(`---------------SIMULATION----------------`, "\n\n");
 	for (let i = 1; i < numUsers + 1; i++) {
-		progress("users", i);
+		u.progress("users", i);
 		const user = generateUser();
 		const { distinct_id, $created, anonymousIds, sessionIds } = user;
 		userProfilesData.push(makeProfile(userProps, user));
 		const mutations = chance.integer({ min: 1, max: 10 });
 		scdTableData.push(makeSCD(scdProps, distinct_id, mutations, $created));
 		const numEventsThisUser = Math.round(
-			chance.normal({ mean: avgEvPerUser, dev: avgEvPerUser / integer(3, 7) })
+			chance.normal({ mean: avgEvPerUser, dev: avgEvPerUser / u.integer(3, 7) })
 		);
 
 		if (firstEvents.length) {
@@ -189,7 +158,7 @@ async function main(config) {
 		const groupCardinality = groupPair[1];
 		const groupProfiles = [];
 		for (let i = 1; i < groupCardinality + 1; i++) {
-			progress("groups", i);
+			u.progress("groups", i);
 			const group = {
 				[groupKey]: i,
 				...makeProfile(groupProps[groupKey]),
@@ -206,7 +175,7 @@ async function main(config) {
 		const { key, entries, attributes } = lookupTable;
 		const data = [];
 		for (let i = 1; i < entries + 1; i++) {
-			progress("lookups", i);
+			u.progress("lookups", i);
 			const item = {
 				[key]: i,
 				...makeProfile(attributes),
@@ -238,17 +207,19 @@ async function main(config) {
 	log(`-----------------WRITES------------------`, `\n\n`);
 	//write the files
 	if (writeToDisk) {
-		if (verbose) log(`writing files... for ${config.simulationName}`);
-		for (const pair of pairs) {
+		if (verbose) log(`writing files... for ${config.simulationName}\n`);
+		loopFiles: for (const pair of pairs) {
 			const [paths, data] = pair;
+			if (!data.length) continue loopFiles;
 			for (const path of paths) {
 				let datasetsToWrite;
 				if (data?.[0]?.["key"]) datasetsToWrite = data.map((d) => d.data);
 				else datasetsToWrite = [data];
 				for (const writeData of datasetsToWrite) {
-					if (format === "csv") {
+					//if it's a lookup table, it's always a CSV
+					if (format === "csv" || path.includes("-LOOKUP.csv")) {
 						log(`writing ${path}`);
-						const columns = getUniqueKeys(writeData);
+						const columns = u.getUniqueKeys(writeData);
 						//papa parse needs nested JSON stringified
 						writeData.forEach((e) => {
 							for (const key in e) {
@@ -329,6 +300,9 @@ async function main(config) {
 	};
 }
 
+
+
+
 function makeProfile(props, defaults) {
 	//build the spec
 	const profile = {
@@ -347,6 +321,7 @@ function makeProfile(props, defaults) {
 }
 
 function makeSCD(props, distinct_id, mutations, $created) {
+	if (JSON.stringify(props) === "{}") return [];
 	const scdEntries = [];
 	let lastInserted = dayjs($created);
 	const deltaDays = dayjs().diff(lastInserted, "day");
@@ -355,12 +330,12 @@ function makeSCD(props, distinct_id, mutations, $created) {
 		if (lastInserted.isAfter(dayjs())) break;
 		const scd = makeProfile(props, { distinct_id });
 		scd.startTime = lastInserted.toISOString();
-		lastInserted = lastInserted.add(integer(1, 1000), "seconds");
+		lastInserted = lastInserted.add(u.integer(1, 1000), "seconds");
 		scd.insertTime = lastInserted.toISOString();
 		scdEntries.push({ ...scd });
 		lastInserted = lastInserted
-			.add(integer(0, deltaDays), "day")
-			.subtract(integer(1, 1000), "seconds");
+			.add(u.integer(0, deltaDays), "day")
+			.subtract(u.integer(1, 1000), "seconds");
 	}
 
 	return scdEntries;
@@ -378,7 +353,7 @@ function makeSCD(props, distinct_id, mutations, $created) {
  * @param  {Boolean} isFirstEvent=false
  */
 function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, events, superProps, groupKeys, isFirstEvent = false) {
-	let chosenEvent = events.pickOne();
+	let chosenEvent = chance.pickone(events);
 
 	//allow for a string shorthand
 	if (typeof chosenEvent === "string") {
@@ -393,11 +368,11 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, events,
 
 	//event time
 	if (isFirstEvent) event.time = dayjs.unix(earliestTime).toISOString();
-	if (!isFirstEvent) event.time = AKsTimeSoup(earliestTime, NOW, PEAK_DAYS);
+	if (!isFirstEvent) event.time = AKsTimeSoup(earliestTime, NOW);
 
 	// anonymous and session ids
-	if (global?.config.anonIds) event.$device_id = chance.pickone(anonymousIds);
-	if (global?.config.sessionIds) event.$session_id = chance.pickone(sessionIds);
+	if (global.MP_SIMULATION_CONFIG?.anonIds) event.$device_id = chance.pickone(anonymousIds);
+	if (global.MP_SIMULATION_CONFIG?.sessionIds) event.$session_id = chance.pickone(sessionIds);
 
 	//sometimes have a $user_id
 	if (!isFirstEvent && chance.bool({ likelihood: 42 })) event.$user_id = distinct_id;
@@ -410,7 +385,7 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, events,
 	//iterate through custom properties
 	for (const key in props) {
 		try {
-			event[key] = choose(props[key]);
+			event[key] = u.choose(props[key]);
 		} catch (e) {
 			console.error(`error with ${key} in ${chosenEvent.event} event`, e);
 			debugger;
@@ -422,7 +397,7 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, events,
 		const groupKey = groupPair[0];
 		const groupCardinality = groupPair[1];
 
-		event[groupKey] = weightedRange(1, groupCardinality).pickOne();
+		event[groupKey] = u.pick(u.weightedRange(1, groupCardinality));
 	}
 
 	//make $insert_id
@@ -440,9 +415,9 @@ function buildFileNames(config) {
 	if (config.writeToDisk) writeDir = mkdir("./data");
 
 	const writePaths = {
-		eventFiles: [path.join(writeDir, `events-${simName}.${extension}`)],
-		userFiles: [path.join(writeDir, `users-${simName}.${extension}`)],
-		scdFiles: [path.join(writeDir, `scd-${simName}.${extension}`)],
+		eventFiles: [path.join(writeDir, `${simName}-EVENTS.${extension}`)],
+		userFiles: [path.join(writeDir, `${simName}-USERS.${extension}`)],
+		scdFiles: [path.join(writeDir, `${simName}-SCD.${extension}`)],
 		groupFiles: [],
 		lookupFiles: [],
 		folder: writeDir,
@@ -451,88 +426,28 @@ function buildFileNames(config) {
 	for (const groupPair of groupKeys) {
 		const groupKey = groupPair[0];
 		writePaths.groupFiles.push(
-			path.join(writeDir, `group-${groupKey}-${simName}.${extension}`)
+			path.join(writeDir, `${simName}-${groupKey}-GROUP.${extension}`)
 		);
 	}
 
 	for (const lookupTable of lookupTables) {
 		const { key } = lookupTable;
 		writePaths.lookupFiles.push(
-			path.join(writeDir, `lookup-${key}-${simName}.${extension}`)
+			//lookups are always CSVs
+			path.join(writeDir, `${simName}-${key}-LOOKUP.csv`)
 		);
 	}
 
 	return writePaths;
 }
 
-/**
- * essentially, a timestamp generator with a twist
- * @param {number} earliestTime - The earliest timestamp in Unix format.
- * @param {number} latestTime - The latest timestamp in Unix format.
- * @param {Array} peakDays - Array of Unix timestamps representing the start of peak days.
- * @returns {number} - The generated event timestamp in Unix format.
- */
-function AKsTimeSoup(earliestTime, latestTime = NOW, peakDays = PEAK_DAYS) {
-	let chosenTime;
-	let eventTime;
-	let validTime = false;
-
-	if (typeof earliestTime !== "number") {
-		if (parseInt(earliestTime) > 0) earliestTime = parseInt(earliestTime);
-		if (dayjs(earliestTime).isValid()) earliestTime = dayjs(earliestTime).unix();
-	}
-
-	while (!validTime) {
-
-		// Define business hours
-		const peakStartHour = 4; // 4 AM
-		const peakEndHour = 23; // 11 PM
-		const likelihoodOfPeakDay = chance.integer({ min: integer(5, 42), max: integer(43, 69) }); // Randomize likelihood with CHAOS!~~
-
-		// Select a day, with a preference for peak days
-		let selectedDay;
-		if (chance.bool({ likelihood: likelihoodOfPeakDay })) { // Randomized likelihood to pick a peak day
-			selectedDay = peakDays.length > 0 ? chance.pickone(peakDays) : integer(earliestTime, latestTime);
-		} else {
-			// Introduce minor peaks by allowing some events to still occur during business hours
-			selectedDay = chance.bool({ likelihood: integer(1, 42) })
-				? chance.pickone(peakDays)
-				: integer(earliestTime, latestTime);
-		}
-
-		// Normalize selectedDay to the start of the day
-		selectedDay = dayjs.unix(selectedDay).startOf('day').unix();
-
-		// Generate a random time within business hours with a higher concentration in the middle of the period
-		const businessStart = dayjs.unix(selectedDay).hour(peakStartHour).minute(0).second(0).unix();
-		const businessEnd = dayjs.unix(selectedDay).hour(peakEndHour).minute(0).second(0).unix();
-
-		if (selectedDay === peakDays[0]) {
-			// Use a skewed distribution for peak days
-			eventTime = chance.normal({ mean: (businessEnd + businessStart) / integer(1, 4), dev: (businessEnd - businessStart) / integer(2, 8) });
-		} else {
-			// For non-peak days, use a uniform distribution to add noise
-			eventTime = integer(integer(businessStart, businessEnd), integer(businessStart, businessEnd));
-		}
-
-		// usually, ensure the event time is within business hours
-		if (chance.bool({ likelihood: 42 })) eventTime = Math.min(Math.max(eventTime, businessStart), businessEnd);
-
-		if (eventTime > 0) validTime = true;
-		const parsedTime = dayjs.unix(eventTime).toISOString();
-		if (!parsedTime.startsWith('20')) validTime = false;
-
-	}
-	chosenTime = dayjs.unix(eventTime).toISOString();
-	return chosenTime;
-}
 
 
 
 // this is for CLI
 if (require.main === module) {
 	const args = cliParams();
-	const { token, seed, format, numDays, numUsers, numEvents, region, writeToDisk } = args;
+	const { token, seed, format, numDays, numUsers, numEvents, region, writeToDisk, complex = false } = args;
 	const suppliedConfig = args._[0];
 
 	//if the user specifics an separate config file
@@ -540,9 +455,18 @@ if (require.main === module) {
 	if (suppliedConfig) {
 		log(`using ${suppliedConfig} for data\n`);
 		config = require(path.resolve(suppliedConfig));
-	} else {
-		log(`... using default configuration ...\n`);
-		config = require("./default.js");
+	}
+	else {
+		if (complex) {
+			log(`... using default COMPLEX configuration [everything] ...\n`);
+			log(`... for more simple data, don't use the --complex flag ...\n`);
+			config = require("./examples/complex.js");
+		}
+		else {
+			log(`... using default SIMPLE configuration [events + users] ...\n`);
+			log(`... for more complex data, use the --complex flag ...\n`);
+			config = require("./examples/simple.js");
+		}
 	}
 
 	//override config with cli params
@@ -589,9 +513,10 @@ if (require.main === module) {
 		})
 		.finally(() => {
 			log("have a wonderful day :)");
-			openFinder(path.resolve("./data"));
+			u.openFinder(path.resolve("./data"));
 		});
 } else {
+	main.utils = { ...u };
 	main.timeSoup = AKsTimeSoup;
 	module.exports = main;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "make-mp-data",
-  "version": "1.1.18",
+  "version": "1.2.0",
   "description": "builds all mixpanel primitives for a given project",
   "main": "index.js",
   "types": "types.d.ts",
@@ -9,7 +9,7 @@
     "prune": "rm ./data/*",
     "go": "sh ./scripts/go.sh",
     "post": "npm publish",
-    "test": "jest",
+    "test": "jest --runInBand",
     "deps": "sh ./scripts/deps.sh"
   },
   "repository": {

package/tests/e2e.test.js

@@ -8,55 +8,94 @@ require('dotenv').config();
 const { execSync } = require("child_process");
 const u = require('ak-tools');
 
+const simple = require('../examples/simple');
+const complex = require('../examples/complex');
+const deep = require('../examples/deepNest');
+
 const timeout = 60000;
 const testToken = process.env.TEST_TOKEN;
 
+describe('module', () => {
 
-describe('e2e', () => {
-
-	test('works as module', async () => {
+	test('works as module (no config)', async () => {
 		console.log('MODULE TEST');
-		const results = await generate({ writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const results = await generate({ verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
 		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
 		expect(groupProfilesData.length).toBe(0);
 		expect(lookupTableData.length).toBe(0);
 		expect(scdTableData.length).toBeGreaterThan(200);
+		expect(userProfilesData.length).toBe(100);		
+
+	}, timeout);
+
+	test('works as module (simple)', async () => {
+		console.log('MODULE TEST: SIMPLE');
+		const results = await generate({ ...simple, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
+		expect(eventData.length).toBeGreaterThan(980);
+		expect(groupProfilesData.length).toBe(0);
+		expect(lookupTableData.length).toBe(0);
+		expect(scdTableData.length).toBe(0);
 		expect(userProfilesData.length).toBe(100);
 
 	}, timeout);
 
-	test('works as CLI', async () => {
-		console.log('CLI TEST');
-		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it"`);
-		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
-		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
-		expect(csvs.length).toBe(5);
+	test('works as module (complex)', async () => {
+		console.log('MODULE TEST: COMPLEX');
+		const results = await generate({ ...complex, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
+		expect(eventData.length).toBeGreaterThan(980);
+		expect(groupProfilesData[0]?.data?.length).toBe(350);
+		expect(lookupTableData.length).toBe(1);
+		expect(lookupTableData[0].data.length).toBe(1000);
+		expect(scdTableData.length).toBeGreaterThan(200);
+		expect(userProfilesData.length).toBe(100);
+
 	}, timeout);
 
-	test('sends data to mixpanel', async () => {
-		console.log('NETWORK TEST');
-		const results = await generate({ writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it", token: testToken });
-		const { events, users, groups } = results.import;
-		expect(events.success).toBeGreaterThan(980);
-		expect(users.success).toBe(100);
-		expect(groups.length).toBe(0);
+	test('works as module (deep nest)', async () => {
+		console.log('MODULE TEST: DEEP NEST');
+		const results = await generate({ ...deep, verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
+		expect(eventData.length).toBeGreaterThan(980);
+		expect(groupProfilesData.length).toBe(0);
+		expect(lookupTableData.length).toBe(0);
+		expect(scdTableData.length).toBeGreaterThan(200);
+		expect(userProfilesData.length).toBe(100);
+
 	}, timeout);
 
-	test('every record is valid', async () => {
-		console.log('VALIDATION TEST');
-		const results = await generate({ writeToDisk: false, numEvents: 10000, numUsers: 500 });
-		const { eventData, userProfilesData } = results;
-		const areEventsValid = eventData.every(validateEvent);
-		const areUsersValid = userProfilesData.every(validateUser);
 
-		const invalidEvents = eventData.filter(e => !validateEvent(e));
-		const invalidUsers = userProfilesData.filter(u => !validateUser(u));
+});
 
-		expect(areEventsValid).toBe(true);
-		expect(areUsersValid).toBe(true);
+describe('cli', () => {
+	test('works as CLI (complex)', async () => {
+		console.log('COMPLEX CLI TEST');
+		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --complex`);
+		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
+		expect(csvs.length).toBe(5);
+		clearData();
 	}, timeout);
 
+	test('works as CLI (simple)', async () => {
+		console.log('simple CLI TEST');
+		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it"`);
+		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
+		expect(csvs.length).toBe(2);
+		clearData();
+	}, timeout);
+
+	test('works as CLI (custom)', async () => {
+		console.log('custom CLI TEST');
+		const run = execSync(`node ./index.js ./examples/deepNest.js`);
+		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
+		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
+		expect(csvs.length).toBe(3);
+		clearData();
+	}, timeout);
 
 });
 
@@ -91,16 +130,47 @@ describe('options + tweaks', () => {
 		expect(anonIds.length).toBe(0);
 	}, timeout);
 
+	test('sends data to mixpanel', async () => {
+		console.log('NETWORK TEST');
+		const results = await generate({ verbose: true, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it", token: testToken });
+		const { events, users, groups } = results.import;
+		expect(events.success).toBeGreaterThan(980);
+		expect(users.success).toBe(100);
+		expect(groups.length).toBe(0);
+	}, timeout);
 
-});
+	test('every record is valid', async () => {
+		console.log('VALIDATION TEST');
+		const results = await generate({ verbose: true, writeToDisk: false, numEvents: 10000, numUsers: 500 });
+		const { eventData, userProfilesData } = results;
+		const areEventsValid = eventData.every(validateEvent);
+		const areUsersValid = userProfilesData.every(validateUser);
 
+		const invalidEvents = eventData.filter(e => !validateEvent(e));
+		const invalidUsers = userProfilesData.filter(u => !validateUser(u));
 
+		expect(areEventsValid).toBe(true);
+		expect(areUsersValid).toBe(true);
+	}, timeout);
 
-afterEach(() => {
+	test('every date is valid', async () => {
+		console.log('DATE TEST');
+		const results = await generate({ ...simple, writeToDisk: false, verbose: true });
+		const { eventData } = results;
+		const invalidDates = eventData.filter(e => !validateTime(e.time));
+		expect(eventData.every(e => validateTime(e.time))).toBe(true);
+
+	}, timeout);
 
 });
 
 afterAll(() => {
+	clearData();
+});
+
+//helpers
+
+function clearData() {
 	try {
 		console.log('clearing...');
 		execSync(`npm run prune`);
@@ -109,9 +179,7 @@ afterAll(() => {
 	catch (err) {
 		console.log('error clearing files');
 	}
-});
-
-//helpers
+}
 
 function validateEvent(event) {
 	if (!event.event) return false;
@@ -128,4 +196,12 @@ function validateUser(user) {
 	if (!user.$email) return false;
 	if (!user.$created) return false;
 	return true;
+}
+
+
+function validateTime(str) {
+	if (!str) return false;
+	if (str.startsWith('-')) return false;
+	if (!str.startsWith('20')) return false;
+	return true;
 }