make-mp-data 1.1.17 → 1.1.19

package/index.js

@@ -22,22 +22,20 @@ const {
 	makeProducts,
 	date,
 	progress,
-	person,
 	choose,
 	range,
 	exhaust,
 	openFinder,
 	applySkew,
 	boxMullerRandom,
-	getUniqueKeys
+	getUniqueKeys,
+	person
 } = require("./utils.js");
 const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const cliParams = require("./cli.js");
 const { makeName, md5 } = require('ak-tools');
-
-Array.prototype.pickOne = pick;
 const NOW = dayjs().unix();
 let VERBOSE = false;
 
@@ -329,6 +327,9 @@ async function main(config) {
 	};
 }
 
+
+
+
 function makeProfile(props, defaults) {
 	//build the spec
 	const profile = {
@@ -378,7 +379,7 @@ function makeSCD(props, distinct_id, mutations, $created) {
  * @param  {Boolean} isFirstEvent=false
  */
 function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, events, superProps, groupKeys, isFirstEvent = false) {
-	let chosenEvent = events.pickOne();
+	let chosenEvent = chance.pickone(events);
 
 	//allow for a string shorthand
 	if (typeof chosenEvent === "string") {
@@ -422,7 +423,7 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, events,
 		const groupKey = groupPair[0];
 		const groupCardinality = groupPair[1];
 
-		event[groupKey] = weightedRange(1, groupCardinality).pickOne();
+		event[groupKey] = pick(weightedRange(1, groupCardinality))
 	}
 
 	//make $insert_id
@@ -479,8 +480,8 @@ function AKsTimeSoup(earliestTime, latestTime = NOW, peakDays = PEAK_DAYS) {
 
 	if (typeof earliestTime !== "number") {
 		if (parseInt(earliestTime) > 0) earliestTime = parseInt(earliestTime);
-		if (dayjs(earliestTime).isValid()) earliestTime = dayjs(earliestTime).unix();		
-	} 
+		if (dayjs(earliestTime).isValid()) earliestTime = dayjs(earliestTime).unix();
+	}
 
 	while (!validTime) {
 
@@ -506,7 +507,7 @@ function AKsTimeSoup(earliestTime, latestTime = NOW, peakDays = PEAK_DAYS) {
 		// Generate a random time within business hours with a higher concentration in the middle of the period
 		const businessStart = dayjs.unix(selectedDay).hour(peakStartHour).minute(0).second(0).unix();
 		const businessEnd = dayjs.unix(selectedDay).hour(peakEndHour).minute(0).second(0).unix();
-		
+
 		if (selectedDay === peakDays[0]) {
 			// Use a skewed distribution for peak days
 			eventTime = chance.normal({ mean: (businessEnd + businessStart) / integer(1, 4), dev: (businessEnd - businessStart) / integer(2, 8) });
@@ -519,9 +520,13 @@ function AKsTimeSoup(earliestTime, latestTime = NOW, peakDays = PEAK_DAYS) {
 		if (chance.bool({ likelihood: 42 })) eventTime = Math.min(Math.max(eventTime, businessStart), businessEnd);
 
 		if (eventTime > 0) validTime = true;
+		const parsedTime = dayjs.unix(eventTime).toISOString();
+		if (!parsedTime.startsWith('20')) validTime = false;
 
 	}
 	chosenTime = dayjs.unix(eventTime).toISOString();
+	if (eventTime < 0) debugger;
+	if (!chosenTime.startsWith('20')) debugger;
 	return chosenTime;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "make-mp-data",
-  "version": "1.1.17",
+  "version": "1.1.19",
   "description": "builds all mixpanel primitives for a given project",
   "main": "index.js",
   "types": "types.d.ts",

package/tests/e2e.test.js

@@ -8,6 +8,8 @@ require('dotenv').config();
 const { execSync } = require("child_process");
 const u = require('ak-tools');
 
+const simple = require('../examples/simple');
+
 const timeout = 60000;
 const testToken = process.env.TEST_TOKEN;
 
@@ -16,7 +18,7 @@ describe('e2e', () => {
 
 	test('works as module', async () => {
 		console.log('MODULE TEST');
-		const results = await generate({ writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
+		const results = await generate({ verbose: false, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it" });
 		const { eventData, groupProfilesData, lookupTableData, scdTableData, userProfilesData } = results;
 		expect(eventData.length).toBeGreaterThan(980);
 		expect(groupProfilesData.length).toBe(0);
@@ -28,7 +30,7 @@ describe('e2e', () => {
 
 	test('works as CLI', async () => {
 		console.log('CLI TEST');
-		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it"`);
+		const run = execSync(`node ./index.js --numEvents 1000 --numUsers 100 --seed "deal with it" --verbose false`);
 		expect(run.toString().trim().includes('have a wonderful day :)')).toBe(true);
 		const csvs = (await u.ls('./data')).filter(a => a.includes('.csv'));
 		expect(csvs.length).toBe(5);
@@ -36,7 +38,7 @@ describe('e2e', () => {
 
 	test('sends data to mixpanel', async () => {
 		console.log('NETWORK TEST');
-		const results = await generate({ writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it", token: testToken });
+		const results = await generate({verbose: false, writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it", token: testToken });
 		const { events, users, groups } = results.import;
 		expect(events.success).toBeGreaterThan(980);
 		expect(users.success).toBe(100);
@@ -45,7 +47,7 @@ describe('e2e', () => {
 
 	test('every record is valid', async () => {
 		console.log('VALIDATION TEST');
-		const results = await generate({ writeToDisk: false, numEvents: 10000, numUsers: 500 });
+		const results = await generate({verbose: false, writeToDisk: false, numEvents: 10000, numUsers: 500 });
 		const { eventData, userProfilesData } = results;
 		const areEventsValid = eventData.every(validateEvent);
 		const areUsersValid = userProfilesData.every(validateUser);
@@ -57,6 +59,15 @@ describe('e2e', () => {
 		expect(areUsersValid).toBe(true);
 	}, timeout);
 
+	test('every date is valid', async () => {
+		console.log('DATE TEST');
+		const results = await generate({ ...simple, writeToDisk: false, verbose: false });
+		const { eventData } = results;
+		const invalidDates = eventData.filter(e => !validateTime(e.time));
+		expect(eventData.every(e => validateTime(e.time))).toBe(true);
+
+		
+	}, timeout)
 
 });
 
@@ -128,4 +139,12 @@ function validateUser(user) {
 	if (!user.$email) return false;
 	if (!user.$created) return false;
 	return true;
+}
+
+
+function validateTime(str) {
+	if (!str) return false;
+	if (str.startsWith('-')) return false;
+	if (!str.startsWith('20')) return false;
+	return true;
 }

package/utils.js

@@ -7,18 +7,23 @@ const dayjs = require('dayjs');
 const utc = require('dayjs/plugin/utc');
 dayjs.extend(utc);
 
-function pick() {
-	try {
-		const choice = chance.pickone(this);
-		return choice;
-	}
-	catch (e) {
-		return null;
+function pick(items) {
+	if (!Array.isArray(items)) {
+		try {
+			const choice = chance.pickone(this);
+			return choice;
+		}
+		catch (e) {
+			return null;
+		}
 	}
+	return chance.pickone(items);
 }
 
 function date(inTheLast = 30, isPast = true, format = 'YYYY-MM-DD') {
 	const now = dayjs.utc();
+	// dates must be in the the last 10 years
+	if (Math.abs(inTheLast) > 365 * 10) inTheLast = chance.integer({ min: 1, max: 180 });
 	return function () {
 		try {
 			const when = chance.integer({ min: 0, max: Math.abs(inTheLast) });
@@ -216,48 +221,8 @@ function progress(thing, p) {
 	process.stdout.write(`${thing} processed ... ${comma(p)}`);
 }
 
-function person(bornDaysAgo = 30) {
-	//names and photos
-	const gender = chance.pickone(['male', 'female']);
-	const first = chance.first({ gender });
-	const last = chance.last();
-	const $name = `${first} ${last}`;
-	const $email = `${first[0]}.${last}@${chance.domain()}.com`;
-	const avatarPrefix = `https://randomuser.me/api/portraits`;
-	const randomAvatarNumber = chance.integer({
-		min: 1,
-		max: 99
-	});
-	const avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
-	const $avatar = avatarPrefix + avPath;
-	const $created = date(bornDaysAgo, true, null)();
-
-	//anon Ids
-	const anonymousIds = [];
-	const clusterSize = integer(2, 10);
-	for (let i = 0; i < clusterSize; i++) {
-		anonymousIds.push(uid(42));
-	}
-
-	//session Ids
-	const sessionIds = [];
-	const sessionSize = integer(5, 30);
-	for (let i = 0; i < sessionSize; i++) {
-		sessionIds.push([uid(5), uid(5), uid(5), uid(5)].join("-"));
-	}
-
 
 
-	return {
-		$name,
-		$email,
-		$avatar,
-		$created,
-		anonymousIds,
-		sessionIds
-	};
-}
-
 function range(a, b, step = 1) {
 	step = !step ? 1 : step;
 	b = b / step;
@@ -368,6 +333,48 @@ function generateName() {
 
 }
 
+function person(bornDaysAgo = 30) {
+	//names and photos
+	const gender = chance.pickone(['male', 'female']);
+	const first = chance.first({ gender });
+	const last = chance.last();
+	const $name = `${first} ${last}`;
+	const $email = `${first[0]}.${last}@${chance.domain()}.com`;
+	const avatarPrefix = `https://randomuser.me/api/portraits`;
+	const randomAvatarNumber = chance.integer({
+		min: 1,
+		max: 99
+	});
+	const avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
+	const $avatar = avatarPrefix + avPath;
+	const $created = date(bornDaysAgo, true, null)();
+
+	//anon Ids
+	const anonymousIds = [];
+	const clusterSize = integer(2, 10);
+	for (let i = 0; i < clusterSize; i++) {
+		anonymousIds.push(uid(42));
+	}
+
+	//session Ids
+	const sessionIds = [];
+	const sessionSize = integer(5, 30);
+	for (let i = 0; i < sessionSize; i++) {
+		sessionIds.push([uid(5), uid(5), uid(5), uid(5)].join("-"));
+	}
+
+
+
+	return {
+		$name,
+		$email,
+		$avatar,
+		$created,
+		anonymousIds,
+		sessionIds
+	};
+}
+
 module.exports = {
 	weightedRange,
 	pick,
@@ -376,7 +383,6 @@ module.exports = {
 	makeProducts,
 	date,
 	progress,
-	person,
 	choose,
 	range,
 	exhaust,
@@ -387,4 +393,5 @@ module.exports = {
 	getUniqueKeys,
 	makeHashTags,
 	generateName,
+	person
 };