make-mp-data 1.1.11 → 1.1.13

package/default.js

@@ -19,6 +19,8 @@ const config = {
 	numUsers: 1000, //how many users	
 	format: 'csv', //csv or json
 	region: "US",
+	anonIds: true, //if true, anonymousIds are created for each user
+	sessionIds: true, //if true, sessionIds are created for each user
 
 	events: [
 		{

package/e2e.test.js

@@ -36,14 +36,60 @@ describe('e2e', () => {
 
 	test('sends data to mixpanel', async () => {
 		console.log('NETWORK TEST');
-		const results = await generate({ writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it", token: testToken});
-		const {events, users, groups } = results.import;
+		const results = await generate({ writeToDisk: false, numEvents: 1100, numUsers: 100, seed: "deal with it", token: testToken });
+		const { events, users, groups } = results.import;
 		expect(events.success).toBeGreaterThan(980);
 		expect(users.success).toBe(100);
-		expect(groups.length).toBe(0);		
+		expect(groups.length).toBe(0);
+	}, timeout);
+
+	test('every record is valid', async () => {
+		console.log('VALIDATION TEST');
+		const results = await generate({ writeToDisk: false, numEvents: 10000, numUsers: 500 });
+		const { eventData, userProfilesData } = results;
+		const areEventsValid = eventData.every(validateEvent);
+		const areUsersValid = userProfilesData.every(validateUser);
+
+		const invalidEvents = eventData.filter(e => !validateEvent(e));
+		const invalidUsers = userProfilesData.filter(u => !validateUser(u));
+		
+		expect(areEventsValid).toBe(true);
+		expect(areUsersValid).toBe(true);
+	}, timeout);
+
+
+});
+
+describe('options + tweaks', () => {
+	test('creates sessionIds', async () => {
+		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, sessionIds: true });
+		const { eventData } = results;
+		const sessionIds = eventData.map(a => a.$session_id).filter(a => a);
+		expect(sessionIds.length).toBe(eventData.length);
+	}, timeout);
 
+	test('no sessionIds', async () => {
+		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, sessionIds: false });
+		const { eventData } = results;
+		const sessionIds = eventData.map(a => a.$session_id).filter(a => a);
+		expect(sessionIds.length).toBe(0);
+	}, timeout);
+
+	test('creates anonymousIds', async () => {
+		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, anonIds: true });
+		const { eventData } = results;
+		const anonIds = eventData.map(a => a.$device_id).filter(a => a);
+		const userIds = eventData.map(a => a.$user_id).filter(a => a);
+		expect(anonIds.length).toBe(eventData.length);
+		expect(userIds.length).toBeLessThan(anonIds.length);
+	}, timeout);
+
+	test('no anonymousIds', async () => {
+		const results = await generate({ writeToDisk: false, numEvents: 1000, numUsers: 100, anonIds: false });
+		const { eventData } = results;
+		const anonIds = eventData.map(a => a.$device_id).filter(a => a);
+		expect(anonIds.length).toBe(0);
 	}, timeout);
-	
 
 
 });
@@ -55,7 +101,31 @@ afterEach(() => {
 });
 
 afterAll(() => {
-	console.log('clearing...');
-	execSync(`npm run prune`);
-	console.log('...files cleared 👍');
-});
+	try {
+		console.log('clearing...');
+		execSync(`npm run prune`);
+		console.log('...files cleared 👍');
+	}
+	catch (err) {
+		console.log('error clearing files');
+	}
+});
+
+//helpers
+
+function validateEvent(event) {
+	if (!event.event) return false;
+	if (!event.$device_id && !event.$user_id) return false;
+	if (!event.time) return false;
+	if (!event.$insert_id) return false;
+	return true;
+}
+
+
+function validateUser(user) {
+	if (!user.distinct_id) return false;
+	if (!user.$name) return false;
+	if (!user.$email) return false;
+	if (!user.$created) return false;
+	return true;
+}

package/index.js

@@ -35,7 +35,7 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const cliParams = require("./cli.js");
-const { makeName } = require('ak-tools');
+const { makeName, md5 } = require('ak-tools');
 
 Array.prototype.pickOne = pick;
 const NOW = dayjs().unix();
@@ -78,6 +78,8 @@ async function main(config) {
 		groupKeys = [],
 		groupProps = {},
 		lookupTables = [],
+		anonIds = true,
+		sessionIds = true,
 		format = "csv",
 		token = null,
 		region = "US",
@@ -86,6 +88,7 @@ async function main(config) {
 	} = config;
 	VERBOSE = verbose;
 	config.simulationName = makeName();
+	global.config = config;
 	const uuidChance = new Chance(seed);
 	log(`------------------SETUP------------------`);
 	log(`\nyour data simulation will heretofore be known as: \n\n\t${config.simulationName.toUpperCase()}...\n`);
@@ -93,8 +96,8 @@ async function main(config) {
 	log(`------------------SETUP------------------`, "\n");
 
 
-	//the function which generates $distinct_id + $created, skewing towards the present
-	function uuid() {
+	//the function which generates $distinct_id + $anonymoud_ids, $session_ids, and $created, skewing towards the present
+	function generateUser() {
 		const distinct_id = uuidChance.guid();
 		let z = boxMullerRandom();
 		const skew = chance.normal({ mean: 10, dev: 3 });
@@ -136,8 +139,8 @@ async function main(config) {
 	log(`---------------SIMULATION----------------`, `\n\n`);
 	for (let i = 1; i < numUsers + 1; i++) {
 		progress("users", i);
-		const user = uuid();
-		const { distinct_id, $created, anonymousIds } = user;
+		const user = generateUser();
+		const { distinct_id, $created, anonymousIds, sessionIds } = user;
 		userProfilesData.push(makeProfile(userProps, user));
 		const mutations = chance.integer({ min: 1, max: 10 });
 		scdTableData.push(makeSCD(scdProps, distinct_id, mutations, $created));
@@ -150,6 +153,7 @@ async function main(config) {
 				makeEvent(
 					distinct_id,
 					anonymousIds,
+					sessionIds,
 					dayjs($created).unix(),
 					firstEvents,
 					superProps,
@@ -165,6 +169,7 @@ async function main(config) {
 				makeEvent(
 					distinct_id,
 					anonymousIds,
+					sessionIds,
 					dayjs($created).unix(),
 					weightedEvents,
 					superProps,
@@ -220,7 +225,7 @@ async function main(config) {
 		[lookupFiles, lookupTableData],
 	];
 	log("\n", `---------------SIMULATION----------------`, "\n");
-	
+
 	if (!writeToDisk && !token) {
 		return {
 			eventData,
@@ -255,7 +260,7 @@ async function main(config) {
 						log(`\tdone\n`);
 					} else {
 						const ndjson = data.map((d) => JSON.stringify(d)).join("\n");
-						await touch(path, ndjson, true);
+						await touch(path, ndjson, false);
 					}
 				}
 			}
@@ -365,29 +370,41 @@ function makeSCD(props, distinct_id, mutations, $created) {
  * creates a random event
  * @param  {string} distinct_id
  * @param  {string[]} anonymousIds
+ * @param  {string[]} sessionIds
  * @param  {number} earliestTime
  * @param  {Object[]} events
  * @param  {Object} superProps
  * @param  {Object} groupKeys
  * @param  {Boolean} isFirstEvent=false
  */
-function makeEvent(distinct_id, anonymousIds, earliestTime, events, superProps, groupKeys, isFirstEvent = false) {
-
+function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, events, superProps, groupKeys, isFirstEvent = false) {
 	let chosenEvent = events.pickOne();
-	if (typeof chosenEvent === "string")
+
+	//allow for a string shorthand
+	if (typeof chosenEvent === "string") {
 		chosenEvent = { event: chosenEvent, properties: {} };
+	}
+
+	//event model
 	const event = {
 		event: chosenEvent.event,
-		$device_id: chance.pickone(anonymousIds), // always have a $device_id		
 		$source: "AKsTimeSoup",
 	};
 
+	//event time
 	if (isFirstEvent) event.time = dayjs.unix(earliestTime).toISOString();
 	if (!isFirstEvent) event.time = AKsTimeSoup(earliestTime, NOW, PEAK_DAYS);
 
+	// anonymous and session ids
+	if (global?.config.anonIds) event.$device_id = chance.pickone(anonymousIds);
+	if (global?.config.sessionIds) event.$session_id = chance.pickone(sessionIds);
+
 	//sometimes have a $user_id
 	if (!isFirstEvent && chance.bool({ likelihood: 42 })) event.$user_id = distinct_id;
 
+	// ensure that there is a $user_id or $device_id
+	if (!event.$user_id && !event.$device_id) event.$user_id = distinct_id;
+
 	const props = { ...chosenEvent.properties, ...superProps };
 
 	//iterate through custom properties
@@ -408,6 +425,9 @@ function makeEvent(distinct_id, anonymousIds, earliestTime, events, superProps,
 		event[groupKey] = weightedRange(1, groupCardinality).pickOne();
 	}
 
+	//make $insert_id
+	event.$insert_id = md5(JSON.stringify(event));
+
 	return event;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "make-mp-data",
-  "version": "1.1.11",
+  "version": "1.1.13",
   "description": "builds all mixpanel primitives for a given project",
   "main": "index.js",
   "types": "types.d.ts",
@@ -38,7 +38,7 @@
   },
   "homepage": "https://github.com/ak--47/make-mp-data#readme",
   "dependencies": {
-    "ak-tools": "^1.0.52",
+    "ak-tools": "^1.0.53",
     "chance": "^1.1.7",
     "dayjs": "^1.11.10",
     "mixpanel-import": "^2.5.5",

package/types.d.ts

@@ -21,6 +21,8 @@ export interface Config {
   writeToDisk?: boolean;
   simulationName?: string;
   verbose?: boolean;
+  anonIds?: boolean;
+  sessionIds?: boolean;
 }
 
 interface EventConfig {

package/utils.js

@@ -231,18 +231,30 @@ function person(bornDaysAgo = 30) {
 	const avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
 	const $avatar = avatarPrefix + avPath;
 	const $created = date(bornDaysAgo, true, null)();
+
+	//anon Ids
 	const anonymousIds = [];
 	const clusterSize = integer(2, 10);
 	for (let i = 0; i < clusterSize; i++) {
 		anonymousIds.push(uid(42));
 	}
 
+	//session Ids
+	const sessionIds = [];
+	const sessionSize = integer(5, 30);
+	for (let i = 0; i < sessionSize; i++) {
+		sessionIds.push([uid(5), uid(5), uid(5), uid(5)].join("-"));
+	}
+
+
+
 	return {
 		$name,
 		$email,
 		$avatar,
 		$created,
-		anonymousIds
+		anonymousIds,
+		sessionIds
 	};
 }
 
@@ -333,7 +345,7 @@ function generateName() {
 		"splintering", "crescendoing", "whirling", "bursting", "shining", "gushing", "emerging", "revealing",
 		"emerging", "unfolding", "unveiling", "emerging", "surrounding", "unveiling", "materializing", "revealing"
 	];
-	
+
 	var adverbs = [
 		"gracefully", "softly", "smoothly", "gently", "tenderly", "quietly", "serenely", "peacefully",
 		"delicately", "effortlessly", "subtly", "tranquilly", "majestically", "silently", "calmly", "harmoniously",
@@ -343,16 +355,16 @@ function generateName() {
 		"seductively", "envelopingly", "ensnaringly", "entrancingly", "intoxicatingly", "irresistibly", "transcendentally",
 		"envelopingly", "rapturously", "intimately", "intensely", "tangibly", "vividly", "intensely", "deeply"
 	];
-	
+
 
 	// ? http://stackoverflow.com/a/17516862/103058
-	var adj = adjs[Math.floor(Math.random() * adjs.length)]; 
+	var adj = adjs[Math.floor(Math.random() * adjs.length)];
 	var noun = nouns[Math.floor(Math.random() * nouns.length)];
 	var verb = verbs[Math.floor(Math.random() * verbs.length)];
 	var adverb = adverbs[Math.floor(Math.random() * adverbs.length)];
-	
-	
-	return adj + '-' + noun + '-' + verb + '-' + adverb
+
+
+	return adj + '-' + noun + '-' + verb + '-' + adverb;
 
 }