make-mp-data 1.4.5 → 1.5.0

package/scripts/new.sh

@@ -13,23 +13,39 @@ const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 require("dotenv").config();
-const u = require("../core/utils");
+const u = require("../src/utils");
 const v = require("ak-tools");
 const chance = u.initChance(SEED);
+const num_users = 25_000
+const days = 100
 
 /** @type {import("../types").Config} */
 const config = {
     token: "",
     seed: SEED,
-    numDays: 100, //how many days worth of data
-    numEvents: 100000, //how many events
-    numUsers: 1000, //how many users    
-    format: "csv", //csv or json
-    region: "US",
-    makeChart: false,
-    anonIds: false,
-    sessionIds: false,
-    writeToDisk: false,
+    numDays: days, 
+    numEvents: num_users * 100, 
+    numUsers: num_users, 
+	hasAnonIds: true, 
+	hasSessionIds: true, 
+
+	hasLocation: true,
+	hasAndroidDevices: true,
+	hasIOSDevices: true,
+	hasDesktopDevices: true,
+	hasBrowser: true,
+	hasCampaigns: true,
+	isAnonymous: false,
+	hasAdSpend: true,
+	
+	hasAvatar: true,
+	makeChart: false,
+
+	batchSize: 500_000,
+	concurrency: 500,
+	writeToDisk: false,
+	
+	funnels: [],
     events: [],
     superProps: {},
     userProps: {},

package/{core → src}/cli.js

@@ -15,7 +15,6 @@ by ak@mixpanel.com
 
 function cliParams() {
 	console.log(hero);
-	// @ts-ignore
 	const args = yargs(process.argv.splice(2))
 		.scriptName("make-mp-data")
 		.usage(`\nusage:\nnpx $0 [dataModel.js] [options]
@@ -85,6 +84,13 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			describe: 'either US or EU',
 			type: 'string'
 		})
+		.option('concurrency', {
+			alias: 'conn',
+			default: 500,
+			demandOption: false,
+			describe: 'concurrency level for data generation',
+			type: 'number'
+		})
 		.options("complex", {
 			demandOption: false,
 			default: false,
@@ -101,7 +107,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
-		.option("sessionIds", {
+		.option("hasSessionIds", {
 			demandOption: false,
 			default: false,
 			describe: 'create session ids in the data',
@@ -109,7 +115,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
-		.option("anonIds", {
+		.option("hasAnonIds", {
 			demandOption: false,
 			default: false,
 			describe: 'create anonymous ids in the data',
@@ -187,6 +193,7 @@ DATA MODEL: https://github.com/ak--47/make-mp-data/blob/main/default.js
 			type: 'boolean',
 			coerce: boolCoerce
 		})
+		
 		.help()
 		.wrap(null)
 		.argv;

package/{core → src}/utils.js

@@ -15,7 +15,7 @@ const { domainSuffix, domainPrefix } = require('./defaults');
 /** @typedef {import('../types').Config} Config */
 /** @typedef {import('../types').EventConfig} EventConfig */
 /** @typedef {import('../types').ValueValid} ValueValid */
-/** @typedef {import('../types').EnrichedArray} hookArray */
+/** @typedef {import('../types').HookedArray} hookArray */
 /** @typedef {import('../types').hookArrayOptions} hookArrayOptions */
 /** @typedef {import('../types').Person} Person */
 /** @typedef {import('../types').Funnel} Funnel */
@@ -40,7 +40,6 @@ function initChance(seed) {
 	if (process.env.SEED) seed = process.env.SEED;  // Override seed with environment variable if available
 	if (!chanceInitialized) {
 		globalChance = new Chance(seed);
-		if (global.MP_SIMULATION_CONFIG) global.MP_SIMULATION_CONFIG.chance = globalChance;
 		chanceInitialized = true;
 	}
 	return globalChance;
@@ -52,11 +51,11 @@ function initChance(seed) {
  */
 function getChance() {
 	if (!chanceInitialized) {
-		const seed = process.env.SEED || global.MP_SIMULATION_CONFIG?.seed;
+		const seed = process.env.SEED || "";
 		if (!seed) {
-			return new Chance();
+			return new Chance(); // this is a new RNG and therefore not deterministic
 		}
-		return initChance(seed);		
+		return initChance(seed);
 	}
 	return globalChance;
 }
@@ -312,52 +311,6 @@ function range(a, b, step = 1) {
 };
 
 
-/**
- * create funnels out of random events
- * @param {EventConfig[]} events
- */
-function inferFunnels(events) {
-	const createdFunnels = [];
-	const firstEvents = events.filter((e) => e.isFirstEvent).map((e) => e.event);
-	const usageEvents = events.filter((e) => !e.isFirstEvent).map((e) => e.event);
-	const numFunnelsToCreate = Math.ceil(usageEvents.length);
-	/** @type {Funnel} */
-	const funnelTemplate = {
-		sequence: [],
-		conversionRate: 50,
-		order: 'sequential',
-		requireRepeats: false,
-		props: {},
-		timeToConvert: 1,
-		isFirstFunnel: false,
-		weight: 1
-	};
-	if (firstEvents.length) {
-		for (const event of firstEvents) {
-			createdFunnels.push({ ...clone(funnelTemplate), sequence: [event], isFirstFunnel: true, conversionRate: 100 });
-		}
-	}
-
-	//at least one funnel with all usage events
-	createdFunnels.push({ ...clone(funnelTemplate), sequence: usageEvents });
-
-	//for the rest, make random funnels
-	followUpFunnels: for (let i = 1; i < numFunnelsToCreate; i++) {
-		/** @type {Funnel} */
-		const funnel = { ...clone(funnelTemplate) };
-		funnel.conversionRate = integer(25, 75);
-		funnel.timeToConvert = integer(1, 10);
-		funnel.weight = integer(1, 10);
-		const sequence = shuffleArray(usageEvents).slice(0, integer(2, usageEvents.length));
-		funnel.sequence = sequence;
-		funnel.order = 'random';
-		createdFunnels.push(funnel);
-	}
-
-	return createdFunnels;
-
-}
-
 
 /*
 ----
@@ -674,7 +627,7 @@ function validateEventConfig(events) {
 	return cleanEventConfig;
 }
 
-function validateTime(chosenTime, earliestTime, latestTime) {
+function validTime(chosenTime, earliestTime, latestTime) {
 	if (!earliestTime) earliestTime = global.NOW - (60 * 60 * 24 * 30); // 30 days ago
 	if (!latestTime) latestTime = global.NOW;
 
@@ -691,6 +644,17 @@ function validateTime(chosenTime, earliestTime, latestTime) {
 	return false;
 }
 
+function validEvent(row) {
+	if (!row) return false;
+	if (!row.event) return false;
+	if (!row.time) return false;
+	if (!row.device_id && !row.user_id) return false;
+	if (!row.insert_id) return false;
+	if (!row.source) return false;
+	if (typeof row.time !== 'string') return false;
+	return true;
+}
+
 
 /*
 ----
@@ -698,68 +662,7 @@ META
 ----
 */
 
-/** 
- * our meta programming function which lets you mutate items as they are pushed into an array
- * @param  {any[]} arr
- * @param  {hookArrayOptions} opts
- * @returns {hookArray}}
- */
-function hookArray(arr = [], opts = {}) {
-	const { hook = a => a, type = "", ...rest } = opts;
-
-	function transformThenPush(item) {
-		if (item === null) return false;
-		if (item === undefined) return false;
-		if (typeof item === 'object') {
-			if (Object.keys(item).length === 0) return false;
-		}
-
-		//hook is passed an array 
-		if (Array.isArray(item)) {
-			for (const i of item) {
-				try {
-					const enriched = hook(i, type, rest);
-					if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
-					else arr.push(enriched);
-
-				}
-				catch (e) {
-					console.error(`\n\nyour hook had an error\n\n`, e);
-					arr.push(i);
-					return false;
-				}
-
-			}
-			return true;
-		}
-
-		//hook is passed a single item
-		else {
-			try {
-				const enriched = hook(item, type, rest);
-				if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
-				else arr.push(enriched);
-				return true;
-			}
-			catch (e) {
-				console.error(`\n\nyour hook had an error\n\n`, e);
-				arr.push(item);
-				return false;
-			}
-		}
-
-	}
-
-	/** @type {hookArray} */
-	// @ts-ignore
-	const enrichedArray = arr;
-
 
-	enrichedArray.hookPush = transformThenPush;
-
-
-	return enrichedArray;
-};
 
 /**
  * @param  {Config} config
@@ -770,7 +673,7 @@ function buildFileNames(config) {
 	extension = format === "csv" ? "csv" : "json";
 	// const current = dayjs.utc().format("MM-DD-HH");
 	let simName = config.simulationName;
-	let writeDir = "./";
+	let writeDir = typeof config.writeToDisk === 'string' ? config.writeToDisk : "./";
 	if (config.writeToDisk) {
 		const dataFolder = path.resolve("./data");
 		if (existsSync(dataFolder)) writeDir = dataFolder;
@@ -835,7 +738,6 @@ function buildFileNames(config) {
  * @param  {[string, number][]} arrayOfArrays
  */
 function progress(arrayOfArrays) {
-	// @ts-ignore
 	readline.cursorTo(process.stdout, 0);
 	let message = "";
 	for (const status of arrayOfArrays) {
@@ -873,8 +775,9 @@ CORE
 */
 
 //the function which generates $distinct_id + $anonymous_ids, $session_ids, and created, skewing towards the present
-function generateUser(user_id, numDays, amplitude = 1, frequency = 1, skew = 1) {
+function generateUser(user_id, opts, amplitude = 1, frequency = 1, skew = 1) {
 	const chance = getChance();
+	const { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds } = opts;
 	// Uniformly distributed `u`, then skew applied
 	let u = Math.pow(chance.random(), skew);
 
@@ -886,16 +789,18 @@ function generateUser(user_id, numDays, amplitude = 1, frequency = 1, skew = 1)
 
 	// Clamp values to ensure they are within the desired range
 	daysAgoBorn = Math.min(daysAgoBorn, numDays);
+	const props = person(user_id, daysAgoBorn, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds);
 
 	const user = {
 		distinct_id: user_id,
-		...person(numDays),
+		...props,
 	};
 
 
 	return user;
 }
 
+let soupHits = 0;
 /**
  * build sign waves basically
  * @param  {number} [earliestTime]
@@ -921,8 +826,9 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 	let isValidTime = false;
 	do {
 		iterations++;
+		soupHits++;
 		offset = chance.normal({ mean: mean, dev: chunkSize / deviation });
-		isValidTime = validateTime(chunkMid + offset, earliestTime, latestTime);
+		isValidTime = validTime(chunkMid + offset, earliestTime, latestTime);
 		if (iterations > 25000) {
 			throw `${iterations} iterations... exceeded`;
 		}
@@ -946,15 +852,17 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
  * @param {string} userId
  * @param  {number} bornDaysAgo=30
  * @param {boolean} isAnonymous
+ * @param {boolean} hasAvatar
+ * @param {boolean} hasAnonIds
+ * @param {boolean} hasSessionIds
  * @return {Person}
  */
-function person(userId, bornDaysAgo = 30, isAnonymous = false) {
+function person(userId, bornDaysAgo = 30, isAnonymous = false, hasAvatar = false, hasAnonIds = false, hasSessionIds = false) {
 	const chance = getChance();
 	//names and photos
 	const l = chance.letter.bind(chance);
 	let gender = chance.pickone(['male', 'female']);
 	if (!gender) gender = "female";
-	// @ts-ignore
 	let first = chance.first({ gender });
 	let last = chance.last();
 	let name = `${first} ${last}`;
@@ -982,21 +890,23 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false) {
 		user.name = "Anonymous User";
 		user.email = l() + l() + `*`.repeat(integer(3, 6)) + l() + `@` + l() + `*`.repeat(integer(3, 6)) + l() + `.` + choose(domainSuffix);
 		delete user.avatar;
-
 	}
 
+	if (!hasAvatar) delete user.avatar;
+
 	//anon Ids
-	if (global.MP_SIMULATION_CONFIG?.anonIds) {
+	if (hasAnonIds) {
 		const clusterSize = integer(2, 10);
 		for (let i = 0; i < clusterSize; i++) {
 			const anonId = uid(42);
 			user.anonymousIds.push(anonId);
 		}
-
 	}
 
+	if (!hasAnonIds) delete user.anonymousIds;
+
 	//session Ids
-	if (global.MP_SIMULATION_CONFIG?.sessionIds) {
+	if (hasSessionIds) {
 		const sessionSize = integer(5, 30);
 		for (let i = 0; i < sessionSize; i++) {
 			const sessionId = [uid(5), uid(5), uid(5), uid(5)].join("-");
@@ -1004,6 +914,8 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false) {
 		}
 	}
 
+	if (!hasSessionIds) delete user.sessionIds;
+
 	return user;
 };
 
@@ -1078,7 +990,10 @@ module.exports = {
 
 	initChance,
 	getChance,
-	validateTime,
+
+	validTime,
+	validEvent,
+
 	boxMullerRandom,
 	applySkew,
 	mapToRange,
@@ -1100,12 +1015,10 @@ module.exports = {
 	shuffleOutside,
 	interruptArray,
 	generateUser,
-	hookArray,
 	optimizedBoxMuller,
 	buildFileNames,
 	streamJSON,
 	streamCSV,
-	inferFunnels,
 	datesBetween,
 	weighChoices
 };

package/tests/benchmark/concurrency.mjs

@@ -0,0 +1,52 @@
+/*
+----
+TO DOs
+----
+*/
+
+//!feature: fixedTimeFunnel? if set this funnel will occur for all users at the same time ['cards charged', 'charge complete']
+//!feature: churn ... is churnFunnel, possible to return, etc
+//!feature: send SCD data to mixpanel (blocked on dev)
+//!feature: send and map lookup tables to mixpanel (also blocked on dev)
+//!bug: using --mc flag reverts to --complex for some reason
+
+
+import main from "../../index.js";
+import simple from '../../schemas/simple.js';
+
+/** @typedef {import('../../types').Config} Config */
+
+/** @type {Config} */
+const noWrites = {
+	...simple,
+	numUsers: 10_000,
+	numEvents: 250_000,
+	writeToDisk: false,
+};
+
+/** @type {Config} */
+const yesWrites = {
+	...noWrites,
+	writeToDisk: true
+};
+
+console.log('concurrency benchmarking');
+
+const concurrency = [1, 2, 3, 4, 5];
+
+const results = [];
+for (const concurrent of concurrency) {
+	console.log(`concurrency: ${concurrent}`);
+	// @ts-ignore
+	const test = await main({ ...noWrites, concurrency: concurrent });
+	results.push({ human: test.time.human, concurrency: concurrent });
+	console.log(`\t\tdone: ${test.time.human}\n\n`);
+}
+
+const display = results.map((r) => {
+	return `concurrency: ${r.concurrency} | duration: ${r.human}`;
+});
+
+console.log(display.join('\n\n'));
+
+debugger;