make-mp-data 1.4.1 → 1.4.3

package/{index.js → core/index.js}

@@ -7,31 +7,43 @@ ak@mixpanel.com
 */
 
 //todo: churn ... is churnFunnel, possible to return, etc
-//todo: fixedTimeFunnel? if set this funnel will occur for all users at the same time ['cart charged', 'charge complete']
-//todo defaults!!!
-//todo ads-data
-		
-		
+//todo: fixedTimeFunnel? if set this funnel will occur for all users at the same time ['cards charged', 'charge complete']
+//todo: send SCD data to mixpanel
+//todo: send and map lookup tables to mixpanel
+
+/** @typedef {import('../types').Config} Config */
+/** @typedef {import('../types').EventConfig} EventConfig */
+/** @typedef {import('../types').Funnel} Funnel */
+/** @typedef {import('../types').Person} Person */
+/** @typedef {import('../types').SCDTableRow} SCDTableRow */
+/** @typedef {import('../types').UserProfile} UserProfile */
+/** @typedef {import('../types').EventSpec} EventSpec */
+
 const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
 dayjs.extend(utc);
 const NOW = dayjs('2024-02-02').unix(); //this is a FIXED POINT and we will shift it later
 global.NOW = NOW;
-const mp = require("mixpanel-import");
+
+const os = require("os");
 const path = require("path");
 const { comma, bytesHuman, makeName, md5, clone, tracker, uid } = require("ak-tools");
 const { generateLineChart } = require('./chart.js');
-const { version } = require('./package.json');
-const os = require("os");
+const { version } = require('../package.json');
+const mp = require("mixpanel-import");
 const metrics = tracker("make-mp-data", "db99eb8f67ae50949a13c27cacf57d41", os.userInfo().username);
 
 
 const u = require("./utils.js");
-const cliParams = require("./cli.js");
+const getCliParams = require("./cli.js");
+const { campaigns, devices, locations } = require('./defaults.js');
 
 let VERBOSE = false;
 let isCLI = false;
+/** @type {Config} */
 let CONFIG;
+let CAMPAIGNS;
+let DEFAULTS;
 require('dotenv').config();
 
 
@@ -40,25 +52,20 @@ function track(name, props, ...rest) {
 	metrics(name, props, ...rest);
 }
 
-/** @typedef {import('./types.d.ts').Config} Config */
-/** @typedef {import('./types.d.ts').EventConfig} EventConfig */
-/** @typedef {import('./types.d.ts').Funnel} Funnel */
-/** @typedef {import('./types.d.ts').Person} Person */
-/** @typedef {import('./types.d.ts').SCDTableRow} SCDTableRow */
-/** @typedef {import('./types.d.ts').UserProfile} UserProfile */
-/** @typedef {import('./types.d.ts').EventSpec} EventSpec */
+
 
 /**
  * generates fake mixpanel data
  * @param  {Config} config
  */
 async function main(config) {
-	//PARAMS
+
+	//seed the random number generator
+	// ^ this is critical; same seed = same data; seed can be passed in as an env var or in the config
 	const seedWord = process.env.SEED || config.seed || "hello friend!";
 	config.seed = seedWord;
 	u.initChance(seedWord);
-	const chance = u.getChance();
-	config.chance = chance;
+	const chance = u.getChance(); // ! this is the only safe way to get the chance instance
 	let {
 		seed,
 		numEvents = 100000,
@@ -67,10 +74,9 @@ async function main(config) {
 		epochStart = 0,
 		epochEnd = dayjs().unix(),
 		events = [{ event: "foo" }, { event: "bar" }, { event: "baz" }],
-		superProps = { platform: ["web", "iOS", "Android"] },
+		superProps = { luckyNumber: [2, 2, 4, 4, 42, 42, 42, 2, 2, 4, 4, 42, 42, 42, 420] },
 		funnels = [],
 		userProps = {
-			favoriteColor: ["red", "green", "blue", "yellow"],
 			spiritAnimal: chance.animal.bind(chance),
 		},
 		scdProps = {},
@@ -88,10 +94,20 @@ async function main(config) {
 		makeChart = false,
 		soup = {},
 		hook = (record) => record,
+		hasAdSpend = false,
+		hasCampaigns = false,
+		hasLocation = false,
+		isAnonymous = false,
+		hasBrowser = false,
+		hasAndroidDevices = false,
+		hasDesktopDevices = false,
+		hasIOSDevices = false
 	} = config;
+
 	if (!config.superProps) config.superProps = superProps;
 	if (!config.userProps || Object.keys(config?.userProps)) config.userProps = userProps;
-	VERBOSE = verbose;
+
+
 	config.simulationName = makeName();
 	const { simulationName } = config;
 	if (epochStart && !numDays) numDays = dayjs.unix(epochEnd).diff(dayjs.unix(epochStart), "day");
@@ -123,13 +139,34 @@ async function main(config) {
 	config.makeChart = makeChart;
 	config.soup = soup;
 	config.hook = hook;
+	config.hasAdSpend = hasAdSpend;
+	config.hasCampaigns = hasCampaigns;
+	config.hasLocation = hasLocation;
+	config.isAnonymous = isAnonymous;
+	config.hasBrowser = hasBrowser;
+	config.hasAndroidDevices = hasAndroidDevices;
+	config.hasDesktopDevices = hasDesktopDevices;
+	config.hasIOSDevices = hasIOSDevices;
 
 	//event validation 
 	const validatedEvents = u.validateEventConfig(events);
 	events = validatedEvents;
 	config.events = validatedEvents;
+
+	//globals
 	global.MP_SIMULATION_CONFIG = config;
 	CONFIG = config;
+	VERBOSE = verbose;
+	CAMPAIGNS = campaigns;
+	DEFAULTS = {
+		locations: u.pickAWinner(locations, 0),
+		iOSDevices: u.pickAWinner(devices.iosDevices, 0),
+		androidDevices: u.pickAWinner(devices.androidDevices, 0),
+		desktopDevices: u.pickAWinner(devices.desktopDevices, 0),
+		browsers: u.pickAWinner(devices.browsers, 0),
+		campaigns: u.pickAWinner(campaigns, 0),
+	};
+
 	const runId = uid(42);
 	let trackingParams = { runId, seed, numEvents, numUsers, numDays, anonIds, sessionIds, format, targetToken: token, region, writeToDisk, isCLI, version };
 	track('start simulation', trackingParams);
@@ -142,6 +179,7 @@ async function main(config) {
 	//setup all the data structures we will push into
 	const eventData = u.enrichArray([], { hook, type: "event", config });
 	const userProfilesData = u.enrichArray([], { hook, type: "user", config });
+	const adSpendData = u.enrichArray([], { hook, type: "ad-spend", config });
 	const scdTableKeys = Object.keys(scdProps);
 	const scdTableData = [];
 	for (const [index, key] of scdTableKeys.entries()) {
@@ -153,7 +191,7 @@ async function main(config) {
 
 	// if no funnels, make some out of events...
 	if (!funnels || !funnels.length) {
-		funnels = inferFunnels(events);
+		funnels = u.inferFunnels(events);
 		config.funnels = funnels;
 		CONFIG = config;
 	}
@@ -161,13 +199,21 @@ async function main(config) {
 	//user loop
 	log(`---------------SIMULATION----------------`, "\n\n");
 	loopUsers: for (let i = 1; i < numUsers + 1; i++) {
-		u.progress("users", i);
+		u.progress([["users", i], ["events", eventData.length]]);
 		const userId = chance.guid();
-		// const user = u.generateUser(userId, numDays, amp, freq, skew);
-		const user = u.generateUser(userId, numDays);
+		const user = u.person(userId, numDays, isAnonymous);
 		const { distinct_id, created, anonymousIds, sessionIds } = user;
 		let numEventsPreformed = 0;
 
+		if (hasLocation) {
+			const location = u.choose(clone(DEFAULTS.locations()).map(l => { delete l.country; return l; }));
+			for (const key in location) {
+				user[key] = location[key];
+			}
+		}
+
+
+
 		// profile creation
 		const profile = makeProfile(userProps, user);
 		userProfilesData.hookPush(profile);
@@ -226,6 +272,17 @@ async function main(config) {
 		// end individual user loop
 	}
 
+	if (hasAdSpend) {
+		const days = u.datesBetween(epochStart, epochEnd);
+		for (const day of days) {
+			const dailySpendData = makeAdSpend(day);
+			for (const spendEvent of dailySpendData) {
+				adSpendData.hookPush(spendEvent);
+			}
+		}
+
+	}
+
 	//flatten SCD tables
 	scdTableData.forEach((table, index) => scdTableData[index] = table.flat());
 
@@ -237,7 +294,7 @@ async function main(config) {
 		const groupCardinality = groupPair[1];
 		const groupProfiles = [];
 		for (let i = 1; i < groupCardinality + 1; i++) {
-			u.progress("groups", i);
+			u.progress([["groups", i]]);
 			const group = {
 				[groupKey]: i,
 				...makeProfile(groupProps[groupKey])
@@ -254,7 +311,7 @@ async function main(config) {
 		const { key, entries, attributes } = lookupTable;
 		const data = [];
 		for (let i = 1; i < entries + 1; i++) {
-			u.progress("lookups", i);
+			u.progress([["lookups", i]]);
 			const item = {
 				[key]: i,
 				...makeProfile(attributes),
@@ -309,11 +366,12 @@ async function main(config) {
 		}
 	}
 
-	const { eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, mirrorFiles, folder } =
+	const { eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, mirrorFiles, folder, adSpendFiles } =
 		u.buildFileNames(config);
 	const pairs = [
 		[eventFiles, [eventData]],
 		[userFiles, [userProfilesData]],
+		[adSpendFiles, [adSpendData]],
 		[scdFiles, scdTableData],
 		[groupFiles, groupProfilesData],
 		[lookupFiles, lookupTableData],
@@ -379,7 +437,7 @@ async function main(config) {
 			fixData: true,
 			verbose: false,
 			forceStream: true,
-			strict: true,
+			strict: false, //! sometimes we get events in the future... it happens
 			dryRun: false,
 			abridged: false,
 			fixJson: true,
@@ -388,7 +446,7 @@ async function main(config) {
 
 		if (eventData) {
 			log(`importing events to mixpanel...\n`);
-			const imported = await mp(creds, eventData, {
+			const imported = await mp(creds, clone(eventData), {
 				recordType: "event",
 				...commonOpts,
 			});
@@ -397,19 +455,28 @@ async function main(config) {
 		}
 		if (userProfilesData) {
 			log(`importing user profiles to mixpanel...\n`);
-			const imported = await mp(creds, userProfilesData, {
+			const imported = await mp(creds, clone(userProfilesData), {
 				recordType: "user",
 				...commonOpts,
 			});
 			log(`\tsent ${comma(imported.success)} user profiles\n`);
 			importResults.users = imported;
 		}
+		if (adSpendData) {
+			log(`importing ad spend data to mixpanel...\n`);
+			const imported = await mp(creds, clone(adSpendData), {
+				recordType: "event",
+				...commonOpts,
+			});
+			log(`\tsent ${comma(imported.success)} ad spend events\n`);
+			importResults.adSpend = imported;
+		}
 		if (groupProfilesData) {
 			for (const groupProfiles of groupProfilesData) {
 				const groupKey = groupProfiles.key;
 				const data = groupProfiles.data;
 				log(`importing ${groupKey} profiles to mixpanel...\n`);
-				const imported = await mp({ token, groupKey }, data, {
+				const imported = await mp({ token, groupKey }, clone(data), {
 					recordType: "group",
 					...commonOpts,
 
@@ -432,6 +499,7 @@ async function main(config) {
 		groupProfilesData,
 		lookupTableData,
 		mirrorEventData,
+		adSpendData
 	};
 }
 
@@ -452,27 +520,42 @@ async function main(config) {
  * @param  {Boolean} isFirstEvent=false
  */
 function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, chosenEvent, superProps, groupKeys, isFirstEvent = false) {
-	const { mean = 0, dev = 2, peaks = 5 } = CONFIG.soup;
+	const chance = u.getChance();
+	const { mean = 0, deviation = 2, peaks = 5 } = CONFIG.soup;
+	const { hasAndroidDevices, hasBrowser, hasCampaigns, hasDesktopDevices, hasIOSDevices, hasLocation } = CONFIG;
 	//event model
 	const eventTemplate = {
 		event: chosenEvent.event,
 		source: "dm4",
 	};
 
+	let defaultProps = {};
+	let devicePool = [];
+	if (hasLocation) defaultProps.location = clone(DEFAULTS.locations()).map(l => { delete l.country_code; return l; });
+	if (hasBrowser) defaultProps.browser = DEFAULTS.browsers();
+	if (hasAndroidDevices) devicePool.push(DEFAULTS.androidDevices());
+	if (hasIOSDevices) devicePool.push(DEFAULTS.iOSDevices());
+	if (hasDesktopDevices) devicePool.push(DEFAULTS.desktopDevices());
+	// we don't always have campaigns, because of attribution
+	if (hasCampaigns && chance.bool({ likelihood: 25 })) defaultProps.campaigns = DEFAULTS.campaigns();
+	const devices = devicePool.flat();
+	if (devices.length) defaultProps.device = devices;
+
+
 	//event time
 	if (earliestTime > NOW) {
 		earliestTime = dayjs.unix(NOW).subtract(2, 'd').unix();
 	};
 
 	if (isFirstEvent) eventTemplate.time = dayjs.unix(earliestTime).toISOString();
-	if (!isFirstEvent) eventTemplate.time = u.TimeSoup(earliestTime, NOW, peaks, dev, mean);
+	if (!isFirstEvent) eventTemplate.time = u.TimeSoup(earliestTime, NOW, peaks, deviation, mean);
 
 	// anonymous and session ids
-	if (CONFIG?.anonIds) eventTemplate.device_id = CONFIG.chance.pickone(anonymousIds);
-	if (CONFIG?.sessionIds) eventTemplate.session_id = CONFIG.chance.pickone(sessionIds);
+	if (CONFIG?.anonIds) eventTemplate.device_id = chance.pickone(anonymousIds);
+	if (CONFIG?.sessionIds) eventTemplate.session_id = chance.pickone(sessionIds);
 
 	//sometimes have a user_id
-	if (!isFirstEvent && CONFIG.chance.bool({ likelihood: 42 })) eventTemplate.user_id = distinct_id;
+	if (!isFirstEvent && chance.bool({ likelihood: 42 })) eventTemplate.user_id = distinct_id;
 
 	// ensure that there is a user_id or device_id
 	if (!eventTemplate.user_id && !eventTemplate.device_id) eventTemplate.user_id = distinct_id;
@@ -489,6 +572,43 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, chosenEv
 		}
 	}
 
+	//iterate through default properties
+	for (const key in defaultProps) {
+		if (Array.isArray(defaultProps[key])) {
+			const choice = u.choose(defaultProps[key]);
+			if (typeof choice === "string") {
+				if (!eventTemplate[key]) eventTemplate[key] = choice;
+			}
+
+			else if (Array.isArray(choice)) {
+				for (const subChoice of choice) {
+					if (!eventTemplate[key]) eventTemplate[key] = subChoice;
+				}
+			}
+
+			else if (typeof choice === "object") {
+				for (const subKey in choice) {
+					if (typeof choice[subKey] === "string") {
+						if (!eventTemplate[subKey]) eventTemplate[subKey] = choice[subKey];
+					}
+					else if (Array.isArray(choice[subKey])) {
+						const subChoice = u.choose(choice[subKey]);
+						if (!eventTemplate[subKey]) eventTemplate[subKey] = subChoice;
+					}
+
+					else if (typeof choice[subKey] === "object") {
+						for (const subSubKey in choice[subKey]) {
+							if (!eventTemplate[subSubKey]) eventTemplate[subSubKey] = choice[subKey][subSubKey];
+						}
+					}
+
+				}
+			}
+
+
+		}
+	}
+
 	//iterate through groups
 	for (const groupPair of groupKeys) {
 		const groupKey = groupPair[0];
@@ -518,6 +638,7 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, chosenEv
  * @return {[EventSpec[], Boolean]}
  */
 function makeFunnel(funnel, user, profile, scd, firstEventTime, config) {
+	const chance = u.getChance();
 	const { hook } = config;
 	hook(funnel, "funnel-pre", { user, profile, scd, funnel, config });
 	let {
@@ -532,6 +653,7 @@ function makeFunnel(funnel, user, profile, scd, firstEventTime, config) {
 	const { superProps, groupKeys } = config;
 	const { name, email } = profile;
 
+	//choose the properties for this funnel
 	const chosenFunnelProps = { ...props, ...superProps };
 	for (const key in props) {
 		try {
@@ -563,7 +685,7 @@ function makeFunnel(funnel, user, profile, scd, firstEventTime, config) {
 		.reduce((acc, step) => {
 			if (!requireRepeats) {
 				if (acc.find(e => e.event === step.event)) {
-					if (config.chance.bool({ likelihood: 50 })) {
+					if (chance.bool({ likelihood: 50 })) {
 						conversionRate = Math.floor(conversionRate * 1.25); //increase conversion rate
 						acc.push(step);
 					}
@@ -583,7 +705,7 @@ function makeFunnel(funnel, user, profile, scd, firstEventTime, config) {
 			return acc;
 		}, []);
 
-	let doesUserConvert = config.chance.bool({ likelihood: conversionRate });
+	let doesUserConvert = chance.bool({ likelihood: conversionRate });
 	let numStepsUserWillTake = sequence.length;
 	if (!doesUserConvert) numStepsUserWillTake = u.integer(1, sequence.length - 1);
 	const funnelTotalRelativeTimeInHours = timeToConvert / numStepsUserWillTake;
@@ -677,49 +799,6 @@ function makeFunnel(funnel, user, profile, scd, firstEventTime, config) {
 }
 
 
-function inferFunnels(events) {
-	const createdFunnels = [];
-	const firstEvents = events.filter((e) => e.isFirstEvent).map((e) => e.event);
-	const usageEvents = events.filter((e) => !e.isFirstEvent).map((e) => e.event);
-	const numFunnelsToCreate = Math.ceil(usageEvents.length);
-	/** @type {Funnel} */
-	const funnelTemplate = {
-		sequence: [],
-		conversionRate: 50,
-		order: 'sequential',
-		requireRepeats: false,
-		props: {},
-		timeToConvert: 1,
-		isFirstFunnel: false,
-		weight: 1
-	};
-	if (firstEvents.length) {
-		for (const event of firstEvents) {
-			createdFunnels.push({ ...clone(funnelTemplate), sequence: [event], isFirstFunnel: true, conversionRate: 100 });
-		}
-	}
-
-	//at least one funnel with all usage events
-	createdFunnels.push({ ...clone(funnelTemplate), sequence: usageEvents });
-
-	//for the rest, make random funnels
-	followUpFunnels: for (let i = 1; i < numFunnelsToCreate; i++) {
-		/** @type {Funnel} */
-		const funnel = { ...clone(funnelTemplate) };
-		funnel.conversionRate = u.integer(25, 75);
-		funnel.timeToConvert = u.integer(1, 10);
-		funnel.weight = u.integer(1, 10);
-		const sequence = u.shuffleArray(usageEvents).slice(0, u.integer(2, usageEvents.length));
-		funnel.sequence = sequence;
-		funnel.order = 'random';
-		createdFunnels.push(funnel);
-	}
-
-	return createdFunnels;
-
-}
-
-
 function makeProfile(props, defaults) {
 	//build the spec
 	const profile = {
@@ -742,7 +821,7 @@ function makeProfile(props, defaults) {
 }
 
 /**
- * @param  {import('./types.d.ts').ValueValid} prop
+ * @param  {import('../types').ValueValid} prop
  * @param  {string} scdKey
  * @param  {string} distinct_id
  * @param  {number} mutations
@@ -770,8 +849,59 @@ function makeSCD(prop, scdKey, distinct_id, mutations, created) {
 	return scdEntries;
 }
 
+//todo
+function makeAdSpend(day) {
+	const chance = u.getChance();
+	const adSpendEvents = [];
+	for (const network of CAMPAIGNS) {
+		const campaigns = network.utm_campaign;
+		loopCampaigns: for (const campaign of campaigns) {
+			if (campaign === "$organic") continue loopCampaigns;
+
+			const CAC = u.integer(42, 420); //todo: get the # of users created in this day from eventData
+			// Randomly generating cost
+			const cost = chance.floating({ min: 10, max: 250, fixed: 2 });
+
+			// Ensuring realistic CPC and CTR
+			const avgCPC = chance.floating({ min: 0.33, max: 2.00, fixed: 4 });
+			const avgCTR = chance.floating({ min: 0.05, max: 0.25, fixed: 4 });
+
+			// Deriving impressions from cost and avg CPC
+			const clicks = Math.floor(cost / avgCPC);
+			const impressions = Math.floor(clicks / avgCTR);
+			const views = Math.floor(impressions * avgCTR);
+
+			//tags 
+			const utm_medium = u.choose(u.pickAWinner(network.utm_medium)());
+			const utm_content = u.choose(u.pickAWinner(network.utm_content)());
+			const utm_term = u.choose(u.pickAWinner(network.utm_term)());
+			//each of these is a campaign
+			const adSpendEvent = {
+				event: "Ad Data",
+				time: day,
+				source: 'dm4',
+				utm_campaign: campaign,
+				campaign_id: md5(network.utm_source[0] + '-' + campaign),
+				network: network.utm_source[0].toUpperCase(),
+				distinct_id: network.utm_source[0].toUpperCase(),
+				utm_source: network.utm_source[0],
+				utm_medium,
+				utm_content,
+				utm_term,
+
+				clicks,
+				views,
+				impressions,
+				cost,
+				date: dayjs(day).format("YYYY-MM-DD"),
+			};
+			adSpendEvents.push(adSpendEvent);
+		}
 
 
+	}
+	return adSpendEvents;
+}
 
 
 
@@ -782,7 +912,7 @@ function makeSCD(prop, scdKey, distinct_id, mutations, created) {
 // this is for CLI
 if (require.main === module) {
 	isCLI = true;
-	const args = cliParams();
+	const args = getCliParams();
 	// @ts-ignore
 	let { token, seed, format, numDays, numUsers, numEvents, region, writeToDisk, complex = false, sessionIds, anonIds } = args;
 	// @ts-ignore
@@ -799,12 +929,12 @@ if (require.main === module) {
 			console.log(`... using default COMPLEX configuration [everything] ...\n`);
 			console.log(`... for more simple data, don't use the --complex flag ...\n`);
 			console.log(`... or specify your own js config file (see docs or --help) ...\n`);
-			config = require(path.resolve(__dirname, "./schemas/complex.js"));
+			config = require(path.resolve(__dirname, "../schemas/complex.js"));
 		}
 		else {
 			console.log(`... using default SIMPLE configuration [events + users] ...\n`);
 			console.log(`... for more complex data, use the --complex flag ...\n`);
-			config = require(path.resolve(__dirname, "./schemas/simple.js"));
+			config = require(path.resolve(__dirname, "../schemas/simple.js"));
 		}
 	}