make-mp-data 1.4.2 → 1.4.4

package/{index.js → core/index.js}

@@ -6,18 +6,18 @@ by AK
 ak@mixpanel.com
 */
 
-//todo: ads-data
-//todo: cart analysis
 //todo: churn ... is churnFunnel, possible to return, etc
-//todo: fixedTimeFunnel? if set this funnel will occur for all users at the same time ['cart charged', 'charge complete']
-
-/** @typedef {import('./types.d.ts').Config} Config */
-/** @typedef {import('./types.d.ts').EventConfig} EventConfig */
-/** @typedef {import('./types.d.ts').Funnel} Funnel */
-/** @typedef {import('./types.d.ts').Person} Person */
-/** @typedef {import('./types.d.ts').SCDTableRow} SCDTableRow */
-/** @typedef {import('./types.d.ts').UserProfile} UserProfile */
-/** @typedef {import('./types.d.ts').EventSpec} EventSpec */
+//todo: fixedTimeFunnel? if set this funnel will occur for all users at the same time ['cards charged', 'charge complete']
+//todo: send SCD data to mixpanel
+//todo: send and map lookup tables to mixpanel
+
+/** @typedef {import('../types').Config} Config */
+/** @typedef {import('../types').EventConfig} EventConfig */
+/** @typedef {import('../types').Funnel} Funnel */
+/** @typedef {import('../types').Person} Person */
+/** @typedef {import('../types').SCDTableRow} SCDTableRow */
+/** @typedef {import('../types').UserProfile} UserProfile */
+/** @typedef {import('../types').EventSpec} EventSpec */
 
 const dayjs = require("dayjs");
 const utc = require("dayjs/plugin/utc");
@@ -29,10 +29,11 @@ const os = require("os");
 const path = require("path");
 const { comma, bytesHuman, makeName, md5, clone, tracker, uid } = require("ak-tools");
 const { generateLineChart } = require('./chart.js');
-const { version } = require('./package.json');
+const { version } = require('../package.json');
 const mp = require("mixpanel-import");
 const metrics = tracker("make-mp-data", "db99eb8f67ae50949a13c27cacf57d41", os.userInfo().username);
 
+
 const u = require("./utils.js");
 const getCliParams = require("./cli.js");
 const { campaigns, devices, locations } = require('./defaults.js');
@@ -41,7 +42,7 @@ let VERBOSE = false;
 let isCLI = false;
 /** @type {Config} */
 let CONFIG;
-
+let CAMPAIGNS;
 let DEFAULTS;
 require('dotenv').config();
 
@@ -156,6 +157,7 @@ async function main(config) {
 	global.MP_SIMULATION_CONFIG = config;
 	CONFIG = config;
 	VERBOSE = verbose;
+	CAMPAIGNS = campaigns;
 	DEFAULTS = {
 		locations: u.pickAWinner(locations, 0),
 		iOSDevices: u.pickAWinner(devices.iosDevices, 0),
@@ -177,6 +179,7 @@ async function main(config) {
 	//setup all the data structures we will push into
 	const eventData = u.enrichArray([], { hook, type: "event", config });
 	const userProfilesData = u.enrichArray([], { hook, type: "user", config });
+	const adSpendData = u.enrichArray([], { hook, type: "ad-spend", config });
 	const scdTableKeys = Object.keys(scdProps);
 	const scdTableData = [];
 	for (const [index, key] of scdTableKeys.entries()) {
@@ -196,14 +199,14 @@ async function main(config) {
 	//user loop
 	log(`---------------SIMULATION----------------`, "\n\n");
 	loopUsers: for (let i = 1; i < numUsers + 1; i++) {
-		u.progress("users", i);
+		u.progress([["users", i], ["events", eventData.length]]);
 		const userId = chance.guid();
 		const user = u.person(userId, numDays, isAnonymous);
 		const { distinct_id, created, anonymousIds, sessionIds } = user;
 		let numEventsPreformed = 0;
 
 		if (hasLocation) {
-			const location = u.choose(DEFAULTS.locations().map(l => { delete l.country; return l; }));
+			const location = u.choose(clone(DEFAULTS.locations()).map(l => { delete l.country; return l; }));
 			for (const key in location) {
 				user[key] = location[key];
 			}
@@ -269,6 +272,17 @@ async function main(config) {
 		// end individual user loop
 	}
 
+	if (hasAdSpend) {
+		const days = u.datesBetween(epochStart, epochEnd);
+		for (const day of days) {
+			const dailySpendData = makeAdSpend(day);
+			for (const spendEvent of dailySpendData) {
+				adSpendData.hookPush(spendEvent);
+			}
+		}
+
+	}
+
 	//flatten SCD tables
 	scdTableData.forEach((table, index) => scdTableData[index] = table.flat());
 
@@ -280,7 +294,7 @@ async function main(config) {
 		const groupCardinality = groupPair[1];
 		const groupProfiles = [];
 		for (let i = 1; i < groupCardinality + 1; i++) {
-			u.progress("groups", i);
+			u.progress([["groups", i]]);
 			const group = {
 				[groupKey]: i,
 				...makeProfile(groupProps[groupKey])
@@ -297,7 +311,7 @@ async function main(config) {
 		const { key, entries, attributes } = lookupTable;
 		const data = [];
 		for (let i = 1; i < entries + 1; i++) {
-			u.progress("lookups", i);
+			u.progress([["lookups", i]]);
 			const item = {
 				[key]: i,
 				...makeProfile(attributes),
@@ -352,11 +366,12 @@ async function main(config) {
 		}
 	}
 
-	const { eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, mirrorFiles, folder } =
+	const { eventFiles, userFiles, scdFiles, groupFiles, lookupFiles, mirrorFiles, folder, adSpendFiles } =
 		u.buildFileNames(config);
 	const pairs = [
 		[eventFiles, [eventData]],
 		[userFiles, [userProfilesData]],
+		[adSpendFiles, [adSpendData]],
 		[scdFiles, scdTableData],
 		[groupFiles, groupProfilesData],
 		[lookupFiles, lookupTableData],
@@ -431,7 +446,7 @@ async function main(config) {
 
 		if (eventData) {
 			log(`importing events to mixpanel...\n`);
-			const imported = await mp(creds, eventData, {
+			const imported = await mp(creds, clone(eventData), {
 				recordType: "event",
 				...commonOpts,
 			});
@@ -440,19 +455,28 @@ async function main(config) {
 		}
 		if (userProfilesData) {
 			log(`importing user profiles to mixpanel...\n`);
-			const imported = await mp(creds, userProfilesData, {
+			const imported = await mp(creds, clone(userProfilesData), {
 				recordType: "user",
 				...commonOpts,
 			});
 			log(`\tsent ${comma(imported.success)} user profiles\n`);
 			importResults.users = imported;
 		}
+		if (adSpendData) {
+			log(`importing ad spend data to mixpanel...\n`);
+			const imported = await mp(creds, clone(adSpendData), {
+				recordType: "event",
+				...commonOpts,
+			});
+			log(`\tsent ${comma(imported.success)} ad spend events\n`);
+			importResults.adSpend = imported;
+		}
 		if (groupProfilesData) {
 			for (const groupProfiles of groupProfilesData) {
 				const groupKey = groupProfiles.key;
 				const data = groupProfiles.data;
 				log(`importing ${groupKey} profiles to mixpanel...\n`);
-				const imported = await mp({ token, groupKey }, data, {
+				const imported = await mp({ token, groupKey }, clone(data), {
 					recordType: "group",
 					...commonOpts,
 
@@ -475,6 +499,7 @@ async function main(config) {
 		groupProfilesData,
 		lookupTableData,
 		mirrorEventData,
+		adSpendData
 	};
 }
 
@@ -506,7 +531,7 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, chosenEv
 
 	let defaultProps = {};
 	let devicePool = [];
-	if (hasLocation) defaultProps.location = DEFAULTS.locations().map(l => { delete l.country_code; return l; });
+	if (hasLocation) defaultProps.location = clone(DEFAULTS.locations()).map(l => { delete l.country_code; return l; });
 	if (hasBrowser) defaultProps.browser = DEFAULTS.browsers();
 	if (hasAndroidDevices) devicePool.push(DEFAULTS.androidDevices());
 	if (hasIOSDevices) devicePool.push(DEFAULTS.iOSDevices());
@@ -563,11 +588,20 @@ function makeEvent(distinct_id, anonymousIds, sessionIds, earliestTime, chosenEv
 
 			else if (typeof choice === "object") {
 				for (const subKey in choice) {
-					if (Array.isArray(choice[subKey])) {
+					if (typeof choice[subKey] === "string") {
+						if (!eventTemplate[subKey]) eventTemplate[subKey] = choice[subKey];
+					}
+					else if (Array.isArray(choice[subKey])) {
 						const subChoice = u.choose(choice[subKey]);
 						if (!eventTemplate[subKey]) eventTemplate[subKey] = subChoice;
 					}
-					if (!eventTemplate[subKey]) eventTemplate[subKey] = choice[subKey];
+
+					else if (typeof choice[subKey] === "object") {
+						for (const subSubKey in choice[subKey]) {
+							if (!eventTemplate[subSubKey]) eventTemplate[subSubKey] = choice[subKey][subSubKey];
+						}
+					}
+
 				}
 			}
 
@@ -787,7 +821,7 @@ function makeProfile(props, defaults) {
 }
 
 /**
- * @param  {import('./types.d.ts').ValueValid} prop
+ * @param  {import('../types').ValueValid} prop
  * @param  {string} scdKey
  * @param  {string} distinct_id
  * @param  {number} mutations
@@ -816,68 +850,58 @@ function makeSCD(prop, scdKey, distinct_id, mutations, created) {
 }
 
 //todo
-// function makeAdSpend(spec, usersCreated, specialDays = []) {
-//     const networks = [
-//         ...new Set(
-//             attribution
-//                 .filter(camp => !["$organic", "$referral"].includes(camp.utm_source.join()))
-//                 .map(network => network.utm_source)
-//                 .flat()
-//         )
-//     ];
-
-//     const campaigns = attribution.slice().pop().utm_campaign;
-// 	const campaignParams = attribution.slice().pop();
-//     const { startDate, endDate } = spec;
-//     const days = getDeltaDays(startDate, endDate);
-//     const numDays = days.length;
-//     const data = [];
-//     for (const network of networks) {
-//         for (const day of days) {
-//             //cost per acquisition ~ $10-50
-//             let CAC = chance.integer({ min: 10, max: 50 });
-//             // daily spend is total users / total days * CAC / num networks
-//             let cost = Math.floor(((usersCreated / numDays) * CAC) / networks.length);
-//             // CTR ~ 0.5-10%
-//             let clicks = Math.floor(cost / chance.floating({ min: 0.5, max: 10 }));
-//             //boost CTR on "special days" ~ 15-50%
-//             if (day in specialDays) clicks *= Math.floor(clicks * chance.floating({ min: 1.15, max: 1.5 }));
-//             //impressions ~100-500 * cost
-//             let impressions = Math.floor(cost * chance.integer({ min: 100, max: 500 }));
-//             // views ~ 25-80% of impressions
-//             let views = Math.floor(impressions * chance.floating({ min: 0.25, max: 0.8 }));
-
-//             let campaign_name = chance.pickone(campaigns);
-//             let campaign_id = md5(campaign_name);
-//             data.push({
-//                 event: "ad data",
-//                 properties: {
-//                     cost,
-//                     clicks,
-//                     impressions,
-//                     campaign_name,
-//                     campaign_id,
-//                     network,
-//                     views,
-
-//                     //addendum
-//                     utm_campaign: campaign_name,
-// 					utm_source: network,
-// 					utm_medium: campaignParams.utm_medium,                    
-// 					utm_content: campaignParams.utm_content,
-// 					utm_term: campaignParams.utm_term,
-
-//                     source: network,
-//                     $insert_id: campaign_id,
-//                     time: dayjs.utc(day).hour(12).unix(),
-//                     $source: "DM3",
-//                     $mp_lib: "DM3"
-//                 }
-//             });
-//         }
-//     }
-//     return data;
-// }
+function makeAdSpend(day) {
+	const chance = u.getChance();
+	const adSpendEvents = [];
+	for (const network of CAMPAIGNS) {
+		const campaigns = network.utm_campaign;
+		loopCampaigns: for (const campaign of campaigns) {
+			if (campaign === "$organic") continue loopCampaigns;
+
+			const CAC = u.integer(42, 420); //todo: get the # of users created in this day from eventData
+			// Randomly generating cost
+			const cost = chance.floating({ min: 10, max: 250, fixed: 2 });
+
+			// Ensuring realistic CPC and CTR
+			const avgCPC = chance.floating({ min: 0.33, max: 2.00, fixed: 4 });
+			const avgCTR = chance.floating({ min: 0.05, max: 0.25, fixed: 4 });
+
+			// Deriving impressions from cost and avg CPC
+			const clicks = Math.floor(cost / avgCPC);
+			const impressions = Math.floor(clicks / avgCTR);
+			const views = Math.floor(impressions * avgCTR);
+
+			//tags 
+			const utm_medium = u.choose(u.pickAWinner(network.utm_medium)());
+			const utm_content = u.choose(u.pickAWinner(network.utm_content)());
+			const utm_term = u.choose(u.pickAWinner(network.utm_term)());
+			//each of these is a campaign
+			const adSpendEvent = {
+				event: "Ad Data",
+				time: day,
+				source: 'dm4',
+				utm_campaign: campaign,
+				campaign_id: md5(network.utm_source[0] + '-' + campaign),
+				network: network.utm_source[0].toUpperCase(),
+				distinct_id: network.utm_source[0].toUpperCase(),
+				utm_source: network.utm_source[0],
+				utm_medium,
+				utm_content,
+				utm_term,
+
+				clicks,
+				views,
+				impressions,
+				cost,
+				date: dayjs(day).format("YYYY-MM-DD"),
+			};
+			adSpendEvents.push(adSpendEvent);
+		}
+
+
+	}
+	return adSpendEvents;
+}
 
 
 
@@ -905,12 +929,12 @@ if (require.main === module) {
 			console.log(`... using default COMPLEX configuration [everything] ...\n`);
 			console.log(`... for more simple data, don't use the --complex flag ...\n`);
 			console.log(`... or specify your own js config file (see docs or --help) ...\n`);
-			config = require(path.resolve(__dirname, "./schemas/complex.js"));
+			config = require(path.resolve(__dirname, "../schemas/complex.js"));
 		}
 		else {
 			console.log(`... using default SIMPLE configuration [events + users] ...\n`);
 			console.log(`... for more complex data, use the --complex flag ...\n`);
-			config = require(path.resolve(__dirname, "./schemas/simple.js"));
+			config = require(path.resolve(__dirname, "../schemas/simple.js"));
 		}
 	}
 

package/{utils.js → core/utils.js}

@@ -7,20 +7,24 @@ const dayjs = require('dayjs');
 const utc = require('dayjs/plugin/utc');
 const path = require('path');
 const { mkdir } = require('ak-tools');
+const { existsSync } = require('fs');
 dayjs.extend(utc);
 require('dotenv').config();
+const { domainSuffix, domainPrefix } = require('./defaults');
 
-/** @typedef {import('./types').Config} Config */
-/** @typedef {import('./types').EventConfig} EventConfig */
-/** @typedef {import('./types').ValueValid} ValueValid */
-/** @typedef {import('./types').EnrichedArray} EnrichArray */
-/** @typedef {import('./types').EnrichArrayOptions} EnrichArrayOptions */
-/** @typedef {import('./types').Person} Person */
-/** @typedef {import('./types').Funnel} Funnel */
+/** @typedef {import('../types').Config} Config */
+/** @typedef {import('../types').EventConfig} EventConfig */
+/** @typedef {import('../types').ValueValid} ValueValid */
+/** @typedef {import('../types').EnrichedArray} EnrichArray */
+/** @typedef {import('../types').EnrichArrayOptions} EnrichArrayOptions */
+/** @typedef {import('../types').Person} Person */
+/** @typedef {import('../types').Funnel} Funnel */
 
 let globalChance;
 let chanceInitialized = false;
 
+
+
 /*
 ----
 RNG
@@ -127,6 +131,21 @@ function dates(inTheLast = 30, numPairs = 5, format = 'YYYY-MM-DD') {
 	return pairs;
 };
 
+function datesBetween(start, end) {
+	const result = [];
+	if (typeof start === 'number') start = dayjs.unix(start).utc();
+	if (typeof start !== 'number') start = dayjs(start).utc();
+	if (typeof end === 'number') end = dayjs.unix(end).utc();
+	if (typeof end !== 'number') end = dayjs(end).utc();
+	const diff = end.diff(start, 'day');
+	for (let i = 0; i < diff; i++) {
+		const day = start.add(i, 'day').startOf('day').add(12, 'hour');
+		result.push(day.toISOString());
+	}
+
+	return result;
+}
+
 /**
  * returns a random date
  * @param  {any} start
@@ -223,77 +242,66 @@ function integer(min = 1, max = 100) {
 };
 
 
+/**
+ * Creates a function that generates a weighted list of items
+ * with a higher likelihood of picking a specified index and clear second and third place indices.
+ * 
+ * @param {Array} items - The list of items to pick from.
+ * @param {number} [mostChosenIndex] - The index of the item to be most favored.
+ * @returns {function} - A function that returns a weighted list of items.
+ */
 function pickAWinner(items, mostChosenIndex) {
 	const chance = getChance();
-	if (!mostChosenIndex) mostChosenIndex = integer(0, items.length);
-	if (mostChosenIndex > items.length) mostChosenIndex = items.length;
+
+	// Ensure mostChosenIndex is within the bounds of the items array
+	if (!items) return () => { return ""; };
+	if (!items.length) return () => { return ""; };
+	if (!mostChosenIndex) mostChosenIndex = chance.integer({ min: 0, max: items.length - 1 });
+	if (mostChosenIndex >= items.length) mostChosenIndex = items.length - 1;
+
+	// Calculate second and third most chosen indices
+	const secondMostChosenIndex = (mostChosenIndex + 1) % items.length;
+	const thirdMostChosenIndex = (mostChosenIndex + 2) % items.length;
+
+	// Return a function that generates a weighted list
 	return function () {
 		const weighted = [];
 		for (let i = 0; i < 10; i++) {
-			if (chance.bool({ likelihood: integer(10, 35) })) {
+			const rand = chance.d10(); // Random number between 1 and 10
+
+			// 35% chance to favor the most chosen index
+			if (chance.bool({ likelihood: 35 })) {
+				// 50% chance to slightly alter the index
 				if (chance.bool({ likelihood: 50 })) {
 					weighted.push(items[mostChosenIndex]);
-				}
-				else {
-					const rand = chance.d10();
+				} else {
 					const addOrSubtract = chance.bool({ likelihood: 50 }) ? -rand : rand;
 					let newIndex = mostChosenIndex + addOrSubtract;
+
+					// Ensure newIndex is within bounds
 					if (newIndex < 0) newIndex = 0;
-					if (newIndex > items.length) newIndex = items.length;
+					if (newIndex >= items.length) newIndex = items.length - 1;
 					weighted.push(items[newIndex]);
 				}
 			}
+			// 25% chance to favor the second most chosen index
+			else if (chance.bool({ likelihood: 25 })) {
+				weighted.push(items[secondMostChosenIndex]);
+			}
+			// 15% chance to favor the third most chosen index
+			else if (chance.bool({ likelihood: 15 })) {
+				weighted.push(items[thirdMostChosenIndex]);
+			}
+			// Otherwise, pick a random item from the list
 			else {
 				weighted.push(chance.pickone(items));
 			}
 		}
 		return weighted;
-
 	};
 }
 
 
-function inferFunnels(events) {
-	const createdFunnels = [];
-	const firstEvents = events.filter((e) => e.isFirstEvent).map((e) => e.event);
-	const usageEvents = events.filter((e) => !e.isFirstEvent).map((e) => e.event);
-	const numFunnelsToCreate = Math.ceil(usageEvents.length);
-	/** @type {Funnel} */
-	const funnelTemplate = {
-		sequence: [],
-		conversionRate: 50,
-		order: 'sequential',
-		requireRepeats: false,
-		props: {},
-		timeToConvert: 1,
-		isFirstFunnel: false,
-		weight: 1
-	};
-	if (firstEvents.length) {
-		for (const event of firstEvents) {
-			createdFunnels.push({ ...clone(funnelTemplate), sequence: [event], isFirstFunnel: true, conversionRate: 100 });
-		}
-	}
-
-	//at least one funnel with all usage events
-	createdFunnels.push({ ...clone(funnelTemplate), sequence: usageEvents });
-
-	//for the rest, make random funnels
-	followUpFunnels: for (let i = 1; i < numFunnelsToCreate; i++) {
-		/** @type {Funnel} */
-		const funnel = { ...clone(funnelTemplate) };
-		funnel.conversionRate = integer(25, 75);
-		funnel.timeToConvert = integer(1, 10);
-		funnel.weight = integer(1, 10);
-		const sequence = shuffleArray(usageEvents).slice(0, integer(2, usageEvents.length));
-		funnel.sequence = sequence;
-		funnel.order = 'random';
-		createdFunnels.push(funnel);
-	}
-
-	return createdFunnels;
-
-}
 
 /*
 ----
@@ -360,6 +368,53 @@ function range(a, b, step = 1) {
 };
 
 
+/**
+ * create funnels out of random events
+ * @param {EventConfig[]} events
+ */
+function inferFunnels(events) {
+	const createdFunnels = [];
+	const firstEvents = events.filter((e) => e.isFirstEvent).map((e) => e.event);
+	const usageEvents = events.filter((e) => !e.isFirstEvent).map((e) => e.event);
+	const numFunnelsToCreate = Math.ceil(usageEvents.length);
+	/** @type {Funnel} */
+	const funnelTemplate = {
+		sequence: [],
+		conversionRate: 50,
+		order: 'sequential',
+		requireRepeats: false,
+		props: {},
+		timeToConvert: 1,
+		isFirstFunnel: false,
+		weight: 1
+	};
+	if (firstEvents.length) {
+		for (const event of firstEvents) {
+			createdFunnels.push({ ...clone(funnelTemplate), sequence: [event], isFirstFunnel: true, conversionRate: 100 });
+		}
+	}
+
+	//at least one funnel with all usage events
+	createdFunnels.push({ ...clone(funnelTemplate), sequence: usageEvents });
+
+	//for the rest, make random funnels
+	followUpFunnels: for (let i = 1; i < numFunnelsToCreate; i++) {
+		/** @type {Funnel} */
+		const funnel = { ...clone(funnelTemplate) };
+		funnel.conversionRate = integer(25, 75);
+		funnel.timeToConvert = integer(1, 10);
+		funnel.weight = integer(1, 10);
+		const sequence = shuffleArray(usageEvents).slice(0, integer(2, usageEvents.length));
+		funnel.sequence = sequence;
+		funnel.order = 'random';
+		createdFunnels.push(funnel);
+	}
+
+	return createdFunnels;
+
+}
+
+
 /*
 ----
 STREAMERS
@@ -425,8 +480,11 @@ function weighFunnels(acc, funnel) {
 /**
  * a utility function to generate a range of numbers within a given skew
  * Skew = 0.5: The values are more concentrated towards the extremes (both ends of the range) with a noticeable dip in the middle. The distribution appears more "U" shaped. Larger sizes result in smoother distributions but maintain the overall shape.
+ * 
  * Skew = 1: This represents the default normal distribution without skew. The values are normally distributed around the mean. Larger sizes create a clearer bell-shaped curve.
+ * 
  * Skew = 2: The values are more concentrated towards the mean, with a steeper drop-off towards the extremes. The distribution appears more peaked, resembling a "sharper" bell curve. Larger sizes enhance the clarity of this peaked distribution.
+ * 
  * Size represents the size of the pool to choose from; Larger sizes result in smoother distributions but maintain the overall shape.
  * @param  {number} min
  * @param  {number} max
@@ -517,6 +575,7 @@ function shuffleOutside(array) {
 }
 
 /**
+ * given a funnel, shuffle the events in the sequence with random events
  * @param  {EventConfig[]} funnel
  * @param  {EventConfig[]} possibles
  */
@@ -662,13 +721,18 @@ function buildFileNames(config) {
 	// const current = dayjs.utc().format("MM-DD-HH");
 	const simName = config.simulationName;
 	let writeDir = "./";
-	if (config.writeToDisk) writeDir = mkdir("./data");
+	if (config.writeToDisk) {
+		const dataFolder = path.resolve("./data");
+		if (existsSync(dataFolder)) writeDir = dataFolder;
+		else writeDir = path.resolve("./");
+	}
 	if (typeof writeDir !== "string") throw new Error("writeDir must be a string");
 	if (typeof simName !== "string") throw new Error("simName must be a string");
 
 	const writePaths = {
 		eventFiles: [path.join(writeDir, `${simName}-EVENTS.${extension}`)],
 		userFiles: [path.join(writeDir, `${simName}-USERS.${extension}`)],
+		adSpendFiles: [path.join(writeDir, `${simName}-AD-SPEND.${extension}`)],
 		scdFiles: [],
 		mirrorFiles: [],
 		groupFiles: [],
@@ -713,11 +777,19 @@ function buildFileNames(config) {
 	return writePaths;
 }
 
-
-function progress(thing, p) {
+/**
+ * @param  {[string, number][]} arrayOfArrays
+ */
+function progress(arrayOfArrays) {
 	// @ts-ignore
 	readline.cursorTo(process.stdout, 0);
-	process.stdout.write(`${thing} processed ... ${comma(p)}`);
+	let message = "";
+	for (const status of arrayOfArrays) {
+		const [thing, p] = status;
+		message += `${thing} processed: ${comma(p)}\t\t`;
+	}
+
+	process.stdout.write(message);
 };
 
 
@@ -823,19 +895,16 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 function person(userId, bornDaysAgo = 30, isAnonymous = false) {
 	const chance = getChance();
 	//names and photos
-	const l = chance.letter;
+	const l = chance.letter.bind(chance);
 	let gender = chance.pickone(['male', 'female']);
 	if (!gender) gender = "female";
 	// @ts-ignore
 	let first = chance.first({ gender });
 	let last = chance.last();
 	let name = `${first} ${last}`;
-	let email = `${first[0]}.${last}@${chance.domain()}.com`;
+	let email = `${first[0]}.${last}@${choose(domainPrefix)}.${choose(domainSuffix)}`;
 	let avatarPrefix = `https://randomuser.me/api/portraits`;
-	let randomAvatarNumber = chance.integer({
-		min: 1,
-		max: 99
-	});
+	let randomAvatarNumber = integer(1, 99);
 	let avPath = gender === 'male' ? `/men/${randomAvatarNumber}.jpg` : `/women/${randomAvatarNumber}.jpg`;
 	let avatar = avatarPrefix + avPath;
 	let created = dayjs.unix(global.NOW).subtract(bornDaysAgo, 'day').format('YYYY-MM-DD');
@@ -855,8 +924,8 @@ function person(userId, bornDaysAgo = 30, isAnonymous = false) {
 
 	if (isAnonymous) {
 		user.name = "Anonymous User";
-		user.email = `${l()}${l()}****${l()}${l()}@${l()}**${l()}*.com`;		
-		delete user.avatar;		
+		user.email = l() + l() + `*`.repeat(integer(3, 6)) + l() + `@` + l() + `*`.repeat(integer(3, 6)) + l() + `.` + choose(domainSuffix);
+		delete user.avatar;
 
 	}
 
@@ -980,5 +1049,6 @@ module.exports = {
 	buildFileNames,
 	streamJSON,
 	streamCSV,
-	inferFunnels
+	inferFunnels,
+	datesBetween
 };