make-mp-data 1.5.1 → 1.5.3

package/index.js

@@ -10,6 +10,8 @@ ak@mixpanel.com
 //todo: regular interval events (like 'card charged')
 //todo: SCDs send to mixpanel
 //todo: decent 'new dungeon' workflow
+//todo: validation that funnel events exist
+//todo: ability to catch events not in funnels and make them random...
 
 
 //TIME
@@ -20,16 +22,17 @@ const FIXED_NOW = dayjs('2024-02-02').unix();
 global.FIXED_NOW = FIXED_NOW;
 // ^ this creates a FIXED POINT in time; we will shift it later
 let FIXED_BEGIN = dayjs.unix(FIXED_NOW).subtract(90, 'd').unix();
+global.FIXED_BEGIN = FIXED_BEGIN;
 const actualNow = dayjs();
 const timeShift = actualNow.diff(dayjs.unix(FIXED_NOW), "seconds");
 const daysShift = actualNow.diff(dayjs.unix(FIXED_NOW), "days");
 
 // UTILS
-const { existsSync } = require("fs");
+const { existsSync, writeFileSync } = require("fs");
 const pLimit = require('p-limit');
 const os = require("os");
 const path = require("path");
-const { comma, bytesHuman, makeName, md5, clone, tracker, uid, timer, ls, rm } = require("ak-tools");
+const { comma, bytesHuman, makeName, md5, clone, tracker, uid, timer, ls, rm, touch, load, sLog } = require("ak-tools");
 const jobTimer = timer('job');
 const { generateLineChart } = require('./components/chart.js');
 const { version } = require('./package.json');
@@ -37,6 +40,11 @@ const mp = require("mixpanel-import");
 const u = require("./components/utils.js");
 const getCliParams = require("./components/cli.js");
 const metrics = tracker("make-mp-data", "db99eb8f67ae50949a13c27cacf57d41", os.userInfo().username);
+const t = require('ak-tools');
+
+
+//CLOUD
+const functions = require('@google-cloud/functions-framework');
 
 // DEFAULTS
 const { campaigns, devices, locations } = require('./components/defaults.js');
@@ -48,13 +56,17 @@ let STORAGE;
 let CONFIG;
 require('dotenv').config();
 
+const { NODE_ENV = "unknown" } = process.env;
+
+
+
 
 // RUN STATE
 let VERBOSE = false;
 let isCLI = false;
 // if we are running in batch mode, we MUST write to disk before we can send to mixpanel
 let isBATCH_MODE = false;
-let BATCH_SIZE = 500_000;
+let BATCH_SIZE = 1_000_000;
 
 //todo: these should be moved into the hookedArrays
 let operations = 0;
@@ -95,7 +107,7 @@ async function main(config) {
 
 	//TRACKING
 	const runId = uid(42);
-	const { events, superProps, userProps, scdProps, groupKeys, groupProps, lookupTables, soup, hook, mirrorProps, ...trackingParams } = config;
+	const { events, superProps, userProps, scdProps, groupKeys, groupProps, lookupTables, soup, hook, mirrorProps, token: source_proj_token, ...trackingParams } = config;
 	let { funnels } = config;
 	trackingParams.runId = runId;
 	trackingParams.version = version;
@@ -106,11 +118,14 @@ async function main(config) {
 	const eventData = await makeHookArray([], { hook, type: "event", config, format, filepath: `${simulationName}-EVENTS` });
 	const userProfilesData = await makeHookArray([], { hook, type: "user", config, format, filepath: `${simulationName}-USERS` });
 	const adSpendData = await makeHookArray([], { hook, type: "ad-spend", config, format, filepath: `${simulationName}-AD-SPEND` });
+	const groupEventData = await makeHookArray([], { hook, type: "group-event", config, format, filepath: `${simulationName}-GROUP-EVENTS` });
 
 	// SCDs, Groups, + Lookups may have multiple tables
 	const scdTableKeys = Object.keys(scdProps);
 	const scdTableData = await Promise.all(scdTableKeys.map(async (key) =>
-		await makeHookArray([], { hook, type: "scd", config, format, scdKey: key, filepath: `${simulationName}-SCD-${key}` })
+		//todo don't assume everything is a string... lol
+		// @ts-ignore
+		await makeHookArray([], { hook, type: "scd", config, format, scdKey: key, entityType: config.scdProps[key].type, dataType: "string", filepath: `${simulationName}-${scdProps[key]?.type || "user"}-SCD-${key}` })
 	));
 	const groupTableKeys = Object.keys(groupKeys);
 	const groupProfilesData = await Promise.all(groupTableKeys.map(async (key, index) => {
@@ -126,7 +141,17 @@ async function main(config) {
 
 	const mirrorEventData = await makeHookArray([], { hook, type: "mirror", config, format, filepath: `${simulationName}-MIRROR` });
 
-	STORAGE = { eventData, userProfilesData, scdTableData, groupProfilesData, lookupTableData, mirrorEventData, adSpendData };
+	STORAGE = {
+		eventData,
+		userProfilesData,
+		scdTableData,
+		groupProfilesData,
+		lookupTableData,
+		mirrorEventData,
+		adSpendData,
+		groupEventData
+
+	};
 
 
 	track('start simulation', trackingParams);
@@ -159,22 +184,72 @@ async function main(config) {
 	log("\n");
 
 	//GROUP PROFILES
+	const groupSCDs = t.objFilter(scdProps, (scd) => scd.type !== 'user');
 	for (const [index, groupPair] of groupKeys.entries()) {
 		const groupKey = groupPair[0];
 		const groupCardinality = groupPair[1];
 		for (let i = 1; i < groupCardinality + 1; i++) {
 			if (VERBOSE) u.progress([["groups", i]]);
-			const props = await makeProfile(groupProps[groupKey]);
+
+			const props = await makeProfile(groupProps[groupKey], { created: () => { return dayjs().subtract(u.integer(0, CONFIG.numDays || 30), 'd').toISOString(); } });
 			const group = {
 				[groupKey]: i,
 				...props,
 			};
 			group["distinct_id"] = i.toString();
 			await groupProfilesData[index].hookPush(group);
+
+			//SCDs
+			const thisGroupSCD = t.objFilter(groupSCDs, (scd) => scd.type === groupKey);
+			const groupSCDKeys = Object.keys(thisGroupSCD);
+			const groupSCD = {};
+			for (const [index, key] of groupSCDKeys.entries()) {
+				const { max = 100 } = groupSCDs[key];
+				const mutations = chance.integer({ min: 2, max });
+				const changes = await makeSCD(scdProps[key], key, i.toString(), mutations, group.created);
+				groupSCD[key] = changes;
+				const scdTable = scdTableData
+					.filter(hookArr => hookArr.scdKey === key);
+
+				await config.hook(changes, 'scd-pre', { profile: group, type: groupKey, scd: { [key]: groupSCDs[key] }, config, allSCDs: groupSCD });
+				await scdTable[0].hookPush(changes, { profile: group, type: groupKey });
+			}
+
+
 		}
 	}
 	log("\n");
 
+	//GROUP EVENTS
+	if (config.groupEvents) {
+		for (const groupEvent of config.groupEvents) {
+			const { frequency, group_key, attribute_to_user, group_size, ...normalEvent } = groupEvent;
+			for (const group_num of Array.from({ length: group_size }, (_, i) => i + 1)) {
+				const groupProfile = groupProfilesData.find(groups => groups.groupKey === group_key).find(group => group[group_key] === group_num);
+				const { created, distinct_id } = groupProfile;
+				normalEvent[group_key] = distinct_id;
+				const random_user_id = chance.pick(eventData.filter(a => a.user_id)).user_id;
+				if (!random_user_id) debugger;
+				const deltaDays = actualNow.diff(dayjs(created), "day");
+				const numIntervals = Math.floor(deltaDays / frequency);
+				const eventsForThisGroup = [];
+				for (let i = 0; i < numIntervals; i++) {
+					const event = await makeEvent(random_user_id, null, normalEvent, [], [], {}, [], false, true);
+					if (!attribute_to_user) delete event.user_id;
+					event[group_key] = distinct_id;
+					event.time = dayjs(created).add(i * frequency, "day").toISOString();
+					delete event.distinct_id;
+					//always skip the first event
+					if (i !== 0) {
+						eventsForThisGroup.push(event);
+					}
+				}
+				await groupEventData.hookPush(eventsForThisGroup, { profile: groupProfile });
+			}
+		}
+	}
+
+
 	//LOOKUP TABLES
 	for (const [index, lookupTable] of lookupTables.entries()) {
 		const { key, entries, attributes } = lookupTable;
@@ -200,14 +275,14 @@ async function main(config) {
 	log(`---------------SIMULATION----------------`, "\n");
 
 	// draw charts
-	const { makeChart } = config;
+	const { makeChart = false } = config;
 	if (makeChart) {
 		const bornEvents = config.events?.filter((e) => e?.isFirstEvent)?.map(e => e.event) || [];
 		const bornFunnels = config.funnels?.filter((f) => f.isFirstFunnel)?.map(f => f.sequence[0]) || [];
 		const bornBehaviors = [...bornEvents, ...bornFunnels];
 		const chart = await generateLineChart(eventData, bornBehaviors, makeChart);
 	}
-	const { writeToDisk, token } = config;
+	const { writeToDisk = true, token } = config;
 	if (!writeToDisk && !token) {
 		jobTimer.stop(false);
 		const { start, end, delta, human } = jobTimer.report(false);
@@ -246,6 +321,7 @@ async function main(config) {
 	jobTimer.stop(false);
 	const { start, end, delta, human } = jobTimer.report(false);
 
+	if (process.env.NODE_ENV === 'dev') debugger;
 	return {
 		...STORAGE,
 		importResults,
@@ -256,6 +332,53 @@ async function main(config) {
 
 
 
+functions.http('entry', async (req, res) => {
+	const reqTimer = timer('request');
+	reqTimer.start();
+	let response = {};
+	let script = req.body || "";
+	let writePath;
+	try {
+		sLog("DM4: start");
+		if (!script) throw new Error("no script");
+
+		// Replace require("../ with require("./
+		script = script.replace(/require\("\.\.\//g, 'require("./');
+		// ^ need to replace this because of the way the script is passed in... this is sketch
+
+		/** @type {Config} */
+		const config = eval(script);
+		sLog("DM4: eval ok");
+
+		const { token } = config;
+		if (!token) throw new Error("no token");
+
+		/** @type {Config} */
+		const optionsYouCantChange = {
+			verbose: false,
+			writeToDisk: false,
+
+		};
+		const result = await main({
+			...config,
+			...optionsYouCantChange,
+		});
+		await rm(writePath);
+		reqTimer.stop(false);
+		const { start, end, delta, human } = jobTimer.report(false);
+		sLog(`DM4: end (${human})`, { ms: delta });
+	}
+	catch (e) {
+		sLog("DM4: error", { error: e.message });
+		response = { error: e.message };
+		res.status(500);
+		await rm(writePath);
+	}
+	finally {
+		res.send(response);
+	}
+});
+
 
 /*
 ------
@@ -275,7 +398,7 @@ MODELS
  * @param  {Boolean} [isFirstEvent]
  * @return {Promise<EventSchema>}
  */
-async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, sessionIds, superProps, groupKeys, isFirstEvent) {
+async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, sessionIds, superProps, groupKeys, isFirstEvent, skipDefaults = false) {
 	operations++;
 	eventCount++;
 	if (!distinct_id) throw new Error("no distinct_id");
@@ -307,11 +430,13 @@ async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, s
 
 	let defaultProps = {};
 	let devicePool = [];
+
 	if (hasLocation) defaultProps.location = DEFAULTS.locationsEvents();
 	if (hasBrowser) defaultProps.browser = DEFAULTS.browsers();
 	if (hasAndroidDevices) devicePool.push(DEFAULTS.androidDevices());
 	if (hasIOSDevices) devicePool.push(DEFAULTS.iOSDevices());
 	if (hasDesktopDevices) devicePool.push(DEFAULTS.desktopDevices());
+
 	// we don't always have campaigns, because of attribution
 	if (hasCampaigns && chance.bool({ likelihood: 25 })) defaultProps.campaigns = DEFAULTS.campaigns();
 	const devices = devicePool.flat();
@@ -319,13 +444,10 @@ async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, s
 
 
 	//event time
-	// if (earliestTime > FIXED_NOW) {
-	// 	earliestTime = dayjs(u.TimeSoup(global.FIXED_BEGIN)).unix();
-	// };
-
-	if (isFirstEvent) eventTemplate.time = dayjs.unix(earliestTime).toISOString();
-	if (!isFirstEvent) eventTemplate.time = u.TimeSoup(earliestTime, FIXED_NOW, peaks, deviation, mean);
-	// eventTemplate.time = u.TimeSoup(earliestTime, FIXED_NOW, peaks, deviation, mean);
+	if (earliestTime) {
+		if (isFirstEvent) eventTemplate.time = dayjs.unix(earliestTime).toISOString();
+		if (!isFirstEvent) eventTemplate.time = u.TimeSoup(earliestTime, FIXED_NOW, peaks, deviation, mean);
+	}
 
 	// anonymous and session ids
 	if (anonymousIds.length) eventTemplate.device_id = chance.pickone(anonymousIds);
@@ -350,39 +472,39 @@ async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, s
 	}
 
 	//iterate through default properties
-	for (const key in defaultProps) {
-		if (Array.isArray(defaultProps[key])) {
-			const choice = u.choose(defaultProps[key]);
-			if (typeof choice === "string") {
-				if (!eventTemplate[key]) eventTemplate[key] = choice;
-			}
-
-			else if (Array.isArray(choice)) {
-				for (const subChoice of choice) {
-					if (!eventTemplate[key]) eventTemplate[key] = subChoice;
+	if (!skipDefaults) {
+		for (const key in defaultProps) {
+			if (Array.isArray(defaultProps[key])) {
+				const choice = u.choose(defaultProps[key]);
+				if (typeof choice === "string") {
+					if (!eventTemplate[key]) eventTemplate[key] = choice;
 				}
-			}
 
-			else if (typeof choice === "object") {
-				for (const subKey in choice) {
-					if (typeof choice[subKey] === "string") {
-						if (!eventTemplate[subKey]) eventTemplate[subKey] = choice[subKey];
-					}
-					else if (Array.isArray(choice[subKey])) {
-						const subChoice = u.choose(choice[subKey]);
-						if (!eventTemplate[subKey]) eventTemplate[subKey] = subChoice;
+				else if (Array.isArray(choice)) {
+					for (const subChoice of choice) {
+						if (!eventTemplate[key]) eventTemplate[key] = subChoice;
 					}
+				}
 
-					else if (typeof choice[subKey] === "object") {
-						for (const subSubKey in choice[subKey]) {
-							if (!eventTemplate[subSubKey]) eventTemplate[subSubKey] = choice[subKey][subSubKey];
+				else if (typeof choice === "object") {
+					for (const subKey in choice) {
+						if (typeof choice[subKey] === "string") {
+							if (!eventTemplate[subKey]) eventTemplate[subKey] = choice[subKey];
+						}
+						else if (Array.isArray(choice[subKey])) {
+							const subChoice = u.choose(choice[subKey]);
+							if (!eventTemplate[subKey]) eventTemplate[subKey] = subChoice;
+						}
+
+						else if (typeof choice[subKey] === "object") {
+							for (const subSubKey in choice[subKey]) {
+								if (!eventTemplate[subSubKey]) eventTemplate[subSubKey] = choice[subKey][subSubKey];
+							}
 						}
-					}
 
+					}
 				}
 			}
-
-
 		}
 	}
 
@@ -401,8 +523,10 @@ async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, s
 	eventTemplate.insert_id = md5(JSON.stringify(eventTemplate));
 
 	// move time forward
-	const timeShifted = dayjs(eventTemplate.time).add(timeShift, "seconds").toISOString();
-	eventTemplate.time = timeShifted;
+	if (earliestTime) {
+		const timeShifted = dayjs(eventTemplate.time).add(timeShift, "seconds").toISOString();
+		eventTemplate.time = timeShifted;
+	}
 
 
 	return eventTemplate;
@@ -492,6 +616,8 @@ async function makeFunnel(funnel, user, firstEventTime, profile, scd, config) {
 			return acc;
 		}, []);
 
+	if (conversionRate > 100) conversionRate = 100;
+	if (conversionRate < 0) conversionRate = 0;
 	let doesUserConvert = chance.bool({ likelihood: conversionRate });
 	let numStepsUserWillTake = sequence.length;
 	if (!doesUserConvert) numStepsUserWillTake = u.integer(1, sequence.length - 1);
@@ -604,12 +730,23 @@ async function makeProfile(props, defaults) {
 		...defaults,
 	};
 
+	for (const key in profile) {
+		try {
+			profile[key] = u.choose(profile[key]);
+		}
+		catch (e) {
+			// never gets here
+			debugger;
+		}
+	}
+
+
 	for (const key in props) {
 		try {
 			profile[key] = u.choose(props[key]);
 		} catch (e) {
 			// never gets here
-			// debugger;
+			debugger;
 		}
 	}
 
@@ -617,42 +754,67 @@ async function makeProfile(props, defaults) {
 }
 
 /**
- * @param  {ValueValid} prop
+ * @param  {SCDProp} scdProp
  * @param  {string} scdKey
  * @param  {string} distinct_id
  * @param  {number} mutations
  * @param  {string} created
  * @return {Promise<SCDSchema[]>}
  */
-async function makeSCD(prop, scdKey, distinct_id, mutations, created) {
-	if (JSON.stringify(prop) === "{}" || JSON.stringify(prop) === "[]") return [];
+async function makeSCD(scdProp, scdKey, distinct_id, mutations, created) {
+	if (Array.isArray(scdProp)) scdProp = { values: scdProp, frequency: 'week', max: 10, timing: 'fuzzy', type: 'user' };
+	const { frequency, max, timing, values, type } = scdProp;
+	if (JSON.stringify(values) === "{}" || JSON.stringify(values) === "[]") return [];
 	const scdEntries = [];
 	let lastInserted = dayjs(created);
 	const deltaDays = dayjs().diff(lastInserted, "day");
+	const uuidKeyName = type === 'user' ? 'distinct_id' : type;
 
 	for (let i = 0; i < mutations; i++) {
 		if (lastInserted.isAfter(dayjs())) break;
-		let scd = await makeProfile({ [scdKey]: prop }, { distinct_id });
+		let scd = await makeProfile({ [scdKey]: values }, { [uuidKeyName]: distinct_id });
 
 		// Explicitly constructing SCDSchema object with all required properties
 		const scdEntry = {
 			...scd, // spread existing properties
-			distinct_id: scd.distinct_id || distinct_id, // ensure distinct_id is set
-			insertTime: lastInserted.add(u.integer(1, 1000), "seconds").toISOString(),
-			startTime: lastInserted.toISOString()
+			[uuidKeyName]: scd.distinct_id || distinct_id, // ensure distinct_id is set			
+			startTime: null,
+			insertTime: null
 		};
 
+		if (timing === 'fixed') {
+			if (frequency === "day") scdEntry.startTime = lastInserted.add(1, "day").startOf('day').toISOString();
+			if (frequency === "week") scdEntry.startTime = lastInserted.add(1, "week").startOf('week').toISOString();
+			if (frequency === "month") scdEntry.startTime = lastInserted.add(1, "month").startOf('month').toISOString();
+		}
+
+		if (timing === 'fuzzy') {
+			scdEntry.startTime = lastInserted.toISOString();
+		}
+
+		const insertTime = lastInserted.add(u.integer(1, 9000), "seconds");
+		scdEntry.insertTime = insertTime.toISOString();
+
+
+
 		// Ensure TypeScript sees all required properties are set
 		if (scdEntry.hasOwnProperty('insertTime') && scdEntry.hasOwnProperty('startTime')) {
 			scdEntries.push(scdEntry);
 		}
 
+		//advance time for next entry
 		lastInserted = lastInserted
 			.add(u.integer(0, deltaDays), "day")
-			.subtract(u.integer(1, 1000), "seconds");
+			.subtract(u.integer(1, 9000), "seconds");
 	}
 
-	return scdEntries;
+	//de-dupe on startTime
+	const deduped = scdEntries.filter((entry, index, self) =>
+		index === self.findIndex((t) => (
+			t.startTime === entry.startTime
+		))
+	);
+	return deduped;
 }
 
 
@@ -806,24 +968,27 @@ async function userLoop(config, storage, concurrency = 1) {
 		userProps,
 		scdProps,
 		numDays,
+		percentUsersBornInDataset = 5,
 	} = config;
 	const { eventData, userProfilesData, scdTableData } = storage;
 	const avgEvPerUser = numEvents / numUsers;
+	const startTime = Date.now();
 
 	for (let i = 0; i < numUsers; i++) {
 		await USER_CONN(async () => {
 			userCount++;
-			if (verbose) u.progress([["users", userCount], ["events", eventCount]]);
+			const eps = Math.floor(eventCount / ((Date.now() - startTime) / 1000));
+			if (verbose) u.progress([["users", userCount], ["events", eventCount], ["eps", eps]]);
 			const userId = chance.guid();
 			const user = u.generateUser(userId, { numDays, isAnonymous, hasAvatar, hasAnonIds, hasSessionIds });
 			const { distinct_id, created } = user;
-			const userIsBornInDataset = chance.bool({ likelihood: 5 });
+			const userIsBornInDataset = chance.bool({ likelihood: percentUsersBornInDataset });
 			let numEventsPreformed = 0;
 			if (!userIsBornInDataset) delete user.created;
 			const adjustedCreated = userIsBornInDataset ? dayjs(created).subtract(daysShift, 'd') : dayjs.unix(global.FIXED_BEGIN);
 
 			if (hasLocation) {
-				const location = u.choose(DEFAULTS.locationsUsers);
+				const location = u.shuffleArray(u.choose(DEFAULTS.locationsUsers)).pop();
 				for (const key in location) {
 					user[key] = location[key];
 				}
@@ -831,16 +996,21 @@ async function userLoop(config, storage, concurrency = 1) {
 
 			// Profile creation
 			const profile = await makeProfile(userProps, user);
-			await userProfilesData.hookPush(profile);
+
 
 			// SCD creation
-			const scdTableKeys = Object.keys(scdProps);
+			const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user');
+			const scdTableKeys = Object.keys(scdUserTables);
+
+
 			const userSCD = {};
 			for (const [index, key] of scdTableKeys.entries()) {
-				const mutations = chance.integer({ min: 1, max: 10 }); //todo: configurable mutations?
+				// @ts-ignore
+				const { max = 100 } = scdProps[key];
+				const mutations = chance.integer({ min: 1, max });
 				const changes = await makeSCD(scdProps[key], key, distinct_id, mutations, created);
 				userSCD[key] = changes;
-				await scdTableData[index].hookPush(changes);
+				await config.hook(changes, "scd-pre", { profile, type: 'user', scd: { [key]: scdProps[key] }, config, allSCDs: userSCD });
 			}
 
 			let numEventsThisUserWillPreform = Math.floor(chance.normal({
@@ -860,6 +1030,7 @@ async function userLoop(config, storage, concurrency = 1) {
 
 			const secondsInDay = 86400;
 			const noise = () => chance.integer({ min: 0, max: secondsInDay });
+			let usersEvents = [];
 
 			if (firstFunnels.length && userIsBornInDataset) {
 				const firstFunnel = chance.pickone(firstFunnels, user);
@@ -868,7 +1039,8 @@ async function userLoop(config, storage, concurrency = 1) {
 				const [data, userConverted] = await makeFunnel(firstFunnel, user, firstTime, profile, userSCD, config);
 				userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
 				numEventsPreformed += data.length;
-				await eventData.hookPush(data);
+				// await eventData.hookPush(data, { profile });
+				usersEvents.push(...data);
 				if (!userConverted) {
 					if (verbose) u.progress([["users", userCount], ["events", eventCount]]);
 					return;
@@ -884,14 +1056,35 @@ async function userLoop(config, storage, concurrency = 1) {
 					const currentFunnel = chance.pickone(usageFunnels);
 					const [data, userConverted] = await makeFunnel(currentFunnel, user, userFirstEventTime, profile, userSCD, config);
 					numEventsPreformed += data.length;
-					await eventData.hookPush(data);
+					usersEvents.push(...data);
+					// await eventData.hookPush(data, { profile });
 				} else {
 					const data = await makeEvent(distinct_id, userFirstEventTime, u.choose(config.events), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true);
 					numEventsPreformed++;
-					await eventData.hookPush(data);
+					usersEvents.push(data);
+					// await eventData.hookPush(data);
 				}
 			}
 
+			// NOW ADD ALL OUR DATA FOR THIS USER
+			if (config.hook) {
+				const newEvents = await config.hook(usersEvents, "everything", { profile, scd: userSCD, config, userIsBornInDataset });
+				if (Array.isArray(newEvents)) usersEvents = newEvents;
+			}
+
+			await userProfilesData.hookPush(profile);
+
+			if (Object.keys(userSCD).length) {
+				for (const [key, changesArray] of Object.entries(userSCD)) {
+					for (const changes of changesArray) {
+						const target = scdTableData.filter(arr => arr.scdKey === key).pop();
+						await target.hookPush(changes, { profile, type: 'user' });
+					}
+				}
+			}
+			await eventData.hookPush(usersEvents, { profile });
+
+
 			if (verbose) u.progress([["users", userCount], ["events", eventCount]]);
 		});
 	}
@@ -906,8 +1099,18 @@ async function userLoop(config, storage, concurrency = 1) {
  * @param  {Storage} storage
  */
 async function sendToMixpanel(config, storage) {
-	const { adSpendData, eventData, groupProfilesData, lookupTableData, mirrorEventData, scdTableData, userProfilesData } = storage;
-	const { token, region, writeToDisk } = config;
+	const {
+		adSpendData,
+		eventData,
+		groupProfilesData,
+		lookupTableData,
+		mirrorEventData,
+		scdTableData,
+		userProfilesData,
+		groupEventData
+
+	} = storage;
+	const { token, region, writeToDisk = true } = config;
 	const importResults = { events: {}, users: {}, groups: [] };
 
 	/** @type {import('mixpanel-import').Creds} */
@@ -924,7 +1127,7 @@ async function sendToMixpanel(config, storage) {
 		dryRun: false,
 		abridged: false,
 		fixJson: true,
-		showProgress: true,
+		showProgress: NODE_ENV === "dev" ? true : false,
 		streamFormat: mpImportFormat
 	};
 
@@ -960,7 +1163,7 @@ async function sendToMixpanel(config, storage) {
 		log(`\tsent ${comma(imported.success)} user profiles\n`);
 		importResults.users = imported;
 	}
-	if (adSpendData || isBATCH_MODE) {
+	if (groupEventData || isBATCH_MODE) {
 		log(`importing ad spend data to mixpanel...\n`);
 		let adSpendDataToImport = clone(adSpendData);
 		if (isBATCH_MODE) {
@@ -996,11 +1199,65 @@ async function sendToMixpanel(config, storage) {
 		}
 	}
 
+	if (groupEventData || isBATCH_MODE) {
+		log(`importing group events to mixpanel...\n`);
+		let groupEventDataToImport = clone(groupEventData);
+		if (isBATCH_MODE) {
+			const writeDir = groupEventData.getWriteDir();
+			const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+			groupEventDataToImport = files.filter(f => f.includes('-GROUP-EVENTS-'));
+		}
+		const imported = await mp(creds, groupEventDataToImport, {
+			recordType: "event",
+			...commonOpts,
+			strict: false
+		});
+		log(`\tsent ${comma(imported.success)} group events\n`);
+		importResults.groupEvents = imported;
+	}
+	const { serviceAccount, projectId, serviceSecret } = config;
+	if (serviceAccount && projectId && serviceSecret) {
+		if (scdTableData || isBATCH_MODE) {
+			log(`importing SCD data to mixpanel...\n`);
+			for (const scdEntity of scdTableData) {
+				const scdKey = scdEntity?.scdKey;
+				log(`importing ${scdKey} SCD data to mixpanel...\n`);
+				let scdDataToImport = clone(scdEntity);
+				if (isBATCH_MODE) {
+					const writeDir = scdEntity.getWriteDir();
+					const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+					scdDataToImport = files.filter(f => f.includes(`-SCD-${scdKey}`));
+				}
+
+				const options = {
+					recordType: "scd",
+					scdKey,
+					scdType: scdEntity.dataType,
+					scdLabel: `${scdKey}-scd`,
+					...commonOpts,
+				};
+				if (scdEntity.entityType !== "user") options.groupKey = scdEntity.entityType;
+				const imported = await mp(
+					{
+						token,
+						acct: serviceAccount,
+						pass: serviceSecret,
+						project: projectId
+					},
+					scdDataToImport,
+					// @ts-ignore
+					options);
+				log(`\tsent ${comma(imported.success)} ${scdKey} SCD data\n`);
+				importResults[`${scdKey}_scd`] = imported;
+			}
+		}
+	}
+
 	//if we are in batch mode, we need to delete the files
 	if (!writeToDisk && isBATCH_MODE) {
 		const writeDir = eventData?.getWriteDir() || userProfilesData?.getWriteDir();
 		const listDir = await ls(writeDir.split(path.basename(writeDir)).join(""));
-		const files = listDir.filter(f => f.includes('-EVENTS-') || f.includes('-USERS-') || f.includes('-AD-SPEND-') || f.includes('-GROUPS-'));
+		const files = listDir.filter(f => f.includes('-EVENTS-') || f.includes('-USERS-') || f.includes('-AD-SPEND-') || f.includes('-GROUPS-') || f.includes('-GROUP-EVENTS-'));
 		for (const file of files) {
 			await rm(file);
 		}
@@ -1093,6 +1350,43 @@ function validateDungeonConfig(config) {
 		funnels = [...funnels, ...inferredFunnels];
 	}
 
+
+	const eventContainedInFunnels = Array.from(funnels.reduce((acc, f) => {
+		const events = f.sequence;
+		events.forEach(event => acc.add(event));
+		return acc;
+	}, new Set()));
+
+	const eventsNotInFunnels = events
+		.filter(e => !e.isFirstEvent)
+		.filter(e => !eventContainedInFunnels.includes(e.event)).map(e => e.event);
+	if (eventsNotInFunnels.length) {
+		// const biggestWeight = funnels.reduce((acc, f) => {
+		// 	if (f.weight > acc) return f.weight;
+		// 	return acc;
+		// }, 0);
+		// const smallestWeight = funnels.reduce((acc, f) => {
+		// 	if (f.weight < acc) return f.weight;
+		// 	return acc;
+		// }, 0);
+		// const weight = u.integer(smallestWeight, biggestWeight) * 2;
+
+		const sequence = u.shuffleArray(eventsNotInFunnels.flatMap(event => {
+			const evWeight = config.events.find(e => e.event === event)?.weight || 1;
+			return Array(evWeight).fill(event);
+		}));
+
+
+
+		funnels.push({
+			sequence,
+			conversionRate: 50,
+			order: 'random',
+			timeToConvert: 24 * 14,
+			requireRepeats: false,
+		});
+	}
+
 	config.concurrency = concurrency;
 	config.funnels = funnels;
 	config.batchSize = batchSize;
@@ -1156,6 +1450,8 @@ async function makeHookArray(arr = [], opts = {}) {
 	if (existsSync(dataFolder)) writeDir = dataFolder;
 	else writeDir = path.resolve("./");
 
+	if (NODE_ENV === "prod") writeDir = path.resolve(os.tmpdir());
+
 	function getWritePath() {
 		if (isBATCH_MODE) {
 			return path.join(writeDir, `${filepath}-part-${batch.toString()}.${format}`);
@@ -1169,14 +1465,14 @@ async function makeHookArray(arr = [], opts = {}) {
 		return path.join(writeDir, `${filepath}.${format}`);
 	}
 
-	async function transformThenPush(item) {
+	async function transformThenPush(item, meta) {
 		if (item === null || item === undefined) return false;
 		if (typeof item === 'object' && Object.keys(item).length === 0) return false;
-
+		const allMetaData = { ...rest, ...meta };
 		if (Array.isArray(item)) {
 			for (const i of item) {
 				try {
-					const enriched = await hook(i, type, rest);
+					const enriched = await hook(i, type, allMetaData);
 					if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
 					else arr.push(enriched);
 				} catch (e) {
@@ -1186,7 +1482,7 @@ async function makeHookArray(arr = [], opts = {}) {
 			}
 		} else {
 			try {
-				const enriched = await hook(item, type, rest);
+				const enriched = await hook(item, type, allMetaData);
 				if (Array.isArray(enriched)) enriched.forEach(e => arr.push(e));
 				else arr.push(enriched);
 			} catch (e) {
@@ -1301,92 +1597,94 @@ CLI
 ----
 */
 
-if (require.main === module) {
-	isCLI = true;
-	const args = /** @type {Config} */ (getCliParams());
-	let { token, seed, format, numDays, numUsers, numEvents, region, writeToDisk, complex = false, hasSessionIds, hasAnonIds } = args;
-	const suppliedConfig = args._[0];
-
-	//if the user specifies an separate config file
-	let config = null;
-	if (suppliedConfig) {
-		console.log(`using ${suppliedConfig} for data\n`);
-		config = require(path.resolve(suppliedConfig));
-	}
-	else {
-		if (complex) {
-			console.log(`... using default COMPLEX configuration [everything] ...\n`);
-			console.log(`... for more simple data, don't use the --complex flag ...\n`);
-			console.log(`... or specify your own js config file (see docs or --help) ...\n`);
-			config = require(path.resolve(__dirname, "./schemas/complex.js"));
+if (NODE_ENV !== "prod") {
+	if (require.main === module) {
+		isCLI = true;
+		const args = /** @type {Config} */ (getCliParams());
+		let { token, seed, format, numDays, numUsers, numEvents, region, writeToDisk, complex = false, hasSessionIds, hasAnonIds } = args;
+		const suppliedConfig = args._[0];
+
+		//if the user specifies an separate config file
+		let config = null;
+		if (suppliedConfig) {
+			console.log(`using ${suppliedConfig} for data\n`);
+			config = require(path.resolve(suppliedConfig));
 		}
 		else {
-			console.log(`... using default SIMPLE configuration [events + users] ...\n`);
-			console.log(`... for more complex data, use the --complex flag ...\n`);
-			config = require(path.resolve(__dirname, "./schemas/simple.js"));
+			if (complex) {
+				console.log(`... using default COMPLEX configuration [everything] ...\n`);
+				console.log(`... for more simple data, don't use the --complex flag ...\n`);
+				console.log(`... or specify your own js config file (see docs or --help) ...\n`);
+				config = require(path.resolve(__dirname, "./dungeons/complex.js"));
+			}
+			else {
+				console.log(`... using default SIMPLE configuration [events + users] ...\n`);
+				console.log(`... for more complex data, use the --complex flag ...\n`);
+				config = require(path.resolve(__dirname, "./dungeons/simple.js"));
+			}
 		}
-	}
 
-	//override config with cli params
-	if (token) config.token = token;
-	if (seed) config.seed = seed;
-	if (format === "csv" && config.format === "json") format = "json";
-	if (format) config.format = format;
-	if (numDays) config.numDays = numDays;
-	if (numUsers) config.numUsers = numUsers;
-	if (numEvents) config.numEvents = numEvents;
-	if (region) config.region = region;
-	if (writeToDisk) config.writeToDisk = writeToDisk;
-	if (writeToDisk === 'false') config.writeToDisk = false;
-	if (hasSessionIds) config.hasSessionIds = hasSessionIds;
-	if (hasAnonIds) config.hasAnonIds = hasAnonIds;
-	config.verbose = true;
-
-	main(config)
-		.then((data) => {
-			//todo: rethink summary
-			log(`-----------------SUMMARY-----------------`);
-			const d = { success: 0, bytes: 0 };
-			const darr = [d];
-			const { events = d, groups = darr, users = d } = data?.importResults || {};
-			const files = data.files;
-			const folder = files?.[0]?.split(path.basename(files?.[0]))?.shift();
-			const groupBytes = groups.reduce((acc, group) => {
-				return acc + group.bytes;
-			}, 0);
-			const groupSuccess = groups.reduce((acc, group) => {
-				return acc + group.success;
-			}, 0);
-			const bytes = events.bytes + groupBytes + users.bytes;
-			const stats = {
-				events: comma(events.success || 0),
-				users: comma(users.success || 0),
-				groups: comma(groupSuccess || 0),
-				bytes: bytesHuman(bytes || 0),
-			};
-			if (bytes > 0) console.table(stats);
-			log(`\nfiles written to ${folder || "no where; we didn't write anything"} ...`);
-			log("  " + files?.flat().join("\n  "));
-			log(`\n----------------SUMMARY-----------------\n\n\n`);
-		})
-		.catch((e) => {
-			log(`------------------ERROR------------------`);
-			console.error(e);
-			log(`------------------ERROR------------------`);
-			debugger;
-		})
-		.finally(() => {
-			log("enjoy your data! :)");
-			u.openFinder(path.resolve("./data"));
-		});
-} else {
-	main.generators = { makeEvent, makeFunnel, makeProfile, makeSCD, makeAdSpend, makeMirror };
-	main.orchestrators = { userLoop, validateDungeonConfig, sendToMixpanel };
-	main.meta = { inferFunnels, hookArray: makeHookArray };
-	module.exports = main;
+		//override config with cli params
+		if (token) config.token = token;
+		if (seed) config.seed = seed;
+		if (format === "csv" && config.format === "json") format = "json";
+		if (format) config.format = format;
+		if (numDays) config.numDays = numDays;
+		if (numUsers) config.numUsers = numUsers;
+		if (numEvents) config.numEvents = numEvents;
+		if (region) config.region = region;
+		if (writeToDisk) config.writeToDisk = writeToDisk;
+		if (writeToDisk === 'false') config.writeToDisk = false;
+		if (hasSessionIds) config.hasSessionIds = hasSessionIds;
+		if (hasAnonIds) config.hasAnonIds = hasAnonIds;
+		config.verbose = true;
+
+		main(config)
+			.then((data) => {
+				log(`-----------------SUMMARY-----------------`);
+				const d = { success: 0, bytes: 0 };
+				const darr = [d];
+				const { events = d, groups = darr, users = d } = data?.importResults || {};
+				const files = data.files;
+				const folder = files?.[0]?.split(path.basename(files?.[0]))?.shift() || "./data";
+				const groupBytes = groups.reduce((acc, group) => {
+					return acc + group.bytes;
+				}, 0);
+				const groupSuccess = groups.reduce((acc, group) => {
+					return acc + group.success;
+				}, 0);
+				const bytes = events.bytes + groupBytes + users.bytes;
+				const stats = {
+					events: comma(events.success || 0),
+					users: comma(users.success || 0),
+					groups: comma(groupSuccess || 0),
+					bytes: bytesHuman(bytes || 0),
+				};
+				if (bytes > 0) console.table(stats);
+				log(`\nlog written to ${folder} ...`);
+				writeFileSync(path.join(folder, "log.txt"), JSON.stringify(data?.importResults, null, 2));
+				// log("  " + files?.flat().join("\n  "));
+				log(`\n----------------SUMMARY-----------------\n\n\n`);
+			})
+			.catch((e) => {
+				log(`------------------ERROR------------------`);
+				console.error(e);
+				log(`------------------ERROR------------------`);
+				debugger;
+			})
+			.finally(() => {
+				log("enjoy your data! :)");
+			});
+	} else {
+		main.generators = { makeEvent, makeFunnel, makeProfile, makeSCD, makeAdSpend, makeMirror };
+		main.orchestrators = { userLoop, validateDungeonConfig, sendToMixpanel };
+		main.meta = { inferFunnels, hookArray: makeHookArray };
+		module.exports = main;
+	}
 }
 
 
+
 /*
 ----
 HELPERS
@@ -1411,7 +1709,7 @@ function track(name, props, ...rest) {
 }
 
 
-/** @typedef {import('./types.js').Config} Config */
+/** @typedef {import('./types.js').Dungeon} Config */
 /** @typedef {import('./types.js').AllData} AllData */
 /** @typedef {import('./types.js').EventConfig} EventConfig */
 /** @typedef {import('./types.js').Funnel} Funnel */
@@ -1424,4 +1722,5 @@ function track(name, props, ...rest) {
 /** @typedef {import('./types.js').ValueValid} ValueValid */
 /** @typedef {import('./types.js').HookedArray} hookArray */
 /** @typedef {import('./types.js').hookArrayOptions} hookArrayOptions */
-/** @typedef {import('./types.js').GroupProfileSchema} GroupProfile */
+/** @typedef {import('./types.js').GroupProfileSchema} GroupProfile */
+/** @typedef {import('./types.js').SCDProp} SCDProp */