make-mp-data 1.5.52 → 1.5.54

package/dungeons/governance.js

@@ -0,0 +1,288 @@
+const run = require("../index.js");
+const SEED = "my-seed";
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+require("dotenv").config();
+const u = require("../components/utils");
+const v = require("ak-tools");
+const { projectId, serviceAccount, serviceSecret } = require("./adspend");
+const chance = u.initChance(SEED);
+const num_users = 1_500;
+const days = 91;
+
+const GOVERNANCE_TOKEN = process.env.GOVERNANCE_TOKEN;
+const GOVERNANCE_ID = process.env.GOVERNANCE_ID;
+const GOVERNANCE_ACCT = process.env.GOVERNANCE_ACCT;
+const GOVERNANCE_SECRET = process.env.GOVERNANCE_SECRET;
+
+const URLS = [
+	"/",
+	"/refinance-student-loan/",
+	"/private-student-loans/",
+	"/personal-loans/",
+	"/home-loans/",
+	"/invest/",
+	"/banking/",
+	"/relay/",
+	"/life-insurance/",
+	"/learn/",
+	"/faq/",
+	"/career-advisory/",
+	"/member-benefits/",
+	"/experiences/",
+	"/member-stories/",
+	"/contact-us/",
+	"/jobs/",
+	"/on-the-money/",
+	"/about-sofi/",
+	"/management-team/",
+	"/eligibility-criteria/",
+	"/referral-program/",
+	"/sofi-at-work/"
+];
+
+/** @type {Config} */
+const commonOpts = {
+	verbose: true,
+	token: GOVERNANCE_TOKEN,
+	projectId: GOVERNANCE_ID,
+	serviceAccount: GOVERNANCE_ACCT,
+	serviceSecret: GOVERNANCE_SECRET,
+	numDays: days,
+	numEvents: num_users * 100,
+	numUsers: num_users,
+	hasAnonIds: false,
+	hasSessionIds: true,
+	format: "json",
+	alsoInferFunnels: true,
+	hasLocation: true,
+	hasAndroidDevices: false,
+	hasIOSDevices: false,
+	hasDesktopDevices: false,
+	hasBrowser: true,
+	hasCampaigns: false,
+	isAnonymous: false,
+	hasAdSpend: false,
+
+	hasAvatar: true,
+	makeChart: false,
+
+	batchSize: 1_500_000,
+	concurrency: 1,
+	writeToDisk: false,
+	percentUsersBornInDataset: 15
+	
+};
+
+/** @typedef  {import("../types").Dungeon} Config */
+
+/** @type {Config} */
+const diffNameSameMeaning = {
+	...commonOpts,
+	name: "same-meaning",
+	hasAdSpend: true,
+	seed: "foo",
+	description: "A dungeon with really unclear event names",
+	funnels: [],
+	events: [
+		//@ts-ignore
+		" login", "login ", " sign in", "login", "Log In", "Sign In", "sign in", "signin", "page view", "page viewed", "viewed page", "view page", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "page_viewed", "page"
+	],
+	superProps: {
+		"theme": "diff name; same meaning",
+		"url": u.pickAWinner(URLS, 0)
+	},
+	userProps: {},	 
+	hook: (a, type) => {
+		if (type === "everything") {
+			return a;
+		}
+
+		return a;
+	}
+};
+
+
+/** @type {Config} */
+const sameNameDiffMeaning = {
+	...commonOpts,
+	name: "diff-meanings",
+	seed: "bar",
+	description: "a dataset with a single event that means different things",
+	funnels: [],
+	events: [
+		{
+			event: "button click",
+			properties: {
+				"button name": ["start now", "register", "submit", "learn more", "try now", "free trial"],
+				"form name": u.pickAWinner(["mortgage", "student loan", "personal loan ", "credit card ", "savings account", "subscribe form"]),
+				"cta": ["free for 30d", "decision in 5min", "chat with a human", "try it in the app"],
+			}
+		}
+	],
+	superProps: {
+		"theme": "same name; diff meaning",
+		"url": u.pickAWinner(URLS, 0)
+	},
+	userProps: {},
+	scdProps: {},
+	hook: (a, type) => {
+		if (type === "everything") {
+			return a;
+		}
+
+		return a;
+	}
+};
+
+/** @type {Config} */
+const Borrowing = {
+	...commonOpts,
+	percentUsersBornInDataset: 100,
+	alsoInferFunnels: false,
+	name: "borrowing",
+	seed: "baz",
+	description: "a dataset which requires borrowing",
+	funnels: [
+		{
+			"sequence": ["page viewed", "click!", "sign up"],
+			"isFirstFunnel": true,
+			"order": "sequential"
+		},
+		{
+			"sequence": ["start application", "submit application"],
+			"props": {
+				"product": u.pickAWinner(["mortgage", "student loan", "personal loan ", "credit card ", "savings account", "investing account"]),
+				"application Id": () => { return v.uid(10); },
+			}
+		}
+	],
+	events: [
+		{
+			event: "page viewed",
+			properties: {
+				"marketing channel": u.pickAWinner(["organic", "meta", "google", "x", "snapchat", "youtube", "instagram"], 0),
+			}
+		},
+		{
+			event: "click!",
+			properties: {
+				"CTA clicked": u.pickAWinner(["start now", "register", "submit", "learn more", "try now", "free trial"]),
+			}
+		},
+		{
+			event: "sign up",
+			properties: {
+				"A/B test": ["fast onboarding", "custom colors", "localization", "new design"],
+				"Variant": ["A", "B", "C", "Control"]
+			},
+			isFirstEvent: true
+		},
+		{
+			event: "start application",
+
+		},
+		{
+			event: "submit application",
+		},
+	],
+	superProps: {
+		"theme": "requires borrows",
+		"url": u.pickAWinner(URLS, 0)
+	},
+	userProps: {},
+	scdProps: {},
+	hook: (a, type) => {
+		if (type === "everything") {
+			return a;
+		}
+
+		return a;
+	}
+};
+
+
+/** @type {Config} */
+const Modeling = {
+	...commonOpts,
+	percentUsersBornInDataset: 100,
+	alsoInferFunnels: false,
+	name: "bad shapes",
+	seed: "qux",
+	description: "a data set with bad modeling",
+	funnels: [
+
+	],
+	events: [
+		{
+			event: "nested nightmare",
+			properties: {
+				data: buildAStupidNestedObject
+			}
+		},
+	],
+	superProps: {
+		"theme": "deeply nested",
+		"url": u.pickAWinner(URLS, 0)
+	},
+	userProps: {},
+	scdProps: {},
+	hook: (a, type) => {
+		if (type === "everything") {
+			return a;
+		}
+
+		return a;
+	}
+};
+
+const metaValues = u.pickAWinner(["foo", "bar", "baz", "qux", "quux", "corge", "grault", "garply", "waldo", "fred", "plugh", "xyzzy", "thud"]);
+function buildAStupidNestedObject(depth = 0) {
+	let obj = {};
+	const keyPrefix = ["A", "B", "C", "D", "E", "F", "G", "H", "I"];
+	const keySuffix = ["1", "2", "3", "4", "5", "6", "7", "8", "9"];
+
+	let numKeys = chance.integer({ min: 1, max: 5 });
+	for (let i = 0; i < numKeys; i++) {
+		if (depth <= 3 && chance.bool({ likelihood: 10 })) {
+			let key = chance.pickone(keyPrefix) + ">" + chance.pickone(keySuffix);
+			obj[key] = buildAStupidNestedObject();
+		}
+		else {
+			let key = chance.pickone(keyPrefix) + "|" + chance.pickone(keySuffix);
+			let value = u.choose(metaValues);
+			obj[key] = value;
+		}
+	}
+	return obj;
+}
+
+
+const dungeons = [
+	diffNameSameMeaning,
+	sameNameDiffMeaning,
+	Borrowing,
+	Modeling
+];
+
+async function runDungeons() {
+	const results = [];
+	for (let dungeon of dungeons) {
+		const job = await run(dungeon);
+		results.push(job);
+	}
+	return results;
+}
+
+runDungeons()
+	.then(results => {
+		debugger;
+	})
+	.catch(e => {
+		console.error(e);
+		debugger;
+
+	});
+
+module.exports = {};

package/index.js

@@ -23,7 +23,7 @@ global.FIXED_NOW = FIXED_NOW;
 // ^ this creates a FIXED POINT in time; we will shift it later
 let FIXED_BEGIN = dayjs.unix(FIXED_NOW).subtract(90, 'd').unix();
 global.FIXED_BEGIN = FIXED_BEGIN;
-const actualNow = dayjs();
+const actualNow = dayjs().add(2, "day");
 const timeShift = actualNow.diff(dayjs.unix(FIXED_NOW), "seconds");
 const daysShift = actualNow.diff(dayjs.unix(FIXED_NOW), "days");
 
@@ -104,6 +104,16 @@ async function main(config) {
 		campaigns: u.pickAWinner(campaigns, 0),
 	};
 
+	if (config.singleCountry) {
+		DEFAULTS.locationsEvents = u.pickAWinner(clone(locations)
+			.filter(l => l.country === config.singleCountry)
+			.map(l => { delete l.country; return l; }), 0);
+
+		DEFAULTS.locationsUsers = u.pickAWinner(clone(locations)
+			.filter(l => l.country === config.singleCountry)
+			.map(l => { delete l.country_code; return l; }), 0);
+	}
+
 
 	//TRACKING
 	const runId = uid(42);
@@ -124,8 +134,8 @@ async function main(config) {
 	const scdTableKeys = Object.keys(scdProps);
 	const scdTableData = await Promise.all(scdTableKeys.map(async (key) =>
 		//todo don't assume everything is a string... lol
-		// @ts-ignore
-		await makeHookArray([], { hook, type: "scd", config, format, scdKey: key, entityType: config.scdProps[key].type, dataType: "string", filepath: `${simulationName}-${scdProps[key]?.type || "user"}-SCD-${key}` })
+		// @ts-ignore		
+		await makeHookArray([], { hook, type: "scd", config, format, scdKey: key, entityType: config.scdProps[key]?.type || "user", dataType: "string", filepath: `${simulationName}-${scdProps[key]?.type || "user"}-SCD-${key}` })
 	));
 	const groupTableKeys = Object.keys(groupKeys);
 	const groupProfilesData = await Promise.all(groupTableKeys.map(async (key, index) => {
@@ -431,16 +441,20 @@ async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, s
 	let defaultProps = {};
 	let devicePool = [];
 
-	if (hasLocation) defaultProps.location = DEFAULTS.locationsEvents();
-	if (hasBrowser) defaultProps.browser = DEFAULTS.browsers();
+	if (hasLocation) defaultProps.location = u.shuffleArray(DEFAULTS.locationsEvents()).pop();
+	if (hasBrowser) defaultProps.browser = u.choose(DEFAULTS.browsers());
 	if (hasAndroidDevices) devicePool.push(DEFAULTS.androidDevices());
 	if (hasIOSDevices) devicePool.push(DEFAULTS.iOSDevices());
 	if (hasDesktopDevices) devicePool.push(DEFAULTS.desktopDevices());
 
 	// we don't always have campaigns, because of attribution
-	if (hasCampaigns && chance.bool({ likelihood: 25 })) defaultProps.campaigns = DEFAULTS.campaigns();
+	if (hasCampaigns && chance.bool({ likelihood: 25 })) defaultProps.campaigns = u.shuffleArray(DEFAULTS.campaigns()).pop();
 	const devices = devicePool.flat();
-	if (devices.length) defaultProps.device = devices;
+	if (devices.length) defaultProps.device = u.shuffleArray(devices).pop();
+
+
+
+
 
 
 	//event time
@@ -505,6 +519,33 @@ async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, s
 					}
 				}
 			}
+			else if (typeof (defaultProps[key]) === "object") {
+				const obj = defaultProps[key];
+				for (const subKey in obj) {
+					if (Array.isArray(obj[subKey])) {
+						const subChoice = u.choose(obj[subKey]);
+						if (Array.isArray(subChoice)) {
+							for (const subSubChoice of subChoice) {
+								if (!eventTemplate[subKey]) eventTemplate[subKey] = subSubChoice;
+							}
+						}
+						else if (typeof subChoice === "object") {
+							for (const subSubKey in subChoice) {
+								if (!eventTemplate[subSubKey]) eventTemplate[subSubKey] = subChoice[subSubKey];
+							}
+						}
+						else {
+							if (!eventTemplate[subKey]) eventTemplate[subKey] = subChoice;
+						}
+					}
+					else {
+						if (!eventTemplate[subKey]) eventTemplate[subKey] = obj[subKey];
+					}
+				}
+			}
+			else {
+				if (!eventTemplate[key]) eventTemplate[key] = defaultProps[key];
+			}
 		}
 	}
 
@@ -725,12 +766,14 @@ async function makeFunnel(funnel, user, firstEventTime, profile, scd, config) {
  */
 async function makeProfile(props, defaults) {
 	operations++;
+	const keysToNotChoose = ["anonymousIds", "sessionIds"];
 
 	const profile = {
 		...defaults,
 	};
 
 	for (const key in profile) {
+		if (keysToNotChoose.includes(key)) continue;
 		try {
 			profile[key] = u.choose(profile[key]);
 		}
@@ -763,7 +806,7 @@ async function makeProfile(props, defaults) {
  */
 async function makeSCD(scdProp, scdKey, distinct_id, mutations, created) {
 	if (Array.isArray(scdProp)) scdProp = { values: scdProp, frequency: 'week', max: 10, timing: 'fuzzy', type: 'user' };
-	const { frequency, max, timing, values, type } = scdProp;
+	const { frequency, max, timing, values, type = "user" } = scdProp;
 	if (JSON.stringify(values) === "{}" || JSON.stringify(values) === "[]") return [];
 	const scdEntries = [];
 	let lastInserted = dayjs(created);
@@ -999,7 +1042,7 @@ async function userLoop(config, storage, concurrency = 1) {
 
 
 			// SCD creation
-			const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user');
+			const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user' || !scd.type);
 			const scdTableKeys = Object.keys(scdUserTables);
 
 
@@ -1123,7 +1166,8 @@ async function sendToMixpanel(config, storage) {
 		fixData: true,
 		verbose: false,
 		forceStream: true,
-		strict: false,
+		strict: true, //false,
+		epochEnd: dayjs().unix(), //is this chill?
 		dryRun: false,
 		abridged: false,
 		fixJson: true,
@@ -1338,6 +1382,11 @@ function validateDungeonConfig(config) {
 	config.simulationName = name || makeName();
 	config.name = config.simulationName;
 
+	//events
+	if (!events || !events.length) events = [{ event: "foo" }, { event: "bar" }, { event: "baz" }];
+	// @ts-ignore
+	if (typeof events[0] === "string") events = events.map(e => ({ event: e }));
+
 	//max batch size
 	if (batchSize > 0) BATCH_SIZE = batchSize;
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "make-mp-data",
-	"version": "1.5.052",
+	"version": "1.5.054",
 	"description": "builds all mixpanel primitives for a given project",
 	"main": "index.js",
 	"types": "types.d.ts",

package/scratch.mjs

@@ -13,17 +13,17 @@ TO DOs
 
 import main from "./index.js";
 
-import simple from './schemas/simple.js';
-import funnels from './schemas/funnels.js';
-import foobar from './schemas/foobar.js';
-import complex from './schemas/complex.js';
-import adspend from './schemas/adspend.js'
+import simple from './dungeons/simple.js';
+import funnels from './dungeons/funnels.js';
+import foobar from './dungeons/foobar.js';
+import complex from './dungeons/complex.js';
+import adspend from './dungeons/adspend.js'
 
-import anon from './schemas/anon.js';
+import anon from './dungeons/anon.js';
 import execSync from 'child_process';
-import mirror from './schemas/mirror.js'
+import mirror from './dungeons/mirror.js'
 // import mds from './dungeons/modern-data-stack.js'
-import big from './schemas/big.js'
+import big from './dungeons/big.js'
 
 const numEvents = 1000;
 
@@ -33,6 +33,8 @@ const spec = {
 	writeToDisk: true,
 	verbose: true,
 	makeChart: false,
+	hasAnonIds: true,
+	hasSessionIds: true
 	// format: "csv",
 	// numEvents,
 	// numUsers: numEvents / 100,

package/types.d.ts

@@ -50,6 +50,7 @@ declare namespace main {
     hasSessionIds?: boolean;
     alsoInferFunnels?: boolean;
     makeChart?: boolean | string;
+	singleCountry?: string;
 
     //models
     events?: EventConfig[]; //| string[]; //can also be a array of strings
@@ -73,7 +74,7 @@ declare namespace main {
   }
 
   export type complexSCDProp = {
-    type: string;
+    type?: string | "user_id" | "company_id";
     frequency: "day" | "week" | "month" | "year";
     values: ValueValid;
     timing: "fixed" | "fuzzy";