make-mp-data 1.5.51 → 1.5.53

package/components/utils.js

@@ -193,9 +193,20 @@ function choose(value) {
 			return chance.pickone(value);
 		}
 
-		// [[{}],[{}],[{}]] should return all		
-		if (Array.isArray(value) && typeof value[0] === "object" && hasSameKeys(value)) {
-			return value;
+		// Now, if the resolved value is an array, use chance.pickone
+		if (Array.isArray(value) && value.every(item => typeof item === 'string')) {
+			return chance.pickone(value);
+		}
+
+		if (Array.isArray(value) && value.every(item => typeof item === 'number')) {
+			return chance.pickone(value);
+		}
+
+		if (Array.isArray(value) && value.every(item => typeof item === 'object')) {
+			if (hasSameKeys(value)) return value;
+			else {
+				if (process.env.NODE_ENV === "dev") debugger;
+			}
 		}
 
 		// ["","",""] should pick-a-winner
@@ -234,27 +245,27 @@ function choose(value) {
 
 function hasSameKeys(arr) {
 	if (arr.length <= 1) {
-	  return true; // An empty array or an array with one object always has the same keys
+		return true; // An empty array or an array with one object always has the same keys
 	}
-  
+
 	const firstKeys = Object.keys(arr[0]);
-  
+
 	for (let i = 1; i < arr.length; i++) {
-	  const currentKeys = Object.keys(arr[i]);
-  
-	  if (currentKeys.length !== firstKeys.length) {
-		return false; // Different number of keys
-	  }
-  
-	  for (const key of firstKeys) {
-		if (!currentKeys.includes(key)) {
-		  return false; // Key missing in current object
+		const currentKeys = Object.keys(arr[i]);
+
+		if (currentKeys.length !== firstKeys.length) {
+			return false; // Different number of keys
+		}
+
+		for (const key of firstKeys) {
+			if (!currentKeys.includes(key)) {
+				return false; // Key missing in current object
+			}
 		}
-	  }
 	}
-  
+
 	return true; // All objects have the same keys
-  }
+}
 
 /**
  * keeps picking from an array until the array is exhausted

package/dungeons/governance.js

@@ -0,0 +1,288 @@
+const run = require("../index.js");
+const SEED = "my-seed";
+const dayjs = require("dayjs");
+const utc = require("dayjs/plugin/utc");
+dayjs.extend(utc);
+require("dotenv").config();
+const u = require("../components/utils");
+const v = require("ak-tools");
+const { projectId, serviceAccount, serviceSecret } = require("./adspend");
+const chance = u.initChance(SEED);
+const num_users = 1_500;
+const days = 91;
+
+const GOVERNANCE_TOKEN = process.env.GOVERNANCE_TOKEN;
+const GOVERNANCE_ID = process.env.GOVERNANCE_ID;
+const GOVERNANCE_ACCT = process.env.GOVERNANCE_ACCT;
+const GOVERNANCE_SECRET = process.env.GOVERNANCE_SECRET;
+
+const URLS = [
+	"/",
+	"/refinance-student-loan/",
+	"/private-student-loans/",
+	"/personal-loans/",
+	"/home-loans/",
+	"/invest/",
+	"/banking/",
+	"/relay/",
+	"/life-insurance/",
+	"/learn/",
+	"/faq/",
+	"/career-advisory/",
+	"/member-benefits/",
+	"/experiences/",
+	"/member-stories/",
+	"/contact-us/",
+	"/jobs/",
+	"/on-the-money/",
+	"/about-sofi/",
+	"/management-team/",
+	"/eligibility-criteria/",
+	"/referral-program/",
+	"/sofi-at-work/"
+];
+
+/** @type {Config} */
+const commonOpts = {
+	verbose: true,
+	token: GOVERNANCE_TOKEN,
+	projectId: GOVERNANCE_ID,
+	serviceAccount: GOVERNANCE_ACCT,
+	serviceSecret: GOVERNANCE_SECRET,
+	numDays: days,
+	numEvents: num_users * 100,
+	numUsers: num_users,
+	hasAnonIds: false,
+	hasSessionIds: true,
+	format: "json",
+	alsoInferFunnels: true,
+	hasLocation: true,
+	hasAndroidDevices: false,
+	hasIOSDevices: false,
+	hasDesktopDevices: false,
+	hasBrowser: true,
+	hasCampaigns: false,
+	isAnonymous: false,
+	hasAdSpend: false,
+
+	hasAvatar: true,
+	makeChart: false,
+
+	batchSize: 1_500_000,
+	concurrency: 1,
+	writeToDisk: false,
+	percentUsersBornInDataset: 15
+	
+};
+
+/** @typedef  {import("../types").Dungeon} Config */
+
+/** @type {Config} */
+const diffNameSameMeaning = {
+	...commonOpts,
+	name: "same-meaning",
+	hasAdSpend: true,
+	seed: "foo",
+	description: "A dungeon with really unclear event names",
+	funnels: [],
+	events: [
+		//@ts-ignore
+		" login", "login ", " sign in", "login", "Log In", "Sign In", "sign in", "signin", "page view", "page viewed", "viewed page", "view page", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "pageviewed", "page viewed", "page view", "pageview", "page viewed", "page view", "page_viewed", "page"
+	],
+	superProps: {
+		"theme": "diff name; same meaning",
+		"url": u.pickAWinner(URLS, 0)
+	},
+	userProps: {},	 
+	hook: (a, type) => {
+		if (type === "everything") {
+			return a;
+		}
+
+		return a;
+	}
+};
+
+
+/** @type {Config} */
+const sameNameDiffMeaning = {
+	...commonOpts,
+	name: "diff-meanings",
+	seed: "bar",
+	description: "a dataset with a single event that means different things",
+	funnels: [],
+	events: [
+		{
+			event: "button click",
+			properties: {
+				"button name": ["start now", "register", "submit", "learn more", "try now", "free trial"],
+				"form name": u.pickAWinner(["mortgage", "student loan", "personal loan ", "credit card ", "savings account", "subscribe form"]),
+				"cta": ["free for 30d", "decision in 5min", "chat with a human", "try it in the app"],
+			}
+		}
+	],
+	superProps: {
+		"theme": "same name; diff meaning",
+		"url": u.pickAWinner(URLS, 0)
+	},
+	userProps: {},
+	scdProps: {},
+	hook: (a, type) => {
+		if (type === "everything") {
+			return a;
+		}
+
+		return a;
+	}
+};
+
+/** @type {Config} */
+const Borrowing = {
+	...commonOpts,
+	percentUsersBornInDataset: 100,
+	alsoInferFunnels: false,
+	name: "borrowing",
+	seed: "baz",
+	description: "a dataset which requires borrowing",
+	funnels: [
+		{
+			"sequence": ["page viewed", "click!", "sign up"],
+			"isFirstFunnel": true,
+			"order": "sequential"
+		},
+		{
+			"sequence": ["start application", "submit application"],
+			"props": {
+				"product": u.pickAWinner(["mortgage", "student loan", "personal loan ", "credit card ", "savings account", "investing account"]),
+				"application Id": () => { return v.uid(10); },
+			}
+		}
+	],
+	events: [
+		{
+			event: "page viewed",
+			properties: {
+				"marketing channel": u.pickAWinner(["organic", "meta", "google", "x", "snapchat", "youtube", "instagram"], 0),
+			}
+		},
+		{
+			event: "click!",
+			properties: {
+				"CTA clicked": u.pickAWinner(["start now", "register", "submit", "learn more", "try now", "free trial"]),
+			}
+		},
+		{
+			event: "sign up",
+			properties: {
+				"A/B test": ["fast onboarding", "custom colors", "localization", "new design"],
+				"Variant": ["A", "B", "C", "Control"]
+			},
+			isFirstEvent: true
+		},
+		{
+			event: "start application",
+
+		},
+		{
+			event: "submit application",
+		},
+	],
+	superProps: {
+		"theme": "requires borrows",
+		"url": u.pickAWinner(URLS, 0)
+	},
+	userProps: {},
+	scdProps: {},
+	hook: (a, type) => {
+		if (type === "everything") {
+			return a;
+		}
+
+		return a;
+	}
+};
+
+
+/** @type {Config} */
+const Modeling = {
+	...commonOpts,
+	percentUsersBornInDataset: 100,
+	alsoInferFunnels: false,
+	name: "bad shapes",
+	seed: "qux",
+	description: "a data set with bad modeling",
+	funnels: [
+
+	],
+	events: [
+		{
+			event: "nested nightmare",
+			properties: {
+				data: buildAStupidNestedObject
+			}
+		},
+	],
+	superProps: {
+		"theme": "deeply nested",
+		"url": u.pickAWinner(URLS, 0)
+	},
+	userProps: {},
+	scdProps: {},
+	hook: (a, type) => {
+		if (type === "everything") {
+			return a;
+		}
+
+		return a;
+	}
+};
+
+const metaValues = u.pickAWinner(["foo", "bar", "baz", "qux", "quux", "corge", "grault", "garply", "waldo", "fred", "plugh", "xyzzy", "thud"]);
+function buildAStupidNestedObject(depth = 0) {
+	let obj = {};
+	const keyPrefix = ["A", "B", "C", "D", "E", "F", "G", "H", "I"];
+	const keySuffix = ["1", "2", "3", "4", "5", "6", "7", "8", "9"];
+
+	let numKeys = chance.integer({ min: 1, max: 5 });
+	for (let i = 0; i < numKeys; i++) {
+		if (depth <= 3 && chance.bool({ likelihood: 10 })) {
+			let key = chance.pickone(keyPrefix) + ">" + chance.pickone(keySuffix);
+			obj[key] = buildAStupidNestedObject();
+		}
+		else {
+			let key = chance.pickone(keyPrefix) + "|" + chance.pickone(keySuffix);
+			let value = u.choose(metaValues);
+			obj[key] = value;
+		}
+	}
+	return obj;
+}
+
+
+const dungeons = [
+	diffNameSameMeaning,
+	sameNameDiffMeaning,
+	Borrowing,
+	Modeling
+];
+
+async function runDungeons() {
+	const results = [];
+	for (let dungeon of dungeons) {
+		const job = await run(dungeon);
+		results.push(job);
+	}
+	return results;
+}
+
+runDungeons()
+	.then(results => {
+		debugger;
+	})
+	.catch(e => {
+		console.error(e);
+		debugger;
+
+	});
+
+module.exports = {};

package/index.js

@@ -124,8 +124,8 @@ async function main(config) {
 	const scdTableKeys = Object.keys(scdProps);
 	const scdTableData = await Promise.all(scdTableKeys.map(async (key) =>
 		//todo don't assume everything is a string... lol
-		// @ts-ignore
-		await makeHookArray([], { hook, type: "scd", config, format, scdKey: key, entityType: config.scdProps[key].type, dataType: "string", filepath: `${simulationName}-${scdProps[key]?.type || "user"}-SCD-${key}` })
+		// @ts-ignore		
+		await makeHookArray([], { hook, type: "scd", config, format, scdKey: key, entityType: config.scdProps[key]?.type || "user", dataType: "string", filepath: `${simulationName}-${scdProps[key]?.type || "user"}-SCD-${key}` })
 	));
 	const groupTableKeys = Object.keys(groupKeys);
 	const groupProfilesData = await Promise.all(groupTableKeys.map(async (key, index) => {
@@ -431,16 +431,20 @@ async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, s
 	let defaultProps = {};
 	let devicePool = [];
 
-	if (hasLocation) defaultProps.location = DEFAULTS.locationsEvents();
-	if (hasBrowser) defaultProps.browser = DEFAULTS.browsers();
+	if (hasLocation) defaultProps.location = u.shuffleArray(DEFAULTS.locationsEvents()).pop();
+	if (hasBrowser) defaultProps.browser = u.choose(DEFAULTS.browsers());
 	if (hasAndroidDevices) devicePool.push(DEFAULTS.androidDevices());
 	if (hasIOSDevices) devicePool.push(DEFAULTS.iOSDevices());
 	if (hasDesktopDevices) devicePool.push(DEFAULTS.desktopDevices());
 
 	// we don't always have campaigns, because of attribution
-	if (hasCampaigns && chance.bool({ likelihood: 25 })) defaultProps.campaigns = DEFAULTS.campaigns();
+	if (hasCampaigns && chance.bool({ likelihood: 25 })) defaultProps.campaigns = u.shuffleArray(DEFAULTS.campaigns()).pop();
 	const devices = devicePool.flat();
-	if (devices.length) defaultProps.device = devices;
+	if (devices.length) defaultProps.device = u.shuffleArray(devices).pop();
+
+
+
+
 
 
 	//event time
@@ -505,6 +509,33 @@ async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, s
 					}
 				}
 			}
+			else if (typeof (defaultProps[key]) === "object") {
+				const obj = defaultProps[key];
+				for (const subKey in obj) {
+					if (Array.isArray(obj[subKey])) { 
+						const subChoice = u.choose(obj[subKey]);
+						if (Array.isArray(subChoice)) {
+							for (const subSubChoice of subChoice) {
+								if (!eventTemplate[subKey]) eventTemplate[subKey] = subSubChoice;
+							}
+						}
+						else if (typeof subChoice === "object") {
+							for (const subSubKey in subChoice) {
+								if (!eventTemplate[subSubKey]) eventTemplate[subSubKey] = subChoice[subSubKey];
+							}
+						}
+						else {
+						if (!eventTemplate[subKey]) eventTemplate[subKey] = subChoice;
+						}
+					}
+					else {
+						if (!eventTemplate[subKey]) eventTemplate[subKey] = obj[subKey];
+					}
+				}
+			}
+			else {
+				if (!eventTemplate[key]) eventTemplate[key] = defaultProps[key];
+			}
 		}
 	}
 
@@ -725,12 +756,14 @@ async function makeFunnel(funnel, user, firstEventTime, profile, scd, config) {
  */
 async function makeProfile(props, defaults) {
 	operations++;
+	const keysToNotChoose = ["anonymousIds", "sessionIds"];
 
 	const profile = {
 		...defaults,
 	};
 
 	for (const key in profile) {
+		if (keysToNotChoose.includes(key)) continue;
 		try {
 			profile[key] = u.choose(profile[key]);
 		}
@@ -763,7 +796,7 @@ async function makeProfile(props, defaults) {
  */
 async function makeSCD(scdProp, scdKey, distinct_id, mutations, created) {
 	if (Array.isArray(scdProp)) scdProp = { values: scdProp, frequency: 'week', max: 10, timing: 'fuzzy', type: 'user' };
-	const { frequency, max, timing, values, type } = scdProp;
+	const { frequency, max, timing, values, type = "user" } = scdProp;
 	if (JSON.stringify(values) === "{}" || JSON.stringify(values) === "[]") return [];
 	const scdEntries = [];
 	let lastInserted = dayjs(created);
@@ -999,7 +1032,7 @@ async function userLoop(config, storage, concurrency = 1) {
 
 
 			// SCD creation
-			const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user');
+			const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user' || !scd.type);
 			const scdTableKeys = Object.keys(scdUserTables);
 
 
@@ -1338,6 +1371,11 @@ function validateDungeonConfig(config) {
 	config.simulationName = name || makeName();
 	config.name = config.simulationName;
 
+	//events
+	if (!events || !events.length) events = [{ event: "foo" }, { event: "bar" }, { event: "baz" }];
+	// @ts-ignore
+	if (typeof events[0] === "string") events = events.map(e => ({ event: e }));
+
 	//max batch size
 	if (batchSize > 0) BATCH_SIZE = batchSize;
 
@@ -1664,9 +1702,9 @@ if (NODE_ENV !== "prod") {
 					bytes: bytesHuman(bytes || 0),
 				};
 				if (bytes > 0) console.table(stats);
-				if (Object.keys(data?.importResults).length) {
-					log(`\nlog written to log.json\n`);
-					writeFileSync(path.resolve(folder, "log.json"), JSON.stringify(data?.importResults, null, 2));
+				if (Object.keys(data?.importResults || {}).length) {
+				log(`\nlog written to log.json\n`);
+				writeFileSync(path.resolve(folder, "log.json"), JSON.stringify(data?.importResults, null, 2));
 				}
 				// log("  " + files?.flat().join("\n  "));
 				log(`\n----------------SUMMARY-----------------\n\n\n`);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "make-mp-data",
-	"version": "1.5.051",
+	"version": "1.5.053",
 	"description": "builds all mixpanel primitives for a given project",
 	"main": "index.js",
 	"types": "types.d.ts",

package/scratch.mjs

@@ -13,17 +13,17 @@ TO DOs
 
 import main from "./index.js";
 
-import simple from './schemas/simple.js';
-import funnels from './schemas/funnels.js';
-import foobar from './schemas/foobar.js';
-import complex from './schemas/complex.js';
-import adspend from './schemas/adspend.js'
+import simple from './dungeons/simple.js';
+import funnels from './dungeons/funnels.js';
+import foobar from './dungeons/foobar.js';
+import complex from './dungeons/complex.js';
+import adspend from './dungeons/adspend.js'
 
-import anon from './schemas/anon.js';
+import anon from './dungeons/anon.js';
 import execSync from 'child_process';
-import mirror from './schemas/mirror.js'
+import mirror from './dungeons/mirror.js'
 // import mds from './dungeons/modern-data-stack.js'
-import big from './schemas/big.js'
+import big from './dungeons/big.js'
 
 const numEvents = 1000;
 
@@ -33,6 +33,8 @@ const spec = {
 	writeToDisk: true,
 	verbose: true,
 	makeChart: false,
+	hasAnonIds: true,
+	hasSessionIds: true
 	// format: "csv",
 	// numEvents,
 	// numUsers: numEvents / 100,

package/types.d.ts

@@ -73,7 +73,7 @@ declare namespace main {
   }
 
   export type complexSCDProp = {
-    type: string;
+    type?: string | "user_id" | "company_id";
     frequency: "day" | "week" | "month" | "year";
     values: ValueValid;
     timing: "fixed" | "fuzzy";