make-mp-data 1.5.54 → 1.5.55

package/.vscode/launch.json

@@ -15,7 +15,7 @@
             "request": "launch",
             "name": "run dungeon",
             "runtimeExecutable": "nodemon",
-            "runtimeArgs": ["--inspect"],
+            "runtimeArgs": ["--inspect", "--ignore", "./data"],
             "program": "${workspaceFolder}/index.js",
 			"args": ["--ignore", "./data/*", "${file}"],
             "restart": true,
@@ -23,6 +23,7 @@
             "internalConsoleOptions": "neverOpen",
             "skipFiles": ["<node_internals>/**"],
 			"preLaunchTask": "npm: prune",
+			
         },
 		{
 			"type": "node",
@@ -35,7 +36,8 @@
 			"internalConsoleOptions": "neverOpen",
 			"env": {
 				"NODE_ENV": "dev"
-			}
+			},
+			"runtimeArgs": ["--ignore", "./data"],
 		},
 		{
 			"command": "npm run func:local",

package/components/ai.js

@@ -83,21 +83,35 @@ function validator(schema) {
 
 
 if (require.main === module) {
-	generateSchema(` a payments platform called popPay PopPay platform, enables facial recognition-based transactions for contact-free payments, entry, and verification. Users sign up with a selfie, and their image becomes a secure digital key stored in PopID’s cloud, allowing for streamlined interactions at participating businesses. This system aims to replace IDs, passwords, and loyalty cards, focusing on security and user control.
+	generateSchema(`https://apps.apple.com/us/app/call-guardian-for-us-cellular/id1228680023  call guardian is an app for blocking spam calls made by TNS 
 		
-		Customers choose
-to be recognized
-Guests opt-in by standing directly in front of camera or notifying cashier. PopID never identifies guests without their permission.
-
-Personalized ordering experience
-Personalized
-ordering experience
-Guests can be automatically signed in to loyalty programs, shown past orders, and offered customized recommendations.
-
-Contact-free payment
-Contact-free
-payment
-PopPay’s eWallet links to guest’s payment card or checking account and offers lower processing fees than credit and debit cards.`)
+this is the list of events we want: 
+
+Onboarding Started
+Onboarding Completed (Basic)
+Onboarding Completed (Premium)
+Page Views (all "screens" within the app")
+Enable/Disable High Risk Blocking
+Enable/Disable Medium Risk Blocking
+Enable/Disable Neighborhood Spoof Blocking
+Call Blocked (Spam)
+Call Blocked (Custom List)
+Branded Call w/o Logo Received
+Branded Call w/ Logo Received
+Branded Call Answered
+Branded Call Blocked
+Enable/Disable Text Spam
+Reverse Number Lookup
+Report as Spam
+Report as Not Spam
+Custom Block List Number Add
+Custom Block List Number Remove
+Call Arrives Before Push
+Error Scenarios
+User Can't Authenticate
+Xfinity Services Can't Connect
+Verizon Services Can't Connect
+Deep Links into app`)
 		.then((result) => {
 			if (NODE_ENV === "dev") debugger;
 		})

package/components/utils.js

@@ -6,7 +6,7 @@ const { spawn } = require('child_process');
 const dayjs = require('dayjs');
 const utc = require('dayjs/plugin/utc');
 const path = require('path');
-const { mkdir } = require('ak-tools');
+const { mkdir, parseGCSUri } = require('ak-tools');
 const { existsSync } = require('fs');
 dayjs.extend(utc);
 require('dotenv').config();
@@ -26,6 +26,11 @@ let chanceInitialized = false;
 const ACTUAL_NOW = dayjs.utc();
 
 
+const { Storage: cloudStorage } = require('@google-cloud/storage');
+const projectId = 'YOUR_PROJECT_ID';
+const storage = new cloudStorage({ projectId });
+
+
 /*
 ----
 RNG
@@ -184,10 +189,10 @@ function choose(value) {
 	try {
 		// Keep resolving the value if it's a function
 		while (typeof value === 'function') {
-			value = value();		
+			value = value();
 		}
 
-		
+
 		// [[],[],[]] should pick one
 		if (Array.isArray(value) && Array.isArray(value[0])) {
 			return chance.pickone(value);
@@ -434,23 +439,38 @@ STREAMERS
 ----
 */
 
-function streamJSON(path, data) {
+function streamJSON(filePath, data) {
 	return new Promise((resolve, reject) => {
-		const writeStream = fs.createWriteStream(path, { encoding: 'utf8' });
+		let writeStream;
+		if (filePath?.startsWith('gs://')) {
+			const { uri, bucket, file } = parseGCSUri(filePath);
+			writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
+		}
+		else {
+			writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
+		}
 		data.forEach(item => {
 			writeStream.write(JSON.stringify(item) + '\n');
 		});
 		writeStream.end();
 		writeStream.on('finish', () => {
-			resolve(path);
+			resolve(filePath);
 		});
 		writeStream.on('error', reject);
 	});
 }
 
-function streamCSV(path, data) {
+function streamCSV(filePath, data) {
 	return new Promise((resolve, reject) => {
-		const writeStream = fs.createWriteStream(path, { encoding: 'utf8' });
+		let writeStream;
+		if (filePath?.startsWith('gs://')) {
+			const { uri, bucket, file } = parseGCSUri(filePath);
+			writeStream = storage.bucket(bucket).file(file).createWriteStream({ gzip: true });
+		}
+		else {
+			writeStream = fs.createWriteStream(filePath, { encoding: 'utf8' });
+		}
+
 		// Extract all unique keys from the data array
 		const columns = getUniqueKeys(data);  // Assuming getUniqueKeys properly retrieves all keys
 
@@ -469,7 +489,7 @@ function streamCSV(path, data) {
 
 		writeStream.end();
 		writeStream.on('finish', () => {
-			resolve(path);
+			resolve(filePath);
 		});
 		writeStream.on('error', reject);
 	});

package/dungeons/big.js

@@ -1,15 +1,4 @@
-/**
- * This is the default configuration file for the data generator in SIMPLE mode
- * notice how the config object is structured, and see it's type definition in ./types.d.ts
- * feel free to modify this file to customize the data you generate
- * see helper functions in utils.js for more ways to generate data
- */
-
-
-/* cSpell:disable */
-
-
-
+const seed = "lets go big";
 const Chance = require('chance');
 const chance = new Chance();
 const dayjs = require("dayjs");
@@ -18,17 +7,19 @@ dayjs.extend(utc);
 const { uid, comma, makeName } = require('ak-tools');
 const { pickAWinner, weighNumRange, integer, date, choose } = require('../components/utils');
 
+
 const eventsPerQuarter = 5_000_000_000 // ~5 billion
-const numQuarters = 8;
-const totalEvents = eventsPerQuarter * numQuarters;
+const numQuarters = 8; // 24 months
+const parallelism = 5000; 
+const totalEvents = Math.floor((eventsPerQuarter * numQuarters) / parallelism);
 const eventPerUser = 500;
-const totalUsers = totalEvents / eventPerUser;
+const totalUsers = Math.floor(totalEvents / eventPerUser);
 const totalDays = (numQuarters * 90) + 10;
 
 /** @type {import('../types').Dungeon} */
 const config = {
-	token: "64b48bc361f4477634bdfafb78ef39ad",
-	seed: "lets go big",
+	token: "",
+	seed: seed,
 	numDays: totalDays,
 	numEvents: totalEvents,
 	numUsers: totalUsers,
@@ -39,14 +30,14 @@ const config = {
 	hasLocation: true,
 	hasAndroidDevices: false,
 	alsoInferFunnels: false,
-	batchSize: 5_000_000,
-	hasAvatar: true,
+	batchSize: 2_000_000,
+	hasAvatar: false,
 	hasAdSpend: false,
 	hasBrowser: false,
 	hasCampaigns: false,
 	hasDesktopDevices: false,
 	hasIOSDevices: false,
-	writeToDisk: false,
+	writeToDisk: "gs://dungeon_master_4/big_data",
 	funnels: [
 		{
 			"sequence": ["foo", "bar", "baz", "qux", "garply", "durtle", "linny", "fonk", "crumn", "yak"],

package/dungeons/simple.js

@@ -118,8 +118,7 @@ const config = {
 			}
 		}
 	],
-	superProps: {
-		platform: ["web", "mobile", "web", "mobile", "web", "web", "kiosk", "smartTV"],
+	superProps: {		
 		currentTheme: weighChoices(["light", "dark", "custom", "light", "dark"]),
 	},
 	/*

package/index.js

@@ -45,6 +45,22 @@ const t = require('ak-tools');
 
 //CLOUD
 const functions = require('@google-cloud/functions-framework');
+const { GoogleAuth } = require('google-auth-library');
+const CONCURRENCY = 1_000;
+let RUNTIME_URL = "https://dm4-lmozz6xkha-uc.a.run.app"; // IMPORTANT: this is what allows the service to call itself
+// const functionName = process.env.FUNCTION_NAME || process.env.K_SERVICE;
+
+// const region = process.env.REGION; // Optionally, you can get the region too
+// const GCP_PROJECT = process.env.GCLOUD_PROJECT; // Project ID is also available as an environment variable
+// const isCloudFunction = !!process.env.FUNCTION_NAME || !!process.env.FUNCTION_TARGET;
+// if (isCloudFunction) {
+// 	RUNTIME_URL = `https://${region}-${GCP_PROJECT}.cloudfunctions.net/${functionName}`;
+// }
+// else {
+// 	RUNTIME_URL = `http://localhost:8080`;
+// }
+
+
 
 // DEFAULTS
 const { campaigns, devices, locations } = require('./components/defaults.js');
@@ -74,7 +90,6 @@ let eventCount = 0;
 let userCount = 0;
 
 
-
 /**
  * generates fake mixpanel data
  * @param  {Config} config
@@ -336,60 +351,156 @@ async function main(config) {
 		...STORAGE,
 		importResults,
 		files,
+		operations,
+		eventCount,
+		userCount,
 		time: { start, end, delta, human },
 	};
 }
 
-
-
 functions.http('entry', async (req, res) => {
 	const reqTimer = timer('request');
 	reqTimer.start();
 	let response = {};
 	let script = req.body || "";
-	let writePath;
+	const params = { replicate: 1, is_replica: "false", runId: "", seed: "", ...req.query };
+	const replicate = Number(params.replicate);
+	// @ts-ignore
+	if (params?.is_replica === "true") params.is_replica = true;
+	// @ts-ignore
+	else params.is_replica = false;
+	const isReplica = params.is_replica;
+	isBATCH_MODE = true;
+	if (!params.runId) params.runId = uid(42);
 	try {
-		sLog("DM4: start");
 		if (!script) throw new Error("no script");
 
 		// Replace require("../ with require("./
-		script = script.replace(/require\("\.\.\//g, 'require("./');
+		// script = script.replace(/require\("\.\.\//g, 'require("./');
 		// ^ need to replace this because of the way the script is passed in... this is sketch
 
 		/** @type {Config} */
 		const config = eval(script);
-		sLog("DM4: eval ok");
-
-		const { token } = config;
-		if (!token) throw new Error("no token");
+		if (isReplica) {
+			const newSeed = (Math.random() / Math.random() / Math.random() / Math.random() / Math.random() / Math.random()).toString();
+			config.seed = newSeed;
+			params.seed = newSeed;
+		}
 
 		/** @type {Config} */
 		const optionsYouCantChange = {
-			verbose: false,
-			writeToDisk: false,
-
+			verbose: false
 		};
-		const result = await main({
-			...config,
-			...optionsYouCantChange,
-		});
-		await rm(writePath);
-		reqTimer.stop(false);
-		const { start, end, delta, human } = jobTimer.report(false);
-		sLog(`DM4: end (${human})`, { ms: delta });
+
+		if (replicate <= 1 || isReplica) {
+			if (isReplica) sLog("DM4: worker start", params);
+			// @ts-ignore
+			const { files = [], operations = 0, eventCount = 0, userCount = 0 } = await main({
+				...config,
+				...optionsYouCantChange,
+			});
+			reqTimer.stop(false);
+			response = { files, operations, eventCount, userCount };
+		}
+
+		else {
+			sLog(`DM4: job start (${replicate} workers)`, params);
+			const results = await spawn_file_workers(replicate, script, params);
+			response = results;
+		}
 	}
 	catch (e) {
-		sLog("DM4: error", { error: e.message });
+		sLog("DM4: error", { error: e.message, stack: e.stack }, "ERROR");
 		response = { error: e.message };
 		res.status(500);
-		await rm(writePath);
 	}
+
 	finally {
+		reqTimer.stop(false);
+		const { start, end, delta, human } = reqTimer.report(false);
+		if (!isReplica) {
+			sLog(`DM4: job end (${human})`, { human, delta, ...params, ...response });
+		}
+		if (isReplica) {
+			const eps = Math.floor(((response?.eventCount || 0) / delta) * 1000);
+			sLog(`DM4: worker end (${human})`, { human, delta, eps, ...params, ...response });
+		}
+		response = { ...response, start, end, delta, human, ...params };
 		res.send(response);
+		return;
 	}
 });
 
 
+/**
+ * @typedef {import('mixpanel-import').ImportResults} ImportResults
+ */
+async function spawn_file_workers(numberWorkers, payload, params) {
+	const auth = new GoogleAuth();
+	let client;
+	if (RUNTIME_URL.includes('localhost')) {
+		client = await auth.getClient();
+	}
+	else {
+		client = await auth.getIdTokenClient(RUNTIME_URL);
+	}
+	const limit = pLimit(CONCURRENCY);
+	const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+	const requestPromises = Array.from({ length: numberWorkers }, async (_, index) => {
+		index = index + 1;
+		await delay(index * 108);
+		return limit(() => build_request(client, payload, index, params, numberWorkers));
+	});
+	const complete = await Promise.allSettled(requestPromises);
+	const results = {
+		jobs_success: complete.filter((p) => p.status === "fulfilled").length,
+		jobs_fail: complete.filter((p) => p.status === "rejected").length
+	};
+
+	return results;
+}
+
+
+async function build_request(client, payload, index, params, total) {
+	let retryAttempt = 0;
+	sLog(`DM4: summoning worker #${index} of ${total}`, params);
+	try {
+		const req = await client.request({
+			url: RUNTIME_URL + `?replicate=1&is_replica=true&runId=${params.runId || "no run id"}`,
+			method: "POST",
+			data: payload,
+			headers: {
+				"Content-Type": "text/plain",
+			},
+			timeout: 3600 * 1000 * 10,
+			retryConfig: {
+				retry: 3,
+				onRetryAttempt: (error) => {
+					const statusCode = error?.response?.status?.toString() || "";
+					retryAttempt++;
+					sLog(`DM4: summon worker ${index} retry #${retryAttempt}`, { statusCode, message: error.message, stack: error.stack, ...params }, "DEBUG");
+				},
+				retryDelay: 1000,
+				shouldRetry: (error) => {
+					if (error.code === 'ECONNRESET') return true;
+					const statusCode = error?.response?.status;
+					if (statusCode >= 500) return true;
+					if (statusCode === 429) return true;
+				}
+			},
+		});
+		sLog(`DM4: worker #${index} responded`, params);
+		const { data } = req;
+		return data;
+	} catch (error) {
+		sLog(`DM4: worker #${index} failed to respond`, { message: error.message, stack: error.stack, code: error.code, retries: retryAttempt, ...params }, "ERROR");
+		return {};
+	}
+}
+
+
+
 /*
 ------
 MODELS
@@ -1393,9 +1504,9 @@ function validateDungeonConfig(config) {
 	// funnels
 
 	// FUNNEL INFERENCE
-	if (!funnels || !funnels.length) {
-		funnels = inferFunnels(events);
-	}
+	// if (!funnels || !funnels.length) {
+	// 	funnels = inferFunnels(events);
+	// }
 
 	if (alsoInferFunnels) {
 		const inferredFunnels = inferFunnels(events);
@@ -1502,13 +1613,21 @@ async function makeHookArray(arr = [], opts = {}) {
 	if (existsSync(dataFolder)) writeDir = dataFolder;
 	else writeDir = path.resolve("./");
 
-	if (NODE_ENV === "prod") writeDir = path.resolve(os.tmpdir());
+	// ! decide where to write the files in prod
+	if (NODE_ENV === "prod") {
+		writeDir = path.resolve(os.tmpdir());
+	}
+	if (typeof rest?.config?.writeToDisk === "string" && rest?.config?.writeToDisk?.startsWith('gs://')) {
+		writeDir = rest.config.writeToDisk;
+	}
 
 	function getWritePath() {
 		if (isBATCH_MODE) {
+			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}-part-${batch.toString()}.${format}`;
 			return path.join(writeDir, `${filepath}-part-${batch.toString()}.${format}`);
 		}
 		else {
+			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}.${format}`;
 			return path.join(writeDir, `${filepath}.${format}`);
 		}
 	}
@@ -1570,6 +1689,7 @@ async function makeHookArray(arr = [], opts = {}) {
 		}
 		if (isBATCH_MODE) data.length = 0;
 		return writeResult;
+
 	}
 
 	async function flush() {