make-mp-data 1.5.53 → 1.5.55

package/index.js

@@ -23,7 +23,7 @@ global.FIXED_NOW = FIXED_NOW;
 // ^ this creates a FIXED POINT in time; we will shift it later
 let FIXED_BEGIN = dayjs.unix(FIXED_NOW).subtract(90, 'd').unix();
 global.FIXED_BEGIN = FIXED_BEGIN;
-const actualNow = dayjs();
+const actualNow = dayjs().add(2, "day");
 const timeShift = actualNow.diff(dayjs.unix(FIXED_NOW), "seconds");
 const daysShift = actualNow.diff(dayjs.unix(FIXED_NOW), "days");
 
@@ -45,6 +45,22 @@ const t = require('ak-tools');
 
 //CLOUD
 const functions = require('@google-cloud/functions-framework');
+const { GoogleAuth } = require('google-auth-library');
+const CONCURRENCY = 1_000;
+let RUNTIME_URL = "https://dm4-lmozz6xkha-uc.a.run.app"; // IMPORTANT: this is what allows the service to call itself
+// const functionName = process.env.FUNCTION_NAME || process.env.K_SERVICE;
+
+// const region = process.env.REGION; // Optionally, you can get the region too
+// const GCP_PROJECT = process.env.GCLOUD_PROJECT; // Project ID is also available as an environment variable
+// const isCloudFunction = !!process.env.FUNCTION_NAME || !!process.env.FUNCTION_TARGET;
+// if (isCloudFunction) {
+// 	RUNTIME_URL = `https://${region}-${GCP_PROJECT}.cloudfunctions.net/${functionName}`;
+// }
+// else {
+// 	RUNTIME_URL = `http://localhost:8080`;
+// }
+
+
 
 // DEFAULTS
 const { campaigns, devices, locations } = require('./components/defaults.js');
@@ -74,7 +90,6 @@ let eventCount = 0;
 let userCount = 0;
 
 
-
 /**
  * generates fake mixpanel data
  * @param  {Config} config
@@ -104,6 +119,16 @@ async function main(config) {
 		campaigns: u.pickAWinner(campaigns, 0),
 	};
 
+	if (config.singleCountry) {
+		DEFAULTS.locationsEvents = u.pickAWinner(clone(locations)
+			.filter(l => l.country === config.singleCountry)
+			.map(l => { delete l.country; return l; }), 0);
+
+		DEFAULTS.locationsUsers = u.pickAWinner(clone(locations)
+			.filter(l => l.country === config.singleCountry)
+			.map(l => { delete l.country_code; return l; }), 0);
+	}
+
 
 	//TRACKING
 	const runId = uid(42);
@@ -326,60 +351,156 @@ async function main(config) {
 		...STORAGE,
 		importResults,
 		files,
+		operations,
+		eventCount,
+		userCount,
 		time: { start, end, delta, human },
 	};
 }
 
-
-
 functions.http('entry', async (req, res) => {
 	const reqTimer = timer('request');
 	reqTimer.start();
 	let response = {};
 	let script = req.body || "";
-	let writePath;
+	const params = { replicate: 1, is_replica: "false", runId: "", seed: "", ...req.query };
+	const replicate = Number(params.replicate);
+	// @ts-ignore
+	if (params?.is_replica === "true") params.is_replica = true;
+	// @ts-ignore
+	else params.is_replica = false;
+	const isReplica = params.is_replica;
+	isBATCH_MODE = true;
+	if (!params.runId) params.runId = uid(42);
 	try {
-		sLog("DM4: start");
 		if (!script) throw new Error("no script");
 
 		// Replace require("../ with require("./
-		script = script.replace(/require\("\.\.\//g, 'require("./');
+		// script = script.replace(/require\("\.\.\//g, 'require("./');
 		// ^ need to replace this because of the way the script is passed in... this is sketch
 
 		/** @type {Config} */
 		const config = eval(script);
-		sLog("DM4: eval ok");
-
-		const { token } = config;
-		if (!token) throw new Error("no token");
+		if (isReplica) {
+			const newSeed = (Math.random() / Math.random() / Math.random() / Math.random() / Math.random() / Math.random()).toString();
+			config.seed = newSeed;
+			params.seed = newSeed;
+		}
 
 		/** @type {Config} */
 		const optionsYouCantChange = {
-			verbose: false,
-			writeToDisk: false,
-
+			verbose: false
 		};
-		const result = await main({
-			...config,
-			...optionsYouCantChange,
-		});
-		await rm(writePath);
-		reqTimer.stop(false);
-		const { start, end, delta, human } = jobTimer.report(false);
-		sLog(`DM4: end (${human})`, { ms: delta });
+
+		if (replicate <= 1 || isReplica) {
+			if (isReplica) sLog("DM4: worker start", params);
+			// @ts-ignore
+			const { files = [], operations = 0, eventCount = 0, userCount = 0 } = await main({
+				...config,
+				...optionsYouCantChange,
+			});
+			reqTimer.stop(false);
+			response = { files, operations, eventCount, userCount };
+		}
+
+		else {
+			sLog(`DM4: job start (${replicate} workers)`, params);
+			const results = await spawn_file_workers(replicate, script, params);
+			response = results;
+		}
 	}
 	catch (e) {
-		sLog("DM4: error", { error: e.message });
+		sLog("DM4: error", { error: e.message, stack: e.stack }, "ERROR");
 		response = { error: e.message };
 		res.status(500);
-		await rm(writePath);
 	}
+
 	finally {
+		reqTimer.stop(false);
+		const { start, end, delta, human } = reqTimer.report(false);
+		if (!isReplica) {
+			sLog(`DM4: job end (${human})`, { human, delta, ...params, ...response });
+		}
+		if (isReplica) {
+			const eps = Math.floor(((response?.eventCount || 0) / delta) * 1000);
+			sLog(`DM4: worker end (${human})`, { human, delta, eps, ...params, ...response });
+		}
+		response = { ...response, start, end, delta, human, ...params };
 		res.send(response);
+		return;
 	}
 });
 
 
+/**
+ * @typedef {import('mixpanel-import').ImportResults} ImportResults
+ */
+async function spawn_file_workers(numberWorkers, payload, params) {
+	const auth = new GoogleAuth();
+	let client;
+	if (RUNTIME_URL.includes('localhost')) {
+		client = await auth.getClient();
+	}
+	else {
+		client = await auth.getIdTokenClient(RUNTIME_URL);
+	}
+	const limit = pLimit(CONCURRENCY);
+	const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+	const requestPromises = Array.from({ length: numberWorkers }, async (_, index) => {
+		index = index + 1;
+		await delay(index * 108);
+		return limit(() => build_request(client, payload, index, params, numberWorkers));
+	});
+	const complete = await Promise.allSettled(requestPromises);
+	const results = {
+		jobs_success: complete.filter((p) => p.status === "fulfilled").length,
+		jobs_fail: complete.filter((p) => p.status === "rejected").length
+	};
+
+	return results;
+}
+
+
+async function build_request(client, payload, index, params, total) {
+	let retryAttempt = 0;
+	sLog(`DM4: summoning worker #${index} of ${total}`, params);
+	try {
+		const req = await client.request({
+			url: RUNTIME_URL + `?replicate=1&is_replica=true&runId=${params.runId || "no run id"}`,
+			method: "POST",
+			data: payload,
+			headers: {
+				"Content-Type": "text/plain",
+			},
+			timeout: 3600 * 1000 * 10,
+			retryConfig: {
+				retry: 3,
+				onRetryAttempt: (error) => {
+					const statusCode = error?.response?.status?.toString() || "";
+					retryAttempt++;
+					sLog(`DM4: summon worker ${index} retry #${retryAttempt}`, { statusCode, message: error.message, stack: error.stack, ...params }, "DEBUG");
+				},
+				retryDelay: 1000,
+				shouldRetry: (error) => {
+					if (error.code === 'ECONNRESET') return true;
+					const statusCode = error?.response?.status;
+					if (statusCode >= 500) return true;
+					if (statusCode === 429) return true;
+				}
+			},
+		});
+		sLog(`DM4: worker #${index} responded`, params);
+		const { data } = req;
+		return data;
+	} catch (error) {
+		sLog(`DM4: worker #${index} failed to respond`, { message: error.message, stack: error.stack, code: error.code, retries: retryAttempt, ...params }, "ERROR");
+		return {};
+	}
+}
+
+
+
 /*
 ------
 MODELS
@@ -512,7 +633,7 @@ async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, s
 			else if (typeof (defaultProps[key]) === "object") {
 				const obj = defaultProps[key];
 				for (const subKey in obj) {
-					if (Array.isArray(obj[subKey])) { 
+					if (Array.isArray(obj[subKey])) {
 						const subChoice = u.choose(obj[subKey]);
 						if (Array.isArray(subChoice)) {
 							for (const subSubChoice of subChoice) {
@@ -525,7 +646,7 @@ async function makeEvent(distinct_id, earliestTime, chosenEvent, anonymousIds, s
 							}
 						}
 						else {
-						if (!eventTemplate[subKey]) eventTemplate[subKey] = subChoice;
+							if (!eventTemplate[subKey]) eventTemplate[subKey] = subChoice;
 						}
 					}
 					else {
@@ -1156,7 +1277,8 @@ async function sendToMixpanel(config, storage) {
 		fixData: true,
 		verbose: false,
 		forceStream: true,
-		strict: false,
+		strict: true, //false,
+		epochEnd: dayjs().unix(), //is this chill?
 		dryRun: false,
 		abridged: false,
 		fixJson: true,
@@ -1382,9 +1504,9 @@ function validateDungeonConfig(config) {
 	// funnels
 
 	// FUNNEL INFERENCE
-	if (!funnels || !funnels.length) {
-		funnels = inferFunnels(events);
-	}
+	// if (!funnels || !funnels.length) {
+	// 	funnels = inferFunnels(events);
+	// }
 
 	if (alsoInferFunnels) {
 		const inferredFunnels = inferFunnels(events);
@@ -1491,13 +1613,21 @@ async function makeHookArray(arr = [], opts = {}) {
 	if (existsSync(dataFolder)) writeDir = dataFolder;
 	else writeDir = path.resolve("./");
 
-	if (NODE_ENV === "prod") writeDir = path.resolve(os.tmpdir());
+	// ! decide where to write the files in prod
+	if (NODE_ENV === "prod") {
+		writeDir = path.resolve(os.tmpdir());
+	}
+	if (typeof rest?.config?.writeToDisk === "string" && rest?.config?.writeToDisk?.startsWith('gs://')) {
+		writeDir = rest.config.writeToDisk;
+	}
 
 	function getWritePath() {
 		if (isBATCH_MODE) {
+			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}-part-${batch.toString()}.${format}`;
 			return path.join(writeDir, `${filepath}-part-${batch.toString()}.${format}`);
 		}
 		else {
+			if (writeDir?.startsWith('gs://')) return `${writeDir}/${filepath}.${format}`;
 			return path.join(writeDir, `${filepath}.${format}`);
 		}
 	}
@@ -1559,6 +1689,7 @@ async function makeHookArray(arr = [], opts = {}) {
 		}
 		if (isBATCH_MODE) data.length = 0;
 		return writeResult;
+
 	}
 
 	async function flush() {
@@ -1703,8 +1834,8 @@ if (NODE_ENV !== "prod") {
 				};
 				if (bytes > 0) console.table(stats);
 				if (Object.keys(data?.importResults || {}).length) {
-				log(`\nlog written to log.json\n`);
-				writeFileSync(path.resolve(folder, "log.json"), JSON.stringify(data?.importResults, null, 2));
+					log(`\nlog written to log.json\n`);
+					writeFileSync(path.resolve(folder, "log.json"), JSON.stringify(data?.importResults, null, 2));
 				}
 				// log("  " + files?.flat().join("\n  "));
 				log(`\n----------------SUMMARY-----------------\n\n\n`);