make-mp-data 2.0.0 → 2.0.2

package/lib/utils/utils.js

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import Chance from 'chance';
 import readline from 'readline';
-import { comma, uid, clone } from 'ak-tools';
+import { comma, uid } from 'ak-tools';
 import { spawn } from 'child_process';
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc.js';
@@ -186,10 +186,48 @@ function day(start, end) {
 function choose(value) {
 	const chance = getChance();
 
+	// most of the time this will receive a list of strings; 
+	// when that is the case, we need to ensure some 'keywords' like 'variant' or 'test' aren't in the array
+	// next we want to see if the array is unweighted ... i.e. no dupe strings and each string only occurs once ['a', 'b', 'c', 'd']
+	// if all these are true we will pickAWinner(value)()
+	if (Array.isArray(value) && value.length > 0 && value.every(item => typeof item === 'string')) {
+		// ensure terms 'variant' 'group' 'experiment' or 'population' are NOT in any of the items
+		if (!value.some(item => item.includes('variant') || item.includes('group') || item.includes('experiment') || item.includes('population'))) {
+			// check to make sure that each element in the array only occurs once...
+			const uniqueItems = new Set(value);
+			if (uniqueItems.size === value.length) {
+				// Array has no duplicates, use pickAWinner
+				return pickAWinner(value, 0)();
+			}
+
+		}
+
+
+
+	}
+
+
 	try {
-		// Keep resolving the value if it's a function
+		// Keep resolving the value if it's a function (with caching)
 		while (typeof value === 'function') {
-			value = value();
+			const funcString = value.toString();
+
+			// Check cache for weighted array functions
+			if (typeof global.weightedArrayCache === 'undefined') {
+				global.weightedArrayCache = new Map();
+			}
+
+			if (global.weightedArrayCache.has(funcString)) {
+				value = global.weightedArrayCache.get(funcString);
+				break;
+			}
+
+			const result = value();
+			if (Array.isArray(result) && result.length > 10) {
+				// Cache large arrays (likely weighted arrays)
+				global.weightedArrayCache.set(funcString, result);
+			}
+			value = result;
 		}
 
 		if (Array.isArray(value) && value.length === 0) {
@@ -201,13 +239,12 @@ function choose(value) {
 			return chance.pickone(value);
 		}
 
-		// Now, if the resolved value is an array, use chance.pickone
-		if (Array.isArray(value) && value.every(item => typeof item === 'string')) {
-			return chance.pickone(value);
-		}
-
-		if (Array.isArray(value) && value.every(item => typeof item === 'number')) {
-			return chance.pickone(value);
+		// PERFORMANCE: Optimized array handling - check first item type instead of every()
+		if (Array.isArray(value) && value.length > 0) {
+			const firstType = typeof value[0];
+			if (firstType === 'string' || firstType === 'number') {
+				return chance.pickone(value);
+			}
 		}
 
 		if (Array.isArray(value) && value.every(item => typeof item === 'object')) {
@@ -666,6 +703,21 @@ function pickAWinner(items, mostChosenIndex) {
 	};
 }
 
+function quickHash(str, seed = 0) {
+	let h1 = 0xdeadbeef ^ seed, h2 = 0x41c6ce57 ^ seed;
+	for (let i = 0, ch; i < str.length; i++) {
+		ch = str.charCodeAt(i);
+		h1 = Math.imul(h1 ^ ch, 2654435761);
+		h2 = Math.imul(h2 ^ ch, 1597334677);
+	}
+	h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507);
+	h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909);
+	h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507);
+	h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909);
+
+	return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString();
+};
+
 /*
 ----
 SHUFFLERS
@@ -677,11 +729,20 @@ function shuffleArray(array) {
 	const chance = getChance();
 	for (let i = array.length - 1; i > 0; i--) {
 		const j = chance.integer({ min: 0, max: i });
-		[array[i], array[j]] = [array[j], array[i]];
+		const temp = array[i];
+		array[i] = array[j];
+		array[j] = temp;
 	}
 	return array;
 }
 
+function pickRandom(array) {
+	if (!array || array.length === 0) return undefined;
+	// PERFORMANCE: Use Math.random() instead of chance.integer() for simple cases
+	const randomIndex = Math.floor(Math.random() * array.length);
+	return array[randomIndex];
+}
+
 function shuffleExceptFirst(array) {
 	if (array.length <= 1) return array;
 	const restShuffled = shuffleArray(array.slice(1));
@@ -877,16 +938,26 @@ function buildFileNames(config) {
  * @param  {[string, number][]} arrayOfArrays
  */
 function progress(arrayOfArrays) {
+	const terminalWidth = process.stdout.columns || 120;
+
+	// Clear the entire line
 	readline.cursorTo(process.stdout, 0);
-	let message = "";
-	for (const status of arrayOfArrays) {
-		const [thing, p] = status;
-		message += `${thing} processed: ${comma(p)}\t\t`;
-	}
+	readline.clearLine(process.stdout, 0);
 
-	process.stdout.write(message);
-};
+	// Build message with better formatting
+	const items = arrayOfArrays.map(([thing, p]) => {
+		return `${thing}: ${comma(p)}`;
+	});
+
+	const message = items.join(' │ ');
 
+	// Ensure we don't exceed terminal width
+	const finalMessage = message.length > terminalWidth
+		? message.substring(0, terminalWidth - 3) + '...'
+		: message.padEnd(terminalWidth, ' ');
+
+	process.stdout.write(finalMessage);
+}
 
 function openFinder(path, callback) {
 	path = path || '/';
@@ -906,7 +977,6 @@ function getUniqueKeys(data) {
 };
 
 
-
 /*
 ----
 CORE
@@ -953,7 +1023,7 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 	let totalRange = latestTime - earliestTime;
 	if (totalRange <= 0 || earliestTime > latestTime) {
 		//just flip earliest and latest
-		let tempEarly = latestTime
+		let tempEarly = latestTime;
 		let tempLate = earliestTime;
 		earliestTime = tempEarly;
 		latestTime = tempLate;
@@ -967,23 +1037,22 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 	const chunkEnd = chunkStart + chunkSize;
 	const chunkMid = (chunkStart + chunkEnd) / 2;
 
-	// Generate a single timestamp within this chunk using a normal distribution centered at chunkMid
-	let offset;
-	let iterations = 0;
-	let isValidTime = false;
-	do {
-		iterations++;
-		soupHits++;
-		offset = chance.normal({ mean: mean, dev: chunkSize / deviation });
-		isValidTime = validTime(chunkMid + offset, earliestTime, latestTime);
-		if (iterations > 25000) {
-			if (process.env?.NODE_ENV === 'dev') debugger;
-			throw `${iterations} iterations... exceeded`;
-		}
-	} while (chunkMid + offset < chunkStart || chunkMid + offset > chunkEnd);
+	// Optimized timestamp generation - clamp to valid range instead of looping
+	const maxDeviation = chunkSize / deviation;
+	let offset = chance.normal({ mean: mean, dev: maxDeviation });
+
+	// Clamp to chunk boundaries to prevent infinite loops
+	const proposedTime = chunkMid + offset;
+	const clampedTime = Math.max(chunkStart, Math.min(chunkEnd, proposedTime));
+
+	// Ensure it's within the overall valid range
+	const finalTime = Math.max(earliestTime, Math.min(latestTime, clampedTime));
+
+	// Update soup hits counter (keep for compatibility)
+	soupHits++;
 
 	try {
-		return dayjs.unix(chunkMid + offset).toISOString();
+		return dayjs.unix(finalTime).toISOString();
 	}
 
 	catch (e) {
@@ -994,8 +1063,6 @@ function TimeSoup(earliestTime, latestTime, peaks = 5, deviation = 2, mean = 0)
 }
 
 
-
-
 /**
  * @param {string} userId
  * @param  {number} bornDaysAgo=30
@@ -1151,6 +1218,61 @@ function generateEmoji(max = 10, array = false) {
 	};
 };
 
+function deepClone(thing, opts) {
+	// Handle primitives first (most common case)
+	if (thing === null || thing === undefined) return thing;
+
+	const type = typeof thing;
+	if (type !== 'object' && type !== 'function') {
+		if (type === 'symbol') {
+			return Symbol(thing.description);
+		}
+		return thing;
+	}
+
+	// Handle arrays (common case)
+	if (Array.isArray(thing)) {
+		const result = new Array(thing.length);
+		for (let i = 0; i < thing.length; i++) {
+			result[i] = deepClone(thing[i], opts);
+		}
+		return result;
+	}
+
+	// Handle other object types
+	if (thing instanceof Date) return new Date(thing.getTime());
+	if (thing instanceof RegExp) return new RegExp(thing.source, thing.flags);
+	if (thing instanceof Function) {
+		return opts && opts.newFns ?
+			new Function('return ' + thing.toString())() :
+			thing;
+	}
+
+	// Handle plain objects
+	if (thing.constructor === Object) {
+		const newObject = {};
+		const keys = Object.keys(thing);
+		for (let i = 0; i < keys.length; i++) {
+			const key = keys[i];
+			newObject[key] = deepClone(thing[key], opts);
+		}
+		return newObject;
+	}
+
+	// Handle other object types
+	try {
+		return new thing.constructor(thing);
+	} catch (e) {
+		// Fallback for objects that can't be constructed this way
+		const newObject = Object.create(Object.getPrototypeOf(thing));
+		const keys = Object.keys(thing);
+		for (let i = 0; i < keys.length; i++) {
+			const key = keys[i];
+			newObject[key] = deepClone(thing[key], opts);
+		}
+		return newObject;
+	}
+};
 
 
 export {
@@ -1159,13 +1281,14 @@ export {
 	dates,
 	day,
 	choose,
+	pickRandom,
 	exhaust,
 	integer,
 	TimeSoup,
 	companyName,
 	generateEmoji,
 	hasSameKeys as haveSameKeys,
-
+	deepClone,
 	initChance,
 	getChance,
 
@@ -1182,6 +1305,7 @@ export {
 	getUniqueKeys,
 	person,
 	pickAWinner,
+	quickHash,
 	weighArray,
 	weighFunnels,
 	validateEventConfig,

package/package.json

@@ -1,10 +1,20 @@
 {
 	"name": "make-mp-data",
-	"version": "2.0.0",
+	"version": "2.0.2",
 	"description": "builds all mixpanel primitives for a given project",
 	"type": "module",
 	"main": "index.js",
 	"types": "types.d.ts",
+	"files": [
+		"index.js",
+		"cli.js",
+		"types.d.ts",
+		"lib/",
+		"dungeons/",
+		"!dungeons/customers/",
+		"package.json",
+		"README.md"
+	],
 	"scripts": {
 		"start": "node ./index.js",
 		"dev": "nodemon scratch.mjs --ignore ./data/*",
@@ -12,12 +22,16 @@
 		"post": "npm publish",
 		"deps": "./scripts/update-deps.sh",
 		"test": "NODE_ENV=test vitest run",
-		"test:watch": "NODE_ENV=test vitest",
-		"test:ui": "NODE_ENV=test vitest --ui",
 		"coverage": "vitest run --coverage && open ./coverage/index.html",
+		"typecheck": "tsc --noEmit",
+		"typecheck:build": "tsc --noEmit --project tsconfig.build.json",
+		"typecheck:strict": "echo '⚠️  Running pedantic strict checks (many false positives expected)...' && tsc --noEmit --project tsconfig.build.json --strict",
+		"typecheck:summary": "echo '🔍 Type-checking all shipped files...' && npm run typecheck:build && echo '✅ All TypeScript checks passed!'",
 		"new:dungeon": "./scripts/new-dungeon.sh",
 		"new:project": "node ./scripts/new-project.mjs",
-		"exp:benchmark": "node --no-warnings --experimental-vm-modules ./tests/benchmark/concurrency.mjs",
+		"exp:benchmark": "node ./tests/benchmark/concurrency.mjs",
+		"benchmark:phase1": "node ./tests/benchmark/phase1-performance.mjs",
+		"test:quick": "node ./tests/benchmark/quick-test.mjs",
 		"exp:soup": "node ./tests/testSoup.mjs",
 		"func:local": "functions-framework --target=entry",
 		"func:deploy": "./scripts/deploy.sh"
@@ -27,7 +41,7 @@
 		"url": "git+https://github.com/ak--47/make-mp-data.git"
 	},
 	"bin": {
-		"make-mp-data": "./index.js"
+		"make-mp-data": "./entry.js"
 	},
 	"keywords": [
 		"mixpanel",
@@ -50,10 +64,9 @@
 	"dependencies": {
 		"@google-cloud/functions-framework": "^3.4.2",
 		"@google-cloud/storage": "^7.14.0",
-		"@google/generative-ai": "^0.24.1",
-		"ak-fetch": "^2.0.1",
-		"ak-gemini": "^1.0.55",
-		"ak-tools": "^1.1.0",
+		"ak-fetch": "^2.0.12",
+		"ak-gemini": "^1.0.59",
+		"ak-tools": "^1.1.1",
 		"chance": "^1.1.11",
 		"chart.js": "^3.9.1",
 		"chartjs-node-canvas": "^4.1.6",
@@ -67,6 +80,7 @@
 	"devDependencies": {
 		"@vitest/ui": "^2.1.9",
 		"nodemon": "^3.1.3",
+		"typescript": "^5.6.0",
 		"vitest": "^2.1.9"
 	},
 	"nodemonConfig": {