make-mp-data 2.0.0 → 2.0.2

package/lib/orchestrators/mixpanel-sender.js

@@ -7,7 +7,8 @@
 
 import dayjs from "dayjs";
 import path from "path";
-import { clone, comma, ls, rm } from "ak-tools";
+import { comma, ls, rm } from "ak-tools";
+import * as u from "../utils/utils.js";
 import mp from "mixpanel-import";
 
 /**
@@ -67,10 +68,11 @@ export async function sendToMixpanel(context) {
     // Import events
     if (eventData || isBATCH_MODE) {
         log(`importing events to mixpanel...\n`);
-        let eventDataToImport = clone(eventData);
+        let eventDataToImport = u.deepClone(eventData);
         if (isBATCH_MODE) {
             const writeDir = eventData.getWriteDir();
             const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+            // @ts-ignore
             eventDataToImport = files.filter(f => f.includes('-EVENTS-'));
         }
         const imported = await mp(creds, eventDataToImport, {
@@ -84,10 +86,11 @@ export async function sendToMixpanel(context) {
     // Import user profiles
     if (userProfilesData || isBATCH_MODE) {
         log(`importing user profiles to mixpanel...\n`);
-        let userProfilesToImport = clone(userProfilesData);
+        let userProfilesToImport = u.deepClone(userProfilesData);
         if (isBATCH_MODE) {
             const writeDir = userProfilesData.getWriteDir();
             const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+            // @ts-ignore
             userProfilesToImport = files.filter(f => f.includes('-USERS-'));
         }
         const imported = await mp(creds, userProfilesToImport, {
@@ -101,10 +104,11 @@ export async function sendToMixpanel(context) {
     // Import ad spend data  
     if (groupEventData || isBATCH_MODE) {
         log(`importing ad spend data to mixpanel...\n`);
-        let adSpendDataToImport = clone(adSpendData);
+        let adSpendDataToImport = u.deepClone(adSpendData);
         if (isBATCH_MODE) {
             const writeDir = adSpendData.getWriteDir();
             const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+            // @ts-ignore
             adSpendDataToImport = files.filter(f => f.includes('-AD-SPEND-'));
         }
         const imported = await mp(creds, adSpendDataToImport, {
@@ -120,10 +124,11 @@ export async function sendToMixpanel(context) {
         for (const groupEntity of groupProfilesData) {
             const groupKey = groupEntity?.groupKey;
             log(`importing ${groupKey} profiles to mixpanel...\n`);
-            let groupProfilesToImport = clone(groupEntity);
+            let groupProfilesToImport = u.deepClone(groupEntity);
             if (isBATCH_MODE) {
                 const writeDir = groupEntity.getWriteDir();
                 const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+                // @ts-ignore
                 groupProfilesToImport = files.filter(f => f.includes(`-GROUPS-${groupKey}`));
             }
             const imported = await mp({ token, groupKey }, groupProfilesToImport, {
@@ -138,10 +143,11 @@ export async function sendToMixpanel(context) {
     // Import group events
     if (groupEventData || isBATCH_MODE) {
         log(`importing group events to mixpanel...\n`);
-        let groupEventDataToImport = clone(groupEventData);
+        let groupEventDataToImport = u.deepClone(groupEventData);
         if (isBATCH_MODE) {
             const writeDir = groupEventData.getWriteDir();
             const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+            // @ts-ignore
             groupEventDataToImport = files.filter(f => f.includes('-GROUP-EVENTS-'));
         }
         const imported = await mp(creds, groupEventDataToImport, {
@@ -160,10 +166,11 @@ export async function sendToMixpanel(context) {
             for (const scdEntity of scdTableData) {
                 const scdKey = scdEntity?.scdKey;
                 log(`importing ${scdKey} SCD data to mixpanel...\n`);
-                let scdDataToImport = clone(scdEntity);
+                let scdDataToImport = u.deepClone(scdEntity);
                 if (isBATCH_MODE) {
                     const writeDir = scdEntity.getWriteDir();
                     const files = await ls(writeDir.split(path.basename(writeDir)).join(""));
+                    // @ts-ignore
                     scdDataToImport = files.filter(f => f.includes(`-SCD-${scdKey}`));
                 }
 
@@ -198,6 +205,7 @@ export async function sendToMixpanel(context) {
     if (!writeToDisk && isBATCH_MODE) {
         const writeDir = eventData?.getWriteDir() || userProfilesData?.getWriteDir();
         const listDir = await ls(writeDir.split(path.basename(writeDir)).join(""));
+        // @ts-ignore
         const files = listDir.filter(f => 
             f.includes('-EVENTS-') || 
             f.includes('-USERS-') || 

package/lib/orchestrators/user-loop.js

@@ -7,6 +7,7 @@
 
 import dayjs from "dayjs";
 import pLimit from 'p-limit';
+import os from 'os';
 import * as u from "../utils/utils.js";
 import * as t from 'ak-tools';
 import { makeEvent } from "../generators/events.js";
@@ -22,7 +23,7 @@ import { makeSCD } from "../generators/scd.js";
 export async function userLoop(context) {
     const { config, storage, defaults } = context;
     const chance = u.getChance();
-    const concurrency = config?.concurrency || 1;
+    const concurrency = config?.concurrency || Math.min(os.cpus().length * 2, 16);
     const USER_CONN = pLimit(concurrency);
     
     const {
@@ -45,8 +46,12 @@ export async function userLoop(context) {
     const avgEvPerUser = numEvents / numUsers;
     const startTime = Date.now();
 
+    // Create batches for parallel processing
+    const batchSize = Math.max(1, Math.ceil(numUsers / concurrency));
+    const userPromises = [];
+    
     for (let i = 0; i < numUsers; i++) {
-        await USER_CONN(async () => {
+        const userPromise = USER_CONN(async () => {
             context.incrementUserCount();
             const eps = Math.floor(context.getEventCount() / ((Date.now() - startTime) / 1000));
             
@@ -73,7 +78,7 @@ export async function userLoop(context) {
                 : dayjs.unix(global.FIXED_BEGIN);
 
             if (hasLocation) {
-                const location = u.shuffleArray(u.choose(defaults.locationsUsers)).pop();
+                const location = u.pickRandom(u.choose(defaults.locationsUsers));
                 for (const key in location) {
                     user[key] = location[key];
                 }
@@ -83,6 +88,7 @@ export async function userLoop(context) {
             const profile = await makeUserProfile(context, userProps, user);
 
             // SCD creation
+            // @ts-ignore
             const scdUserTables = t.objFilter(scdProps, (scd) => scd.type === 'user' || !scd.type);
             const scdTableKeys = Object.keys(scdUserTables);
 
@@ -129,12 +135,12 @@ export async function userLoop(context) {
                 const timeShift = context.getTimeShift();
                 userFirstEventTime = dayjs(data[0].time).subtract(timeShift, 'seconds').unix();
                 numEventsPreformed += data.length;
-                usersEvents.push(...data);
+                usersEvents = usersEvents.concat(data);
                 
                 if (!userConverted) {
-                    if (verbose) {
-                        u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
-                    }
+                    // if (verbose) {
+                    //     u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
+                    // }
                     return;
                 }
             } else {
@@ -146,11 +152,11 @@ export async function userLoop(context) {
                     const currentFunnel = chance.pickone(usageFunnels);
                     const [data, userConverted] = await makeFunnel(context, currentFunnel, user, userFirstEventTime, profile, userSCD);
                     numEventsPreformed += data.length;
-                    usersEvents.push(...data);
+                    usersEvents = usersEvents.concat(data);
                 } else {
                     const data = await makeEvent(context, distinct_id, userFirstEventTime, u.pick(config.events), user.anonymousIds, user.sessionIds, {}, config.groupKeys, true);
                     numEventsPreformed++;
-                    usersEvents.push(data);
+                    usersEvents = usersEvents.concat(data);
                 }
             }
 
@@ -187,8 +193,13 @@ export async function userLoop(context) {
             await eventData.hookPush(usersEvents, { profile });
 
             if (verbose) {
-                u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
+                // u.progress([["users", context.getUserCount()], ["events", context.getEventCount()]]);
             }
         });
+        
+        userPromises.push(userPromise);
     }
+    
+    // Wait for all users to complete
+    await Promise.all(userPromises);
 }

package/lib/orchestrators/worker-manager.js

@@ -3,6 +3,9 @@
  * Handles distributed processing across multiple cloud function workers
  */
 
+/** @typedef {import('../../types.js').Context} Context */
+/** @typedef {import('../../types.js').Dungeon} Dungeon */
+
 import pLimit from 'p-limit';
 import { GoogleAuth } from 'google-auth-library';
 import { timer, uid, sLog } from 'ak-tools';
@@ -138,7 +141,7 @@ export async function handleCloudFunctionEntry(req, res, mainFunction) {
     try {
         if (!script) throw new Error("no script");
 
-        /** @type {Config} */
+        /** @type {Dungeon} */
         const config = eval(script);
         
         if (isReplica) {
@@ -148,7 +151,7 @@ export async function handleCloudFunctionEntry(req, res, mainFunction) {
             params.seed = newSeed;
         }
 
-        /** @type {Config} */
+        /** @type {Dungeon} */
         const optionsYouCantChange = {
             verbose: false
         };

package/lib/utils/ai.js

@@ -1,89 +1,62 @@
-import { GoogleGenerativeAI } from "@google/generative-ai";
-import * as u from "ak-tools";
 
-import 'dotenv/config';
+/** @typedef {import('../../types.js').Dungeon} Dungeon */
+/** @typedef {import('../../types.js').EventConfig} EventConfig */
 
+import * as u from "ak-tools";
+import 'dotenv/config';
 const { GEMINI_API_KEY: API_KEY, NODE_ENV = "unknown" } = process.env;
 if (!API_KEY) throw new Error("Please provide a Gemini API key");
+import AITransformer from 'ak-gemini';
 
-async function generateSchema(userInput) {
-	const gemini = new GoogleGenerativeAI(API_KEY);
-	const model = gemini.getGenerativeModel({ model: "gemini-2.0-flash" });
-	const PROOMPTY = await u.load("./components/prompt.txt");
-	const prompt = `
-Given the following information about a website or app:
+`build me a dungeon stream with these events and structure
 
-${userInput}
+{ "event": "impression", "carousel": [{"product": "big mac"}] }
+{ "event": "viewed", "product_viewed": "big mac" }
+{ "event": "add to basket", "product_added": "big mac" }
+{ "event": "customized", "product_customized": "big mac" }
+{ "event": "checked out", "cart": [{"item": "big mac"}] }
 
-${PROOMPTY}
 
-REMEMBER, YOUR INPUT IS:
+but use all the different mcdonalds products as a possible values`
+let CURRENT_PROMPT = `
 
-${userInput}
-`.trim();
+build me a dungeon with 250 unique event names ... they can be anything i don't really care make them from many verticals
 
-	let schema;
-	let schemaIsValid = false;
-	let attempts = 0;
-	do {
-		attempts++;
-		const result = await model.generateContent(prompt);
-		const response = await result.response;
-		const text = response.text();
-		schema = processResponse(text);
-		schemaIsValid = validator(schema);
-	} while (!schemaIsValid);
 
-	return schema;
-}
+`;
 
-function processResponse(text) {
-	let json;
-	// check for ```json
-	const start = text.indexOf("```json");
-	const end = text.indexOf("```", start + 1);
 
-	if (start === -1 || end === -1) {
-		const start = text.indexOf("{");
-		const end = text.lastIndexOf("}");
-		json = text.slice(start, end + 1).trim();
-	}
+async function main(params) {
+	const { prompt } = params;
+	if (!prompt) throw new Error("Please provide a prompt");
+	const INSTRUCTIONS = await u.load('./lib/utils/instructions.txt', false);
+	const ai = new AITransformer({
+		apiKey: API_KEY,
+		onlyJSON: false,
+		systemInstructions: INSTRUCTIONS?.trim(),
 
-	json = text.slice(start + 7, end).trim();
 
-	try {
-		return JSON.parse(json);
-	}
-	catch (e) {
-		return null;
-	}
+	});
+	await ai.init();
+	const response = await ai.message(prompt)
 
+	if (NODE_ENV === "dev") {
+		debugger;
+	}
 
-}
-
-function validator(schema) {
-	let valid = true;
-
-	//null schema are always invalid
-	if (!schema) valid = false;
-
-	//must have 3 or more events
-	if (schema?.events?.length < 3) valid = false;
-
-	//must have 2 or more superProps
-	if (Object.keys(schema.superProps).length < 2) valid = false;
-
-	//must have 2 or more userProps
-	if (Object.keys(schema.userProps).length < 2) valid = false;
+	return response;
 
-	return valid;
 }
 
 
-export { generateSchema };
+export default main;
 
 if (import.meta.url === `file://${process.argv[1]}`) {
-	generateSchema(`metube, a video streaming company like youutube, where users watch videos, search, like, comment, subscribe, share, create playlists, etc...`)
+	main(
+		{
+			prompt: CURRENT_PROMPT || "Generate a dungeon spec for a simple e-commerce site with checkout and add to cart events."
+		}
+	)
 		.then((result) => {
 			if (NODE_ENV === "dev") debugger;
 		})

package/lib/utils/chart.js

@@ -1,3 +1,7 @@
+/** @typedef {import('../../types.js').EventSchema} EventSchema */
+/** @typedef {import('../../types.js').Result} Result */
+/** @typedef {import('../../types.js').Context} Context */
+
 import { ChartJSNodeCanvas } from 'chartjs-node-canvas';
 import fs from 'fs';
 import * as u from 'ak-tools';
@@ -179,6 +183,7 @@ async function generateLineChart(rawData, signupEvents = ["sign up"], fileName)
 	const imageBuffer = await chartJSNodeCanvas.renderToBuffer(configuration);
 	const filePath = path.join(tempDir, `${fileName}.png`);
 	const removed = await u.rm(filePath);
+	// @ts-ignore - imageBuffer is a Buffer but touch accepts it
 	const file = await u.touch(filePath, imageBuffer);
 
 	console.log(`Chart saved as ${fileName}.png`);