bulltrackers-module 1.0.127 → 1.0.129

package/functions/task-engine/handler_creator.js

@@ -1,26 +1,16 @@
 /**
  * @fileoverview Main pipe for the Task Engine.
- * REFACTORED: This file is now 'handler_creator.js' in name only.
- * It exports the main 'handleRequest' pipe function.
- *
- * --- MODIFICATION ---
- * The `handleRequest` function is now designed to receive a *batch*
- * of tasks (e.g., { tasks: [...] }) from the Dispatcher. It loops
- * through all tasks in a single function instance, allowing the
- * FirestoreBatchManager to accumulate state. It then explicitly
- * calls `batchManager.flushBatches()` in the `finally` block
- * to commit all changes.
- * --- V2 MODIFICATION ---
- * - Loads username map at start.
- * - Sorts tasks into "run now" (username known) and "lookup needed".
- * - Runs a batch lookup for missing usernames.
- * - Runs all update tasks *after* lookups are complete.
+ * REFACTORED: This function is the high-level "manual of processes"
+ * that orchestrates the task engine logic.
  */
 
-const { handleDiscover } = require('./helpers/discover_helpers');
-const { handleVerify   } = require('./helpers/verify_helpers');
-// --- MODIFIED: Import both helpers from update_helpers ---
-const { handleUpdate, lookupUsernames } = require('./helpers/update_helpers');
+// Import the new utility functions that contain the logic
+const {
+    parseTaskPayload,
+    prepareTaskBatches,
+    runUsernameLookups,
+    executeTasks
+} = require('./utils/task_engine_utils'); // <-- Import from our new utils file
 
 /**
  * Main pipe: pipe.taskEngine.handleRequest
@@ -31,131 +21,20 @@ const { handleUpdate, lookupUsernames } = require('./helpers/update_helpers');
  * @param {object} dependencies - Contains all clients: db, pubsub, logger, headerManager, proxyManager, batchManager.
  */
 async function handleRequest(message, context, config, dependencies) {
-    const { logger, batchManager, headerManager } = dependencies;
-
-    if (!message || !message.data) {
-        logger.log('ERROR', '[TaskEngine Module] Received invalid message structure.', { message });
-        return;
-    }
-
-    let payload;
+    const { logger, batchManager } = dependencies;
+    const taskId = `batch-${context.eventId || Date.now()}`;
     try {
-        payload = JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8'));
-    } catch (e) {
-        logger.log('ERROR', '[TaskEngine Module] Failed to parse Pub/Sub message data.', { error: e.message, data: message.data });
-        return;
-    }
-
-    if (!payload.tasks || !Array.isArray(payload.tasks)) {
-        logger.log('ERROR', `[TaskEngine] Received invalid payload. Expected { tasks: [...] }`, { payload });
-        return;
-    }
-
-    if (payload.tasks.length === 0) {
-        logger.log('WARN', '[TaskEngine] Received message with an empty tasks array.');
-        return;
-    }
-
-    const taskId = `batch-${payload.tasks[0]?.type || 'unknown'}-${Date.now()}`;
-    logger.log('INFO', `[TaskEngine/${taskId}] Received batch of ${payload.tasks.length} tasks.`);
-
-    try {
-        // --- START V2 MODIFICATION ---
-
-        // 1. Load the username map into the batch manager's memory
-        await batchManager.loadUsernameMap();
-
-        // 2. Sort tasks
-        const tasksToRun = []; // { task, username }
-        const cidsToLookup = new Map(); // Map<string, object>
-        const otherTasks = []; // for 'discover' and 'verify'
-
-        for (const task of payload.tasks) {
-            if (task.type === 'update') {
-                const username = batchManager.getUsername(task.userId);
-                if (username) {
-                    tasksToRun.push({ task, username });
-                } else {
-                    cidsToLookup.set(String(task.userId), task);
-                }
-            } else if (task.type === 'discover' || task.type === 'verify') {
-                otherTasks.push(task);
-            }
-        }
-
-        logger.log('INFO', `[TaskEngine/${taskId}] Usernames known for ${tasksToRun.length} users. Looking up ${cidsToLookup.size} users. Processing ${otherTasks.length} other tasks.`);
-
-        // 3. Run username lookups if needed
-        if (cidsToLookup.size > 0) {
-            const foundUsers = await lookupUsernames(Array.from(cidsToLookup.keys()), dependencies, config);
-
-            for (const user of foundUsers) {
-                const cidStr = String(user.CID);
-                const username = user.Value.UserName;
-                
-                // Add to batch manager to save this new mapping
-                batchManager.addUsernameMapUpdate(cidStr, username);
-                
-                // Get the original task
-                const task = cidsToLookup.get(cidStr);
-                if (task) {
-                    // Add to the list of tasks to run
-                    tasksToRun.push({ task, username });
-                    // Remove from lookup map
-                    cidsToLookup.delete(cidStr);
-                }
-            }
-
-            if (cidsToLookup.size > 0) {
-                logger.log('WARN', `[TaskEngine/${taskId}] Could not find usernames for ${cidsToLookup.size} CIDs (likely private). They will be skipped.`, { skippedCids: Array.from(cidsToLookup.keys()) });
-            }
-        }
-
-        // 4. Process all tasks
-        
-        // Process discover/verify tasks
-        for (const task of otherTasks) {
-            const subTaskId = `${task.type || 'unknown'}-${task.userType || 'unknown'}-${task.userId || task.cids?.[0] || 'sub'}`;
-            const handlerFunction = {
-                discover: handleDiscover, 
-                verify: handleVerify,    
-            }[task.type];
-
-            if (handlerFunction) {
-                await handlerFunction(task, subTaskId, dependencies, config);
-            } else {
-                logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type in 'otherTasks': ${task.type}`);
-            }
-        }
-        
-        // Process update tasks (now with usernames)
-        for (const { task, username } of tasksToRun) {
-            const subTaskId = `${task.type || 'unknown'}-${task.userType || 'unknown'}-${task.userId || 'sub'}`;
-            try {
-                // Pass the username to handleUpdate
-                await handleUpdate(task, subTaskId, dependencies, config, username);
-            } catch (updateError) {
-                 logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for user ${task.userId}`, { errorMessage: updateError.message, errorStack: updateError.stack });
-            }
-        }
-        
-        logger.log('SUCCESS', `[TaskEngine/${taskId}] Processed all tasks. Done.`);
-        // --- END V2 MODIFICATION ---
-
-    } catch (error) {
-        logger.log('ERROR', `[TaskEngine/${taskId}] Failed during batch processing.`, { errorMessage: error.message, errorStack: error.stack });
-
+        const tasks = parseTaskPayload(message, logger);
+        if (!tasks) {return; }
+        logger.log('INFO', `[TaskEngine/${taskId}] Received batch of ${tasks.length} tasks.`);
+        const { tasksToRun, cidsToLookup, otherTasks } = await prepareTaskBatches(tasks, batchManager, logger);
+        await runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config, batchManager, logger);
+        await executeTasks(tasksToRun, otherTasks, dependencies, config, taskId);
+    } catch (error) {logger.log('ERROR', `[TaskEngine/${taskId}] Failed during batch processing.`, { errorMessage: error.message, errorStack: error.stack });
     } finally {
-         try {
-             logger.log('INFO', `[TaskEngine/${taskId}] Flushing all accumulated batches...`);
-             await batchManager.flushBatches();
-             logger.log('INFO', `[TaskEngine/${taskId}] Final batch and header flush complete.`);
-         } catch (flushError) {
-             logger.log('ERROR', `[TaskEngine/${taskId}] Error during final flush attempt.`, { error: flushError.message });
-         }
-    }
+        try {logger.log('INFO', `[TaskEngine/${taskId}] Flushing all accumulated batches...`);
+        await batchManager.flushBatches();
+        logger.log('INFO', `[TaskEngine/${taskId}] Final batch and header flush complete.`);} catch (flushError) {logger.log('ERROR', `[TaskEngine/${taskId}] Error during final flush attempt.`, { error: flushError.message });}}
 }
 
-module.exports = {
-    handleRequest,
-};
+module.exports = {handleRequest};

package/functions/task-engine/helpers/discover_helpers.js

@@ -1,8 +1,3 @@
-/*
- * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/discover_helpers.js
- * (FIXED: Publishes 'verify' task in the correct batch format { tasks: [...] })
- */
-
 /**
  * @fileoverview Sub-pipe: pipe.taskEngine.handleDiscover
  * REFACTORED: Now stateless and receives dependencies.
@@ -19,120 +14,55 @@
 async function handleDiscover(task, taskId, dependencies, config) {
     const { logger, headerManager, proxyManager, pubsub, batchManager } = dependencies;
     const { cids, blockId, instrument, userType } = task;
-    
     const url = `${config.ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
     const selectedHeader = await headerManager.selectHeader();
     if (!selectedHeader) throw new Error("Could not select a header.");
-
     let wasSuccess = false;
-    try {
-        if (userType === 'speculator') {
-            // Use the batchManager from dependencies
-            batchManager.addProcessedSpeculatorCids(cids);
-            logger.log('INFO', `[DISCOVER] Added ${cids.length} speculator CIDs to the in-memory set to be flushed.`);
-        }
-        
-        // Use the proxyManager from dependencies
-        const response = await proxyManager.fetch(url, {
-            method: 'POST',
-            headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },
-            body: JSON.stringify(cids),
-        });
-        if (!response.ok) {
-            throw new Error(`API status ${response.status}`);
-        }
-        wasSuccess = true;
 
+    // Step 1. Discover Speculators
+    try {if (userType === 'speculator') {batchManager.addProcessedSpeculatorCids(cids);logger.log('INFO', `[DISCOVER] Added ${cids.length} speculator CIDs to the in-memory set to be flushed.`);}
+        const response = await proxyManager.fetch(url, {method: 'POST',headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },body: JSON.stringify(cids),});
+        if (!response.ok) {throw new Error(`API status ${response.status}`);}
+        wasSuccess = true;
         const publicUsers = await response.json();
         if (!Array.isArray(publicUsers)) return;
-
         const oneMonthAgo = new Date();
         oneMonthAgo.setMonth(oneMonthAgo.getMonth() - 1);
-
-        const preliminaryActiveUsers = publicUsers.filter(user => 
-            new Date(user.Value.LastActivity) > oneMonthAgo &&
-            user.Value.DailyGain !== 0 &&
-            user.Value.Exposure !== 0 &&
-            user.Value.RiskScore !== 0
-        );
-        
+        // Step 2. We know a user is "active" if their risk score is NOT 0 and they logged in, in the last month. This covers the edge case that a user IS active but does NOT hold positions (cannot hold no positions and have 0 risk score)
+        const preliminaryActiveUsers = publicUsers.filter(user => new Date(user.Value.LastActivity) > oneMonthAgo &&user.Value.DailyGain !== 0 &&user.Value.Exposure !== 0 &&user.Value.RiskScore !== 0);
         let finalActiveUsers = [];
         const invalidCidsToLog = [];
-
-        if (userType === 'speculator') {
-            const publicUserCids = new Set(publicUsers.map(u => u.CID));
-            
-            if (publicUserCids.size > 0 && publicUserCids.size < cids.length) {
-                const privateUserCids = cids.filter(cid => !publicUserCids.has(cid));
-                invalidCidsToLog.push(...privateUserCids);
-            }
-
+        // Step 3. Users that were passed INTO the discovery API but did not come OUT OF the discovery API, are private accounts, the API filters this for us, so mark them as such to avoid processing them again.
+        if (userType === 'speculator') {const publicUserCids = new Set(publicUsers.map(u => u.CID));
+            if (publicUserCids.size > 0 && publicUserCids.size < cids.length) {const privateUserCids = cids.filter(cid => !publicUserCids.has(cid));invalidCidsToLog.push(...privateUserCids);}
             const activeUserCids = new Set(preliminaryActiveUsers.map(u => u.CID));
-            const inactiveUserCids = publicUsers
-                .filter(u => !activeUserCids.has(u.CID))
-                .map(u => u.CID);
+            const inactiveUserCids = publicUsers.filter(u => !activeUserCids.has(u.CID)).map(u => u.CID);
             invalidCidsToLog.push(...inactiveUserCids);
-
             logger.log('INFO', `[DISCOVER] Applying new speculator pre-filter to ${preliminaryActiveUsers.length} active users.`);
             const nonSpeculatorCids = [];
-            
+            // Step 4. For the remaining active users, apply heuristic filters to weed out non-speculators.
             for (const user of preliminaryActiveUsers) {
                 const v = user.Value;
                 const totalLeverage = (v.MediumLeveragePct || 0) + (v.HighLeveragePct || 0);
-
-                const isLikelySpeculator = (
-                    (v.Trades || 0) > 500 ||
-                    (v.TotalTradedInstruments || 0) > 50 ||
-                    totalLeverage > 50 ||
-                    (v.WeeklyDD || 0) < -25
-                );
-
-                if (isLikelySpeculator) {
-                    finalActiveUsers.push(user);
-                } else {
-                    nonSpeculatorCids.push(user.CID);
-                }
-            }
-            
+                const isLikelySpeculator = ((v.Trades || 0) > 500 ||(v.TotalTradedInstruments || 0) > 50 ||totalLeverage > 50 ||(v.WeeklyDD || 0) < -25);
+                if (isLikelySpeculator) {finalActiveUsers.push(user);} else {nonSpeculatorCids.push(user.CID);}}
             invalidCidsToLog.push(...nonSpeculatorCids);
             logger.log('INFO', `[DISCOVER] Pre-filter complete. ${finalActiveUsers.length} users passed. ${nonSpeculatorCids.length} users failed heuristic.`);
-            
             if (invalidCidsToLog.length > 0) {
-                // Use pubsub from dependencies
-                await pubsub.topic(config.PUBSUB_TOPIC_INVALID_SPECULATOR_LOG)
-                                  .publishMessage({ json: { invalidCids: invalidCidsToLog } });
+                await pubsub.topic(config.PUBSUB_TOPIC_INVALID_SPECULATOR_LOG).publishMessage({ json: { invalidCids: invalidCidsToLog } });
                 logger.log('INFO', `[DISCOVER] Reported ${invalidCidsToLog.length} invalid (private, inactive, or failed heuristic) speculator IDs.`);
             }
-
-        } else { // 'normal' users
+        // Step 5. Remaining users are public active non speculators
+        } else { 
             finalActiveUsers = preliminaryActiveUsers;
         }
-
+        // Step 6. Publish 'verify' task for all active users found
         if (finalActiveUsers.length > 0) {
-            // --- START MODIFICATION ---
-            // Include the UserName in the task payload for 'verify'
-            const verificationTask = {
-                type: 'verify',
-                users: finalActiveUsers.map(u => ({ 
-                    cid: u.CID, 
-                    isBronze: u.Value.IsBronze,
-                    username: u.Value.UserName // <-- ADD THIS
-                })),
-                blockId,
-                instrument,
-                userType
-            };
-            // --- END MODIFICATION ---
-
-            // --- FIX: WRAP IN BATCH FORMAT ---
-            // Use pubsub from dependencies
-            await pubsub.topic(config.PUBSUB_TOPIC_USER_FETCH)
-                               .publishMessage({ json: { tasks: [verificationTask] } });
+            const verificationTask = {type: 'verify',users: finalActiveUsers.map(u => ({ cid: u.CID, isBronze: u.Value.IsBronze,username: u.Value.UserName})), blockId, instrument, userType};
+            await pubsub.topic(config.PUBSUB_TOPIC_USER_FETCH).publishMessage({ json: { tasks: [verificationTask] } });
             logger.log('INFO', `[DISCOVER] Verification message published was : ${JSON.stringify({ tasks: [verificationTask] })} `);
-            // --- END FIX ---
             logger.log('INFO', `[DISCOVER] Chaining to 'verify' task for ${finalActiveUsers.length} active users.`);
         }
-        
     } finally {
         if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess);
     }