bulltrackers-module 1.0.121 → 1.0.122

package/functions/dispatcher/helpers/dispatch_helpers.js

@@ -1,6 +1,11 @@
 /**
  * @fileoverview Sub-pipe for dispatching tasks.
  * REFACTORED: Now stateless and receives dependencies.
+ *
+ * --- MODIFICATION ---
+ * This file has been modified to publish tasks in batches
+ * within a single Pub/Sub message, rather than one message per task.
+ * This enables the Task Engine to process users in a stateful batch.
  */
 
 const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
@@ -23,12 +28,21 @@ async function dispatchTasksInBatches(tasks, dependencies, config) {
     logger.log('INFO', `[Module Dispatcher] Received ${tasks.length} tasks. Creating batches...`);
 
     for (let i = 0; i < tasks.length; i += batchSize) {
-        const batch = tasks.slice(i, i + batchSize);
+        const batch = tasks.slice(i, i + batchSize); // e.g., batch is [task1, task2, ... task100]
 
         try {
-            await Promise.all(batch.map(task => topic.publishMessage({ json: task })));
+            // --- START MODIFICATION ---
+            // Wrap the entire batch of tasks into a single object payload.
+            // This is the payload the Task Engine will now receive.
+            const messagePayload = { tasks: batch };
+            
+            // Publish the entire batch as a single message
+            await topic.publishMessage({ json: messagePayload });
+            
+            // --- END MODIFICATION ---
+
             totalTasksQueued += batch.length;
-            logger.log('INFO', `[Module Dispatcher] Dispatched batch ${Math.ceil((i + 1) / batchSize)} with ${batch.length} tasks.`);
+            logger.log('INFO', `[Module Dispatcher] Dispatched batch ${Math.ceil((i + 1) / batchSize)} with ${batch.length} tasks as a single message.`);
 
             if (i + batchSize < tasks.length) {
                 await sleep(batchDelayMs);
@@ -44,4 +58,4 @@ async function dispatchTasksInBatches(tasks, dependencies, config) {
 
 module.exports = {
     dispatchTasksInBatches
-};
+};

package/functions/task-engine/handler_creator.js

@@ -2,8 +2,14 @@
  * @fileoverview Main pipe for the Task Engine.
  * REFACTORED: This file is now 'handler_creator.js' in name only.
  * It exports the main 'handleRequest' pipe function.
- * OPTIMIZED: Removed 'batchManager.flushBatches()' from 'finally' block
- * to enable true asynchronous batching.
+ *
+ * --- MODIFICATION ---
+ * The `handleRequest` function is now designed to receive a *batch*
+ * of tasks (e.g., { tasks: [...] }) from the Dispatcher. It loops
+ * through all tasks in a single function instance, allowing the
+ * FirestoreBatchManager to accumulate state. It then explicitly
+ * calls `batchManager.flushBatches()` in the `finally` block
+ * to commit all changes.
  */
 
 const { handleDiscover } = require('./helpers/discover_helpers');
@@ -19,7 +25,7 @@ const { handleUpdate   } = require('./helpers/update_helpers');
  * @param {object} dependencies - Contains all clients: db, pubsub, logger, headerManager, proxyManager, batchManager.
  */
 async function handleRequest(message, context, config, dependencies) {
-    // --- OPTIMIZATION: Destructure headerManager here ---
+    // --- MODIFICATION: Destructure batchManager and headerManager ---
     const { logger, batchManager, headerManager } = dependencies;
 
     if (!message || !message.data) {
@@ -27,51 +33,73 @@ async function handleRequest(message, context, config, dependencies) {
         return;
     }
 
-    let task;
+    let payload;
     try {
-        task = JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8'));
+        payload = JSON.parse(Buffer.from(message.data, 'base64').toString('utf-8'));
     } catch (e) {
         logger.log('ERROR', '[TaskEngine Module] Failed to parse Pub/Sub message data.', { error: e.message, data: message.data });
         return;
     }
 
-    const taskId = `${task.type || 'unknown'}-${task.userType || 'unknown'}-${task.userId || task.cids?.[0] || 'batch'}-${Date.now()}`;
-    logger.log('INFO', `[TaskEngine/${taskId}] Received.`);
+    // --- START MODIFICATION: Handle Batch Payload ---
+
+    // Check if this is a batched task payload
+    if (!payload.tasks || !Array.isArray(payload.tasks)) {
+        logger.log('ERROR', `[TaskEngine] Received invalid payload. Expected { tasks: [...] }`, { payload });
+        return;
+    }
+
+    if (payload.tasks.length === 0) {
+        logger.log('WARN', '[TaskEngine] Received message with an empty tasks array.');
+        return;
+    }
+
+    const taskId = `batch-${payload.tasks[0]?.type || 'unknown'}-${Date.now()}`;
+    logger.log('INFO', `[TaskEngine/${taskId}] Received batch of ${payload.tasks.length} tasks.`);
 
     try {
-        await new Promise(resolve => setTimeout(resolve, Math.random() * 500 + 250));
+        // Loop through all tasks in this single function instance
+        for (const task of payload.tasks) {
+            
+            // Generate a sub-task ID for logging
+            const subTaskId = `${task.type || 'unknown'}-${task.userType || 'unknown'}-${task.userId || task.cids?.[0] || 'sub'}`;
 
-        // --- 3. Use the directly required functions ---
-        // Route to the correct sub-pipe
-        const handlerFunction = {
-            discover: handleDiscover, 
-            verify: handleVerify,    
-            update: handleUpdate     
-        }[task.type];
+            const handlerFunction = {
+                discover: handleDiscover, 
+                verify: handleVerify,    
+                update: handleUpdate     
+            }[task.type];
 
-        if (handlerFunction) {
-            // Pass the dependencies object to the sub-pipe
-            await handlerFunction(task, taskId, dependencies, config);
-            logger.log('SUCCESS', `[TaskEngine/${taskId}] Done.`);
-        } else {
-            logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type received: ${task.type}`);
+            if (handlerFunction) {
+                // We await each one sequentially to manage API load, 
+                // but they all use the *same* batchManager instance.
+                await handlerFunction(task, subTaskId, dependencies, config);
+            } else {
+                logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type in batch: ${task.type}`);
+            }
         }
+        
+        logger.log('SUCCESS', `[TaskEngine/${taskId}] Processed ${payload.tasks.length} tasks. Done.`);
+
     } catch (error) {
-        logger.log('ERROR', `[TaskEngine/${taskId}] Failed.`, { errorMessage: error.message, errorStack: error.stack });
+        logger.log('ERROR', `[TaskEngine/${taskId}] Failed during batch processing.`, { errorMessage: error.message, errorStack: error.stack });
 
     } finally {
          try {
-             // --- OPTIMIZATION START ---
-             // The FirestoreBatchManager will flush itself on its own timer.
-             // We no longer call `await batchManager.flushBatches()` here.
-             // We *only* need to flush the header performance updates at the end of the task.
-             await headerManager.flushPerformanceUpdates();
-             logger.log('INFO', `[TaskEngine/${taskId}] Final header performance flush complete.`);
-             // --- OPTIMIZATION END ---
+             // --- MODIFICATION ---
+             // This is the most critical change.
+             // After the loop (or if an error occurs), explicitly flush
+             // all accumulated writes (portfolios, timestamps, etc.)
+             // from the batchManager.
+             logger.log('INFO', `[TaskEngine/${taskId}] Flushing all accumulated batches...`);
+             await batchManager.flushBatches();
+             logger.log('INFO', `[TaskEngine/${taskId}] Final batch and header flush complete.`);
+             // --- END MODIFICATION ---
          } catch (flushError) {
              logger.log('ERROR', `[TaskEngine/${taskId}] Error during final flush attempt.`, { error: flushError.message });
          }
     }
+    // --- END MODIFICATION ---
 }
 
 module.exports = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.121",
+  "version": "1.0.122",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [