bulltrackers-module 1.0.64 → 1.0.65

package/functions/task-engine/handler_creator.js

@@ -2,6 +2,8 @@
  * @fileoverview Main pipe for the Task Engine.
  * REFACTORED: This file is now 'handler_creator.js' in name only.
  * It exports the main 'handleRequest' pipe function.
+ * OPTIMIZED: Removed 'batchManager.flushBatches()' from 'finally' block
+ * to enable true asynchronous batching.
  */
 
 // --- 1. REMOVE the circular require ---
@@ -21,6 +23,7 @@ const { handleUpdate } = require('./helpers/update_helpers');
  * @param {object} dependencies - Contains all clients: db, pubsub, logger, headerManager, proxyManager, batchManager.
  */
 async function handleRequest(message, context, config, dependencies) {
+    // --- OPTIMIZATION: Destructure headerManager here ---
     const { logger, batchManager, headerManager } = dependencies;
 
     if (!message || !message.data) {
@@ -65,13 +68,16 @@ async function handleRequest(message, context, config, dependencies) {
         // Let's rely on sub-pipes' finally blocks.
     } finally {
          try {
-             // Flush any remaining items in the batch
-             await batchManager.flushBatches();
-             logger.log('INFO', `[TaskEngine/${taskId}] Final batch flush complete.`);
+             // --- OPTIMIZATION START ---
+             // The FirestoreBatchManager will flush itself on its own timer.
+             // We no longer call `await batchManager.flushBatches()` here.
+             // We *only* need to flush the header performance updates at the end of the task.
+             await headerManager.flushPerformanceUpdates();
+             logger.log('INFO', `[TaskEngine/${taskId}] Final header performance flush complete.`);
+             // --- OPTIMIZATION END ---
          } catch (flushError) {
              logger.log('ERROR', `[TaskEngine/${taskId}] Error during final flush attempt.`, { error: flushError.message });
          }
-         // Note: Flushing headerManager is handled by batchManager
     }
 }
 

package/functions/task-engine/helpers/update_helpers.js

@@ -1,6 +1,7 @@
 /**
  * @fileoverview Sub-pipe: pipe.taskEngine.handleUpdate
  * REFACTORED: Now stateless and receives dependencies.
+ * OPTIMIZED: Removed immediate batch commit for speculator timestamp fix.
  */
 const { FieldValue } = require('@google-cloud/firestore');
 
@@ -69,6 +70,7 @@ async function handleUpdate(task, taskId, dependencies, config) {
                 : config.FIRESTORE_DOC_BLOCK_COUNTS);
             // <<< END FIX for private user blockId format >>>
                 
+            // This is a single, immediate write, which is fine for this rare case.
             await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
 
             return;
@@ -82,7 +84,7 @@ async function handleUpdate(task, taskId, dependencies, config) {
 
         const today = new Date().toISOString().slice(0, 10);
         
-        // <<< START FULL CODE FIX >>>
+        // <<< START OPTIMIZATION / FULL CODE FIX >>>
 
         // BUG 1: The blockId format is different for portfolio storage ('1M') 
         // vs. orchestrator logic ('1000000').
@@ -91,9 +93,7 @@ async function handleUpdate(task, taskId, dependencies, config) {
         // This call is correct for storing portfolio data (which uses the 'M' format)
         await batchManager.addToPortfolioBatch(userId, portfolioStorageBlockId, today, portfolioData, userType, instrumentId);
         
-        // This call is correct for 'normal' users, but for 'speculator' users
-        // it writes to a timestamp doc that the orchestrator *does not read*.
-        // We still call it to update 'normal' users correctly.
+        // This call is correct for 'normal' users.
         await batchManager.updateUserTimestamp(userId, userType, instrumentId);
 
         // BUG 2 (The Main Problem): We must *also* update the 'SpeculatorBlocks'
@@ -103,22 +103,13 @@ async function handleUpdate(task, taskId, dependencies, config) {
 
             logger.log('INFO', `[UPDATE] Applying speculator timestamp fix for user ${userId} in block ${orchestratorBlockId}`);
             
-            // Create a new, separate batch to commit this timestamp *immediately*.
-            // This mirrors the logic in 'verify_helpers.js'.
-            const fixBatch = db.batch();
-            const speculatorBlockRef = db.collection(config.FIRESTORE_COLLECTION_SPECULATOR_BLOCKS).doc(orchestratorBlockId);
-            
-            // These are the two fields the orchestrator and cleanup functions read
-            const updates = {
-                [`users.${userId}.lastVerified`]: new Date(),
-                [`users.${userId}.lastHeldSpeculatorAsset`]: new Date()
-            };
-
-            fixBatch.set(speculatorBlockRef, updates, { merge: true });
-            await fixBatch.commit();
-            logger.log('INFO', `[UPDATE] Speculator timestamp fix for user ${userId} committed.`);
+            // --- OPTIMIZATION: Use the batch manager instead of committing immediately ---
+            // This queues the write and allows the function to return,
+            // instead of blocking on `await fixBatch.commit()`.
+            await batchManager.addSpeculatorTimestampFix(userId, orchestratorBlockId);
+            logger.log('INFO', `[UPDATE] Speculator timestamp fix for user ${userId} queued.`);
         }
-        // <<< END FULL CODE FIX >>>
+        // <<< END OPTIMIZATION / FULL CODE FIX >>>
 
     } finally {
         if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess);

package/functions/task-engine/helpers/verify_helpers.js

@@ -1,9 +1,81 @@
 /**
  * @fileoverview Sub-pipe: pipe.taskEngine.handleVerify
  * REFACTORED: Now stateless and receives dependencies.
+ * OPTIMIZED: Fetches all user portfolios in parallel to reduce function runtime.
  */
 const { FieldValue } = require('@google-cloud/firestore');
 
+/**
+ * Internal helper to fetch and process a single user's portfolio.
+ * This allows the main function to run these in parallel.
+ * @param {object} user - The user object { cid, isBronze }
+ * @param {object} dependencies - Contains proxyManager, headerManager
+ * @param {object} config - The configuration object
+ * @param {Set<number>} speculatorInstrumentSet - Pre-built Set of instrument IDs
+ * @returns {Promise<object|null>} A result object for batching, or null on failure.
+ */
+async function fetchAndVerifyUser(user, dependencies, config, speculatorInstrumentSet) {
+    const { logger, headerManager, proxyManager } = dependencies;
+    const userId = user.cid;
+    
+    const portfolioUrl = `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+    const selectedHeader = await headerManager.selectHeader();
+    if (!selectedHeader) return null; // Cannot fetch
+
+    let wasSuccess = false;
+    try {
+        const response = await proxyManager.fetch(portfolioUrl, { headers: selectedHeader.header });
+        if (!response.ok) {
+            wasSuccess = false;
+            return null; // API error or private user
+        }
+        wasSuccess = true;
+        
+        const portfolioData = await response.json();
+
+        // Process verification logic *within* the parallel task
+        if (config.userType === 'speculator') {
+            const matchingInstruments = portfolioData.AggregatedPositions
+                .map(p => p.InstrumentID)
+                .filter(id => speculatorInstrumentSet.has(id));
+
+            if (matchingInstruments.length > 0) {
+                logger.log('INFO', `[VERIFY] Speculator user ${userId} holds assets: ${matchingInstruments.join(', ')}`);
+                // Return data needed for the speculator batch update
+                return {
+                    type: 'speculator',
+                    userId: userId,
+                    isBronze: user.isBronze,
+                    updateData: {
+                        instruments: matchingInstruments,
+                        lastVerified: new Date(),
+                        lastHeldSpeculatorAsset: new Date()
+                    }
+                };
+            } else {
+                 logger.log('INFO', `[VERIFY] Speculator user ${userId} does not hold any speculator assets.`);
+                 return null; // Valid fetch, but failed verification
+            }
+        } else { // Normal user verification
+            // Return data needed for the normal user batch update
+            return {
+                type: 'normal',
+                userId: userId,
+                isBronze: user.isBronze,
+                updateData: {
+                    lastVerified: new Date()
+                }
+            };
+        }
+    } catch (error) {
+        logger.log('WARN', `[VERIFY] Error processing user ${userId}`, { errorMessage: error.message });
+        return null;
+    } finally {
+        if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess);
+    }
+}
+
+
 /**
  * Sub-pipe: pipe.taskEngine.handleVerify
  * @param {object} task The Pub/Sub task payload.
@@ -12,7 +84,7 @@ const { FieldValue } = require('@google-cloud/firestore');
  * @param {object} config The configuration object.
  */
 async function handleVerify(task, taskId, dependencies, config) {
-    const { logger, headerManager, proxyManager, db } = dependencies;
+    const { logger, db } = dependencies;
     const { users, blockId, instrument, userType } = task;
     
     // Use db from dependencies
@@ -20,55 +92,43 @@ async function handleVerify(task, taskId, dependencies, config) {
     let validUserCount = 0;
     const speculatorUpdates = {};
     const normalUserUpdates = {};
-    const bronzeStateUpdates = {};
+    const bronzeStateUpdates = {}; // This can be one object
 
     const speculatorInstrumentSet = new Set(config.SPECULATOR_INSTRUMENTS_ARRAY);
     
-    for (const user of users) {
-        const userId = user.cid;
-        
-        const portfolioUrl = `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
-        const selectedHeader = await headerManager.selectHeader();
-        if (!selectedHeader) continue;
+    // --- OPTIMIZATION: Run all user fetches in parallel ---
+    const verificationPromises = users.map(user => 
+        fetchAndVerifyUser(
+            user, 
+            dependencies, 
+            { ...config, userType: userType }, // Pass userType into the helper config
+            speculatorInstrumentSet
+        )
+    );
 
-        let wasSuccess = false;
-        try {
-            // Use proxyManager from dependencies
-            const response  = await proxyManager.fetch(portfolioUrl, { headers: selectedHeader.header });
-            if (!response.ok) continue;
-            wasSuccess = true;
-            
-            const portfolioData = await response.json();
+    const results = await Promise.allSettled(verificationPromises);
+    // --- END OPTIMIZATION ---
 
-            if (userType === 'speculator') {
-                const matchingInstruments = portfolioData.AggregatedPositions
-                    .map(p => p.InstrumentID)
-                    .filter(id => speculatorInstrumentSet.has(id));
+    // Process results (this is fast, no I/O)
+    results.forEach(result => {
+        if (result.status === 'rejected' || !result.value) {
+            // Log rejection reason if needed: result.reason
+            return;
+        }
 
-                if (matchingInstruments.length > 0) {
-                    speculatorUpdates[`users.${userId}`] = {
-                        instruments: matchingInstruments,
-                        lastVerified: new Date(),
-                        lastHeldSpeculatorAsset: new Date()
-                    };
-                    bronzeStateUpdates[userId] = user.isBronze;
-                    logger.log('INFO', `[VERIFY] Speculator user ${userId} holds speculator assets: ${matchingInstruments.join(', ')}`);
-                    validUserCount++;
-                }
-                else {
-                    logger.log('INFO', `[VERIFY] Speculator user ${userId} does not hold any speculator assets.`);
-                }
-            } else { // Normal user verification
-                normalUserUpdates[`users.${userId}`] = {
-                    lastVerified: new Date()
-                };
-                 bronzeStateUpdates[userId] = user.isBronze;
-                validUserCount++;
-            }
-        } finally {
-            if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess);
+        const data = result.value;
+        
+        if (data.type === 'speculator') {
+            speculatorUpdates[`users.${data.userId}`] = data.updateData;
+            bronzeStateUpdates[data.userId] = data.isBronze;
+            validUserCount++;
+        } else if (data.type === 'normal') {
+            normalUserUpdates[`users.${data.userId}`] = data.updateData;
+            bronzeStateUpdates[data.userId] = data.isBronze;
+            validUserCount++;
         }
-    }
+    });
+
 
     if (Object.keys(speculatorUpdates).length > 0 || Object.keys(normalUserUpdates).length > 0) {
         if (userType === 'speculator') {
@@ -100,4 +160,4 @@ async function handleVerify(task, taskId, dependencies, config) {
     }
 }
 
-module.exports = { handleVerify };
+module.exports = { handleVerify };

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -1,6 +1,7 @@
 /**
  * @fileoverview Utility class to manage stateful Firestore write batches.
  * REFACTORED: Renamed 'firestore' to 'db' for consistency.
+ * OPTIMIZED: Added logic to handle speculator timestamp fixes within the batch.
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
@@ -22,6 +23,7 @@ class FirestoreBatchManager {
         this.portfolioBatch = {};
         this.timestampBatch = {};
         this.processedSpeculatorCids = new Set();
+        this.speculatorTimestampFixBatch = {}; // <-- OPTIMIZATION: ADD THIS
         this.batchTimeout = null;
 
         this.logger.log('INFO', 'FirestoreBatchManager initialized.');
@@ -123,6 +125,28 @@ class FirestoreBatchManager {
         cids.forEach(cid => this.processedSpeculatorCids.add(cid));
     }
 
+    /**
+     * --- OPTIMIZATION: ADD THIS NEW METHOD ---
+     * Adds a speculator timestamp fix to the batch.
+     * @param {string} userId
+     * @param {string} orchestratorBlockId (e.g., "1000000")
+     */
+    async addSpeculatorTimestampFix(userId, orchestratorBlockId) {
+        const docPath = `${this.config.FIRESTORE_COLLECTION_SPECULATOR_BLOCKS}/${orchestratorBlockId}`;
+
+        if (!this.speculatorTimestampFixBatch[docPath]) {
+            this.speculatorTimestampFixBatch[docPath] = {};
+        }
+
+        // These are the two fields the orchestrator reads
+        this.speculatorTimestampFixBatch[docPath][`users.${userId}.lastVerified`] = new Date();
+        this.speculatorTimestampFixBatch[docPath][`users.${userId}.lastHeldSpeculatorAsset`] = new Date();
+        
+        this.logger.log('TRACE', `[BATCH] Queued speculator timestamp fix for user ${userId} in block ${orchestratorBlockId}`);
+        this._scheduleFlush(); // Schedule a flush
+    }
+    // --- END NEW METHOD ---
+
     /**
      * Flushes all pending writes to Firestore and updates header performance.
      */
@@ -206,6 +230,20 @@ class FirestoreBatchManager {
             }
         }
 
+        // --- OPTIMIZATION: ADD THIS LOOP ---
+        for (const docPath in this.speculatorTimestampFixBatch) {
+            const updates = this.speculatorTimestampFixBatch[docPath];
+            if (Object.keys(updates).length > 0) {
+                const docRef = this.db.doc(docPath);
+                
+                // Add to the existing batch
+                firestoreBatch.set(docRef, updates, { merge: true });
+                batchOperationCount++;
+            }
+            delete this.speculatorTimestampFixBatch[docPath];
+        }
+        // --- END OPTIMIZATION ---
+
         if (batchOperationCount > 0) {
             promises.push(firestoreBatch.commit());
         }
@@ -217,4 +255,4 @@ class FirestoreBatchManager {
     }
 }
 
-module.exports = { FirestoreBatchManager };
+module.exports = { FirestoreBatchManager };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.64",
+  "version": "1.0.65",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [