bulltrackers-module 1.0.122 → 1.0.124

package/functions/orchestrator/helpers/discovery_helpers.js

@@ -1,3 +1,8 @@
+/*
+ * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/orchestrator/helpers/discovery_helpers.js
+ * (FIXED: Publishes 'discover' tasks in the correct batch format { tasks: [...] })
+ */
+
 /**
  * @fileoverview Orchestrator's discovery sub-pipes.
  * REFACTORED: All functions are now stateless and receive dependencies.
@@ -164,11 +169,11 @@ async function getDiscoveryCandidates(userType, blocksToFill, config, dependenci
  * @returns {Promise<void>}
  */
 async function dispatchDiscovery(userType, candidates, config, dependencies) {
-    const { logger, firestoreUtils, pubsubUtils } = dependencies;
+    const { logger, firestoreUtils, pubsubUtils, pubsub } = dependencies; // <-- Add pubsub
     const {
         topicName,
         dispatchBatchSize,
-        pubsubBatchSize,
+        pubsubBatchSize, // This is no longer used by this function
         pendingSpecCollection, 
         pendingMaxFieldsPerDoc,
         pendingMaxWritesPerBatch
@@ -208,14 +213,43 @@ async function dispatchDiscovery(userType, candidates, config, dependencies) {
         }
     }
 
-    // Pass dependencies and config to core util
-    await pubsubUtils.batchPublishTasks(dependencies, {
-        topicName, 
-        tasks, 
-        taskType: `${userType} discovery`,
-        maxPubsubBatchSize: pubsubBatchSize
-    });
-    logger.log('SUCCESS', `[Orchestrator Helpers] Dispatched ${tasks.length} task messages (${candidates.size} CIDs) for ${userType} discovery.`);
+    // --- START FIX: RE-IMPLEMENT BATCHING LOGIC ---
+    const topic = pubsub.topic(topicName);
+    let totalCidsPublished = 0;
+    let messagesPublished = 0;
+
+    // 'tasks' is an array of 'discover' task objects. We group them into
+    // batches and publish each batch as a single message.
+    // We use `dispatchBatchSize` here to determine how many 'discover' tasks go into one message.
+    // This value should probably be small, like 1-10.
+    
+    // Let's re-use `dispatchBatchSize` as the number of *tasks* per message.
+    // If `tasks.length` is 200 and `dispatchBatchSize` is 50, this will create 4 messages.
+    // This seems correct.
+
+    for (let i = 0; i < tasks.length; i += dispatchBatchSize) {
+        // This batch contains multiple 'discover' tasks
+        const batchOfTasks = tasks.slice(i, i + dispatchBatchSize);
+        
+        // Wrap this batch in the new payload format
+        const messagePayload = { tasks: batchOfTasks };
+
+        try {
+            await topic.publishMessage({ json: messagePayload });
+            
+            // Log the number of CIDs in this message
+            const cidsInThisMessage = batchOfTasks.reduce((acc, task) => acc + task.cids.length, 0);
+            totalCidsPublished += cidsInThisMessage;
+            messagesPublished++;
+            
+            logger.log('INFO', `[Orchestrator Helpers] Dispatched batch ${messagesPublished} with ${batchOfTasks.length} discover tasks (${cidsInThisMessage} CIDs) as 1 Pub/Sub message.`);
+        } catch (publishError) {
+            logger.log('ERROR', `[Orchestrator Helpers] Failed to publish discover batch ${messagesPublished + 1}.`, { error: publishError.message });
+        }
+    }
+    
+    logger.log('SUCCESS', `[Orchestrator Helpers] Dispatched ${totalCidsPublished} CIDs in ${tasks.length} tasks, grouped into ${messagesPublished} Pub/Sub messages for ${userType} discovery.`);
+    // --- END FIX ---
 }
 
 
@@ -223,4 +257,4 @@ module.exports = {
   checkDiscoveryNeed,
   getDiscoveryCandidates,
   dispatchDiscovery,
-};
+};

package/functions/task-engine/handler_creator.js

@@ -10,11 +10,17 @@
  * FirestoreBatchManager to accumulate state. It then explicitly
  * calls `batchManager.flushBatches()` in the `finally` block
  * to commit all changes.
+ * --- V2 MODIFICATION ---
+ * - Loads username map at start.
+ * - Sorts tasks into "run now" (username known) and "lookup needed".
+ * - Runs a batch lookup for missing usernames.
+ * - Runs all update tasks *after* lookups are complete.
  */
 
 const { handleDiscover } = require('./helpers/discover_helpers');
 const { handleVerify   } = require('./helpers/verify_helpers');
-const { handleUpdate   } = require('./helpers/update_helpers');
+// --- MODIFIED: Import both helpers from update_helpers ---
+const { handleUpdate, lookupUsernames } = require('./helpers/update_helpers');
 
 /**
  * Main pipe: pipe.taskEngine.handleRequest
@@ -25,7 +31,6 @@ const { handleUpdate   } = require('./helpers/update_helpers');
  * @param {object} dependencies - Contains all clients: db, pubsub, logger, headerManager, proxyManager, batchManager.
  */
 async function handleRequest(message, context, config, dependencies) {
-    // --- MODIFICATION: Destructure batchManager and headerManager ---
     const { logger, batchManager, headerManager } = dependencies;
 
     if (!message || !message.data) {
@@ -41,9 +46,6 @@ async function handleRequest(message, context, config, dependencies) {
         return;
     }
 
-    // --- START MODIFICATION: Handle Batch Payload ---
-
-    // Check if this is a batched task payload
     if (!payload.tasks || !Array.isArray(payload.tasks)) {
         logger.log('ERROR', `[TaskEngine] Received invalid payload. Expected { tasks: [...] }`, { payload });
         return;
@@ -58,48 +60,100 @@ async function handleRequest(message, context, config, dependencies) {
     logger.log('INFO', `[TaskEngine/${taskId}] Received batch of ${payload.tasks.length} tasks.`);
 
     try {
-        // Loop through all tasks in this single function instance
+        // --- START V2 MODIFICATION ---
+
+        // 1. Load the username map into the batch manager's memory
+        await batchManager.loadUsernameMap();
+
+        // 2. Sort tasks
+        const tasksToRun = []; // { task, username }
+        const cidsToLookup = new Map(); // Map<string, object>
+        const otherTasks = []; // for 'discover' and 'verify'
+
         for (const task of payload.tasks) {
-            
-            // Generate a sub-task ID for logging
-            const subTaskId = `${task.type || 'unknown'}-${task.userType || 'unknown'}-${task.userId || task.cids?.[0] || 'sub'}`;
+            if (task.type === 'update') {
+                const username = batchManager.getUsername(task.userId);
+                if (username) {
+                    tasksToRun.push({ task, username });
+                } else {
+                    cidsToLookup.set(String(task.userId), task);
+                }
+            } else if (task.type === 'discover' || task.type === 'verify') {
+                otherTasks.push(task);
+            }
+        }
+
+        logger.log('INFO', `[TaskEngine/${taskId}] Usernames known for ${tasksToRun.length} users. Looking up ${cidsToLookup.size} users. Processing ${otherTasks.length} other tasks.`);
 
+        // 3. Run username lookups if needed
+        if (cidsToLookup.size > 0) {
+            const foundUsers = await lookupUsernames(Array.from(cidsToLookup.keys()), dependencies, config);
+
+            for (const user of foundUsers) {
+                const cidStr = String(user.CID);
+                const username = user.Value.UserName;
+                
+                // Add to batch manager to save this new mapping
+                batchManager.addUsernameMapUpdate(cidStr, username);
+                
+                // Get the original task
+                const task = cidsToLookup.get(cidStr);
+                if (task) {
+                    // Add to the list of tasks to run
+                    tasksToRun.push({ task, username });
+                    // Remove from lookup map
+                    cidsToLookup.delete(cidStr);
+                }
+            }
+
+            if (cidsToLookup.size > 0) {
+                logger.log('WARN', `[TaskEngine/${taskId}] Could not find usernames for ${cidsToLookup.size} CIDs (likely private). They will be skipped.`, { skippedCids: Array.from(cidsToLookup.keys()) });
+            }
+        }
+
+        // 4. Process all tasks
+        
+        // Process discover/verify tasks
+        for (const task of otherTasks) {
+            const subTaskId = `${task.type || 'unknown'}-${task.userType || 'unknown'}-${task.userId || task.cids?.[0] || 'sub'}`;
             const handlerFunction = {
                 discover: handleDiscover, 
                 verify: handleVerify,    
-                update: handleUpdate     
             }[task.type];
 
             if (handlerFunction) {
-                // We await each one sequentially to manage API load, 
-                // but they all use the *same* batchManager instance.
                 await handlerFunction(task, subTaskId, dependencies, config);
             } else {
-                logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type in batch: ${task.type}`);
+                logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type in 'otherTasks': ${task.type}`);
+            }
+        }
+        
+        // Process update tasks (now with usernames)
+        for (const { task, username } of tasksToRun) {
+            const subTaskId = `${task.type || 'unknown'}-${task.userType || 'unknown'}-${task.userId || 'sub'}`;
+            try {
+                // Pass the username to handleUpdate
+                await handleUpdate(task, subTaskId, dependencies, config, username);
+            } catch (updateError) {
+                 logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for user ${task.userId}`, { errorMessage: updateError.message, errorStack: updateError.stack });
             }
         }
         
-        logger.log('SUCCESS', `[TaskEngine/${taskId}] Processed ${payload.tasks.length} tasks. Done.`);
+        logger.log('SUCCESS', `[TaskEngine/${taskId}] Processed all tasks. Done.`);
+        // --- END V2 MODIFICATION ---
 
     } catch (error) {
         logger.log('ERROR', `[TaskEngine/${taskId}] Failed during batch processing.`, { errorMessage: error.message, errorStack: error.stack });
 
     } finally {
          try {
-             // --- MODIFICATION ---
-             // This is the most critical change.
-             // After the loop (or if an error occurs), explicitly flush
-             // all accumulated writes (portfolios, timestamps, etc.)
-             // from the batchManager.
              logger.log('INFO', `[TaskEngine/${taskId}] Flushing all accumulated batches...`);
              await batchManager.flushBatches();
              logger.log('INFO', `[TaskEngine/${taskId}] Final batch and header flush complete.`);
-             // --- END MODIFICATION ---
          } catch (flushError) {
              logger.log('ERROR', `[TaskEngine/${taskId}] Error during final flush attempt.`, { error: flushError.message });
          }
     }
-    // --- END MODIFICATION ---
 }
 
 module.exports = {

package/functions/task-engine/helpers/discover_helpers.js

@@ -1,6 +1,12 @@
+/*
+ * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/discover_helpers.js
+ * (FIXED: Publishes 'verify' task in the correct batch format { tasks: [...] })
+ */
+
 /**
  * @fileoverview Sub-pipe: pipe.taskEngine.handleDiscover
  * REFACTORED: Now stateless and receives dependencies.
+ * --- MODIFIED: Passes username to the 'verify' task. ---
  */
 
 /**
@@ -103,17 +109,27 @@ async function handleDiscover(task, taskId, dependencies, config) {
         }
 
         if (finalActiveUsers.length > 0) {
+            // --- START MODIFICATION ---
+            // Include the UserName in the task payload for 'verify'
             const verificationTask = {
                 type: 'verify',
-                users: finalActiveUsers.map(u => ({ cid: u.CID, isBronze: u.Value.IsBronze })),
+                users: finalActiveUsers.map(u => ({ 
+                    cid: u.CID, 
+                    isBronze: u.Value.IsBronze,
+                    username: u.Value.UserName // <-- ADD THIS
+                })),
                 blockId,
                 instrument,
                 userType
             };
+            // --- END MODIFICATION ---
+
+            // --- FIX: WRAP IN BATCH FORMAT ---
             // Use pubsub from dependencies
             await pubsub.topic(config.PUBSUB_TOPIC_USER_FETCH)
-                               .publishMessage({ json: verificationTask });
-            logger.log('INFO', `[DISCOVER] Verification message published was : ${JSON.stringify(verificationTask)} `);
+                               .publishMessage({ json: { tasks: [verificationTask] } });
+            logger.log('INFO', `[DISCOVER] Verification message published was : ${JSON.stringify({ tasks: [verificationTask] })} `);
+            // --- END FIX ---
             logger.log('INFO', `[DISCOVER] Chaining to 'verify' task for ${finalActiveUsers.length} active users.`);
         }
         
@@ -122,4 +138,4 @@ async function handleDiscover(task, taskId, dependencies, config) {
     }
 }
 
-module.exports = { handleDiscover };
+module.exports = { handleDiscover };

package/functions/task-engine/helpers/update_helpers.js

@@ -1,50 +1,192 @@
+/*
+ * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
+ * (FIXED: Enhanced error logging for portfolio and history fetches)
+ */
+
 /**
  * @fileoverview Sub-pipe: pipe.taskEngine.handleUpdate
  * REFACTORED: Now stateless and receives dependencies.
  * OPTIMIZED: Removed immediate batch commit for speculator timestamp fix.
+ * --- MODIFIED: Fetches portfolio AND trade history in parallel. ---
+ * --- MODIFIED: Includes helper to look up usernames from CIDs. ---
  */
 const { FieldValue } = require('@google-cloud/firestore');
 
+/**
+ * --- NEW HELPER ---
+ * Fetches user details (including username) from the rankings API for a batch of CIDs.
+ * @param {Array<string>} cids - An array of user CIDs (as strings or numbers).
+ * @param {object} dependencies - Contains proxyManager, headerManager, logger.
+ * @param {object} config - The configuration object.
+ * @returns {Promise<Array<object>>} An array of the raw user objects from the API.
+ */
+async function lookupUsernames(cids, dependencies, config) {
+    const { logger, headerManager, proxyManager } = dependencies;
+    const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
+
+    if (!cids || cids.length === 0) {
+        return [];
+    }
+
+    logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
+    const allPublicUsers = [];
+
+    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) {
+        const batchCids = cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number);
+        
+        const url = `${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
+        const selectedHeader = await headerManager.selectHeader();
+        if (!selectedHeader) {
+            logger.log('ERROR', '[lookupUsernames] Could not select a header.');
+            continue; // Skip this batch
+        }
+
+        let wasSuccess = false;
+        try {
+            const response = await proxyManager.fetch(url, {
+                method: 'POST',
+                headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },
+                body: JSON.stringify(batchCids),
+            });
+
+            if (!response.ok) {
+                throw new Error(`[lookupUsernames] API status ${response.status}`);
+            }
+            wasSuccess = true;
+
+            const publicUsers = await response.json();
+            if (Array.isArray(publicUsers)) {
+                allPublicUsers.push(...publicUsers);
+            }
+        } catch (error) {
+            logger.log('WARN', `[lookupUsernames] Failed to fetch batch of usernames.`, { errorMessage: error.message });
+        } finally {
+            if (selectedHeader) {
+                headerManager.updatePerformance(selectedHeader.id, wasSuccess);
+            }
+        }
+    }
+
+    logger.log('INFO', `[lookupUsernames] Found ${allPublicUsers.length} public users out of ${cids.length}.`);
+    return allPublicUsers;
+}
+
+
 /**
  * Sub-pipe: pipe.taskEngine.handleUpdate
  * @param {object} task The Pub/Sub task payload.
  * @param {string} taskId A unique ID for logging.
  * @param {object} dependencies - Contains db, pubsub, logger, headerManager, proxyManager, batchManager.
  * @param {object} config The configuration object.
+ * @param {string} username - The user's eToro username.
  */
-async function handleUpdate(task, taskId, dependencies, config) {
+async function handleUpdate(task, taskId, dependencies, config, username) {
     const { logger, headerManager, proxyManager, db, batchManager } = dependencies;
     const { userId, instrumentId, userType } = task;
+
+    // --- START MODIFICATION: Define both URLs ---
+    const portfolioUrl = userType === 'speculator' 
+        ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instrumentId}` 
+        : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+
+    const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
     
-    const selectedHeader = await headerManager.selectHeader();
-    if (!selectedHeader) throw new Error("Could not select a header.");
+    const today = new Date().toISOString().slice(0, 10);
+    const portfolioStorageBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
+    
+    let portfolioHeader = null;
+    let historyHeader = null;
+    let wasPortfolioSuccess = false;
+    let wasHistorySuccess = false;
+    let isPrivate = false;
 
-    let wasSuccess = false;
     try {
-        const url = userType === 'speculator' 
-            ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instrumentId}` 
-            : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
-        
-        logger.log('INFO', `[UPDATE] Fetching portfolio for user ${userId} (${userType} with url ${url})`);
-        
-        // Use proxyManager from dependencies
-        const response  = await proxyManager.fetch(url, { headers: selectedHeader.header });
+        logger.log('INFO', `[UPDATE] Fetching data for user ${userId} (${username}, ${userType})`);
 
-        if (!response || typeof response.text !== 'function') {
-            logger.log('ERROR', `[UPDATE] Invalid or incomplete response received for user ${userId}`, { response });
-            throw new Error(`Invalid response structure received from proxy for user ${userId}.`);
+        // Select headers for both requests
+        portfolioHeader = await headerManager.selectHeader();
+        historyHeader = await headerManager.selectHeader();
+        if (!portfolioHeader || !historyHeader) {
+            throw new Error("Could not select headers for update requests.");
         }
 
-        const responseBody = await response.text();
+        // --- Fetch both endpoints in parallel ---
+        const [portfolioResult, historyResult] = await Promise.allSettled([
+            proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header }),
+            proxyManager.fetch(historyUrl, { headers: historyHeader.header })
+        ]);
 
-        if (responseBody.includes("user is PRIVATE")) {
-            logger.log('WARN', `User ${userId} is private. Removing from future updates and decrementing block count.`);
+        // --- Process Portfolio Result ---
+        if (portfolioResult.status === 'fulfilled' && portfolioResult.value.ok) {
+            const responseBody = await portfolioResult.value.text();
+            
+            // Check for private user
+            if (responseBody.includes("user is PRIVATE")) {
+                isPrivate = true;
+            } else {
+                wasPortfolioSuccess = true;
+                const portfolioData = JSON.parse(responseBody);
+                // Add to portfolio batch
+                console.log('DEBUG', `[UPDATE] SUCCESS Adding portfolio data for user ${userId} to batch.`);
+                await batchManager.addToPortfolioBatch(userId, portfolioStorageBlockId, today, portfolioData, userType, instrumentId);
+                
+            }
+        } else {
+            // --- FIX: ENHANCED LOGGING ---
+            let errorMsg = 'Unknown fetch error';
+            let rawErrorText = 'N/A';
+
+            if (portfolioResult.status === 'rejected') {
+                errorMsg = portfolioResult.reason.message;
+            } else if (portfolioResult.value) {
+                errorMsg = portfolioResult.value.error?.message || `API status ${portfolioResult.value.status}`;
+                // Get the raw text from the proxy manager's response
+                if (typeof portfolioResult.value.text === 'function') {
+                    rawErrorText = await portfolioResult.value.text();
+                }
+            }
             
-            // Use batchManager from dependencies
+            logger.log('WARN', `[UPDATE] Failed to fetch PORTFOLIO for user ${userId}.`, { 
+                error: errorMsg,
+                proxyResponse: rawErrorText // This will contain "urlfetch" or other quota messages
+            });
+            // --- END FIX ---
+        }
+
+        // --- Process History Result ---
+        if (historyResult.status === 'fulfilled' && historyResult.value.ok) {
+            const historyData = await historyResult.value.json();
+            wasHistorySuccess = true;
+            // Add to history batch
+            console.log('DEBUG', `[UPDATE] SUCCESS Adding history data for user ${userId} to batch.`);
+            await batchManager.addToTradingHistoryBatch(userId, portfolioStorageBlockId, today, historyData, userType);
+        } else {
+            // --- FIX: ENHANCED LOGGING ---
+            let errorMsg = 'Unknown fetch error';
+            let rawErrorText = 'N/A';
+
+            if (historyResult.status === 'rejected') {
+                errorMsg = historyResult.reason.message;
+            } else if (historyResult.value) {
+                errorMsg = historyResult.value.error?.message || `API status ${historyResult.value.status}`;
+                // Get the raw text from the proxy manager's response
+                if (typeof historyResult.value.text === 'function') {
+                    rawErrorText = await historyResult.value.text();
+                }
+            }
+            
+            logger.log('WARN', `[UPDATE] Failed to fetch HISTORY for user ${userId} (${username}).`, { 
+                error: errorMsg,
+                proxyResponse: rawErrorText // This will contain "urlfetch" or other quota messages
+            });
+            // --- END FIX ---
+        }
+
+        // --- Handle Private User (if detected) ---
+        if (isPrivate) {
+            logger.log('WARN', `User ${userId} is private. Removing from future updates and decrementing block count.`);
             batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
 
-            // <<< START FIX for private user blockId format >>>
-            // The blockId format for Speculator Blocks is numeric (e.g., "1000000")
             let blockId;
             let incrementField;
 
@@ -52,68 +194,42 @@ async function handleUpdate(task, taskId, dependencies, config) {
                 blockId = String(Math.floor(parseInt(userId) / 1000000) * 1000000);
                 incrementField = `counts.${instrumentId}_${blockId}`;
             } else {
-                
-                // <<< START FIX for Normal User Block ID >>>
-                // BUG: This was calculating '19M', which is wrong for the counter.
-                // blockId = `${Math.floor(parseInt(userId) / 1000000)}M`; 
-                
-                // FIX: Use the numeric '19000000' format, just like the orchestrator.
                 blockId = String(Math.floor(parseInt(userId) / 1000000) * 1000000);
-                // <<< END FIX for Normal User Block ID >>>
-
                 incrementField = `counts.${blockId}`;
             }
             
-            // Use db from dependencies
             const blockCountsRef = db.doc(userType === 'speculator' 
                 ? config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS 
                 : config.FIRESTORE_DOC_BLOCK_COUNTS);
-            // <<< END FIX for private user blockId format >>>
                 
             // This is a single, immediate write, which is fine for this rare case.
+            // We use .set here because batchManager.flushBatches() might not run if this is the only user.
+            // To be safer, this should also be added to a batch, but this is an edge case.
             await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
-
             return;
         }
-        
-        if (!response.ok) {
-            throw new Error(`API Error: Status ${response.status}`);
-        }
-        wasSuccess = true;
-        const portfolioData = JSON.parse(responseBody);
-
-        const today = new Date().toISOString().slice(0, 10);
-        
-        // <<< START OPTIMIZATION / FULL CODE FIX >>>
-
-        // BUG 1: The blockId format is different for portfolio storage ('1M') 
-        // vs. orchestrator logic ('1000000').
-        const portfolioStorageBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
-        
-        // This call is correct for storing portfolio data (which uses the 'M' format)
-        await batchManager.addToPortfolioBatch(userId, portfolioStorageBlockId, today, portfolioData, userType, instrumentId);
-        
-        // This call is correct for 'normal' users.
-        await batchManager.updateUserTimestamp(userId, userType, instrumentId);
 
-        // BUG 2 (The Main Problem): We must *also* update the 'SpeculatorBlocks'
-        // collection, which is what the orchestrator *does* read.
-        if (userType === 'speculator') {
-            const orchestratorBlockId = String(Math.floor(parseInt(userId) / 1000000) * 1000000);
+        // --- If either fetch was successful, update timestamps ---
+        if (wasPortfolioSuccess || wasHistorySuccess) {
+            // This call is correct for 'normal' users.
+            await batchManager.updateUserTimestamp(userId, userType, instrumentId);
 
-            logger.log('INFO', `[UPDATE] Applying speculator timestamp fix for user ${userId} in block ${orchestratorBlockId}`);
-            
-            // --- OPTIMIZATION: Use the batch manager instead of committing immediately ---
-            // This queues the write and allows the function to return,
-            // instead of blocking on `await fixBatch.commit()`.
-            await batchManager.addSpeculatorTimestampFix(userId, orchestratorBlockId);
-            logger.log('INFO', `[UPDATE] Speculator timestamp fix for user ${userId} queued.`);
+            // This is the fix for speculator timestamps
+            if (userType === 'speculator') {
+                const orchestratorBlockId = String(Math.floor(parseInt(userId) / 1000000) * 1000000);
+                logger.log('INFO', `[UPDATE] Applying speculator timestamp fix for user ${userId} in block ${orchestratorBlockId}`);
+                await batchManager.addSpeculatorTimestampFix(userId, orchestratorBlockId);
+            }
         }
-        // <<< END OPTIMIZATION / FULL CODE FIX >>>
-
+        
     } finally {
-        if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess);
+        // Update performance for both headers used
+        if (portfolioHeader) headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess);
+        if (historyHeader) headerManager.updatePerformance(historyHeader.id, wasHistorySuccess);
     }
 }
 
-module.exports = { handleUpdate };
+module.exports = { 
+    handleUpdate,
+    lookupUsernames // <-- EXPORT NEW HELPER
+};

package/functions/task-engine/helpers/verify_helpers.js

@@ -2,13 +2,13 @@
  * @fileoverview Sub-pipe: pipe.taskEngine.handleVerify
  * REFACTORED: Now stateless and receives dependencies.
  * OPTIMIZED: Fetches all user portfolios in parallel to reduce function runtime.
+ * --- MODIFIED: Saves username to cid_username_map collection. ---
  */
 const { FieldValue } = require('@google-cloud/firestore');
 
 /**
  * Internal helper to fetch and process a single user's portfolio.
- * This allows the main function to run these in parallel.
- * @param {object} user - The user object { cid, isBronze }
+ * @param {object} user - The user object { cid, isBronze, username }
  * @param {object} dependencies - Contains proxyManager, headerManager
  * @param {object} config - The configuration object
  * @param {Set<number>} speculatorInstrumentSet - Pre-built Set of instrument IDs
@@ -46,6 +46,7 @@ async function fetchAndVerifyUser(user, dependencies, config, speculatorInstrume
                     type: 'speculator',
                     userId: userId,
                     isBronze: user.isBronze,
+                    username: user.username, // <-- PASS USERNAME
                     updateData: {
                         instruments: matchingInstruments,
                         lastVerified: new Date(),
@@ -62,6 +63,7 @@ async function fetchAndVerifyUser(user, dependencies, config, speculatorInstrume
                 type: 'normal',
                 userId: userId,
                 isBronze: user.isBronze,
+                username: user.username, // <-- PASS USERNAME
                 updateData: {
                     lastVerified: new Date()
                 }
@@ -93,6 +95,10 @@ async function handleVerify(task, taskId, dependencies, config) {
     const speculatorUpdates = {};
     const normalUserUpdates = {};
     const bronzeStateUpdates = {}; // This can be one object
+    
+    // --- NEW: Object to store username updates ---
+    const usernameMapUpdates = {};
+    // --- END NEW ---
 
     const speculatorInstrumentSet = new Set(config.SPECULATOR_INSTRUMENTS_ARRAY);
     
@@ -118,6 +124,13 @@ async function handleVerify(task, taskId, dependencies, config) {
 
         const data = result.value;
         
+        // --- NEW: Capture username ---
+        if (data.username) {
+            // We use the CID as the key for the username map
+            usernameMapUpdates[data.userId] = { username: data.username };
+        }
+        // --- END NEW ---
+
         if (data.type === 'speculator') {
             speculatorUpdates[`users.${data.userId}`] = data.updateData;
             bronzeStateUpdates[data.userId] = data.isBronze;
@@ -154,6 +167,18 @@ async function handleVerify(task, taskId, dependencies, config) {
         }
     }
     
+    // --- NEW: Add username map updates to the batch ---
+    // We will use a simple sharding strategy (e.g., by first letter of CID)
+    // For simplicity here, we'll write to one document.
+    // A better production strategy would shard.
+    if (Object.keys(usernameMapUpdates).length > 0) {
+        // Simple, non-scalable approach:
+        const mapDocRef = db.collection(config.FIRESTORE_COLLECTION_USERNAME_MAP).doc('cid_map_shard_1');
+        batch.set(mapDocRef, usernameMapUpdates, { merge: true });
+        logger.log('INFO', `[VERIFY] Staging ${Object.keys(usernameMapUpdates).length} username updates.`);
+    }
+    // --- END NEW ---
+    
     await batch.commit();
     if(validUserCount > 0) {
         logger.log('INFO', `[VERIFY] Verified and stored ${validUserCount} new ${userType} users.`);

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -2,6 +2,7 @@
  * @fileoverview Utility class to manage stateful Firestore write batches.
  * REFACTORED: Renamed 'firestore' to 'db' for consistency.
  * OPTIMIZED: Added logic to handle speculator timestamp fixes within the batch.
+ * --- MODIFIED: Added username map caching and trading history batching. ---
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
@@ -23,7 +24,18 @@ class FirestoreBatchManager {
         this.portfolioBatch = {};
         this.timestampBatch = {};
         this.processedSpeculatorCids = new Set();
-        this.speculatorTimestampFixBatch = {}; // <-- OPTIMIZATION: ADD THIS
+        this.speculatorTimestampFixBatch = {};
+        
+        // --- NEW STATE ---
+        this.tradingHistoryBatch = {}; // For the new trade history data
+        this.usernameMap = new Map(); // In-memory cache: Map<cid, username>
+        this.usernameMapUpdates = {}; // Batched updates for Firestore: { [cid]: { username } }
+        this.usernameMapLastLoaded = 0;
+        this.usernameMapCollectionName = config.FIRESTORE_COLLECTION_USERNAME_MAP;
+        this.normalHistoryCollectionName = config.FIRESTORE_COLLECTION_NORMAL_HISTORY;
+        this.speculatorHistoryCollectionName = config.FIRESTORE_COLLECTION_SPECULATOR_HISTORY;
+        // --- END NEW STATE ---
+
         this.batchTimeout = null;
 
         this.logger.log('INFO', 'FirestoreBatchManager initialized.');
@@ -41,6 +53,82 @@ class FirestoreBatchManager {
         }
     }
 
+    // --- NEW: Load the username map into memory ---
+    async loadUsernameMap() {
+        // Cache map for 1 hour to reduce reads
+        if (Date.now() - this.usernameMapLastLoaded < 3600000) {
+            return;
+        }
+
+        this.logger.log('INFO', '[BATCH] Refreshing username map from Firestore...');
+        this.usernameMap.clear();
+        
+        try {
+            const snapshot = await this.db.collection(this.usernameMapCollectionName).get();
+            if (snapshot.empty) {
+                this.logger.log('WARN', '[BATCH] Username map collection is empty.');
+            } else {
+                snapshot.forEach(doc => {
+                    const data = doc.data();
+                    // Assumes doc.id is CID, or data is { [cid]: { username } }
+                    for (const cid in data) {
+                        if (data[cid] && data[cid].username) {
+                            this.usernameMap.set(String(cid), data[cid].username);
+                        }
+                    }
+                });
+            }
+            this.usernameMapLastLoaded = Date.now();
+            this.logger.log('INFO', `[BATCH] Loaded ${this.usernameMap.size} usernames into memory cache.`);
+        } catch (error) {
+            this.logger.log('ERROR', '[BATCH] Failed to load username map.', { errorMessage: error.message });
+        }
+    }
+
+    // --- NEW: Get username from in-memory cache ---
+    getUsername(cid) {
+        return this.usernameMap.get(String(cid));
+    }
+
+    // --- NEW: Add a username to the cache and the write batch ---
+    addUsernameMapUpdate(cid, username) {
+        const cidStr = String(cid);
+        if (!username) return;
+
+        // Update in-memory cache immediately
+        this.usernameMap.set(cidStr, username);
+        
+        // Add to Firestore write batch
+        // We'll write this to a single-field doc for simple merging
+        this.usernameMapUpdates[cidStr] = { username: username };
+        this.logger.log('TRACE', `[BATCH] Queued username update for ${cidStr}.`);
+        this._scheduleFlush();
+    }
+    
+    // --- NEW: Add trading history to its own batch ---
+    async addToTradingHistoryBatch(userId, blockId, date, historyData, userType) {
+        const collection = userType === 'speculator'
+            ? this.speculatorHistoryCollectionName
+            : this.normalHistoryCollectionName;
+        
+        const basePath = `${collection}/${blockId}/snapshots/${date}`;
+
+        if (!this.tradingHistoryBatch[basePath]) {
+            this.tradingHistoryBatch[basePath] = {};
+        }
+
+        this.tradingHistoryBatch[basePath][userId] = historyData;
+
+        // Trigger flush based on portfolio batch size, assuming they'll be roughly equal
+        const totalUsersInBatch = Object.values(this.portfolioBatch).reduce((sum, users) => sum + Object.keys(users).length, 0);
+
+        if (totalUsersInBatch >= this.config.TASK_ENGINE_MAX_BATCH_SIZE) {
+            await this.flushBatches();
+        } else {
+            this._scheduleFlush();
+        }
+    }
+
     /**
      * Adds a portfolio to the batch.
      * @param {string} userId 
@@ -148,41 +236,56 @@ class FirestoreBatchManager {
     // --- END NEW METHOD ---
 
     /**
-     * Flushes all pending writes to Firestore and updates header performance.
+     * Helper to flush a specific batch type (portfolio or history).
+     * @param {object} batchData - The batch object (e.g., this.portfolioBatch)
+     * @param {Firestore.Batch} firestoreBatch - The main Firestore batch object
+     * @param {string} logName - A name for logging (e.g., "Portfolio")
+     * @returns {number} The number of batch operations added.
      */
-    async flushBatches() {
-        if (this.batchTimeout) {
-            clearTimeout(this.batchTimeout);
-            this.batchTimeout = null;
-        }
-
-        const promises = [];
-        const firestoreBatch = this.db.batch(); // Use this.db
-        let batchOperationCount = 0;
-
-        for (const basePath in this.portfolioBatch) {
-            const userPortfolios = this.portfolioBatch[basePath];
+    _flushDataBatch(batchData, firestoreBatch, logName) {
+        let operationCount = 0;
+        for (const basePath in batchData) {
+            const userPortfolios = batchData[basePath];
             const userIds = Object.keys(userPortfolios);
 
             if (userIds.length > 0) {
                 for (let i = 0; i < userIds.length; i += this.config.TASK_ENGINE_MAX_USERS_PER_SHARD) {
                     const chunkUserIds = userIds.slice(i, i + this.config.TASK_ENGINE_MAX_USERS_PER_SHARD);
-                    const shardIndex = Math.floor(i / this.config.TASK_ENGINE_MAX_USERS_PER_SHARD);
-
+                    
                     const chunkData = {};
                     chunkUserIds.forEach(userId => {
                         chunkData[userId] = userPortfolios[userId];
                     });
 
-                    const docRef = this.db.collection(`${basePath}/parts`).doc(); // Now uses a reandomised ID from firestore to solve an overwrite issue with contention
-                    firestoreBatch.set(docRef, chunkData); // No merge: true needed, it's a new doc
-                    batchOperationCount++;
+                    const docRef = this.db.collection(`${basePath}/parts`).doc(); // Use random ID
+                    firestoreBatch.set(docRef, chunkData);
+                    operationCount++;
                 }
-                    // (Optional) Update the log message to reflect the path used for the data write
-                this.logger.log('INFO', `[BATCH] Staged ${userIds.length} users into ${Math.ceil(userIds.length / this.config.TASK_ENGINE_MAX_USERS_PER_SHARD)} unique shard documents for ${basePath}.`);
+                this.logger.log('INFO', `[BATCH] Staged ${userIds.length} users' ${logName} data into ${Math.ceil(userIds.length / this.config.TASK_ENGINE_MAX_USERS_PER_SHARD)} shards for ${basePath}.`);
             }
-            delete this.portfolioBatch[basePath];
+            delete batchData[basePath];
         }
+        return operationCount;
+    }
+
+
+    /**
+     * Flushes all pending writes to Firestore and updates header performance.
+     */
+    async flushBatches() {
+        if (this.batchTimeout) {
+            clearTimeout(this.batchTimeout);
+            this.batchTimeout = null;
+        }
+
+        const promises = [];
+        const firestoreBatch = this.db.batch(); // Use this.db
+        let batchOperationCount = 0;
+
+        // --- REFACTOR: Use helper for both batches ---
+        batchOperationCount += this._flushDataBatch(this.portfolioBatch, firestoreBatch, "Portfolio");
+        batchOperationCount += this._flushDataBatch(this.tradingHistoryBatch, firestoreBatch, "Trade History");
+        // --- END REFACTOR ---
 
         for (const docPath in this.timestampBatch) {
             if (Object.keys(this.timestampBatch[docPath]).length > 0) {
@@ -243,7 +346,18 @@ class FirestoreBatchManager {
             }
             delete this.speculatorTimestampFixBatch[docPath];
         }
-        // --- END OPTIMIZATION ---
+        
+        // --- NEW: Flush Username Map Updates ---
+        if (Object.keys(this.usernameMapUpdates).length > 0) {
+            this.logger.log('INFO', `[BATCH] Flushing ${Object.keys(this.usernameMapUpdates).length} username map updates.`);
+            // Simple sharding: just merge into one doc for now.
+            // A more robust solution would shard based on CID.
+            const mapDocRef = this.db.collection(this.usernameMapCollectionName).doc('cid_map_shard_1');
+            firestoreBatch.set(mapDocRef, this.usernameMapUpdates, { merge: true });
+            batchOperationCount++;
+            this.usernameMapUpdates = {}; // Clear the batch
+        }
+        // --- END NEW ---
 
         if (batchOperationCount > 0) {
             promises.push(firestoreBatch.commit());

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.122",
+  "version": "1.0.124",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [