bulltrackers-module 1.0.152 → 1.0.154

package/functions/task-engine/helpers/update_helpers.js

@@ -1,6 +1,8 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
  * (MODIFIED: To conditionally fetch history API once per user per batch)
+ * (MODIFIED: `lookupUsernames` runs batches in parallel)
+ * (MODIFIED: `handleUpdate` fetches history and all portfolios in parallel)
  */
 
 /**
@@ -12,136 +14,108 @@
  * --- MODIFIED: Conditionally fetches history only once per user per batch. ---
  */
 const { FieldValue } = require('@google-cloud/firestore');
+const pLimit = require('p-limit'); // <--- IMPORT p-limit
 
-async function lookupUsernames(cids, { logger, headerManager, proxyManager }, { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL }) {
+/**
+ * (MODIFIED: Runs lookup batches in parallel)
+ */
+async function lookupUsernames(cids, { logger, headerManager, proxyManager }, config) {
     if (!cids?.length) return [];
     logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
-    const allUsers = [];
-    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) {
-        const batch = cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number);
+    const limit = pLimit(config.USERNAME_LOOKUP_CONCURRENCY || 5);
+    const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
+    const batches = [];
+    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) { batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number)); }
+
+    const batchPromises = batches.map(batch => limit(async () => {
         const header = await headerManager.selectHeader();
-        if (!header) { logger.log('ERROR', '[lookupUsernames] Could not select a header.'); continue; }
+        if (!header) { logger.log('ERROR', '[lookupUsernames] Could not select a header.'); return null; }
         let success = false;
         try {
-            const res = await proxyManager.fetch(`${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`, {
-                method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch)
-            });
+            const res = await proxyManager.fetch(`${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`, { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) });
             if (!res.ok) throw new Error(`API status ${res.status}`);
             const data = await res.json();
-            if (Array.isArray(data)) allUsers.push(...data);
             success = true;
-        } catch (err) {
-            logger.log('WARN', `[lookupUsernames] Failed batch`, { error: err.message });
-        } finally { headerManager.updatePerformance(header.id, success); }
-    }
+            logger.log('DEBUG', 'Looked up usernames', { batch: batch.slice(0, 5) }); 
+            return data;
+        } catch (err) { logger.log('WARN', `[lookupUsernames] Failed batch`, { error: err.message }); return null; 
+        } finally { headerManager.updatePerformance(header.id, success); } }));
+    const results = await Promise.allSettled(batchPromises);
+    const allUsers = results.filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value)).flatMap(r => r.value);
     logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
     return allUsers;
 }
 
-// --- START MODIFICATION: Added historyFetchedForUser argument ---
+
+/**
+ * (MODIFIED: Fetches history and all portfolios in parallel)
+ */
 async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username, historyFetchedForUser) {
-    const { userId, instruments, instrumentId, userType } = task; // ⚠️ Now supports both
-    
-    // ⚠️ Support both old (instrumentId) and new (instruments array) format
-    const instrumentsToProcess = userType === 'speculator' 
-        ? (instruments || [instrumentId]) // New format or fallback to old
-        : [undefined]; // Normal users don't have instruments
-    
+    const { userId, instruments, instrumentId, userType } = task;
+    const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
-    
-    let portfolioHeader = await headerManager.selectHeader();
+
     let historyHeader = null;
-    if (!portfolioHeader) throw new Error("Could not select portfolio header.");
-    
-    let wasHistorySuccess = false, isPrivate = false;
-    
-    // --- FIX: 'fetchHistory' MUST be declared here, outside the 'try' block ---
-    let fetchHistory = false;
-    
+    let wasHistorySuccess = false;
+    let historyFetchPromise = null;
+    let isPrivate = false;
+
     try {
-        // Fetch history ONCE per user
-        const promisesToRun = [];
-        // --- 'fetchHistory' is no longer declared here ---
-        
+        // --- 1. Prepare History Fetch (if needed) ---
         if (!historyFetchedForUser.has(userId)) {
             historyHeader = await headerManager.selectHeader();
-            if (historyHeader) {
-                fetchHistory = true; // This now sets the outer variable
-                historyFetchedForUser.add(userId);
+            if (historyHeader) { historyFetchedForUser.add(userId);
                 const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
-                promisesToRun.push(proxyManager.fetch(historyUrl, { headers: historyHeader.header }));
-            }
-        }
-        
-        // Process history result (if fetched)
-        if (fetchHistory) {
-            const results = await Promise.allSettled(promisesToRun);
-            const historyRes = results[0];
-            if (historyRes.status === 'fulfilled' && historyRes.value.ok) {
-                const data = await historyRes.value.json();
-                wasHistorySuccess = true; 
-                await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
-            }
-        }
+                historyFetchPromise = proxyManager.fetch(historyUrl, { headers: historyHeader.header }); } }
+
+        // --- 2. Prepare All Portfolio Fetches ---
+        const portfolioRequests = [];
+        for (const instId of instrumentsToProcess) {
+            const portfolioHeader = await headerManager.selectHeader();
+            if (!portfolioHeader) throw new Error(`Could not select portfolio header for ${userId}`);
+            const portfolioUrl = userType === 'speculator'  ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+            portfolioRequests.push({ instrumentId: instId, url: portfolioUrl, header: portfolioHeader, promise: proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header }) }); }
         
-        // Now fetch portfolio for EACH instrument (speculators) or once (normal)
-        for (const instrumentId of instrumentsToProcess) {
-            const portfolioUrl = userType === 'speculator'
-                ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instrumentId}`
-                : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
-            
+        // --- 3. Execute All API Calls in Parallel ---
+        const allPromises = [ ...(historyFetchPromise ? [historyFetchPromise] : []), ...portfolioRequests.map(r => r.promise) ];
+        const allResults = await Promise.allSettled(allPromises);
+
+        // --- 4. Process History Result ---
+        let resultIndex = 0;
+        if (historyFetchPromise) {
+            const historyRes = allResults[resultIndex++];
+            if (historyRes.status === 'fulfilled' && historyRes.value.ok) { const data = await historyRes.value.json(); wasHistorySuccess = true; await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType); } }
+
+        // --- 5. Process Portfolio Results ---
+        for (let i = 0; i < portfolioRequests.length; i++) {
+            const requestInfo = portfolioRequests[i];
+            const portfolioRes = allResults[resultIndex++];
             let wasPortfolioSuccess = false;
-            
-            const portfolioRes = await proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header });
-            
-            if (portfolioRes.ok) {
-                const body = await portfolioRes.text();
-                if (body.includes("user is PRIVATE")) {
-                    isPrivate = true;
-                    break; // Stop processing this user
-                } else {
-                    wasPortfolioSuccess = true;
-                    await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, instrumentId);
-                }
-            }
-            
-            headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess);
-            
-            // Re-select header for next instrument
-            if (instrumentsToProcess.length > 1 && instrumentId !== instrumentsToProcess[instrumentsToProcess.length - 1]) {
-                portfolioHeader = await headerManager.selectHeader();
-            }
+            if (portfolioRes.status === 'fulfilled' && portfolioRes.value.ok) {
+                const body = await portfolioRes.value.text();
+                if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `User ${userId} is private. Removing from updates.`); break;
+                } else { wasPortfolioSuccess = true; await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, requestInfo.instrumentId); }
+                logger.log('DEBUG', 'Processing portfolio for user', { userId, portfolioUrl: requestInfo.url });
+            } else { logger.log('WARN', `Failed to fetch portfolio`, { userId, url: requestInfo.url, error: portfolioRes.reason || `status ${portfolioRes.value?.status}` }); }
+            headerManager.updatePerformance(requestInfo.header.id, wasPortfolioSuccess);
         }
-        
-        // Handle private user
+
+        // --- 6. Handle Private Users & Timestamps ---
         if (isPrivate) {
             logger.log('WARN', `User ${userId} is private. Removing from updates.`);
-            // Delete for ALL instruments
-            for (const instrumentId of instrumentsToProcess) {
-                await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
-            }
+            for (const instrumentId of instrumentsToProcess) { await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId); }
             const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
             for (const instrumentId of instrumentsToProcess) {
                 const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
-                await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
-            }
-            return;
-        }
-        
-        // Update timestamps
-        for (const instrumentId of instrumentsToProcess) {
-            await batchManager.updateUserTimestamp(userId, userType, instrumentId);
-        }
-        if (userType === 'speculator') {
-            await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
-        }
-        
-    } finally {
-        if (historyHeader && fetchHistory) {
-            headerManager.updatePerformance(historyHeader.id, wasHistorySuccess);
+                await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true }); }
+            return; 
         }
-    }
+
+        for (const instrumentId of instrumentsToProcess) { await batchManager.updateUserTimestamp(userId, userType, instrumentId); }
+        if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
+
+    } finally { if (historyHeader) {  headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } }
 }
 
 module.exports = { handleUpdate, lookupUsernames };

package/functions/task-engine/helpers/verify_helpers.js

@@ -3,27 +3,15 @@ const { FieldValue } = require('@google-cloud/firestore');
 async function fetchAndVerifyUser(user, { logger, headerManager, proxyManager }, { userType, SPECULATOR_INSTRUMENTS_ARRAY }) {
     const selectedHeader = await headerManager.selectHeader();
     if (!selectedHeader) return null;
-
     let wasSuccess = false;
-    try {
-        const res = await proxyManager.fetch(`${process.env.ETORO_API_PORTFOLIO_URL}?cid=${user.cid}`, { headers: selectedHeader.header });
-        if (!res.ok) return null;
-
+    try { const res = await proxyManager.fetch(`${process.env.ETORO_API_PORTFOLIO_URL}?cid=${user.cid}`, { headers: selectedHeader.header }); if (!res.ok) return null;
         wasSuccess = true;
         const portfolioData = await res.json();
-        if (userType === 'speculator') {
-            const instruments = portfolioData.AggregatedPositions.map(p => p.InstrumentID)
-                                .filter(id => SPECULATOR_INSTRUMENTS_ARRAY.includes(id));
-            if (!instruments.length) return null;
-            return { type: 'speculator', userId: user.cid, isBronze: user.isBronze, username: user.username, updateData: { instruments, lastVerified: new Date(), lastHeldSpeculatorAsset: new Date() } };
-        }
+        if (userType === 'speculator') { const instruments = portfolioData.AggregatedPositions.map(p => p.InstrumentID) .filter(id => SPECULATOR_INSTRUMENTS_ARRAY.includes(id));
+            if (!instruments.length) return null; return { type: 'speculator', userId: user.cid, isBronze: user.isBronze, username: user.username, updateData: { instruments, lastVerified: new Date(), lastHeldSpeculatorAsset: new Date() } }; }
         return { type: 'normal', userId: user.cid, isBronze: user.isBronze, username: user.username, updateData: { lastVerified: new Date() } };
-    } catch (err) {
-        logger.log('WARN', `[VERIFY] Error processing user ${user.cid}`, { errorMessage: err.message });
-        return null;
-    } finally {
-        if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess);
-    }
+    } catch (err) { logger.log('WARN', `[VERIFY] Error processing user ${user.cid}`, { errorMessage: err.message }); return null;
+    } finally { if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess); }
 }
 
 async function handleVerify(task, taskId, { db, logger, ...dependencies }, config) {
@@ -31,56 +19,23 @@ async function handleVerify(task, taskId, { db, logger, ...dependencies }, confi
     const batch = db.batch();
     const speculatorUpdates = {}, normalUpdates = {}, bronzeStates = {}, usernameMap = {};
     const specSet = new Set(config.SPECULATOR_INSTRUMENTS_ARRAY);
-
     const results = await Promise.allSettled(users.map(u => fetchAndVerifyUser(u, dependencies, { ...config, userType })));
-
     let validUserCount = 0;
     results.forEach(r => {
-        if (r.status === 'fulfilled' && r.value) {
-            const d = r.value;
-            usernameMap[d.userId] = { username: d.username };
-            bronzeStates[d.userId] = d.isBronze;
-            validUserCount++;
-            if (d.type === 'speculator') speculatorUpdates[`users.${d.userId}`] = d.updateData;
-            else normalUpdates[`users.${d.userId}`] = d.updateData;
-        }
-    });
-
+        if (r.status === 'fulfilled' && r.value) { const d = r.value; usernameMap[d.userId] = { username: d.username }; bronzeStates[d.userId] = d.isBronze; validUserCount++; if (d.type === 'speculator') speculatorUpdates[`users.${d.userId}`] = d.updateData; else normalUpdates[`users.${d.userId}`] = d.updateData; } });
     if (Object.keys(speculatorUpdates).length || Object.keys(normalUpdates).length) {
         const blockRef = db.collection(userType === 'speculator' ? config.FIRESTORE_COLLECTION_SPECULATOR_BLOCKS : config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS).doc(String(blockId));
         batch.set(blockRef, userType === 'speculator' ? speculatorUpdates : normalUpdates, { merge: true });
         const bronzeRef = db.collection(userType === 'speculator' ? config.FIRESTORE_COLLECTION_BRONZE_SPECULATORS : config.FIRESTORE_COLLECTION_BRONZE_NORMAL).doc(String(blockId));
         batch.set(bronzeRef, bronzeStates, { merge: true });
+        if (validUserCount) { const countsRef = db.doc(userType === 'speculator' ? config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS : config.FIRESTORE_DOC_BLOCK_COUNTS); const field = userType === 'speculator' ? `counts.${instrument}_${blockId}` : `counts.${blockId}`; batch.set(countsRef, { [field]: FieldValue.increment(validUserCount) }, { merge: true }); } }
 
-        if (validUserCount) {
-            const countsRef = db.doc(userType === 'speculator' ? config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS : config.FIRESTORE_DOC_BLOCK_COUNTS);
-            const field = userType === 'speculator' ? `counts.${instrument}_${blockId}` : `counts.${blockId}`;
-            batch.set(countsRef, { [field]: FieldValue.increment(validUserCount) }, { merge: true });
-        }
-    }
-
-    // --- START FIX: MODIFIED FOR SHARDING ---
     if (Object.keys(usernameMap).length) {
-        // Group updates by shard
         const shardedUpdates = {};
-        for (const cid in usernameMap) {
-            // Re-implement the sharding logic here
-            const shardId = `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`;
-            if (!shardedUpdates[shardId]) {
-                shardedUpdates[shardId] = {};
-            }
-            shardedUpdates[shardId][cid] = usernameMap[cid];
-        }
-
-        // Write each shard to its own document
-        for (const shardId in shardedUpdates) {
-            const mapRef = db.collection(config.FIRESTORE_COLLECTION_USERNAME_MAP).doc(shardId);
-            batch.set(mapRef, shardedUpdates[shardId], { merge: true });
-        }
-        logger.log('INFO', `[VERIFY] Staging username updates across ${Object.keys(shardedUpdates).length} shards.`);
-    }
-    // --- END FIX ---
-
+        for (const cid in usernameMap) { const shardId = `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`;
+            if (!shardedUpdates[shardId]) { shardedUpdates[shardId] = {}; } shardedUpdates[shardId][cid] = usernameMap[cid]; }
+        for (const shardId in shardedUpdates) { const mapRef = db.collection(config.FIRESTORE_COLLECTION_USERNAME_MAP).doc(shardId); batch.set(mapRef, shardedUpdates[shardId], { merge: true }); }
+        logger.log('INFO', `[VERIFY] Staging username updates across ${Object.keys(shardedUpdates).length} shards.`); }
     await batch.commit();
     if (validUserCount) logger.log('INFO', `[VERIFY] Verified and stored ${validUserCount} new ${userType} users.`);
 }

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -12,7 +12,6 @@ class FirestoreBatchManager {
         this.headerManager = headerManager;
         this.logger = logger;
         this.config = config;
-
         this.portfolioBatch = {};
         this.timestampBatch = {};
         this.tradingHistoryBatch = {};
@@ -21,25 +20,30 @@ class FirestoreBatchManager {
         this.usernameMapUpdates = {};
         this.usernameMapLastLoaded = 0;
         this.processedSpeculatorCids = new Set();
-
         this.usernameMapCollectionName = config.FIRESTORE_COLLECTION_USERNAME_MAP;
         this.normalHistoryCollectionName = config.FIRESTORE_COLLECTION_NORMAL_HISTORY;
         this.speculatorHistoryCollectionName = config.FIRESTORE_COLLECTION_SPECULATOR_HISTORY;
-
         this.batchTimeout = null;
-
         logger.log('INFO', 'FirestoreBatchManager initialized.');
     }
 
-    // --- START FIX: ADDED SHARDING HELPER ---
-    _getUsernameShardId(cid) {
-        // Shard across 10 documents (supports ~200k users assuming 50 bytes/user)
-        return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`;
-    }
-    // --- END FIX ---
+    _getUsernameShardId(cid) { return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`; }
+
+    // _scheduleFlush() { if (!this.batchTimeout) this.batchTimeout = setTimeout(() => this.flushBatches(), this.config.TASK_ENGINE_FLUSH_INTERVAL_MS); } Old version
 
     _scheduleFlush() {
-        if (!this.batchTimeout) this.batchTimeout = setTimeout(() => this.flushBatches(), this.config.TASK_ENGINE_FLUSH_INTERVAL_MS);
+        const totalOps = this._estimateBatchSize();
+        if (totalOps >= 400) { this.flushBatches(); return; }
+        if (!this.batchTimeout) { this.batchTimeout = setTimeout(() => this.flushBatches(),  this.config.TASK_ENGINE_FLUSH_INTERVAL_MS); }
+    }
+
+    _estimateBatchSize() {
+        let ops = 0;
+        ops += Object.keys(this.portfolioBatch).length;
+        ops += Object.keys(this.tradingHistoryBatch).length;
+        ops += Object.keys(this.timestampBatch).length;
+        ops += Object.keys(this.speculatorTimestampFixBatch).length;
+        return ops;
     }
 
     async loadUsernameMap() {
@@ -47,38 +51,25 @@ class FirestoreBatchManager {
         this.usernameMap.clear();
         this.logger.log('INFO', '[BATCH] Refreshing username map from Firestore...');
         try {
-            // This correctly gets ALL documents (shards) in the collection
             const snapshot = await this.db.collection(this.usernameMapCollectionName).get();
-            snapshot.forEach(doc => {
-                const data = doc.data();
-                for (const cid in data) if (data[cid]?.username) this.usernameMap.set(String(cid), data[cid].username);
-            });
+            snapshot.forEach(doc => { const data = doc.data(); for (const cid in data) if (data[cid]?.username) this.usernameMap.set(String(cid), data[cid].username); });
             this.usernameMapLastLoaded = Date.now();
             this.logger.log('INFO', `[BATCH] Loaded ${this.usernameMap.size} usernames.`);
-        } catch (e) {
-            this.logger.log('ERROR', '[BATCH] Failed to load username map.', { errorMessage: e.message });
-        }
+        } catch (e) { this.logger.log('ERROR', '[BATCH] Failed to load username map.', { errorMessage: e.message }); }
     }
 
     getUsername(cid) { return this.usernameMap.get(String(cid)); }
 
-    // --- START FIX: MODIFIED FOR SHARDING ---
     addUsernameMapUpdate(cid, username) {
         if (!username) return;
         const cidStr = String(cid);
         this.usernameMap.set(cidStr, username);
-
-        // Organize updates by shard ID
         const shardId = this._getUsernameShardId(cidStr);
-        if (!this.usernameMapUpdates[shardId]) {
-            this.usernameMapUpdates[shardId] = {};
-        }
+        if (!this.usernameMapUpdates[shardId]) { this.usernameMapUpdates[shardId] = {}; }
         this.usernameMapUpdates[shardId][cidStr] = { username };
-
         this.logger.log('TRACE', `[BATCH] Queued username update for ${cidStr} in ${shardId}.`);
         this._scheduleFlush();
     }
-    // --- END FIX ---
 
     async addToTradingHistoryBatch(userId, blockId, date, historyData, userType) {
         const collection = userType === 'speculator' ? this.speculatorHistoryCollectionName : this.normalHistoryCollectionName;
@@ -108,10 +99,7 @@ class FirestoreBatchManager {
     deleteFromTimestampBatch(userId, userType, instrumentId) {
         const collection = userType === 'speculator' ? this.config.FIRESTORE_COLLECTION_SPECULATOR_PORTFOLIOS : this.config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS;
         const docPath = `${collection}/${userType === 'speculator' ? 'speculators' : 'normal'}`;
-        if (this.timestampBatch[docPath]) {
-            const key = userType === 'speculator' ? `${userId}_${instrumentId}` : userId;
-            delete this.timestampBatch[docPath][key];
-        }
+        if (this.timestampBatch[docPath]) { const key = userType === 'speculator' ? `${userId}_${instrumentId}` : userId; delete this.timestampBatch[docPath][key]; }
     }
 
     addProcessedSpeculatorCids(cids) { cids.forEach(cid => this.processedSpeculatorCids.add(cid)); }
@@ -131,11 +119,7 @@ class FirestoreBatchManager {
             const users = batchData[basePath];
             const userIds = Object.keys(users);
             if (!userIds.length) continue;
-            for (let i = 0; i < userIds.length; i += this.config.TASK_ENGINE_MAX_USERS_PER_SHARD) {
-                const chunkData = Object.fromEntries(userIds.slice(i, i + this.config.TASK_ENGINE_MAX_USERS_PER_SHARD).map(id => [id, users[id]]));
-                firestoreBatch.set(this.db.collection(`${basePath}/parts`).doc(), chunkData);
-                count++;
-            }
+            for (let i = 0; i < userIds.length; i += this.config.TASK_ENGINE_MAX_USERS_PER_SHARD) { const chunkData = Object.fromEntries(userIds.slice(i, i + this.config.TASK_ENGINE_MAX_USERS_PER_SHARD).map(id => [id, users[id]])); firestoreBatch.set(this.db.collection(`${basePath}/parts`).doc(), chunkData); count++; }
             this.logger.log('INFO', `[BATCH] Staged ${userIds.length} ${logName} users in ${Math.ceil(userIds.length / this.config.TASK_ENGINE_MAX_USERS_PER_SHARD)} shards for ${basePath}.`);
             delete batchData[basePath];
         }
@@ -144,13 +128,10 @@ class FirestoreBatchManager {
 
     async flushBatches() {
         if (this.batchTimeout) { clearTimeout(this.batchTimeout); this.batchTimeout = null; }
-
         const firestoreBatch = this.db.batch();
         let batchOps = 0;
-
         batchOps += this._flushDataBatch(this.portfolioBatch, firestoreBatch, 'Portfolio');
         batchOps += this._flushDataBatch(this.tradingHistoryBatch, firestoreBatch, 'Trade History');
-
         for (const docPath in this.timestampBatch) {
             const timestamps = this.timestampBatch[docPath];
             if (!Object.keys(timestamps).length) continue;
@@ -168,39 +149,22 @@ class FirestoreBatchManager {
             delete this.speculatorTimestampFixBatch[docPath];
         }
 
-        // --- START FIX: MODIFIED FOR SHARDING ---
-        // Loop over each shardId in the updates object
         for (const shardId in this.usernameMapUpdates) {
             const updates = this.usernameMapUpdates[shardId];
-            if (updates && Object.keys(updates).length > 0) {
-                firestoreBatch.set(
-                    this.db.collection(this.usernameMapCollectionName).doc(shardId),
-                    updates,
-                    { merge: true }
-                );
-                batchOps++;
-                this.logger.log('INFO', `[BATCH] Flushing ${Object.keys(updates).length} username updates to ${shardId}.`);
-            }
-        }
-        this.usernameMapUpdates = {}; // Clear updates after staging them
-        // --- END FIX ---
+            if (updates && Object.keys(updates).length > 0) { firestoreBatch.set( this.db.collection(this.usernameMapCollectionName).doc(shardId), updates, { merge: true } ); batchOps++; this.logger.log('INFO', `[BATCH] Flushing ${Object.keys(updates).length} username updates to ${shardId}.`); } }
+        this.usernameMapUpdates = {}; 
 
         if (this.processedSpeculatorCids.size) {
             const cids = Array.from(this.processedSpeculatorCids);
             this.processedSpeculatorCids.clear();
             const snapshot = await this.db.collection(this.config.PENDING_SPECULATORS_COLLECTION).get();
-            snapshot.forEach(doc => {
-                const docData = doc.data().users || {};
-                const cidsInDoc = cids.filter(cid => docData[cid]);
-                if (!cidsInDoc.length) return;
-                const delBatch = this.db.batch();
-                const updates = Object.fromEntries(cidsInDoc.map(cid => [`users.${cid}`, FieldValue.delete()]));
-                delBatch.update(doc.ref, updates);
-                delBatch.commit();
-                this.logger.log('INFO', `[BATCH] Deleted ${cidsInDoc.length} CIDs from ${doc.id}`);
-            });
-        }
-
+            snapshot.forEach(doc => { const docData = doc.data().users || {}; const cidsInDoc = cids.filter(cid => docData[cid]); if (!cidsInDoc.length) return;
+            const delBatch = this.db.batch();
+            const updates = Object.fromEntries(cidsInDoc.map(cid => [`users.${cid}`, FieldValue.delete()]));
+            delBatch.update(doc.ref, updates);
+            delBatch.commit();
+            this.logger.log('INFO', `[BATCH] Deleted ${cidsInDoc.length} CIDs from ${doc.id}`); }); }
+            
         if (batchOps) await firestoreBatch.commit();
         await this.headerManager.flushPerformanceUpdates();
         this.logger.log('INFO', '[BATCH] All batches flushed successfully.');

package/functions/task-engine/utils/task_engine_utils.js

@@ -1,6 +1,7 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/utils/task_engine_utils.js
  * (MODIFIED: To pass down a Set to track history fetches)
+ * (MODIFIED: To run all update tasks in parallel with a concurrency limit)
  */
 
 /**
@@ -12,6 +13,7 @@
 const { handleDiscover } = require('../helpers/discover_helpers');
 const { handleVerify } = require('../helpers/verify_helpers');
 const { handleUpdate, lookupUsernames } = require('../helpers/update_helpers');
+const pLimit = require('p-limit'); 
 
 /**
  * Parses Pub/Sub message into task array.
@@ -21,10 +23,7 @@ function parseTaskPayload(message, logger) {
     let payload;
     try { payload = JSON.parse(Buffer.from(message.data, 'base64').toString()); }
     catch (e) { logger.log('ERROR', '[TaskEngine] Failed to parse message', { error: e.message }); return null; }
-    if (!Array.isArray(payload.tasks) || payload.tasks.length === 0) {
-        logger.log(payload.tasks?.length ? 'WARN' : 'ERROR', '[TaskEngine] Invalid or empty tasks array.', { payload });
-        return null;
-    }
+    if (!Array.isArray(payload.tasks) || payload.tasks.length === 0) { logger.log(payload.tasks?.length ? 'WARN' : 'ERROR', '[TaskEngine] Invalid or empty tasks array.', { payload }); return null; }
     return payload.tasks;
 }
 
@@ -35,11 +34,7 @@ async function prepareTaskBatches(tasks, batchManager, logger) {
     const tasksToRun = [], cidsToLookup = new Map(), otherTasks = [];
     await batchManager.loadUsernameMap();
     for (const task of tasks) {
-        if (task.type === 'update') {
-            const username = batchManager.getUsername(task.userId);
-            username ? tasksToRun.push({ task, username }) : cidsToLookup.set(String(task.userId), task);
-        } else otherTasks.push(task);
-    }
+        if (task.type === 'update') { const username = batchManager.getUsername(task.userId); username ? tasksToRun.push({ task, username }) : cidsToLookup.set(String(task.userId), task); } else otherTasks.push(task); }
     logger.log('INFO', `[TaskEngine] Sorting complete. Known: ${tasksToRun.length}, Lookup: ${cidsToLookup.size}, Other: ${otherTasks.length}`);
     return { tasksToRun, cidsToLookup, otherTasks };
 }
@@ -51,12 +46,7 @@ async function runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config
     if (!cidsToLookup.size) return;
     logger.log('INFO', `[TaskEngine] Looking up ${cidsToLookup.size} usernames...`);
     const foundUsers = await lookupUsernames([...cidsToLookup.keys()], dependencies, config);
-    for (const u of foundUsers) {
-        const cid = String(u.CID), username = u.Value.UserName;
-        batchManager.addUsernameMapUpdate(cid, username);
-        const task = cidsToLookup.get(cid);
-        if (task) { tasksToRun.push({ task, username }); cidsToLookup.delete(cid); }
-    }
+    for (const u of foundUsers) { const cid = String(u.CID), username = u.Value.UserName; batchManager.addUsernameMapUpdate(cid, username); const task = cidsToLookup.get(cid); if (task) { tasksToRun.push({ task, username }); cidsToLookup.delete(cid); } }
     if (cidsToLookup.size) logger.log('WARN', `[TaskEngine] Could not find ${cidsToLookup.size} usernames (likely private).`, { skippedCids: [...cidsToLookup.keys()] });
 }
 
@@ -65,35 +55,22 @@ async function runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config
  */
 async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId) {
     const { logger } = dependencies;
-    
-    // --- START MODIFICATION ---
-    // This Set will track history fetches *only for this batch*.
     const historyFetchedForUser = new Set();
-    // --- END MODIFICATION ---
-
-
-    // Process discover/verify tasks
     for (const task of otherTasks) {
         const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId || task.cids?.[0] || 'sub'}`;
         const handler = { discover: handleDiscover, verify: handleVerify }[task.type];
         if (handler) try { await handler(task, subTaskId, dependencies, config); } 
         catch (err) { logger.log('ERROR', `[TaskEngine/${taskId}] Error in ${task.type} for ${subTaskId}`, { errorMessage: err.message }); }
-        else logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type: ${task.type}`);
-    }
-
-    // Process update tasks (all have usernames)
-    for (const { task, username } of tasksToRun) {
+        else logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type: ${task.type}`); }
+    const limit = pLimit(config.TASK_ENGINE_CONCURRENCY || 10);
+    let successCount = 0;
+    let errorCount = 0;
+    const updatePromises = tasksToRun.map(({ task, username }) => {
         const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId}`;
-        try {
-            // --- START MODIFICATION ---
-            // Pass the Set to the update handler
-            await handleUpdate(task, subTaskId, dependencies, config, username, historyFetchedForUser);
-            // --- END MODIFICATION ---
-        }
-        catch (err) { logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message }); }
-    }
-
-    logger.log('SUCCESS', `[TaskEngine/${taskId}] Processed all tasks.`);
+        return limit(() =>  handleUpdate(task, subTaskId, dependencies, config, username, historyFetchedForUser) .catch(err => { logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message }); throw err;  }) ); });
+    const results = await Promise.allSettled(updatePromises);
+    results.forEach(result => { if (result.status === 'fulfilled') { successCount++; } else { errorCount++; } });
+    logger.log( errorCount > 0 ? 'WARN' : 'SUCCESS', `[TaskEngine/${taskId}] Processed all ${tasksToRun.length} update tasks. Success: ${successCount}, Failed: ${errorCount}.` );
 }
 
 module.exports = { parseTaskPayload, prepareTaskBatches, runUsernameLookups, executeTasks };