bulltrackers-module 1.0.153 → 1.0.155

package/functions/core/utils/intelligent_proxy_manager.js

@@ -3,9 +3,13 @@
  * It selects an available (unlocked) proxy for each request and locks it upon failure.
  * * This module is designed to be reusable and receives all dependencies
  * (firestore, logger) and configuration via its constructor.
+ * --- MODIFIED: Now includes exponential backoff and retries specifically for rate-limit errors. ---
  */
 const { FieldValue } = require('@google-cloud/firestore');
 
+// --- NEW: Added sleep utility ---
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
 class IntelligentProxyManager {
     /**
      * @param {object} firestore - An initialized Firestore instance.
@@ -29,6 +33,11 @@ class IntelligentProxyManager {
         this.proxyLockingEnabled = config.proxyLockingEnabled !== false;
         this.proxies = {}; 
         this.configLastLoaded = 0;
+
+        // --- NEW: Retry configuration ---
+        this.MAX_RETRIES = 3;
+        this.INITIAL_BACKOFF_MS = 1000;
+
         if (this.proxyUrls.length === 0) { this.logger.log('WARN', '[ProxyManager] No proxy URLs provided in config.');
         } else { const lockingStatus = this.proxyLockingEnabled ? "Locking Mechanism Enabled" : "Locking Mechanism DISABLED"; this.logger.log('INFO', `[ProxyManager] Initialized with ${this.proxyUrls.length} proxies and ${lockingStatus}.`); }
     }
@@ -81,43 +90,105 @@ class IntelligentProxyManager {
     }
 
     /**
-     * Makes a fetch request using a selected proxy.
+     * --- MODIFIED: Makes a fetch request with exponential backoff for rate limits ---
      * @param {string} targetUrl - The URL to fetch.
      * @param {object} options - Fetch options (e.g., headers).
      * @returns {Promise<object>} A mock Response object.
      */
     async fetch(targetUrl, options = {}) {
         let proxy = null;
-        try { proxy = await this._selectProxy();
-        } catch (error) { return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() }; }
-        const response = await this._fetchViaAppsScript(proxy.url, targetUrl, options);
-        if (!response.ok && response.isUrlFetchError) { await this.lockProxy(proxy.owner); }
-        return response;
+        try {
+            proxy = await this._selectProxy();
+        } catch (error) {
+            // This happens if *all* proxies are locked.
+            return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() };
+        }
+
+        let backoff = this.INITIAL_BACKOFF_MS;
+        let lastResponse = null;
+
+        for (let attempt = 1; attempt <= this.MAX_RETRIES; attempt++) {
+            const response = await this._fetchViaAppsScript(proxy.url, targetUrl, options);
+            lastResponse = response; // Always store the last response
+
+            // 1. Success
+            if (response.ok) {
+                return response;
+            }
+
+            // 2. Rate Limit Error (Retryable)
+            if (response.isRateLimitError) {
+                this.logger.log('WARN', `[ProxyManager] Rate limit hit on proxy ${proxy.owner} (Attempt ${attempt}/${this.MAX_RETRIES}). Backing off for ${backoff}ms...`, { url: targetUrl });
+                await sleep(backoff);
+                backoff *= 2; // Exponential backoff
+                // Continue to the next attempt
+                continue; 
+            }
+
+            // 3. Other Fetch Error (Non-Retryable, Lock Proxy)
+            if (response.isUrlFetchError) {
+                this.logger.log('ERROR', `[ProxyManager] Proxy ${proxy.owner} failed (non-rate-limit). Locking proxy.`, { url: targetUrl, status: response.status });
+                await this.lockProxy(proxy.owner);
+                return response; // Fail fast and return
+            }
+            
+            // 4. Standard Error (e.g., 404, 500 from *target* URL, not proxy)
+            // This was a "successful" proxy fetch of a failing URL. Not retryable.
+            return response;
+        }
+
+        // If loop finishes, all retries failed (likely all were rate-limit errors)
+        this.logger.log('ERROR', `[ProxyManager] Request failed after ${this.MAX_RETRIES} rate-limit retries.`, { url: targetUrl });
+        return lastResponse;
     }
 
+
     /**
      * Internal function to call the Google AppScript proxy.
+     * --- MODIFIED: Now adds `isRateLimitError` flag to response ---
      * @private
      */
     async _fetchViaAppsScript(proxyUrl, targetUrl, options) {
         const payload = { url: targetUrl, ...options };
         try {
             const response = await fetch(proxyUrl, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) });
+            
+            // This is an error with the *proxy function itself* (e.g., 500, 429)
             if (!response.ok) {
                  const errorText = await response.text();
                  this.logger.log('WARN', `[ProxyManager] Proxy infrastructure itself failed.`, {  status: response.status,  proxy: proxyUrl,  error: errorText  });
-                return {  ok: false,  status: response.status, isUrlFetchError: true,  error: { message: `Proxy infrastructure failed with status ${response.status}` }, headers: response.headers, text: () => Promise.resolve(errorText) }; }
+                 const isRateLimit = response.status === 429;
+                 return {  ok: false,  status: response.status, isUrlFetchError: true, isRateLimitError: isRateLimit, error: { message: `Proxy infrastructure failed with status ${response.status}` }, headers: response.headers, text: () => Promise.resolve(errorText) }; 
+            }
+
             const proxyResponse = await response.json();
+            
+            // This is an error *returned by the proxy* (e.g., UrlFetchApp failed)
             if (proxyResponse.error) {
                 const errorMsg = proxyResponse.error.message || '';
+                // --- NEW: Check for AppScript's rate limit error text ---
                 if (errorMsg.toLowerCase().includes('service invoked too many times')) {
                     this.logger.log('WARN', `[ProxyManager] Proxy quota error: ${proxyUrl}`, { error: proxyResponse.error });
-                    return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, headers: new Headers() }; }
-                return { ok: false, status: 500, error: proxyResponse.error, headers: new Headers(), text: () => Promise.resolve(errorMsg) }; }
-            return { ok: proxyResponse.statusCode >= 200 && proxyResponse.statusCode < 300, status: proxyResponse.statusCode, headers: new Headers(proxyResponse.headers || {}), json: () => Promise.resolve(JSON.parse(proxyResponse.body)), text: () => Promise.resolve(proxyResponse.body), };
+                    return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, isRateLimitError: true, headers: new Headers() }; // <-- Set flag
+                }
+                // Other UrlFetchApp error
+                return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, isRateLimitError: false, headers: new Headers(), text: () => Promise.resolve(errorMsg) }; 
+            }
+
+            // Success. The proxy fetched the target URL.
+            return { 
+                ok: proxyResponse.statusCode >= 200 && proxyResponse.statusCode < 300, 
+                status: proxyResponse.statusCode, 
+                headers: new Headers(proxyResponse.headers || {}), 
+                json: () => Promise.resolve(JSON.parse(proxyResponse.body)), 
+                text: () => Promise.resolve(proxyResponse.body),
+                isUrlFetchError: false,
+                isRateLimitError: false 
+            };
         } catch (networkError) {
              this.logger.log('ERROR', `[ProxyManager] Network error calling proxy: ${proxyUrl}`, { errorMessage: networkError.message });
-            return { ok: false, status: 0,  isUrlFetchError: true, error: { message: `Network error: ${networkError.message}` }, headers: new Headers() }; }
+            return { ok: false, status: 0,  isUrlFetchError: true, isRateLimitError: false, error: { message: `Network error: ${networkError.message}` }, headers: new Headers() }; 
+        }
     }
 }
 

package/functions/task-engine/helpers/update_helpers.js

@@ -1,6 +1,9 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
  * (MODIFIED: To conditionally fetch history API once per user per batch)
+ * (MODIFIED: `lookupUsernames` runs batches in parallel)
+ * (MODIFIED: `handleUpdate` fetches history and all portfolios in parallel)
+ * (MODIFIED: `handleUpdate` now uses batchManager for history cache)
  */
 
 /**
@@ -12,70 +15,180 @@
  * --- MODIFIED: Conditionally fetches history only once per user per batch. ---
  */
 const { FieldValue } = require('@google-cloud/firestore');
+const pLimit = require('p-limit');
 
-async function lookupUsernames(cids, { logger, headerManager, proxyManager }, { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL }) {
+/**
+ * (MODIFIED: Runs lookup batches in parallel)
+ */
+async function lookupUsernames(cids, { logger, headerManager, proxyManager }, config) {
     if (!cids?.length) return [];
     logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
-    const allUsers = [];
+    
+    // Use a new config value, falling back to 5
+    const limit = pLimit(config.USERNAME_LOOKUP_CONCURRENCY || 5);
+    const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
+    
+    const batches = [];
     for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) {
-        const batch = cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number);
+        batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number));
+    }
+
+    const batchPromises = batches.map(batch => limit(async () => {
         const header = await headerManager.selectHeader();
-        if (!header) { logger.log('ERROR', '[lookupUsernames] Could not select a header.'); continue; }
+        if (!header) {
+            logger.log('ERROR', '[lookupUsernames] Could not select a header.');
+            return null; // Return null to skip this batch
+        }
+        
         let success = false;
         try {
             const res = await proxyManager.fetch(`${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`, { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) });
             if (!res.ok) throw new Error(`API status ${res.status}`);
             const data = await res.json();
-            if (Array.isArray(data)) allUsers.push(...data);
-            success = true; logger.log('DEBUG', 'Looked up usernames', { batch })
+            success = true;
+            logger.log('DEBUG', 'Looked up usernames', { batch: batch.slice(0, 5) }); // Log only a few
+            return data; // Return data on success
         } catch (err) {
             logger.log('WARN', `[lookupUsernames] Failed batch`, { error: err.message });
-        } finally { headerManager.updatePerformance(header.id, success); }
-    }
+            return null; // Return null on failure
+        } finally {
+            headerManager.updatePerformance(header.id, success);
+        }
+    }));
+
+    const results = await Promise.allSettled(batchPromises);
+    
+    const allUsers = results
+        .filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value))
+        .flatMap(r => r.value); // Flatten all successful batch results
+
     logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
     return allUsers;
 }
 
-async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username, historyFetchedForUser) {
+
+/**
+ * (MODIFIED: Fetches history and all portfolios in parallel)
+ * (MODIFIED: Uses batchManager for history cache)
+ */
+async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) { // <--- REMOVED historyFetchedForUser
     const { userId, instruments, instrumentId, userType } = task;
     const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
-    let portfolioHeader = await headerManager.selectHeader();
+
     let historyHeader = null;
-    if (!portfolioHeader) throw new Error("Could not select portfolio header.");
-    let wasHistorySuccess = false, isPrivate = false;
-    let fetchHistory = false;
+    let wasHistorySuccess = false;
+    let historyFetchPromise = null;
+    let isPrivate = false;
+
     try {
-        const promisesToRun = [];
-        if (!historyFetchedForUser.has(userId)) {
+        // --- 1. Prepare History Fetch (if needed) ---
+        // (MODIFIED: Use batchManager's cross-invocation cache)
+        if (!batchManager.checkAndSetHistoryFetched(userId)) {
+            // This user has NOT been fetched in the last 10 mins (by this instance)
             historyHeader = await headerManager.selectHeader();
-            if (historyHeader) { fetchHistory = true;  historyFetchedForUser.add(userId); const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`; promisesToRun.push(proxyManager.fetch(historyUrl, { headers: historyHeader.header })); } }
-        if (fetchHistory) {
-            const results = await Promise.allSettled(promisesToRun);
-            const historyRes = results[0];
-            if (historyRes.status === 'fulfilled' && historyRes.value.ok) { const data = await historyRes.value.json(); wasHistorySuccess = true; await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType); } }
-        for (const instrumentId of instrumentsToProcess) {
-            const portfolioUrl = userType === 'speculator' ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instrumentId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+            if (historyHeader) {
+                // No need to add to a local set, batchManager did it.
+                const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
+                historyFetchPromise = proxyManager.fetch(historyUrl, { headers: historyHeader.header });
+            } else {
+                logger.log('WARN', `[handleUpdate] Could not select history header for ${userId}. History will be skipped for this task.`);
+            }
+        } else {
+            logger.log('TRACE', `[handleUpdate] History fetch for ${userId} skipped (already fetched by this instance).`);
+        }
+
+        // --- 2. Prepare All Portfolio Fetches ---
+        const portfolioRequests = [];
+        for (const instId of instrumentsToProcess) {
+            const portfolioHeader = await headerManager.selectHeader();
+            if (!portfolioHeader) throw new Error(`Could not select portfolio header for ${userId}`);
+            
+            const portfolioUrl = userType === 'speculator' 
+                ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}`
+                : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+            
+            portfolioRequests.push({
+                instrumentId: instId,
+                url: portfolioUrl,
+                header: portfolioHeader,
+                promise: proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header })
+            });
+        }
+        
+        // --- 3. Execute All API Calls in Parallel ---
+        const allPromises = [
+            ...(historyFetchPromise ? [historyFetchPromise] : []),
+            ...portfolioRequests.map(r => r.promise)
+        ];
+        const allResults = await Promise.allSettled(allPromises);
+
+        // --- 4. Process History Result ---
+        let resultIndex = 0;
+        if (historyFetchPromise) {
+            const historyRes = allResults[resultIndex++];
+            if (historyRes.status === 'fulfilled' && historyRes.value.ok) {
+                const data = await historyRes.value.json();
+                wasHistorySuccess = true;
+                await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
+            } else {
+                logger.log('WARN', `[handleUpdate] History fetch failed for ${userId}`, { error: historyRes.reason || `status ${historyRes.value?.status}` });
+            }
+        }
+
+        // --- 5. Process Portfolio Results ---
+        for (let i = 0; i < portfolioRequests.length; i++) {
+            const requestInfo = portfolioRequests[i];
+            const portfolioRes = allResults[resultIndex++];
             let wasPortfolioSuccess = false;
-            const portfolioRes = await proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header });
-            if (portfolioRes.ok) {
-                const body = await portfolioRes.text();
-                if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `User ${userId} is private. Removing from updates.`) ; break; 
-                } else { wasPortfolioSuccess = true; await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, instrumentId); } }
-                logger.log('DEBUG', 'Processing portfolio for user', { userId, portfolioUrl })
-            headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess);
-            if (instrumentsToProcess.length > 1 && instrumentId !== instrumentsToProcess[instrumentsToProcess.length - 1]) {
-                portfolioHeader = await headerManager.selectHeader(); } }
+
+            if (portfolioRes.status === 'fulfilled' && portfolioRes.value.ok) {
+                const body = await portfolioRes.value.text();
+                if (body.includes("user is PRIVATE")) {
+                    isPrivate = true;
+                    logger.log('WARN', `User ${userId} is private. Removing from updates.`);
+                    break; // Stop processing more portfolios for this private user
+                } else {
+                    wasPortfolioSuccess = true;
+                    await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, requestInfo.instrumentId);
+                }
+                logger.log('DEBUG', 'Processing portfolio for user', { userId, portfolioUrl: requestInfo.url });
+            } else {
+                logger.log('WARN', `Failed to fetch portfolio`, { userId, url: requestInfo.url, error: portfolioRes.reason || `status ${portfolioRes.value?.status}` });
+            }
+            // Update performance for this specific header
+            headerManager.updatePerformance(requestInfo.header.id, wasPortfolioSuccess);
+        }
+
+        // --- 6. Handle Private Users & Timestamps ---
         if (isPrivate) {
             logger.log('WARN', `User ${userId} is private. Removing from updates.`);
-            for (const instrumentId of instrumentsToProcess) { await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId); }
+            for (const instrumentId of instrumentsToProcess) {
+                await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
+            }
             const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
-            for (const instrumentId of instrumentsToProcess) { const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`; await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true }); }
-            return; }
-        for (const instrumentId of instrumentsToProcess) { await batchManager.updateUserTimestamp(userId, userType, instrumentId); }
-        if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
-    } finally { if (historyHeader && fetchHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); }}
+            for (const instrumentId of instrumentsToProcess) {
+                const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
+                // This is not batched, but it's a rare event.
+                await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
+            }
+            return; // Don't update timestamps
+        }
+
+        // If not private, update all timestamps
+        for (const instrumentId of instrumentsToProcess) {
+            await batchManager.updateUserTimestamp(userId, userType, instrumentId);
+        }
+        if (userType === 'speculator') {
+            await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
+        }
+
+    } finally {
+        if (historyHeader) { // historyHeader is only set if a fetch was attempted
+            headerManager.updatePerformance(historyHeader.id, wasHistorySuccess);
+        }
+    }
 }
 
 module.exports = { handleUpdate, lookupUsernames };

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -3,6 +3,7 @@
  * REFACTORED: Renamed 'firestore' to 'db' for consistency.
  * OPTIMIZED: Added logic to handle speculator timestamp fixes within the batch.
  * --- MODIFIED: Added username map caching and trading history batching. ---
+ * --- MODIFIED: Added cross-invocation cache for history fetches. ---
  */
 const { FieldValue } = require('@google-cloud/firestore');
 
@@ -16,9 +17,18 @@ class FirestoreBatchManager {
         this.timestampBatch = {};
         this.tradingHistoryBatch = {};
         this.speculatorTimestampFixBatch = {};
+        
+        // Username map cache
         this.usernameMap = new Map();
         this.usernameMapUpdates = {};
         this.usernameMapLastLoaded = 0;
+        
+        // History fetch cache (NEW)
+        this.historyFetchedUserIds = new Set();
+        this.historyCacheTimestamp = Date.now();
+        // Set a 10-minute TTL on this cache (600,000 ms)
+        this.HISTORY_CACHE_TTL_MS = config.HISTORY_CACHE_TTL_MS || 600000; 
+
         this.processedSpeculatorCids = new Set();
         this.usernameMapCollectionName = config.FIRESTORE_COLLECTION_USERNAME_MAP;
         this.normalHistoryCollectionName = config.FIRESTORE_COLLECTION_NORMAL_HISTORY;
@@ -27,9 +37,47 @@ class FirestoreBatchManager {
         logger.log('INFO', 'FirestoreBatchManager initialized.');
     }
 
+    /* 
+     * NEW: Checks if a user's history has been fetched in the last 10 minutes.
+     * If not, it logs them as fetched and returns false (to trigger a fetch).
+     * @param {string} userId 
+     * @returns {boolean} True if already fetched, false if not.
+     */
+    checkAndSetHistoryFetched(userId) {
+        // Check if the cache is stale
+        if (Date.now() - this.historyCacheTimestamp > this.HISTORY_CACHE_TTL_MS) {
+            this.logger.log('INFO', '[BATCH] History fetch cache (10m TTL) expired. Clearing set.');
+            this.historyFetchedUserIds.clear();
+            this.historyCacheTimestamp = Date.now();
+        }
+
+        if (this.historyFetchedUserIds.has(userId)) {
+            return true; // Yes, already fetched
+        }
+
+        // Not fetched yet. Mark as fetched and return false.
+        this.historyFetchedUserIds.add(userId);
+        return false;
+    }
+
     _getUsernameShardId(cid) { return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`; }
 
-    _scheduleFlush() { if (!this.batchTimeout) this.batchTimeout = setTimeout(() => this.flushBatches(), this.config.TASK_ENGINE_FLUSH_INTERVAL_MS); }
+    // _scheduleFlush() { if (!this.batchTimeout) this.batchTimeout = setTimeout(() => this.flushBatches(), this.config.TASK_ENGINE_FLUSH_INTERVAL_MS); } Old version
+
+    _scheduleFlush() {
+        const totalOps = this._estimateBatchSize();
+        if (totalOps >= 400) { this.flushBatches(); return; }
+        if (!this.batchTimeout) { this.batchTimeout = setTimeout(() => this.flushBatches(),  this.config.TASK_ENGINE_FLUSH_INTERVAL_MS); }
+    }
+
+    _estimateBatchSize() {
+        let ops = 0;
+        ops += Object.keys(this.portfolioBatch).length;
+        ops += Object.keys(this.tradingHistoryBatch).length;
+        ops += Object.keys(this.timestampBatch).length;
+        ops += Object.keys(this.speculatorTimestampFixBatch).length;
+        return ops;
+    }
 
     async loadUsernameMap() {
         if (Date.now() - this.usernameMapLastLoaded < 3600000) return;

package/functions/task-engine/utils/task_engine_utils.js

@@ -1,6 +1,9 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/utils/task_engine_utils.js
  * (MODIFIED: To pass down a Set to track history fetches)
+ * (MODIFIED: To run all update tasks in parallel with a concurrency limit)
+ * (MODIFIED: To use a SINGLE parallel work pool for ALL tasks)
+ * (MODIFIED: To remove local history cache set)
  */
 
 /**
@@ -12,6 +15,7 @@
 const { handleDiscover } = require('../helpers/discover_helpers');
 const { handleVerify } = require('../helpers/verify_helpers');
 const { handleUpdate, lookupUsernames } = require('../helpers/update_helpers');
+const pLimit = require('p-limit'); // <--- IMPORT p-limit
 
 /**
  * Parses Pub/Sub message into task array.
@@ -43,28 +47,72 @@ async function prepareTaskBatches(tasks, batchManager, logger) {
 async function runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config, batchManager, logger) {
     if (!cidsToLookup.size) return;
     logger.log('INFO', `[TaskEngine] Looking up ${cidsToLookup.size} usernames...`);
-    const foundUsers = await lookupUsernames([...cidsToLookup.keys()], dependencies, config);
+    // Pass config to lookupUsernames
+    const foundUsers = await lookupUsernames([...cidsToLookup.keys()], dependencies, config); // <--- PASS FULL CONFIG
     for (const u of foundUsers) { const cid = String(u.CID), username = u.Value.UserName; batchManager.addUsernameMapUpdate(cid, username); const task = cidsToLookup.get(cid); if (task) { tasksToRun.push({ task, username }); cidsToLookup.delete(cid); } }
     if (cidsToLookup.size) logger.log('WARN', `[TaskEngine] Could not find ${cidsToLookup.size} usernames (likely private).`, { skippedCids: [...cidsToLookup.keys()] });
 }
 
 /**
  * Executes all tasks.
+ * (MODIFIED: Runs ALL tasks in a single parallel pool)
  */
 async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId) {
-    const { logger } = dependencies;
-    const historyFetchedForUser = new Set();
+    const { logger, batchManager } = dependencies; // <--- Get batchManager
+    
+    // REMOVED: const historyFetchedForUser = new Set();
+    
+    // Create one unified parallel pool
+    const limit = pLimit(config.TASK_ENGINE_CONCURRENCY || 10);
+    const allTaskPromises = [];
+    let taskCounters = { update: 0, discover: 0, verify: 0, unknown: 0, failed: 0 };
+
+    // 1. Queue 'other' tasks (discover, verify)
     for (const task of otherTasks) {
         const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId || task.cids?.[0] || 'sub'}`;
         const handler = { discover: handleDiscover, verify: handleVerify }[task.type];
-        if (handler) try { await handler(task, subTaskId, dependencies, config); } 
-        catch (err) { logger.log('ERROR', `[TaskEngine/${taskId}] Error in ${task.type} for ${subTaskId}`, { errorMessage: err.message }); }
-        else logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type: ${task.type}`); }
+
+        if (handler) {
+            allTaskPromises.push(
+                limit(() => 
+                    handler(task, subTaskId, dependencies, config)
+                        .then(() => taskCounters[task.type]++)
+                        .catch(err => {
+                            logger.log('ERROR', `[TaskEngine/${taskId}] Error in ${task.type} for ${subTaskId}`, { errorMessage: err.message });
+                            taskCounters.failed++;
+                        })
+                )
+            );
+        } else {
+            logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type: ${task.type}`);
+            taskCounters.unknown++;
+        }
+    }
+
+    // 2. Queue 'update' tasks
     for (const { task, username } of tasksToRun) {
         const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId}`;
-        try { await handleUpdate(task, subTaskId, dependencies, config, username, historyFetchedForUser);}
-        catch (err) { logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message }); } }
-    logger.log('SUCCESS', `[TaskEngine/${taskId}] Processed all tasks.`);
+        allTaskPromises.push(
+            limit(() => 
+                // Pass batchManager instead of the local set
+                handleUpdate(task, subTaskId, dependencies, config, username) // <--- REMOVED historyFetchedForUser
+                    .then(() => taskCounters.update++)
+                    .catch(err => {
+                        logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message });
+                        taskCounters.failed++;
+                    })
+            )
+        );
+    }
+
+    // 3. Wait for ALL tasks to complete
+    await Promise.all(allTaskPromises);
+
+    // 4. Log final summary
+    logger.log(
+        taskCounters.failed > 0 ? 'WARN' : 'SUCCESS', 
+        `[TaskEngine/${taskId}] Processed all tasks. Updates: ${taskCounters.update}, Discovers: ${taskCounters.discover}, Verifies: ${taskCounters.verify}, Unknown: ${taskCounters.unknown}, Failed: ${taskCounters.failed}.`
+    );
 }
 
 module.exports = { parseTaskPayload, prepareTaskBatches, runUsernameLookups, executeTasks };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.153",
+  "version": "1.0.155",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [