bulltrackers-module 1.0.154 → 1.0.156

package/functions/core/utils/intelligent_proxy_manager.js

@@ -3,8 +3,10 @@
  * It selects an available (unlocked) proxy for each request and locks it upon failure.
  * * This module is designed to be reusable and receives all dependencies
  * (firestore, logger) and configuration via its constructor.
+ * --- MODIFIED: Now includes exponential backoff and retries specifically for rate-limit errors. ---
  */
 const { FieldValue } = require('@google-cloud/firestore');
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
 
 class IntelligentProxyManager {
     /**
@@ -29,6 +31,8 @@ class IntelligentProxyManager {
         this.proxyLockingEnabled = config.proxyLockingEnabled !== false;
         this.proxies = {}; 
         this.configLastLoaded = 0;
+        this.MAX_RETRIES = 3;
+        this.INITIAL_BACKOFF_MS = 1000;
         if (this.proxyUrls.length === 0) { this.logger.log('WARN', '[ProxyManager] No proxy URLs provided in config.');
         } else { const lockingStatus = this.proxyLockingEnabled ? "Locking Mechanism Enabled" : "Locking Mechanism DISABLED"; this.logger.log('INFO', `[ProxyManager] Initialized with ${this.proxyUrls.length} proxies and ${lockingStatus}.`); }
     }
@@ -59,10 +63,8 @@ class IntelligentProxyManager {
      */
     async _selectProxy() {
         await this._loadConfig();
-
         const availableProxies = this.proxyLockingEnabled ? Object.values(this.proxies).filter(p => p.status === 'unlocked') : Object.values(this.proxies);
-        if (availableProxies.length === 0) { const errorMsg = this.proxyLockingEnabled  ? "All proxies are locked. No proxy available." : "No proxies are loaded. Cannot make request."; 
-            this.logger.log('ERROR', `[ProxyManager] ${errorMsg}`); throw new Error(errorMsg); }
+        if (availableProxies.length === 0) { const errorMsg = this.proxyLockingEnabled  ? "All proxies are locked. No proxy available." : "No proxies are loaded. Cannot make request.";  this.logger.log('ERROR', `[ProxyManager] ${errorMsg}`); throw new Error(errorMsg); }
         const selected = availableProxies[Math.floor(Math.random() * availableProxies.length)];
         return { owner: selected.owner, url: selected.url };
     }
@@ -75,28 +77,41 @@ class IntelligentProxyManager {
         if (!this.proxyLockingEnabled) { this.logger.log('TRACE', `[ProxyManager] Locking skipped for ${owner} (locking is disabled).`); return; }
         if (this.proxies[owner]) { this.proxies[owner].status = 'locked'; }
         this.logger.log('WARN', `[ProxyManager] Locking proxy: ${owner}`);
-        try { const docRef = this.firestore.doc(this.PERFORMANCE_DOC_PATH);
-            await docRef.set({ locks: { [owner]: { locked: true, lastLocked: FieldValue.serverTimestamp() } } }, { merge: true });
+        try { const docRef = this.firestore.doc(this.PERFORMANCE_DOC_PATH); await docRef.set({ locks: { [owner]: { locked: true, lastLocked: FieldValue.serverTimestamp() } } }, { merge: true });
         } catch (error) { this.logger.log('ERROR', `[ProxyManager] Failed to write lock for ${owner} to Firestore.`, { errorMessage: error.message }); }
     }
 
     /**
-     * Makes a fetch request using a selected proxy.
+     * --- MODIFIED: Makes a fetch request with exponential backoff for rate limits ---
      * @param {string} targetUrl - The URL to fetch.
      * @param {object} options - Fetch options (e.g., headers).
      * @returns {Promise<object>} A mock Response object.
      */
     async fetch(targetUrl, options = {}) {
         let proxy = null;
-        try { proxy = await this._selectProxy();
-        } catch (error) { return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() }; }
-        const response = await this._fetchViaAppsScript(proxy.url, targetUrl, options);
-        if (!response.ok && response.isUrlFetchError) { await this.lockProxy(proxy.owner); }
-        return response;
+        try { proxy = await this._selectProxy(); } catch (error) { return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() }; }
+        let backoff = this.INITIAL_BACKOFF_MS;
+        let lastResponse = null;
+        for (let attempt = 1; attempt <= this.MAX_RETRIES; attempt++) {
+            const response = await this._fetchViaAppsScript(proxy.url, targetUrl, options);
+            lastResponse = response; 
+            // 1. Success
+            if (response.ok) { return response; }
+            // 2. Rate Limit Error (Retryable)
+            if (response.isRateLimitError) { this.logger.log('WARN', `[ProxyManager] Rate limit hit on proxy ${proxy.owner} (Attempt ${attempt}/${this.MAX_RETRIES}). Backing off for ${backoff}ms...`, { url: targetUrl }); await sleep(backoff); backoff *= 2; continue;  }
+            // 3. Other Fetch Error (Non-Retryable, Lock Proxy)
+            if (response.isUrlFetchError) { this.logger.log('ERROR', `[ProxyManager] Proxy ${proxy.owner} failed (non-rate-limit). Locking proxy.`, { url: targetUrl, status: response.status }); await this.lockProxy(proxy.owner); return response;  }
+            // 4. Standard Error (e.g., 404, 500 from *target* URL, not proxy)
+            return response; }
+        // If loop finishes, all retries failed (likely all were rate-limit errors)
+        this.logger.log('ERROR', `[ProxyManager] Request failed after ${this.MAX_RETRIES} rate-limit retries.`, { url: targetUrl });
+        return lastResponse;
     }
 
+
     /**
      * Internal function to call the Google AppScript proxy.
+     * --- MODIFIED: Now adds `isRateLimitError` flag to response ---
      * @private
      */
     async _fetchViaAppsScript(proxyUrl, targetUrl, options) {
@@ -106,19 +121,22 @@ class IntelligentProxyManager {
             if (!response.ok) {
                  const errorText = await response.text();
                  this.logger.log('WARN', `[ProxyManager] Proxy infrastructure itself failed.`, {  status: response.status,  proxy: proxyUrl,  error: errorText  });
-                return {  ok: false,  status: response.status, isUrlFetchError: true,  error: { message: `Proxy infrastructure failed with status ${response.status}` }, headers: response.headers, text: () => Promise.resolve(errorText) }; }
+                 const isRateLimit = response.status === 429;
+                 return {  ok: false,  status: response.status, isUrlFetchError: true, isRateLimitError: isRateLimit, error: { message: `Proxy infrastructure failed with status ${response.status}` }, headers: response.headers, text: () => Promise.resolve(errorText) };  }
             const proxyResponse = await response.json();
             if (proxyResponse.error) {
                 const errorMsg = proxyResponse.error.message || '';
-                if (errorMsg.toLowerCase().includes('service invoked too many times')) {
-                    this.logger.log('WARN', `[ProxyManager] Proxy quota error: ${proxyUrl}`, { error: proxyResponse.error });
-                    return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, headers: new Headers() }; }
-                return { ok: false, status: 500, error: proxyResponse.error, headers: new Headers(), text: () => Promise.resolve(errorMsg) }; }
-            return { ok: proxyResponse.statusCode >= 200 && proxyResponse.statusCode < 300, status: proxyResponse.statusCode, headers: new Headers(proxyResponse.headers || {}), json: () => Promise.resolve(JSON.parse(proxyResponse.body)), text: () => Promise.resolve(proxyResponse.body), };
-        } catch (networkError) {
-             this.logger.log('ERROR', `[ProxyManager] Network error calling proxy: ${proxyUrl}`, { errorMessage: networkError.message });
-            return { ok: false, status: 0,  isUrlFetchError: true, error: { message: `Network error: ${networkError.message}` }, headers: new Headers() }; }
-    }
+                if (errorMsg.toLowerCase().includes('service invoked too many times')) { this.logger.log('WARN', `[ProxyManager] Proxy quota error: ${proxyUrl}`, { error: proxyResponse.error }); return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, isRateLimitError: true, headers: new Headers() }; }
+                return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, isRateLimitError: false, headers: new Headers(), text: () => Promise.resolve(errorMsg) };  }
+            return { 
+                ok: proxyResponse.statusCode >= 200 && proxyResponse.statusCode < 300, 
+                status: proxyResponse.statusCode, 
+                headers: new Headers(proxyResponse.headers || {}), 
+                json: () => Promise.resolve(JSON.parse(proxyResponse.body)), 
+                text: () => Promise.resolve(proxyResponse.body),
+                isUrlFetchError: false,
+                isRateLimitError: false  };
+        } catch (networkError) { this.logger.log('ERROR', `[ProxyManager] Network error calling proxy: ${proxyUrl}`, { errorMessage: networkError.message }); return { ok: false, status: 0,  isUrlFetchError: true, isRateLimitError: false, error: { message: `Network error: ${networkError.message}` }, headers: new Headers() };  } }
 }
 
 module.exports = { IntelligentProxyManager };

package/functions/task-engine/helpers/update_helpers.js

@@ -3,6 +3,7 @@
  * (MODIFIED: To conditionally fetch history API once per user per batch)
  * (MODIFIED: `lookupUsernames` runs batches in parallel)
  * (MODIFIED: `handleUpdate` fetches history and all portfolios in parallel)
+ * (MODIFIED: `handleUpdate` now uses batchManager for history cache)
  */
 
 /**
@@ -14,7 +15,7 @@
  * --- MODIFIED: Conditionally fetches history only once per user per batch. ---
  */
 const { FieldValue } = require('@google-cloud/firestore');
-const pLimit = require('p-limit'); // <--- IMPORT p-limit
+const pLimit = require('p-limit');
 
 /**
  * (MODIFIED: Runs lookup batches in parallel)
@@ -22,26 +23,45 @@ const pLimit = require('p-limit'); // <--- IMPORT p-limit
 async function lookupUsernames(cids, { logger, headerManager, proxyManager }, config) {
     if (!cids?.length) return [];
     logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
+    
+    // Use a new config value, falling back to 5
     const limit = pLimit(config.USERNAME_LOOKUP_CONCURRENCY || 5);
     const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
+    
     const batches = [];
-    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) { batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number)); }
+    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) {
+        batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number));
+    }
 
     const batchPromises = batches.map(batch => limit(async () => {
         const header = await headerManager.selectHeader();
-        if (!header) { logger.log('ERROR', '[lookupUsernames] Could not select a header.'); return null; }
+        if (!header) {
+            logger.log('ERROR', '[lookupUsernames] Could not select a header.');
+            return null; // Return null to skip this batch
+        }
+        
         let success = false;
         try {
             const res = await proxyManager.fetch(`${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`, { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) });
             if (!res.ok) throw new Error(`API status ${res.status}`);
             const data = await res.json();
             success = true;
-            logger.log('DEBUG', 'Looked up usernames', { batch: batch.slice(0, 5) }); 
-            return data;
-        } catch (err) { logger.log('WARN', `[lookupUsernames] Failed batch`, { error: err.message }); return null; 
-        } finally { headerManager.updatePerformance(header.id, success); } }));
+            logger.log('DEBUG', 'Looked up usernames', { batch: batch.slice(0, 5) }); // Log only a few
+            return data; // Return data on success
+        } catch (err) {
+            logger.log('WARN', `[lookupUsernames] Failed batch`, { error: err.message });
+            return null; // Return null on failure
+        } finally {
+            headerManager.updatePerformance(header.id, success);
+        }
+    }));
+
     const results = await Promise.allSettled(batchPromises);
-    const allUsers = results.filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value)).flatMap(r => r.value);
+    
+    const allUsers = results
+        .filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value))
+        .flatMap(r => r.value); // Flatten all successful batch results
+
     logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
     return allUsers;
 }
@@ -49,8 +69,9 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
 
 /**
  * (MODIFIED: Fetches history and all portfolios in parallel)
+ * (MODIFIED: Uses batchManager for history cache)
  */
-async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username, historyFetchedForUser) {
+async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) { // <--- REMOVED historyFetchedForUser
     const { userId, instruments, instrumentId, userType } = task;
     const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
@@ -63,59 +84,111 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
 
     try {
         // --- 1. Prepare History Fetch (if needed) ---
-        if (!historyFetchedForUser.has(userId)) {
+        // (MODIFIED: Use batchManager's cross-invocation cache)
+        if (!batchManager.checkAndSetHistoryFetched(userId)) {
+            // This user has NOT been fetched in the last 10 mins (by this instance)
             historyHeader = await headerManager.selectHeader();
-            if (historyHeader) { historyFetchedForUser.add(userId);
+            if (historyHeader) {
+                // No need to add to a local set, batchManager did it.
                 const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
-                historyFetchPromise = proxyManager.fetch(historyUrl, { headers: historyHeader.header }); } }
+                historyFetchPromise = proxyManager.fetch(historyUrl, { headers: historyHeader.header });
+            } else {
+                logger.log('WARN', `[handleUpdate] Could not select history header for ${userId}. History will be skipped for this task.`);
+            }
+        } else {
+            logger.log('TRACE', `[handleUpdate] History fetch for ${userId} skipped (already fetched by this instance).`);
+        }
 
         // --- 2. Prepare All Portfolio Fetches ---
         const portfolioRequests = [];
         for (const instId of instrumentsToProcess) {
             const portfolioHeader = await headerManager.selectHeader();
             if (!portfolioHeader) throw new Error(`Could not select portfolio header for ${userId}`);
-            const portfolioUrl = userType === 'speculator'  ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
-            portfolioRequests.push({ instrumentId: instId, url: portfolioUrl, header: portfolioHeader, promise: proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header }) }); }
+            
+            const portfolioUrl = userType === 'speculator' 
+                ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}`
+                : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+            
+            portfolioRequests.push({
+                instrumentId: instId,
+                url: portfolioUrl,
+                header: portfolioHeader,
+                promise: proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header })
+            });
+        }
         
         // --- 3. Execute All API Calls in Parallel ---
-        const allPromises = [ ...(historyFetchPromise ? [historyFetchPromise] : []), ...portfolioRequests.map(r => r.promise) ];
+        const allPromises = [
+            ...(historyFetchPromise ? [historyFetchPromise] : []),
+            ...portfolioRequests.map(r => r.promise)
+        ];
         const allResults = await Promise.allSettled(allPromises);
 
         // --- 4. Process History Result ---
         let resultIndex = 0;
         if (historyFetchPromise) {
             const historyRes = allResults[resultIndex++];
-            if (historyRes.status === 'fulfilled' && historyRes.value.ok) { const data = await historyRes.value.json(); wasHistorySuccess = true; await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType); } }
+            if (historyRes.status === 'fulfilled' && historyRes.value.ok) {
+                const data = await historyRes.value.json();
+                wasHistorySuccess = true;
+                await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
+            } else {
+                logger.log('WARN', `[handleUpdate] History fetch failed for ${userId}`, { error: historyRes.reason || `status ${historyRes.value?.status}` });
+            }
+        }
 
         // --- 5. Process Portfolio Results ---
         for (let i = 0; i < portfolioRequests.length; i++) {
             const requestInfo = portfolioRequests[i];
             const portfolioRes = allResults[resultIndex++];
             let wasPortfolioSuccess = false;
+
             if (portfolioRes.status === 'fulfilled' && portfolioRes.value.ok) {
                 const body = await portfolioRes.value.text();
-                if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `User ${userId} is private. Removing from updates.`); break;
-                } else { wasPortfolioSuccess = true; await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, requestInfo.instrumentId); }
+                if (body.includes("user is PRIVATE")) {
+                    isPrivate = true;
+                    logger.log('WARN', `User ${userId} is private. Removing from updates.`);
+                    break; // Stop processing more portfolios for this private user
+                } else {
+                    wasPortfolioSuccess = true;
+                    await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, requestInfo.instrumentId);
+                }
                 logger.log('DEBUG', 'Processing portfolio for user', { userId, portfolioUrl: requestInfo.url });
-            } else { logger.log('WARN', `Failed to fetch portfolio`, { userId, url: requestInfo.url, error: portfolioRes.reason || `status ${portfolioRes.value?.status}` }); }
+            } else {
+                logger.log('WARN', `Failed to fetch portfolio`, { userId, url: requestInfo.url, error: portfolioRes.reason || `status ${portfolioRes.value?.status}` });
+            }
+            // Update performance for this specific header
             headerManager.updatePerformance(requestInfo.header.id, wasPortfolioSuccess);
         }
 
         // --- 6. Handle Private Users & Timestamps ---
         if (isPrivate) {
             logger.log('WARN', `User ${userId} is private. Removing from updates.`);
-            for (const instrumentId of instrumentsToProcess) { await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId); }
+            for (const instrumentId of instrumentsToProcess) {
+                await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
+            }
             const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
             for (const instrumentId of instrumentsToProcess) {
                 const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
-                await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true }); }
-            return; 
+                // This is not batched, but it's a rare event.
+                await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
+            }
+            return; // Don't update timestamps
         }
 
-        for (const instrumentId of instrumentsToProcess) { await batchManager.updateUserTimestamp(userId, userType, instrumentId); }
-        if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
+        // If not private, update all timestamps
+        for (const instrumentId of instrumentsToProcess) {
+            await batchManager.updateUserTimestamp(userId, userType, instrumentId);
+        }
+        if (userType === 'speculator') {
+            await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
+        }
 
-    } finally { if (historyHeader) {  headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } }
+    } finally {
+        if (historyHeader) { // historyHeader is only set if a fetch was attempted
+            headerManager.updatePerformance(historyHeader.id, wasHistorySuccess);
+        }
+    }
 }
 
 module.exports = { handleUpdate, lookupUsernames };

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -3,6 +3,7 @@
  * REFACTORED: Renamed 'firestore' to 'db' for consistency.
  * OPTIMIZED: Added logic to handle speculator timestamp fixes within the batch.
  * --- MODIFIED: Added username map caching and trading history batching. ---
+ * --- MODIFIED: Added cross-invocation cache for history fetches. ---
  */
 const { FieldValue } = require('@google-cloud/firestore');
 
@@ -16,9 +17,18 @@ class FirestoreBatchManager {
         this.timestampBatch = {};
         this.tradingHistoryBatch = {};
         this.speculatorTimestampFixBatch = {};
+        
+        // Username map cache
         this.usernameMap = new Map();
         this.usernameMapUpdates = {};
         this.usernameMapLastLoaded = 0;
+        
+        // History fetch cache (NEW)
+        this.historyFetchedUserIds = new Set();
+        this.historyCacheTimestamp = Date.now();
+        // Set a 10-minute TTL on this cache (600,000 ms)
+        this.HISTORY_CACHE_TTL_MS = config.HISTORY_CACHE_TTL_MS || 600000; 
+
         this.processedSpeculatorCids = new Set();
         this.usernameMapCollectionName = config.FIRESTORE_COLLECTION_USERNAME_MAP;
         this.normalHistoryCollectionName = config.FIRESTORE_COLLECTION_NORMAL_HISTORY;
@@ -27,6 +37,29 @@ class FirestoreBatchManager {
         logger.log('INFO', 'FirestoreBatchManager initialized.');
     }
 
+    /* 
+     * NEW: Checks if a user's history has been fetched in the last 10 minutes.
+     * If not, it logs them as fetched and returns false (to trigger a fetch).
+     * @param {string} userId 
+     * @returns {boolean} True if already fetched, false if not.
+     */
+    checkAndSetHistoryFetched(userId) {
+        // Check if the cache is stale
+        if (Date.now() - this.historyCacheTimestamp > this.HISTORY_CACHE_TTL_MS) {
+            this.logger.log('INFO', '[BATCH] History fetch cache (10m TTL) expired. Clearing set.');
+            this.historyFetchedUserIds.clear();
+            this.historyCacheTimestamp = Date.now();
+        }
+
+        if (this.historyFetchedUserIds.has(userId)) {
+            return true; // Yes, already fetched
+        }
+
+        // Not fetched yet. Mark as fetched and return false.
+        this.historyFetchedUserIds.add(userId);
+        return false;
+    }
+
     _getUsernameShardId(cid) { return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`; }
 
     // _scheduleFlush() { if (!this.batchTimeout) this.batchTimeout = setTimeout(() => this.flushBatches(), this.config.TASK_ENGINE_FLUSH_INTERVAL_MS); } Old version

package/functions/task-engine/utils/task_engine_utils.js

@@ -2,6 +2,8 @@
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/utils/task_engine_utils.js
  * (MODIFIED: To pass down a Set to track history fetches)
  * (MODIFIED: To run all update tasks in parallel with a concurrency limit)
+ * (MODIFIED: To use a SINGLE parallel work pool for ALL tasks)
+ * (MODIFIED: To remove local history cache set)
  */
 
 /**
@@ -13,7 +15,7 @@
 const { handleDiscover } = require('../helpers/discover_helpers');
 const { handleVerify } = require('../helpers/verify_helpers');
 const { handleUpdate, lookupUsernames } = require('../helpers/update_helpers');
-const pLimit = require('p-limit'); 
+const pLimit = require('p-limit'); // <--- IMPORT p-limit
 
 /**
  * Parses Pub/Sub message into task array.
@@ -45,32 +47,72 @@ async function prepareTaskBatches(tasks, batchManager, logger) {
 async function runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config, batchManager, logger) {
     if (!cidsToLookup.size) return;
     logger.log('INFO', `[TaskEngine] Looking up ${cidsToLookup.size} usernames...`);
-    const foundUsers = await lookupUsernames([...cidsToLookup.keys()], dependencies, config);
+    // Pass config to lookupUsernames
+    const foundUsers = await lookupUsernames([...cidsToLookup.keys()], dependencies, config); // <--- PASS FULL CONFIG
     for (const u of foundUsers) { const cid = String(u.CID), username = u.Value.UserName; batchManager.addUsernameMapUpdate(cid, username); const task = cidsToLookup.get(cid); if (task) { tasksToRun.push({ task, username }); cidsToLookup.delete(cid); } }
     if (cidsToLookup.size) logger.log('WARN', `[TaskEngine] Could not find ${cidsToLookup.size} usernames (likely private).`, { skippedCids: [...cidsToLookup.keys()] });
 }
 
 /**
  * Executes all tasks.
+ * (MODIFIED: Runs ALL tasks in a single parallel pool)
  */
 async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId) {
-    const { logger } = dependencies;
-    const historyFetchedForUser = new Set();
+    const { logger, batchManager } = dependencies; // <--- Get batchManager
+    
+    // REMOVED: const historyFetchedForUser = new Set();
+    
+    // Create one unified parallel pool
+    const limit = pLimit(config.TASK_ENGINE_CONCURRENCY || 10);
+    const allTaskPromises = [];
+    let taskCounters = { update: 0, discover: 0, verify: 0, unknown: 0, failed: 0 };
+
+    // 1. Queue 'other' tasks (discover, verify)
     for (const task of otherTasks) {
         const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId || task.cids?.[0] || 'sub'}`;
         const handler = { discover: handleDiscover, verify: handleVerify }[task.type];
-        if (handler) try { await handler(task, subTaskId, dependencies, config); } 
-        catch (err) { logger.log('ERROR', `[TaskEngine/${taskId}] Error in ${task.type} for ${subTaskId}`, { errorMessage: err.message }); }
-        else logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type: ${task.type}`); }
-    const limit = pLimit(config.TASK_ENGINE_CONCURRENCY || 10);
-    let successCount = 0;
-    let errorCount = 0;
-    const updatePromises = tasksToRun.map(({ task, username }) => {
+
+        if (handler) {
+            allTaskPromises.push(
+                limit(() => 
+                    handler(task, subTaskId, dependencies, config)
+                        .then(() => taskCounters[task.type]++)
+                        .catch(err => {
+                            logger.log('ERROR', `[TaskEngine/${taskId}] Error in ${task.type} for ${subTaskId}`, { errorMessage: err.message });
+                            taskCounters.failed++;
+                        })
+                )
+            );
+        } else {
+            logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type: ${task.type}`);
+            taskCounters.unknown++;
+        }
+    }
+
+    // 2. Queue 'update' tasks
+    for (const { task, username } of tasksToRun) {
         const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId}`;
-        return limit(() =>  handleUpdate(task, subTaskId, dependencies, config, username, historyFetchedForUser) .catch(err => { logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message }); throw err;  }) ); });
-    const results = await Promise.allSettled(updatePromises);
-    results.forEach(result => { if (result.status === 'fulfilled') { successCount++; } else { errorCount++; } });
-    logger.log( errorCount > 0 ? 'WARN' : 'SUCCESS', `[TaskEngine/${taskId}] Processed all ${tasksToRun.length} update tasks. Success: ${successCount}, Failed: ${errorCount}.` );
+        allTaskPromises.push(
+            limit(() => 
+                // Pass batchManager instead of the local set
+                handleUpdate(task, subTaskId, dependencies, config, username) // <--- REMOVED historyFetchedForUser
+                    .then(() => taskCounters.update++)
+                    .catch(err => {
+                        logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message });
+                        taskCounters.failed++;
+                    })
+            )
+        );
+    }
+
+    // 3. Wait for ALL tasks to complete
+    await Promise.all(allTaskPromises);
+
+    // 4. Log final summary
+    logger.log(
+        taskCounters.failed > 0 ? 'WARN' : 'SUCCESS', 
+        `[TaskEngine/${taskId}] Processed all tasks. Updates: ${taskCounters.update}, Discovers: ${taskCounters.discover}, Verifies: ${taskCounters.verify}, Unknown: ${taskCounters.unknown}, Failed: ${taskCounters.failed}.`
+    );
 }
 
 module.exports = { parseTaskPayload, prepareTaskBatches, runUsernameLookups, executeTasks };

package/index.js

@@ -1,101 +1,45 @@
 /**
  * @fileoverview Main entry point for the Bulltrackers shared module.
- * This module consolidates all core logic into a single 'pipe' object
- * to enforce a clear naming convention and dependency injection pattern.
+ * Export the pipes!
  */
-
-// --- Core Utilities (Classes and Stateless Helpers) ---
-
-const core = {
-    IntelligentHeaderManager : require('./functions/core/utils/intelligent_header_manager') .IntelligentHeaderManager,
-    IntelligentProxyManager  : require('./functions/core/utils/intelligent_proxy_manager')  .IntelligentProxyManager,
-    FirestoreBatchManager    : require('./functions/task-engine/utils/firestore_batch_manager').FirestoreBatchManager,
-    firestoreUtils           : require('./functions/core/utils/firestore_utils'),
-    pubsubUtils              : require('./functions/core/utils/pubsub_utils'),
-};
-
-// --- Pipe 1: Orchestrator ---
-
-const orchestrator = {
-    // Main Pipes (Entry points for Cloud Functions)
-    runDiscoveryOrchestrator : require('./functions/orchestrator/index').runDiscoveryOrchestrator,
-    runUpdateOrchestrator    : require('./functions/orchestrator/index').runUpdateOrchestrator,
-    
-    // Sub-Pipes (Discovery)
-    checkDiscoveryNeed       : require('./functions/orchestrator/helpers/discovery_helpers').checkDiscoveryNeed,
-    getDiscoveryCandidates   : require('./functions/orchestrator/helpers/discovery_helpers').getDiscoveryCandidates,
-    dispatchDiscovery        : require('./functions/orchestrator/helpers/discovery_helpers').dispatchDiscovery,
-
-    // Sub-Pipes (Updates)
-    getUpdateTargets         : require('./functions/orchestrator/helpers/update_helpers').getUpdateTargets,
-    dispatchUpdates          : require('./functions/orchestrator/helpers/update_helpers').dispatchUpdates,
-};
-
-
-// --- Pipe 2: Dispatcher ---
-
-const dispatcher = {
-    handleRequest            : require('./functions/dispatcher/index').handleRequest,
-    dispatchTasksInBatches   : require('./functions/dispatcher/helpers/dispatch_helpers').dispatchTasksInBatches,
-};
-
-
-// --- Pipe 3: Task Engine ---
-
-const taskEngine = {
-    handleRequest            : require('./functions/task-engine/handler_creator').handleRequest,
-    handleDiscover           : require('./functions/task-engine/helpers/discover_helpers').handleDiscover,
-    handleVerify             : require('./functions/task-engine/helpers/verify_helpers').handleVerify,
-    handleUpdate             : require('./functions/task-engine/helpers/update_helpers').handleUpdate,
-};
-
-
-// --- Pipe 4: Computation System ---
-
-const computationSystem = {
-    runComputationPass       : require('./functions/computation-system/helpers/computation_pass_runner').runComputationPass,    
-    dataLoader               : require('./functions/computation-system/utils/data_loader'),
-    computationUtils         : require('./functions/computation-system/utils/utils'),
-};
-
-
-// --- Pipe 5: API ---
-
-const api = {
-    createApiApp             : require('./functions/generic-api/index').createApiApp,
-    helpers                  : require('./functions/generic-api/helpers/api_helpers'),
-};
-
-
-// --- Pipe 6: Maintenance ---
-
-const maintenance = {
-    runSpeculatorCleanup     : require('./functions/speculator-cleanup-orchestrator/helpers/cleanup_helpers') .runCleanup,
-    handleInvalidSpeculator  : require('./functions/invalid-speculator-handler/helpers/handler_helpers') .handleInvalidSpeculator,
-    runFetchInsights         : require('./functions/fetch-insights/helpers/handler_helpers').fetchAndStoreInsights,
-    runFetchPrices           : require('./functions/etoro-price-fetcher/helpers/handler_helpers').fetchAndStorePrices,
-    runSocialOrchestrator    : require('./functions/social-orchestrator/helpers/orchestrator_helpers') .runSocialOrchestrator,
-    handleSocialTask         : require('./functions/social-task-handler/helpers/handler_helpers') .handleSocialTask,
-    runBackfillAssetPrices   : require('./functions/price-backfill/helpers/handler_helpers') .runBackfillAssetPrices,
-};
-
-
-// --- Pipe 7: Proxy ---
-
-const proxy = {
-    handlePost               : require('./functions/appscript-api/index').handlePost,
-};
-
-
-module.exports = {
-    pipe: {
-        core,
-        orchestrator,
-        dispatcher,
-        taskEngine,
-        computationSystem,
-        api,
-        maintenance,
-        proxy,
-    }
-};
+                            // Core
+const core =              { IntelligentHeaderManager : require('./functions/core/utils/intelligent_header_manager')                .IntelligentHeaderManager,
+                            IntelligentProxyManager  : require('./functions/core/utils/intelligent_proxy_manager')                 .IntelligentProxyManager,
+                            FirestoreBatchManager    : require('./functions/task-engine/utils/firestore_batch_manager')            .FirestoreBatchManager,
+                            firestoreUtils           : require('./functions/core/utils/firestore_utils'),
+                            pubsubUtils              : require('./functions/core/utils/pubsub_utils') };
+                            // Orchestrator
+const orchestrator =      { runDiscoveryOrchestrator : require('./functions/orchestrator/index')                                    .runDiscoveryOrchestrator,
+                            runUpdateOrchestrator    : require('./functions/orchestrator/index')                                    .runUpdateOrchestrator,
+                            checkDiscoveryNeed       : require('./functions/orchestrator/helpers/discovery_helpers')                .checkDiscoveryNeed,
+                            getDiscoveryCandidates   : require('./functions/orchestrator/helpers/discovery_helpers')                .getDiscoveryCandidates,
+                            dispatchDiscovery        : require('./functions/orchestrator/helpers/discovery_helpers')                .dispatchDiscovery,
+                            getUpdateTargets         : require('./functions/orchestrator/helpers/update_helpers')                   .getUpdateTargets,
+                            dispatchUpdates          : require('./functions/orchestrator/helpers/update_helpers')                   .dispatchUpdates };
+                            // Dispatcher
+const dispatcher =        { handleRequest            : require('./functions/dispatcher/index')                                      .handleRequest          ,
+                            dispatchTasksInBatches   : require('./functions/dispatcher/helpers/dispatch_helpers')                   .dispatchTasksInBatches };
+                            // Task Engine
+const taskEngine =        { handleRequest            : require('./functions/task-engine/handler_creator')                           .handleRequest ,
+                            handleDiscover           : require('./functions/task-engine/helpers/discover_helpers')                  .handleDiscover,
+                            handleVerify             : require('./functions/task-engine/helpers/verify_helpers')                    .handleVerify  ,
+                            handleUpdate             : require('./functions/task-engine/helpers/update_helpers')                    .handleUpdate };
+                            // Computation System
+const computationSystem = { runComputationPass       : require('./functions/computation-system/helpers/computation_pass_runner')    .runComputationPass,    
+                            dataLoader               : require('./functions/computation-system/utils/data_loader'),
+                            computationUtils         : require('./functions/computation-system/utils/utils') };
+                            // API
+const api =               { createApiApp             : require('./functions/generic-api/index')                                       .createApiApp,
+                            helpers                  : require('./functions/generic-api/helpers/api_helpers') };
+                            // Maintenance
+const maintenance =       { runSpeculatorCleanup     : require('./functions/speculator-cleanup-orchestrator/helpers/cleanup_helpers') .runCleanup,
+                            handleInvalidSpeculator  : require('./functions/invalid-speculator-handler/helpers/handler_helpers')      .handleInvalidSpeculator,
+                            runFetchInsights         : require('./functions/fetch-insights/helpers/handler_helpers')                  .fetchAndStoreInsights,
+                            runFetchPrices           : require('./functions/etoro-price-fetcher/helpers/handler_helpers')             .fetchAndStorePrices,
+                            runSocialOrchestrator    : require('./functions/social-orchestrator/helpers/orchestrator_helpers')        .runSocialOrchestrator,
+                            handleSocialTask         : require('./functions/social-task-handler/helpers/handler_helpers')             .handleSocialTask,
+                            runBackfillAssetPrices   : require('./functions/price-backfill/helpers/handler_helpers')                  .runBackfillAssetPrices };
+                            // Proxy
+const proxy =             { handlePost               : require('./functions/appscript-api/index')                                      .handlePost };
+
+module.exports = { pipe: { core, orchestrator, dispatcher, taskEngine, computationSystem, api, maintenance, proxy } };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.154",
+  "version": "1.0.156",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [