bulltrackers-module 1.0.153 → 1.0.154

package/functions/task-engine/helpers/update_helpers.js

@@ -1,6 +1,8 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
  * (MODIFIED: To conditionally fetch history API once per user per batch)
+ * (MODIFIED: `lookupUsernames` runs batches in parallel)
+ * (MODIFIED: `handleUpdate` fetches history and all portfolios in parallel)
  */
 
 /**
@@ -12,70 +14,108 @@
  * --- MODIFIED: Conditionally fetches history only once per user per batch. ---
  */
 const { FieldValue } = require('@google-cloud/firestore');
+const pLimit = require('p-limit'); // <--- IMPORT p-limit
 
-async function lookupUsernames(cids, { logger, headerManager, proxyManager }, { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL }) {
+/**
+ * (MODIFIED: Runs lookup batches in parallel)
+ */
+async function lookupUsernames(cids, { logger, headerManager, proxyManager }, config) {
     if (!cids?.length) return [];
     logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
-    const allUsers = [];
-    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) {
-        const batch = cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number);
+    const limit = pLimit(config.USERNAME_LOOKUP_CONCURRENCY || 5);
+    const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
+    const batches = [];
+    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) { batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number)); }
+
+    const batchPromises = batches.map(batch => limit(async () => {
         const header = await headerManager.selectHeader();
-        if (!header) { logger.log('ERROR', '[lookupUsernames] Could not select a header.'); continue; }
+        if (!header) { logger.log('ERROR', '[lookupUsernames] Could not select a header.'); return null; }
         let success = false;
         try {
             const res = await proxyManager.fetch(`${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`, { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) });
             if (!res.ok) throw new Error(`API status ${res.status}`);
             const data = await res.json();
-            if (Array.isArray(data)) allUsers.push(...data);
-            success = true; logger.log('DEBUG', 'Looked up usernames', { batch })
-        } catch (err) {
-            logger.log('WARN', `[lookupUsernames] Failed batch`, { error: err.message });
-        } finally { headerManager.updatePerformance(header.id, success); }
-    }
+            success = true;
+            logger.log('DEBUG', 'Looked up usernames', { batch: batch.slice(0, 5) }); 
+            return data;
+        } catch (err) { logger.log('WARN', `[lookupUsernames] Failed batch`, { error: err.message }); return null; 
+        } finally { headerManager.updatePerformance(header.id, success); } }));
+    const results = await Promise.allSettled(batchPromises);
+    const allUsers = results.filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value)).flatMap(r => r.value);
     logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
     return allUsers;
 }
 
+
+/**
+ * (MODIFIED: Fetches history and all portfolios in parallel)
+ */
 async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username, historyFetchedForUser) {
     const { userId, instruments, instrumentId, userType } = task;
     const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
-    let portfolioHeader = await headerManager.selectHeader();
+
     let historyHeader = null;
-    if (!portfolioHeader) throw new Error("Could not select portfolio header.");
-    let wasHistorySuccess = false, isPrivate = false;
-    let fetchHistory = false;
+    let wasHistorySuccess = false;
+    let historyFetchPromise = null;
+    let isPrivate = false;
+
     try {
-        const promisesToRun = [];
+        // --- 1. Prepare History Fetch (if needed) ---
         if (!historyFetchedForUser.has(userId)) {
             historyHeader = await headerManager.selectHeader();
-            if (historyHeader) { fetchHistory = true;  historyFetchedForUser.add(userId); const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`; promisesToRun.push(proxyManager.fetch(historyUrl, { headers: historyHeader.header })); } }
-        if (fetchHistory) {
-            const results = await Promise.allSettled(promisesToRun);
-            const historyRes = results[0];
+            if (historyHeader) { historyFetchedForUser.add(userId);
+                const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
+                historyFetchPromise = proxyManager.fetch(historyUrl, { headers: historyHeader.header }); } }
+
+        // --- 2. Prepare All Portfolio Fetches ---
+        const portfolioRequests = [];
+        for (const instId of instrumentsToProcess) {
+            const portfolioHeader = await headerManager.selectHeader();
+            if (!portfolioHeader) throw new Error(`Could not select portfolio header for ${userId}`);
+            const portfolioUrl = userType === 'speculator'  ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+            portfolioRequests.push({ instrumentId: instId, url: portfolioUrl, header: portfolioHeader, promise: proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header }) }); }
+        
+        // --- 3. Execute All API Calls in Parallel ---
+        const allPromises = [ ...(historyFetchPromise ? [historyFetchPromise] : []), ...portfolioRequests.map(r => r.promise) ];
+        const allResults = await Promise.allSettled(allPromises);
+
+        // --- 4. Process History Result ---
+        let resultIndex = 0;
+        if (historyFetchPromise) {
+            const historyRes = allResults[resultIndex++];
             if (historyRes.status === 'fulfilled' && historyRes.value.ok) { const data = await historyRes.value.json(); wasHistorySuccess = true; await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType); } }
-        for (const instrumentId of instrumentsToProcess) {
-            const portfolioUrl = userType === 'speculator' ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instrumentId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+
+        // --- 5. Process Portfolio Results ---
+        for (let i = 0; i < portfolioRequests.length; i++) {
+            const requestInfo = portfolioRequests[i];
+            const portfolioRes = allResults[resultIndex++];
             let wasPortfolioSuccess = false;
-            const portfolioRes = await proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header });
-            if (portfolioRes.ok) {
-                const body = await portfolioRes.text();
-                if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `User ${userId} is private. Removing from updates.`) ; break; 
-                } else { wasPortfolioSuccess = true; await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, instrumentId); } }
-                logger.log('DEBUG', 'Processing portfolio for user', { userId, portfolioUrl })
-            headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess);
-            if (instrumentsToProcess.length > 1 && instrumentId !== instrumentsToProcess[instrumentsToProcess.length - 1]) {
-                portfolioHeader = await headerManager.selectHeader(); } }
+            if (portfolioRes.status === 'fulfilled' && portfolioRes.value.ok) {
+                const body = await portfolioRes.value.text();
+                if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `User ${userId} is private. Removing from updates.`); break;
+                } else { wasPortfolioSuccess = true; await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, requestInfo.instrumentId); }
+                logger.log('DEBUG', 'Processing portfolio for user', { userId, portfolioUrl: requestInfo.url });
+            } else { logger.log('WARN', `Failed to fetch portfolio`, { userId, url: requestInfo.url, error: portfolioRes.reason || `status ${portfolioRes.value?.status}` }); }
+            headerManager.updatePerformance(requestInfo.header.id, wasPortfolioSuccess);
+        }
+
+        // --- 6. Handle Private Users & Timestamps ---
         if (isPrivate) {
             logger.log('WARN', `User ${userId} is private. Removing from updates.`);
             for (const instrumentId of instrumentsToProcess) { await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId); }
             const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
-            for (const instrumentId of instrumentsToProcess) { const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`; await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true }); }
-            return; }
+            for (const instrumentId of instrumentsToProcess) {
+                const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
+                await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true }); }
+            return; 
+        }
+
         for (const instrumentId of instrumentsToProcess) { await batchManager.updateUserTimestamp(userId, userType, instrumentId); }
         if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
-    } finally { if (historyHeader && fetchHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); }}
+
+    } finally { if (historyHeader) {  headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } }
 }
 
 module.exports = { handleUpdate, lookupUsernames };

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -29,7 +29,22 @@ class FirestoreBatchManager {
 
     _getUsernameShardId(cid) { return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`; }
 
-    _scheduleFlush() { if (!this.batchTimeout) this.batchTimeout = setTimeout(() => this.flushBatches(), this.config.TASK_ENGINE_FLUSH_INTERVAL_MS); }
+    // _scheduleFlush() { if (!this.batchTimeout) this.batchTimeout = setTimeout(() => this.flushBatches(), this.config.TASK_ENGINE_FLUSH_INTERVAL_MS); } Old version
+
+    _scheduleFlush() {
+        const totalOps = this._estimateBatchSize();
+        if (totalOps >= 400) { this.flushBatches(); return; }
+        if (!this.batchTimeout) { this.batchTimeout = setTimeout(() => this.flushBatches(),  this.config.TASK_ENGINE_FLUSH_INTERVAL_MS); }
+    }
+
+    _estimateBatchSize() {
+        let ops = 0;
+        ops += Object.keys(this.portfolioBatch).length;
+        ops += Object.keys(this.tradingHistoryBatch).length;
+        ops += Object.keys(this.timestampBatch).length;
+        ops += Object.keys(this.speculatorTimestampFixBatch).length;
+        return ops;
+    }
 
     async loadUsernameMap() {
         if (Date.now() - this.usernameMapLastLoaded < 3600000) return;

package/functions/task-engine/utils/task_engine_utils.js

@@ -1,6 +1,7 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/utils/task_engine_utils.js
  * (MODIFIED: To pass down a Set to track history fetches)
+ * (MODIFIED: To run all update tasks in parallel with a concurrency limit)
  */
 
 /**
@@ -12,6 +13,7 @@
 const { handleDiscover } = require('../helpers/discover_helpers');
 const { handleVerify } = require('../helpers/verify_helpers');
 const { handleUpdate, lookupUsernames } = require('../helpers/update_helpers');
+const pLimit = require('p-limit'); 
 
 /**
  * Parses Pub/Sub message into task array.
@@ -60,11 +62,15 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
         if (handler) try { await handler(task, subTaskId, dependencies, config); } 
         catch (err) { logger.log('ERROR', `[TaskEngine/${taskId}] Error in ${task.type} for ${subTaskId}`, { errorMessage: err.message }); }
         else logger.log('ERROR', `[TaskEngine/${taskId}] Unknown task type: ${task.type}`); }
-    for (const { task, username } of tasksToRun) {
+    const limit = pLimit(config.TASK_ENGINE_CONCURRENCY || 10);
+    let successCount = 0;
+    let errorCount = 0;
+    const updatePromises = tasksToRun.map(({ task, username }) => {
         const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId}`;
-        try { await handleUpdate(task, subTaskId, dependencies, config, username, historyFetchedForUser);}
-        catch (err) { logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message }); } }
-    logger.log('SUCCESS', `[TaskEngine/${taskId}] Processed all tasks.`);
+        return limit(() =>  handleUpdate(task, subTaskId, dependencies, config, username, historyFetchedForUser) .catch(err => { logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message }); throw err;  }) ); });
+    const results = await Promise.allSettled(updatePromises);
+    results.forEach(result => { if (result.status === 'fulfilled') { successCount++; } else { errorCount++; } });
+    logger.log( errorCount > 0 ? 'WARN' : 'SUCCESS', `[TaskEngine/${taskId}] Processed all ${tasksToRun.length} update tasks. Success: ${successCount}, Failed: ${errorCount}.` );
 }
 
 module.exports = { parseTaskPayload, prepareTaskBatches, runUsernameLookups, executeTasks };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.153",
+  "version": "1.0.154",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [