bulltrackers-module 1.0.194 → 1.0.195

package/functions/task-engine/helpers/update_helpers.js

@@ -1,14 +1,41 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
- * (REFACTORED: Removed all concurrency from `handleUpdate` and `lookupUsernames`)
- * (REFACTORED: Added node-fetch fallback for all API calls)
- * (REFACTORED: Added verbose, user-centric logging for all operations)
- * (FIXED: Corrected variable name 'instId' to 'instrumentId' in final timestamp loops)
+ * (OPTIMIZED V2: Added "Circuit Breaker" for Proxy failures)
+ * (OPTIMIZED V2: Downgraded verbose per-user logs to TRACE to save costs)
+ * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
 const pLimit = require('p-limit');
 
+// --- CIRCUIT BREAKER STATE ---
+// Persists across function invocations in the same instance.
+// If the Proxy fails 3 times in a row, we stop trying it to save the 5s timeout cost.
+let _consecutiveProxyFailures = 0;
+const MAX_PROXY_FAILURES = 3;
+
+/**
+ * Helper to check if we should attempt the proxy
+ */
+function shouldTryProxy() {
+    return _consecutiveProxyFailures < MAX_PROXY_FAILURES;
+}
+
+/**
+ * Helper to record proxy result
+ */
+function recordProxyOutcome(success) {
+    if (success) {
+        if (_consecutiveProxyFailures > 0) {
+            // Optional: Only log recovery to reduce noise
+            // console.log('[ProxyCircuit] Proxy recovered.');
+        }
+        _consecutiveProxyFailures = 0;
+    } else {
+        _consecutiveProxyFailures++;
+    }
+}
+
 /**
  * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
  */
@@ -21,35 +48,65 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
     const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
     const batches = [];
     for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) { batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number)); }
-    const batchPromises = batches.map((batch, index) => limit(async () => { const batchId = `batch-${index + 1}`;
-        logger.log('INFO', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`);
+    
+    const batchPromises = batches.map((batch, index) => limit(async () => { 
+        const batchId = `batch-${index + 1}`;
+        logger.log('TRACE', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`); // DOWNGRADED TO TRACE
+        
         const header = await headerManager.selectHeader();
         if (!header) { logger.log('ERROR', `[lookupUsernames/${batchId}] Could not select a header.`); return null; }
+        
         let wasSuccess = false;
-        let proxyUsed = true;
+        let proxyUsed = false;
         let response;
         const url = `${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
         const options = { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) };
-        try {
-            logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
-            response = await proxyManager.fetch(url, options);
-            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-            wasSuccess = true; // Yay we win
-            logger.log('INFO', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`);
-        } catch (proxyError) { logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' }); // SHIT we failed...
-            proxyUsed = false; // Don't penalize header for proxy failure
-            try { response = await fetch(url, options);  // Ok let's try again with node, using GCP IP pools
+        
+        // --- 1. Try Proxy (Circuit Breaker Protected) ---
+        if (shouldTryProxy()) {
+            try {
+                logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
+                response = await proxyManager.fetch(url, options);
+                if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
+                
+                wasSuccess = true; 
+                proxyUsed = true;
+                recordProxyOutcome(true); // Reset failure count
+                logger.log('TRACE', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`); // DOWNGRADED TO TRACE
+
+            } catch (proxyError) { 
+                recordProxyOutcome(false); // Increment failure count
+                logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
+                // Fall through to direct...
+            }
+        } else {
+            logger.log('TRACE', `[lookupUsernames/${batchId}] Circuit Breaker Open. Skipping Proxy.`);
+        }
+
+        // --- 2. Direct Fallback ---
+        if (!wasSuccess) {
+            try { 
+                response = await fetch(url, options);  
                 if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); }
-                logger.log('INFO', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`); // Yay we win
-            } catch (fallbackError) { logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, { error: fallbackError.message, source: 'eToro/Network' }); // SHIT, we failed here too
+                logger.log('TRACE', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`); // DOWNGRADED TO TRACE
+                wasSuccess = true; // It worked eventually
+            } catch (fallbackError) { 
+                logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, { error: fallbackError.message, source: 'eToro/Network' }); 
                 return null; // Give up on this batch
             }
-        } finally { if (proxyUsed) { headerManager.updatePerformance(header.id, wasSuccess); } } // If we used Appscript IP Pool and not GCP IP Pool, record performance
-        try { const data = await response.json(); return data;
-        } catch (parseError) { logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message }); return null; } }));
+        } 
+        
+        if (proxyUsed) { headerManager.updatePerformance(header.id, wasSuccess); } 
+        
+        try { 
+            const data = await response.json(); return data;
+        } catch (parseError) { 
+            logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message }); return null; 
+        } 
+    }));
 
     const results  = await Promise.allSettled(batchPromises);
-    const allUsers = results .filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value)) .flatMap(r => r.value); 
+    const allUsers = results.filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value)).flatMap(r => r.value); 
     logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
     return allUsers;
 }
@@ -64,118 +121,140 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
     let isPrivate = false;
-    logger.log('INFO', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
+    
+    // DOWNGRADED TO TRACE
+    logger.log('TRACE', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
 
     // --- 1. Process History Fetch (Sequentially) ---
     let historyHeader = null;
     let wasHistorySuccess = false;
-    let proxyUsedForHistory = true;
+    let proxyUsedForHistory = false;
 
     try {
         if (!batchManager.checkAndSetHistoryFetched(userId)) {
-            logger.log('INFO', `[handleUpdate/${userId}] Attempting history fetch.`);
+            logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch.`);
             historyHeader = await headerManager.selectHeader();
-            if (!historyHeader) { logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
+            if (!historyHeader) { 
+                logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
             } else {
                 const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
                 const options = { headers: historyHeader.header };
                 let response;
-                try { logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
-                    response = await proxyManager.fetch(historyUrl, options);
-                    if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // SHIT we failed here
-                    wasHistorySuccess = true; // Appscript worked, we are very smart
-                    
-                } catch (proxyError) {
-                    logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' }); // SHIT we failed here
-                    proxyUsedForHistory = false;
-                    try { response = await fetch(historyUrl, options);
-                        if (!response.ok) { const errorText = await response.text(); 
-                            throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); } // SHIT we failed here too
-                        wasHistorySuccess = true; // Fallback succeeded, we are so smart
+
+                // --- PROXY ATTEMPT ---
+                if (shouldTryProxy()) {
+                    try { 
+                        logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
+                        response = await proxyManager.fetch(historyUrl, options);
+                        if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
                         
-                    } catch (fallbackError) { logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' }); // We are dumb, everything failed
-                        wasHistorySuccess = false; // Nope we are dumb....
+                        wasHistorySuccess = true; 
+                        proxyUsedForHistory = true;
+                        recordProxyOutcome(true); // Reset
+
+                    } catch (proxyError) {
+                        recordProxyOutcome(false); // Count failure
+                        logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
                     }
                 }
 
-                if (wasHistorySuccess) { logger.log('INFO', `[handleUpdate/${userId}] History fetch successful.`); // Some method worked, we are very smart
+                // --- DIRECT FALLBACK ---
+                if (!wasHistorySuccess) {
+                    try { 
+                        response = await fetch(historyUrl, options);
+                        if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); } 
+                        wasHistorySuccess = true; 
+                        logger.log('TRACE', `[handleUpdate/${userId}] History fetch direct success.`);
+                    } catch (fallbackError) { 
+                        logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' }); 
+                        wasHistorySuccess = false; 
+                    }
+                }
+
+                if (wasHistorySuccess) { 
                     const data = await response.json();
-                    await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);  }
+                    await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);  
+                }
             }
-        } else { logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched by this instance).`); }
-    } catch (err) { logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message }); wasHistorySuccess = false; // We fucked up.
-    } finally { if (historyHeader && proxyUsedForHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } } // If we used appscript proxy, record performance, otherwise fuck off.
+        } else { 
+            logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched).`); 
+        }
+    } catch (err) { 
+        logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message }); wasHistorySuccess = false; 
+    } finally { 
+        if (historyHeader && proxyUsedForHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } 
+    } 
     
     // --- 2. Process Portfolio Fetches (Sequentially) ---
-    logger.log('INFO', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`);
+    logger.log('TRACE', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`); // DOWNGRADED TO TRACE
     
     for (const instId of instrumentsToProcess) { 
         if (isPrivate) { 
-            logger.log('INFO', `[handleUpdate/${userId}] Skipping remaining instruments because user was marked as private.`); 
+            logger.log('TRACE', `[handleUpdate/${userId}] Skipping remaining instruments (User Private).`); 
             break;  
         }
 
         const portfolioHeader = await headerManager.selectHeader();
-        if (!portfolioHeader) { logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`);
-            continue;
-        }
+        if (!portfolioHeader) { logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`); continue; }
 
         const portfolioUrl = userType === 'speculator'  ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
-        
         const options = { headers: portfolioHeader.header };
         let response;
         let wasPortfolioSuccess = false;
-        let proxyUsedForPortfolio = true;
-
-        try {
-            // --- REFACTOR 3: ADD FALLBACK ---
-            logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch for instId ${instId} via AppScript proxy...`);
-            response = await proxyManager.fetch(portfolioUrl, options);
-            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // SHIT we failed here
-            wasPortfolioSuccess = true; // Oh we are smart, worked first time.
-
-        } catch (proxyError) { // try fallback with local node fetch using GCP IP Pools
-            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
-            proxyUsedForPortfolio = false; // We are not using Appscript proxy here as fallback is GCP based, so false
-            
+        let proxyUsedForPortfolio = false;
+
+        // --- PROXY ATTEMPT ---
+        if (shouldTryProxy()) {
             try {
-                response = await fetch(portfolioUrl, options); // Direct node-fetch
-                if (!response.ok) {
-                    const errorText = await response.text();
-                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); // SHIT we failed here
-                }
-                wasPortfolioSuccess = true; // Fallback succeeded we are so smart
+                logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch via AppScript proxy...`);
+                response = await proxyManager.fetch(portfolioUrl, options);
+                if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); 
                 
+                wasPortfolioSuccess = true; 
+                proxyUsedForPortfolio = true;
+                recordProxyOutcome(true); // Reset
+
+            } catch (proxyError) { 
+                recordProxyOutcome(false); // Count failure
+                logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch via Proxy FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
+            }
+        }
+
+        // --- DIRECT FALLBACK ---
+        if (!wasPortfolioSuccess) {
+            try {
+                response = await fetch(portfolioUrl, options); 
+                if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); }
+                wasPortfolioSuccess = true; 
+                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio fetch direct success.`);
             } catch (fallbackError) {
-                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
+                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
                 wasPortfolioSuccess = false;
             }
         }
 
-        // --- 4. Process Portfolio Result (with verbose, raw logging) ---
+        // --- 4. Process Portfolio Result ---
         if (wasPortfolioSuccess) {
             const body = await response.text();
-            if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`);
-                break; // Stop processing more portfolios for this private user
-            }
+            if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`); break; }
 
             try {
                 const portfolioJson = JSON.parse(body);
                 await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
-                if (userType === 'speculator') { logger.log('INFO', `[handleUpdate/${userId}] Successfully processed portfolio for instId ${instId}.`); // Only speculators have an instid, so this is conditional
-                } else { logger.log('INFO', `[handleUpdate/${userId}] Successfully processed full portfolio (normal user).`); } // Normal users
+                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio processed successfully.`); // DOWNGRADED TO TRACE
 
-            } catch (parseError) { // Idk why this would happen, but if it does....log.
-                wasPortfolioSuccess = false; // Mark as failure
-                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, { url: portfolioUrl, parseErrorMessage: parseError.message, rawResponseText: body }); // Return full response
+            } catch (parseError) { 
+                wasPortfolioSuccess = false; 
+                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE.`, { url: portfolioUrl, parseErrorMessage: parseError.message }); 
             }
-        } else { logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}. No response to process.`); }
+        } else { 
+            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}.`); 
+        }
         
         if (proxyUsedForPortfolio) { headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess); }
     } 
 
     // --- 5. Handle Private Users & Timestamps ---
-    // FIXED: Corrected variable naming here from 'instId' to 'instrumentId'
     if (isPrivate) {
         logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
         for (const instrumentId of instrumentsToProcess) { 
@@ -190,15 +269,13 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     }
 
     // If not private, update all timestamps
-    // FIXED: Corrected variable naming here from 'instId' to 'instrumentId'
     for (const instrumentId of instrumentsToProcess) { 
         await batchManager.updateUserTimestamp(userId, userType, instrumentId); 
     }
     
     if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
     
-    logger.log('INFO', `[handleUpdate/${userId}] Update task finished successfully.`);
-    // 'finally' block for header flushing is handled by the main handler_creator.js
+    logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`); // DOWNGRADED TO TRACE
 }
 
 module.exports = { handleUpdate, lookupUsernames };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.194",
+  "version": "1.0.195",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [