bulltrackers-module 1.0.544 → 1.0.545

package/functions/task-engine/helpers/data_storage_helpers.js

@@ -133,16 +133,8 @@ async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, ci
     
     // 2. Store latest posts to user-centric collection (for fallback)
     // Path structure: SignedInUsers/{cid}/posts/{postId}
-    // We need to construct this step-by-step, not use the template directly
-    const { getCollectionPath } = collectionRegistry || {};
-    if (!getCollectionPath) {
-        throw new Error('collectionRegistry.getCollectionPath is required');
-    }
-    
-    // Get the base path template to extract collection name
-    const pathTemplate = getCollectionPath('signedInUsers', 'socialPosts', { cid: String(cid), postId: 'placeholder' });
-    // Extract collection name (first part before /)
-    const collectionName = pathTemplate.split('/')[0]; // Should be 'SignedInUsers'
+    // Construct path directly - we know the structure
+    const collectionName = 'SignedInUsers';
     
     // Store each post individually in user-centric collection
     const batch = db.batch();
@@ -309,16 +301,8 @@ async function storePopularInvestorSocialPosts({ db, logger, collectionRegistry,
     
     // 2. Store latest posts to user-centric collection (for fallback)
     // Path structure: PopularInvestors/{piCid}/posts/{postId}
-    // We need to construct this step-by-step, not use the template directly
-    const { getCollectionPath } = collectionRegistry || {};
-    if (!getCollectionPath) {
-        throw new Error('collectionRegistry.getCollectionPath is required');
-    }
-    
-    // Get the base path template to extract collection name
-    const pathTemplate = getCollectionPath('popularInvestors', 'socialPosts', { piCid: String(cid), postId: 'placeholder' });
-    // Extract collection name (first part before /)
-    const collectionName = pathTemplate.split('/')[0]; // Should be 'PopularInvestors'
+    // Construct path directly - we know the structure
+    const collectionName = 'PopularInvestors';
     
     // Store each post individually in user-centric collection
     const batch = db.batch();

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -19,70 +19,69 @@ const {
 // 1. SHARED INTERNAL HELPERS (The "Engine")
 // ==========================================
 
-async function fetchWithRetry(url, options, proxyManager, logger, label) {
-    // ALWAYS try proxy first, regardless of circuit breaker state
-    // Circuit breaker only prevents retrying after it's open, but we always attempt proxy first
-    let proxyAttempted = false;
-    let proxyFailed = false;
+async function fetchWithRetry(url, options, proxyManager, logger, label, headerManager = null) {
+    // SIMPLIFIED: Always try proxy first, then fallback to direct
+    // Use header manager if provided
+    let finalOptions = { ...options };
+    let headerId = null;
     
-    // Try proxy first (unless circuit breaker is open and we've already failed)
-    if (shouldTryProxy()) {
-        proxyAttempted = true;
+    if (headerManager) {
         try {
-            const res = await proxyManager.fetch(url, options);
-            if (res.ok) {
-                recordProxyOutcome(true);
-                logger.log('TRACE', `[${label}] Proxy fetch succeeded for ${url}`);
-                return res;
-            } else {
-                // Log proxy failure with details
-                const errorText = await res.text().catch(() => 'Unable to read response');
-                logger.log('WARN', `[${label}] Proxy returned status ${res.status} for ${url}. Response: ${errorText.substring(0, 200)}`);
-                recordProxyOutcome(false);
-                proxyFailed = true;
+            const headerResult = await headerManager.selectHeader();
+            if (headerResult && headerResult.header) {
+                finalOptions.headers = {
+                    ...(finalOptions.headers || {}),
+                    ...headerResult.header
+                };
+                headerId = headerResult.id;
             }
         } catch (e) {
-            recordProxyOutcome(false);
-            logger.log('WARN', `[${label}] Proxy failed for ${url}: ${e.message}. Failures: ${getFailureCount()}/${getMaxFailures()}. Falling back to direct fetch.`);
-            proxyFailed = true;
+            logger.log('WARN', `[${label}] Failed to get header from headerManager: ${e.message}`);
         }
-    } else {
-        // Circuit breaker is open - log but still try proxy once more
-        logger.log('INFO', `[${label}] Circuit breaker open (${getFailureCount()}/${getMaxFailures()} failures), but attempting proxy once more for ${url}`);
-        proxyAttempted = true;
+    }
+    
+    // Try proxy first (if circuit breaker allows)
+    if (shouldTryProxy()) {
         try {
-            const res = await proxyManager.fetch(url, options);
+            const res = await proxyManager.fetch(url, finalOptions);
             if (res.ok) {
-                // Success! Reset circuit breaker
                 recordProxyOutcome(true);
-                logger.log('INFO', `[${label}] Proxy succeeded despite circuit breaker being open. Resetting circuit breaker.`);
+                if (headerManager && headerId) {
+                    headerManager.updatePerformance(headerId, true);
+                }
                 return res;
             } else {
-                const errorText = await res.text().catch(() => 'Unable to read response');
-                logger.log('WARN', `[${label}] Proxy failed (circuit breaker open): Status ${res.status}. Response: ${errorText.substring(0, 200)}`);
-                proxyFailed = true;
+                // Proxy returned non-OK status - record failure but don't throw yet
+                const status = res.status;
+                recordProxyOutcome(false);
+                logger.log('WARN', `[${label}] Proxy returned status ${status} for ${url}. Failures: ${getFailureCount()}/${getMaxFailures()}. Will try direct fetch.`);
+                // Continue to fallback - don't throw here
             }
         } catch (e) {
-            logger.log('WARN', `[${label}] Proxy failed (circuit breaker open): ${e.message}`);
-            proxyFailed = true;
+            recordProxyOutcome(false);
+            logger.log('WARN', `[${label}] Proxy exception for ${url}: ${e.message}. Failures: ${getFailureCount()}/${getMaxFailures()}. Will try direct fetch.`);
+            // Continue to fallback - don't throw here
         }
+    } else {
+        logger.log('INFO', `[${label}] Circuit breaker open (${getFailureCount()}/${getMaxFailures()} failures), skipping proxy for ${url}`);
     }
     
-    // Fallback to direct fetch only after proxy fails
-    if (proxyFailed || !proxyAttempted) {
-        logger.log('INFO', `[${label}] Falling back to direct fetch for ${url}${proxyFailed ? ' (proxy failed)' : ''}`);
-        const directFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch');
-        const res = await directFetch(url, options);
-        if (!res.ok) {
-            const errorText = await res.text().catch(() => 'Unable to read response');
-            logger.log('ERROR', `[${label}] Direct fetch failed for ${url}: Status ${res.status}. Response: ${errorText.substring(0, 200)}`);
-            throw new Error(`Fetch failed: ${res.status} ${res.statusText} - ${errorText.substring(0, 100)}`);
-        }
-        return res;
+    // Fallback to direct fetch only if proxy failed or circuit breaker is open
+    logger.log('INFO', `[${label}] Using direct fetch for ${url}`);
+    const directFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch');
+    const res = await directFetch(url, finalOptions);
+    if (!res.ok) {
+        const errorText = await res.text().catch(() => 'Unable to read response');
+        logger.log('ERROR', `[${label}] Direct fetch failed for ${url}: Status ${res.status}. Response: ${errorText.substring(0, 200)}`);
+        throw new Error(`Fetch failed: ${res.status} ${res.statusText} - ${errorText.substring(0, 100)}`);
+    }
+    
+    // Update header manager on success
+    if (headerManager && headerId) {
+        headerManager.updatePerformance(headerId, true);
     }
     
-    // Should not reach here, but just in case
-    throw new Error(`[${label}] Unexpected state: proxy attempted but no response returned`);
+    return res;
 }
 
 async function updateLastUpdated(db, collectionRegistry, cid, userType, dataType, logger) {
@@ -97,13 +96,13 @@ async function updateLastUpdated(db, collectionRegistry, cid, userType, dataType
 }
 
 async function processPortfolio(context, config, taskData, isPI) {
-    const { db, logger, collectionRegistry, proxyManager } = context;
+    const { db, logger, collectionRegistry, proxyManager, headerManager } = context;
     const { cid, username, uuid, today, requestOptions } = taskData;
     const url = `${config.ETORO_API_PORTFOLIO_URL}?cid=${cid}&client_request_id=${uuid}`;
     
     logger.log('INFO', `[Portfolio] Fetching ${isPI ? 'PI' : 'Signed-In User'} portfolio from: ${url}`);
     
-    const res = await fetchWithRetry(url, requestOptions, proxyManager, logger, 'Portfolio');
+    const res = await fetchWithRetry(url, requestOptions, proxyManager, logger, 'Portfolio', headerManager);
     const data = await res.json();
     data.fetchedAt = new Date();
     data.username = username;
@@ -116,7 +115,7 @@ async function processPortfolio(context, config, taskData, isPI) {
             for (const pos of topPositions) {
                 try {
                     const posUrl = `${config.ETORO_API_POSITIONS_URL}?cid=${cid}&InstrumentID=${pos.InstrumentID}&client_request_id=${uuid}`;
-                    const deepRes = await fetchWithRetry(posUrl, requestOptions, proxyManager, logger, 'DeepPos');
+                    const deepRes = await fetchWithRetry(posUrl, requestOptions, proxyManager, logger, 'DeepPos', headerManager);
                     deepPositions.push({ instrumentId: pos.InstrumentID, ...(await deepRes.json()) });
                 } catch (e) {} // Skip failed deep positions
             }
@@ -133,12 +132,12 @@ async function processPortfolio(context, config, taskData, isPI) {
 }
 
 async function processHistory(context, config, taskData, isPI) {
-    const { db, logger, collectionRegistry, proxyManager } = context;
+    const { db, logger, collectionRegistry, proxyManager, headerManager } = context;
     const { cid, uuid, today, requestOptions } = taskData;
     const oneYearAgo = new Date(); oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
     const url = `${config.ETORO_API_HISTORY_URL}?StartTime=${oneYearAgo.toISOString()}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${uuid}`;
 
-    const res = await fetchWithRetry(url, requestOptions, proxyManager, logger, 'History');
+    const res = await fetchWithRetry(url, requestOptions, proxyManager, logger, 'History', headerManager);
     const data = await res.json();
     
     // Filter valid close reasons
@@ -154,7 +153,7 @@ async function processHistory(context, config, taskData, isPI) {
 }
 
 async function processSocial(context, config, taskData, isPI) {
-    const { db, logger, collectionRegistry, proxyManager } = context;
+    const { db, logger, collectionRegistry, proxyManager, headerManager } = context;
     const { cid, username, uuid, today, requestOptions } = taskData;
     const { getGcidForUser } = require('../../social-task-handler/helpers/handler_helpers');
     
@@ -162,7 +161,7 @@ async function processSocial(context, config, taskData, isPI) {
         const gcid = await getGcidForUser(context, config.social || {}, cid, username);
         const url = `${config.social?.userFeedApiUrl || 'https://www.etoro.com/api/edm-streams/v1/feed/user/top/'}${gcid}?take=10&client_request_id=${uuid}`;
         
-        const res = await fetchWithRetry(url, requestOptions, proxyManager, logger, 'Social');
+        const res = await fetchWithRetry(url, requestOptions, proxyManager, logger, 'Social', headerManager);
         const data = await res.json();
         const posts = (data.discussions || []).slice(0, 30).map(d => ({
             id: d.post.id,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.544",
+  "version": "1.0.545",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [