bulltrackers-module 1.0.543 → 1.0.545

package/functions/computation-system/executors/StandardExecutor.js

@@ -18,7 +18,7 @@ class StandardExecutor {
         const dStr   = date.toISOString().slice(0, 10);
         const logger = deps.logger;
 
-        // [FIX] Determine required user types for this batch of calculations
+        // Determine required user types for this batch of calculations
         const requiredUserTypes = new Set();
         calcs.forEach(c => {
             const type = (c.userType || 'ALL').toUpperCase();
@@ -27,7 +27,6 @@ class StandardExecutor {
         const userTypeArray = requiredUserTypes.has('ALL') ? null : Array.from(requiredUserTypes);
         
         // [OPTIMIZATION] Check for Target CID in manifests (On-Demand Optimization)
-        // If present, we will filter all data streams to strictly this user
         const targetCid = calcs.find(c => c.targetCid)?.targetCid || calcs.find(c => c.manifest?.targetCid)?.manifest?.targetCid;
         if (targetCid) {
             logger.log('INFO', `[StandardExecutor] Running in Targeted Mode for CID: ${targetCid}`);
@@ -62,7 +61,6 @@ class StandardExecutor {
             } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); } 
         }
 
-        // Pass targetCid to streamAndProcess
         return await StandardExecutor.streamAndProcess(
             dStr, state, passName, config, deps, fullRoot, 
             rootData.portfolioRefs, rootData.historyRefs, 
@@ -71,7 +69,6 @@ class StandardExecutor {
         );
     }
 
-    // [UPDATED] Added targetCid param
     static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, requiredUserTypes = null, targetCid = null) {
         const { logger } = deps;
         const calcs = Object.values(state).filter(c => c && c.manifest);
@@ -82,11 +79,9 @@ class StandardExecutor {
         // --- 1. Resolve and Filter Portfolio Refs (Today) ---
         let effectivePortfolioRefs = portfolioRefs;
         if (!effectivePortfolioRefs) {
-            // If refs weren't provided by AvailabilityChecker, fetch them now
             effectivePortfolioRefs = await getPortfolioPartRefs(config, deps, dateStr, requiredUserTypes);
         }
         if (targetCid && effectivePortfolioRefs) {
-            // Filter: Keep only refs that match the CID (or Legacy refs without CID)
             effectivePortfolioRefs = effectivePortfolioRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
         }
 
@@ -129,7 +124,6 @@ class StandardExecutor {
         const setupDuration = performance.now() - startSetup;
         Object.keys(executionStats).forEach(name => executionStats[name].timings.setup += setupDuration);
 
-        // Yesterday's Refs are already filtered in run()
         const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
         const prevDateStr = prevDate.toISOString().slice(0, 10);
         
@@ -138,7 +132,6 @@ class StandardExecutor {
             earliestDates = await getEarliestDataDates(config, deps);
         }
 
-        // [FIX] Use effective/filtered refs
         const tP_iter = streamPortfolioData(config, deps, dateStr, effectivePortfolioRefs, requiredUserTypes);
         
         const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
@@ -200,7 +193,6 @@ class StandardExecutor {
         return { successUpdates: aggregatedSuccess, failureReport: aggregatedFailures };
     }
     
-    // ... rest of the file (flushBuffer, mergeReports, executePerUser) ...
     static async flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, mode, skipStatusWrite, isInitialWrite = false) {
         const { logger } = deps;
         const transformedState = {};
@@ -272,35 +264,39 @@ class StandardExecutor {
    static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, loader, stats, earliestDates) {
         const { logger } = deps;
         const targetUserType = metadata.userType; 
-        // [NEW] Always load Global Helpers
+        // [FIX] Always load Global Helpers
         const mappings = await loader.loadMappings();
-        // [FIX] Correct method name: loadPIMasterList (not loadPopularInvestorMasterList)
+        // [FIX] Correct method: loadPIMasterList() (no args needed as loader has context)
         const piMasterList = await loader.loadPIMasterList(); 
         const SCHEMAS = mathLayer.SCHEMAS;
 
         // 1. Load Root Data
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
-        const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerificationProfiles(config, deps) : null;
-        const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadPopularInvestorRankings(config, deps, dateStr) : null;
         
-        // [FIX] Load Yesterday's Rankings if isHistorical is true
+        // [FIX] Correct method: loadVerifications() (no args)
+        const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerifications() : null;
+        
+        // [FIX] Correct method: loadRankings(dateStr) (no config/deps args)
+        const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadRankings(dateStr) : null;
+        
+        // [FIX] Correct method: loadRankings(prevStr)
         let yesterdayRankings = null;
         if (metadata.rootDataDependencies?.includes('rankings') && metadata.isHistorical) {
              const prevDate = new Date(dateStr); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
              const prevStr = prevDate.toISOString().slice(0, 10);
-             yesterdayRankings = await loader.loadPopularInvestorRankings(config, deps, prevStr); 
+             yesterdayRankings = await loader.loadRankings(prevStr); 
         }
 
-        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadDailySocialPostInsights(config, deps, dateStr) : null;
+        // [FIX] Correct method: loadSocial(dateStr)
+        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadSocial(dateStr) : null;
         
-        // [NEW] Load New Root Data Types for Profile Metrics
-        // [FIX] Enforce canHaveMissingRoots
         const allowMissing = metadata.canHaveMissingRoots === true;
         
+        // [FIX] Correct method: loadRatings(dateStr)
         let ratings = null;
         if (metadata.rootDataDependencies?.includes('ratings')) {
             try {
-                ratings = await loader.loadPIRatings(config, deps, dateStr);
+                ratings = await loader.loadRatings(dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'ratings' failed to load for ${metadata.name}: ${e.message}`);
@@ -312,10 +308,11 @@ class StandardExecutor {
             }
         }
         
+        // [FIX] Correct method: loadPageViews(dateStr)
         let pageViews = null;
         if (metadata.rootDataDependencies?.includes('pageViews')) {
             try {
-                pageViews = await loader.loadPIPageViews(config, deps, dateStr);
+                pageViews = await loader.loadPageViews(dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'pageViews' failed to load for ${metadata.name}: ${e.message}`);
@@ -327,10 +324,11 @@ class StandardExecutor {
             }
         }
         
+        // [FIX] Correct method: loadWatchlistMembership(dateStr)
         let watchlistMembership = null;
         if (metadata.rootDataDependencies?.includes('watchlist')) {
             try {
-                watchlistMembership = await loader.loadWatchlistMembership(config, deps, dateStr);
+                watchlistMembership = await loader.loadWatchlistMembership(dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'watchlist' failed to load for ${metadata.name}: ${e.message}`);
@@ -342,10 +340,11 @@ class StandardExecutor {
             }
         }
         
+        // [FIX] Correct method: loadAlertHistory(dateStr)
         let alertHistory = null;
         if (metadata.rootDataDependencies?.includes('alerts')) {
             try {
-                alertHistory = await loader.loadPIAlertHistory(config, deps, dateStr);
+                alertHistory = await loader.loadAlertHistory(dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'alerts' failed to load for ${metadata.name}: ${e.message}`);

package/functions/task-engine/helpers/data_storage_helpers.js

@@ -132,13 +132,9 @@ async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, ci
     }, { merge: false });
     
     // 2. Store latest posts to user-centric collection (for fallback)
-    const { getCollectionPath } = collectionRegistry || {};
-    if (!getCollectionPath) {
-        throw new Error('collectionRegistry.getCollectionPath is required');
-    }
-    const userPostsRef = db.collection(
-        getCollectionPath('signedInUsers', 'socialPosts', { cid })
-    );
+    // Path structure: SignedInUsers/{cid}/posts/{postId}
+    // Construct path directly - we know the structure
+    const collectionName = 'SignedInUsers';
     
     // Store each post individually in user-centric collection
     const batch = db.batch();
@@ -149,7 +145,11 @@ async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, ci
         const postId = post.id || post.postId;
         if (!postId) continue;
         
-        const postRef = userPostsRef.doc(postId);
+        // Construct path step-by-step: SignedInUsers/{cid}/posts/{postId}
+        const postRef = db.collection(collectionName)
+            .doc(String(cid))
+            .collection('posts')
+            .doc(postId);
         batch.set(postRef, {
             ...post,
             fetchedAt: FieldValue.serverTimestamp(),
@@ -300,13 +300,9 @@ async function storePopularInvestorSocialPosts({ db, logger, collectionRegistry,
     }, { merge: false });
     
     // 2. Store latest posts to user-centric collection (for fallback)
-    const { getCollectionPath } = collectionRegistry || {};
-    if (!getCollectionPath) {
-        throw new Error('collectionRegistry.getCollectionPath is required');
-    }
-    const userPostsRef = db.collection(
-        getCollectionPath('popularInvestors', 'socialPosts', { piCid: String(cid) })
-    );
+    // Path structure: PopularInvestors/{piCid}/posts/{postId}
+    // Construct path directly - we know the structure
+    const collectionName = 'PopularInvestors';
     
     // Store each post individually in user-centric collection
     const batch = db.batch();
@@ -317,7 +313,11 @@ async function storePopularInvestorSocialPosts({ db, logger, collectionRegistry,
         const postId = post.id || post.postId;
         if (!postId) continue;
         
-        const postRef = userPostsRef.doc(postId);
+        // Construct path step-by-step: PopularInvestors/{cid}/posts/{postId}
+        const postRef = db.collection(collectionName)
+            .doc(String(cid))
+            .collection('posts')
+            .doc(postId);
         batch.set(postRef, {
             ...post,
             fetchedAt: FieldValue.serverTimestamp(),

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -19,34 +19,68 @@ const {
 // 1. SHARED INTERNAL HELPERS (The "Engine")
 // ==========================================
 
-async function fetchWithRetry(url, options, proxyManager, logger, label) {
-    // Circuit Breaker / Proxy Logic abstracted here
+async function fetchWithRetry(url, options, proxyManager, logger, label, headerManager = null) {
+    // SIMPLIFIED: Always try proxy first, then fallback to direct
+    // Use header manager if provided
+    let finalOptions = { ...options };
+    let headerId = null;
+    
+    if (headerManager) {
+        try {
+            const headerResult = await headerManager.selectHeader();
+            if (headerResult && headerResult.header) {
+                finalOptions.headers = {
+                    ...(finalOptions.headers || {}),
+                    ...headerResult.header
+                };
+                headerId = headerResult.id;
+            }
+        } catch (e) {
+            logger.log('WARN', `[${label}] Failed to get header from headerManager: ${e.message}`);
+        }
+    }
+    
+    // Try proxy first (if circuit breaker allows)
     if (shouldTryProxy()) {
         try {
-            const res = await proxyManager.fetch(url, options);
+            const res = await proxyManager.fetch(url, finalOptions);
             if (res.ok) {
                 recordProxyOutcome(true);
+                if (headerManager && headerId) {
+                    headerManager.updatePerformance(headerId, true);
+                }
                 return res;
             } else {
-                // Log proxy failure with details
-                const errorText = await res.text().catch(() => 'Unable to read response');
-                logger.log('WARN', `[${label}] Proxy returned status ${res.status} for ${url}. Response: ${errorText.substring(0, 200)}`);
+                // Proxy returned non-OK status - record failure but don't throw yet
+                const status = res.status;
                 recordProxyOutcome(false);
+                logger.log('WARN', `[${label}] Proxy returned status ${status} for ${url}. Failures: ${getFailureCount()}/${getMaxFailures()}. Will try direct fetch.`);
+                // Continue to fallback - don't throw here
             }
         } catch (e) {
             recordProxyOutcome(false);
-            logger.log('WARN', `[${label}] Proxy failed for ${url}: ${e.message}. Retrying direct.`);
+            logger.log('WARN', `[${label}] Proxy exception for ${url}: ${e.message}. Failures: ${getFailureCount()}/${getMaxFailures()}. Will try direct fetch.`);
+            // Continue to fallback - don't throw here
         }
+    } else {
+        logger.log('INFO', `[${label}] Circuit breaker open (${getFailureCount()}/${getMaxFailures()} failures), skipping proxy for ${url}`);
     }
     
-    // Fallback
+    // Fallback to direct fetch only if proxy failed or circuit breaker is open
+    logger.log('INFO', `[${label}] Using direct fetch for ${url}`);
     const directFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch');
-    const res = await directFetch(url, options);
+    const res = await directFetch(url, finalOptions);
     if (!res.ok) {
         const errorText = await res.text().catch(() => 'Unable to read response');
         logger.log('ERROR', `[${label}] Direct fetch failed for ${url}: Status ${res.status}. Response: ${errorText.substring(0, 200)}`);
         throw new Error(`Fetch failed: ${res.status} ${res.statusText} - ${errorText.substring(0, 100)}`);
     }
+    
+    // Update header manager on success
+    if (headerManager && headerId) {
+        headerManager.updatePerformance(headerId, true);
+    }
+    
     return res;
 }
 
@@ -62,13 +96,13 @@ async function updateLastUpdated(db, collectionRegistry, cid, userType, dataType
 }
 
 async function processPortfolio(context, config, taskData, isPI) {
-    const { db, logger, collectionRegistry, proxyManager } = context;
+    const { db, logger, collectionRegistry, proxyManager, headerManager } = context;
     const { cid, username, uuid, today, requestOptions } = taskData;
     const url = `${config.ETORO_API_PORTFOLIO_URL}?cid=${cid}&client_request_id=${uuid}`;
     
     logger.log('INFO', `[Portfolio] Fetching ${isPI ? 'PI' : 'Signed-In User'} portfolio from: ${url}`);
     
-    const res = await fetchWithRetry(url, requestOptions, proxyManager, logger, 'Portfolio');
+    const res = await fetchWithRetry(url, requestOptions, proxyManager, logger, 'Portfolio', headerManager);
     const data = await res.json();
     data.fetchedAt = new Date();
     data.username = username;
@@ -81,7 +115,7 @@ async function processPortfolio(context, config, taskData, isPI) {
             for (const pos of topPositions) {
                 try {
                     const posUrl = `${config.ETORO_API_POSITIONS_URL}?cid=${cid}&InstrumentID=${pos.InstrumentID}&client_request_id=${uuid}`;
-                    const deepRes = await fetchWithRetry(posUrl, requestOptions, proxyManager, logger, 'DeepPos');
+                    const deepRes = await fetchWithRetry(posUrl, requestOptions, proxyManager, logger, 'DeepPos', headerManager);
                     deepPositions.push({ instrumentId: pos.InstrumentID, ...(await deepRes.json()) });
                 } catch (e) {} // Skip failed deep positions
             }
@@ -98,12 +132,12 @@ async function processPortfolio(context, config, taskData, isPI) {
 }
 
 async function processHistory(context, config, taskData, isPI) {
-    const { db, logger, collectionRegistry, proxyManager } = context;
+    const { db, logger, collectionRegistry, proxyManager, headerManager } = context;
     const { cid, uuid, today, requestOptions } = taskData;
     const oneYearAgo = new Date(); oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1);
     const url = `${config.ETORO_API_HISTORY_URL}?StartTime=${oneYearAgo.toISOString()}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${cid}&client_request_id=${uuid}`;
 
-    const res = await fetchWithRetry(url, requestOptions, proxyManager, logger, 'History');
+    const res = await fetchWithRetry(url, requestOptions, proxyManager, logger, 'History', headerManager);
     const data = await res.json();
     
     // Filter valid close reasons
@@ -119,7 +153,7 @@ async function processHistory(context, config, taskData, isPI) {
 }
 
 async function processSocial(context, config, taskData, isPI) {
-    const { db, logger, collectionRegistry, proxyManager } = context;
+    const { db, logger, collectionRegistry, proxyManager, headerManager } = context;
     const { cid, username, uuid, today, requestOptions } = taskData;
     const { getGcidForUser } = require('../../social-task-handler/helpers/handler_helpers');
     
@@ -127,7 +161,7 @@ async function processSocial(context, config, taskData, isPI) {
         const gcid = await getGcidForUser(context, config.social || {}, cid, username);
         const url = `${config.social?.userFeedApiUrl || 'https://www.etoro.com/api/edm-streams/v1/feed/user/top/'}${gcid}?take=10&client_request_id=${uuid}`;
         
-        const res = await fetchWithRetry(url, requestOptions, proxyManager, logger, 'Social');
+        const res = await fetchWithRetry(url, requestOptions, proxyManager, logger, 'Social', headerManager);
         const data = await res.json();
         const posts = (data.discussions || []).slice(0, 30).map(d => ({
             id: d.post.id,

package/functions/task-engine/helpers/social_helpers.js

@@ -90,13 +90,59 @@ async function handleSocialFetch(taskData, config, dependencies, batchCounterRef
 
             logger.log('INFO', `[SocialFetch/${taskId}] Requesting URL: ${url}`);
 
+            // Use circuit breaker logic: always try proxy first, fallback to direct only after proxy fails
+            const { shouldTryProxy, recordProxyOutcome, getFailureCount, getMaxFailures } = require('../utils/proxy_circuit_breaker');
+            
             let response;
-            try {
-                response = await proxyManager.fetch(url, { headers: requestHeaders });
-                if (!response.ok) throw new Error(`Status ${response.status}`);
-                headerManager.updatePerformance(selectedHeader.id, true);
-            } catch (err) {
-                logger.log('WARN', `[SocialFetch/${taskId}] Fetch failed for ${url}: ${err.message}. Retrying direct.`);
+            let proxyAttempted = false;
+            let proxyFailed = false;
+            
+            // Always try proxy first (unless circuit breaker is open and we know it will fail)
+            if (shouldTryProxy()) {
+                proxyAttempted = true;
+                try {
+                    response = await proxyManager.fetch(url, { headers: requestHeaders });
+                    if (response.ok) {
+                        recordProxyOutcome(true);
+                        headerManager.updatePerformance(selectedHeader.id, true);
+                        logger.log('TRACE', `[SocialFetch/${taskId}] Proxy fetch succeeded`);
+                    } else {
+                        recordProxyOutcome(false);
+                        proxyFailed = true;
+                        headerManager.updatePerformance(selectedHeader.id, false);
+                        logger.log('WARN', `[SocialFetch/${taskId}] Proxy returned status ${response.status} for ${url}. Failures: ${getFailureCount()}/${getMaxFailures()}`);
+                    }
+                } catch (err) {
+                    recordProxyOutcome(false);
+                    proxyFailed = true;
+                    headerManager.updatePerformance(selectedHeader.id, false);
+                    logger.log('WARN', `[SocialFetch/${taskId}] Proxy failed for ${url}: ${err.message}. Failures: ${getFailureCount()}/${getMaxFailures()}`);
+                }
+            } else {
+                // Circuit breaker is open, but try proxy once more to check if it recovered
+                logger.log('INFO', `[SocialFetch/${taskId}] Circuit breaker open (${getFailureCount()}/${getMaxFailures()} failures), attempting proxy once more`);
+                proxyAttempted = true;
+                try {
+                    response = await proxyManager.fetch(url, { headers: requestHeaders });
+                    if (response.ok) {
+                        recordProxyOutcome(true); // Reset circuit breaker on success
+                        headerManager.updatePerformance(selectedHeader.id, true);
+                        logger.log('INFO', `[SocialFetch/${taskId}] Proxy succeeded despite circuit breaker. Resetting circuit breaker.`);
+                    } else {
+                        proxyFailed = true;
+                        headerManager.updatePerformance(selectedHeader.id, false);
+                        logger.log('WARN', `[SocialFetch/${taskId}] Proxy failed (circuit breaker open): Status ${response.status}`);
+                    }
+                } catch (err) {
+                    proxyFailed = true;
+                    headerManager.updatePerformance(selectedHeader.id, false);
+                    logger.log('WARN', `[SocialFetch/${taskId}] Proxy failed (circuit breaker open): ${err.message}`);
+                }
+            }
+            
+            // Fallback to direct fetch only after proxy fails
+            if (proxyFailed || !proxyAttempted) {
+                logger.log('INFO', `[SocialFetch/${taskId}] Falling back to direct fetch for ${url}${proxyFailed ? ' (proxy failed)' : ''}`);
                 try {
                     const directFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch');
                     response = await directFetch(url, { headers: requestHeaders });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.543",
+  "version": "1.0.545",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [