bulltrackers-module 1.0.543 → 1.0.544

package/functions/computation-system/executors/StandardExecutor.js

@@ -18,7 +18,7 @@ class StandardExecutor {
         const dStr   = date.toISOString().slice(0, 10);
         const logger = deps.logger;
 
-        // [FIX] Determine required user types for this batch of calculations
+        // Determine required user types for this batch of calculations
         const requiredUserTypes = new Set();
         calcs.forEach(c => {
             const type = (c.userType || 'ALL').toUpperCase();
@@ -27,7 +27,6 @@ class StandardExecutor {
         const userTypeArray = requiredUserTypes.has('ALL') ? null : Array.from(requiredUserTypes);
         
         // [OPTIMIZATION] Check for Target CID in manifests (On-Demand Optimization)
-        // If present, we will filter all data streams to strictly this user
         const targetCid = calcs.find(c => c.targetCid)?.targetCid || calcs.find(c => c.manifest?.targetCid)?.manifest?.targetCid;
         if (targetCid) {
             logger.log('INFO', `[StandardExecutor] Running in Targeted Mode for CID: ${targetCid}`);
@@ -62,7 +61,6 @@ class StandardExecutor {
             } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); } 
         }
 
-        // Pass targetCid to streamAndProcess
         return await StandardExecutor.streamAndProcess(
             dStr, state, passName, config, deps, fullRoot, 
             rootData.portfolioRefs, rootData.historyRefs, 
@@ -71,7 +69,6 @@ class StandardExecutor {
         );
     }
 
-    // [UPDATED] Added targetCid param
     static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, requiredUserTypes = null, targetCid = null) {
         const { logger } = deps;
         const calcs = Object.values(state).filter(c => c && c.manifest);
@@ -82,11 +79,9 @@ class StandardExecutor {
         // --- 1. Resolve and Filter Portfolio Refs (Today) ---
         let effectivePortfolioRefs = portfolioRefs;
         if (!effectivePortfolioRefs) {
-            // If refs weren't provided by AvailabilityChecker, fetch them now
             effectivePortfolioRefs = await getPortfolioPartRefs(config, deps, dateStr, requiredUserTypes);
         }
         if (targetCid && effectivePortfolioRefs) {
-            // Filter: Keep only refs that match the CID (or Legacy refs without CID)
             effectivePortfolioRefs = effectivePortfolioRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
         }
 
@@ -129,7 +124,6 @@ class StandardExecutor {
         const setupDuration = performance.now() - startSetup;
         Object.keys(executionStats).forEach(name => executionStats[name].timings.setup += setupDuration);
 
-        // Yesterday's Refs are already filtered in run()
         const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
         const prevDateStr = prevDate.toISOString().slice(0, 10);
         
@@ -138,7 +132,6 @@ class StandardExecutor {
             earliestDates = await getEarliestDataDates(config, deps);
         }
 
-        // [FIX] Use effective/filtered refs
         const tP_iter = streamPortfolioData(config, deps, dateStr, effectivePortfolioRefs, requiredUserTypes);
         
         const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
@@ -200,7 +193,6 @@ class StandardExecutor {
         return { successUpdates: aggregatedSuccess, failureReport: aggregatedFailures };
     }
     
-    // ... rest of the file (flushBuffer, mergeReports, executePerUser) ...
     static async flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, mode, skipStatusWrite, isInitialWrite = false) {
         const { logger } = deps;
         const transformedState = {};
@@ -272,35 +264,39 @@ class StandardExecutor {
    static async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps, config, deps, loader, stats, earliestDates) {
         const { logger } = deps;
         const targetUserType = metadata.userType; 
-        // [NEW] Always load Global Helpers
+        // [FIX] Always load Global Helpers
         const mappings = await loader.loadMappings();
-        // [FIX] Correct method name: loadPIMasterList (not loadPopularInvestorMasterList)
+        // [FIX] Correct method: loadPIMasterList() (no args needed as loader has context)
         const piMasterList = await loader.loadPIMasterList(); 
         const SCHEMAS = mathLayer.SCHEMAS;
 
         // 1. Load Root Data
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
-        const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerificationProfiles(config, deps) : null;
-        const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadPopularInvestorRankings(config, deps, dateStr) : null;
         
-        // [FIX] Load Yesterday's Rankings if isHistorical is true
+        // [FIX] Correct method: loadVerifications() (no args)
+        const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerifications() : null;
+        
+        // [FIX] Correct method: loadRankings(dateStr) (no config/deps args)
+        const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadRankings(dateStr) : null;
+        
+        // [FIX] Correct method: loadRankings(prevStr)
         let yesterdayRankings = null;
         if (metadata.rootDataDependencies?.includes('rankings') && metadata.isHistorical) {
              const prevDate = new Date(dateStr); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
              const prevStr = prevDate.toISOString().slice(0, 10);
-             yesterdayRankings = await loader.loadPopularInvestorRankings(config, deps, prevStr); 
+             yesterdayRankings = await loader.loadRankings(prevStr); 
         }
 
-        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadDailySocialPostInsights(config, deps, dateStr) : null;
+        // [FIX] Correct method: loadSocial(dateStr)
+        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadSocial(dateStr) : null;
         
-        // [NEW] Load New Root Data Types for Profile Metrics
-        // [FIX] Enforce canHaveMissingRoots
         const allowMissing = metadata.canHaveMissingRoots === true;
         
+        // [FIX] Correct method: loadRatings(dateStr)
         let ratings = null;
         if (metadata.rootDataDependencies?.includes('ratings')) {
             try {
-                ratings = await loader.loadPIRatings(config, deps, dateStr);
+                ratings = await loader.loadRatings(dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'ratings' failed to load for ${metadata.name}: ${e.message}`);
@@ -312,10 +308,11 @@ class StandardExecutor {
             }
         }
         
+        // [FIX] Correct method: loadPageViews(dateStr)
         let pageViews = null;
         if (metadata.rootDataDependencies?.includes('pageViews')) {
             try {
-                pageViews = await loader.loadPIPageViews(config, deps, dateStr);
+                pageViews = await loader.loadPageViews(dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'pageViews' failed to load for ${metadata.name}: ${e.message}`);
@@ -327,10 +324,11 @@ class StandardExecutor {
             }
         }
         
+        // [FIX] Correct method: loadWatchlistMembership(dateStr)
         let watchlistMembership = null;
         if (metadata.rootDataDependencies?.includes('watchlist')) {
             try {
-                watchlistMembership = await loader.loadWatchlistMembership(config, deps, dateStr);
+                watchlistMembership = await loader.loadWatchlistMembership(dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'watchlist' failed to load for ${metadata.name}: ${e.message}`);
@@ -342,10 +340,11 @@ class StandardExecutor {
             }
         }
         
+        // [FIX] Correct method: loadAlertHistory(dateStr)
         let alertHistory = null;
         if (metadata.rootDataDependencies?.includes('alerts')) {
             try {
-                alertHistory = await loader.loadPIAlertHistory(config, deps, dateStr);
+                alertHistory = await loader.loadAlertHistory(dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'alerts' failed to load for ${metadata.name}: ${e.message}`);

package/functions/task-engine/helpers/data_storage_helpers.js

@@ -132,13 +132,17 @@ async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, ci
     }, { merge: false });
     
     // 2. Store latest posts to user-centric collection (for fallback)
+    // Path structure: SignedInUsers/{cid}/posts/{postId}
+    // We need to construct this step-by-step, not use the template directly
     const { getCollectionPath } = collectionRegistry || {};
     if (!getCollectionPath) {
         throw new Error('collectionRegistry.getCollectionPath is required');
     }
-    const userPostsRef = db.collection(
-        getCollectionPath('signedInUsers', 'socialPosts', { cid })
-    );
+    
+    // Get the base path template to extract collection name
+    const pathTemplate = getCollectionPath('signedInUsers', 'socialPosts', { cid: String(cid), postId: 'placeholder' });
+    // Extract collection name (first part before /)
+    const collectionName = pathTemplate.split('/')[0]; // Should be 'SignedInUsers'
     
     // Store each post individually in user-centric collection
     const batch = db.batch();
@@ -149,7 +153,11 @@ async function storeSignedInUserSocialPosts({ db, logger, collectionRegistry, ci
         const postId = post.id || post.postId;
         if (!postId) continue;
         
-        const postRef = userPostsRef.doc(postId);
+        // Construct path step-by-step: SignedInUsers/{cid}/posts/{postId}
+        const postRef = db.collection(collectionName)
+            .doc(String(cid))
+            .collection('posts')
+            .doc(postId);
         batch.set(postRef, {
             ...post,
             fetchedAt: FieldValue.serverTimestamp(),
@@ -300,13 +308,17 @@ async function storePopularInvestorSocialPosts({ db, logger, collectionRegistry,
     }, { merge: false });
     
     // 2. Store latest posts to user-centric collection (for fallback)
+    // Path structure: PopularInvestors/{piCid}/posts/{postId}
+    // We need to construct this step-by-step, not use the template directly
     const { getCollectionPath } = collectionRegistry || {};
     if (!getCollectionPath) {
         throw new Error('collectionRegistry.getCollectionPath is required');
     }
-    const userPostsRef = db.collection(
-        getCollectionPath('popularInvestors', 'socialPosts', { piCid: String(cid) })
-    );
+    
+    // Get the base path template to extract collection name
+    const pathTemplate = getCollectionPath('popularInvestors', 'socialPosts', { piCid: String(cid), postId: 'placeholder' });
+    // Extract collection name (first part before /)
+    const collectionName = pathTemplate.split('/')[0]; // Should be 'PopularInvestors'
     
     // Store each post individually in user-centric collection
     const batch = db.batch();
@@ -317,7 +329,11 @@ async function storePopularInvestorSocialPosts({ db, logger, collectionRegistry,
         const postId = post.id || post.postId;
         if (!postId) continue;
         
-        const postRef = userPostsRef.doc(postId);
+        // Construct path step-by-step: PopularInvestors/{cid}/posts/{postId}
+        const postRef = db.collection(collectionName)
+            .doc(String(cid))
+            .collection('posts')
+            .doc(postId);
         batch.set(postRef, {
             ...post,
             fetchedAt: FieldValue.serverTimestamp(),

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -20,34 +20,69 @@ const {
 // ==========================================
 
 async function fetchWithRetry(url, options, proxyManager, logger, label) {
-    // Circuit Breaker / Proxy Logic abstracted here
+    // ALWAYS try proxy first, regardless of circuit breaker state
+    // Circuit breaker only prevents retrying after it's open, but we always attempt proxy first
+    let proxyAttempted = false;
+    let proxyFailed = false;
+    
+    // Try proxy first (unless circuit breaker is open and we've already failed)
     if (shouldTryProxy()) {
+        proxyAttempted = true;
         try {
             const res = await proxyManager.fetch(url, options);
             if (res.ok) {
                 recordProxyOutcome(true);
+                logger.log('TRACE', `[${label}] Proxy fetch succeeded for ${url}`);
                 return res;
             } else {
                 // Log proxy failure with details
                 const errorText = await res.text().catch(() => 'Unable to read response');
                 logger.log('WARN', `[${label}] Proxy returned status ${res.status} for ${url}. Response: ${errorText.substring(0, 200)}`);
                 recordProxyOutcome(false);
+                proxyFailed = true;
             }
         } catch (e) {
             recordProxyOutcome(false);
-            logger.log('WARN', `[${label}] Proxy failed for ${url}: ${e.message}. Retrying direct.`);
+            logger.log('WARN', `[${label}] Proxy failed for ${url}: ${e.message}. Failures: ${getFailureCount()}/${getMaxFailures()}. Falling back to direct fetch.`);
+            proxyFailed = true;
+        }
+    } else {
+        // Circuit breaker is open - log but still try proxy once more
+        logger.log('INFO', `[${label}] Circuit breaker open (${getFailureCount()}/${getMaxFailures()} failures), but attempting proxy once more for ${url}`);
+        proxyAttempted = true;
+        try {
+            const res = await proxyManager.fetch(url, options);
+            if (res.ok) {
+                // Success! Reset circuit breaker
+                recordProxyOutcome(true);
+                logger.log('INFO', `[${label}] Proxy succeeded despite circuit breaker being open. Resetting circuit breaker.`);
+                return res;
+            } else {
+                const errorText = await res.text().catch(() => 'Unable to read response');
+                logger.log('WARN', `[${label}] Proxy failed (circuit breaker open): Status ${res.status}. Response: ${errorText.substring(0, 200)}`);
+                proxyFailed = true;
+            }
+        } catch (e) {
+            logger.log('WARN', `[${label}] Proxy failed (circuit breaker open): ${e.message}`);
+            proxyFailed = true;
         }
     }
     
-    // Fallback
-    const directFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch');
-    const res = await directFetch(url, options);
-    if (!res.ok) {
-        const errorText = await res.text().catch(() => 'Unable to read response');
-        logger.log('ERROR', `[${label}] Direct fetch failed for ${url}: Status ${res.status}. Response: ${errorText.substring(0, 200)}`);
-        throw new Error(`Fetch failed: ${res.status} ${res.statusText} - ${errorText.substring(0, 100)}`);
+    // Fallback to direct fetch only after proxy fails
+    if (proxyFailed || !proxyAttempted) {
+        logger.log('INFO', `[${label}] Falling back to direct fetch for ${url}${proxyFailed ? ' (proxy failed)' : ''}`);
+        const directFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch');
+        const res = await directFetch(url, options);
+        if (!res.ok) {
+            const errorText = await res.text().catch(() => 'Unable to read response');
+            logger.log('ERROR', `[${label}] Direct fetch failed for ${url}: Status ${res.status}. Response: ${errorText.substring(0, 200)}`);
+            throw new Error(`Fetch failed: ${res.status} ${res.statusText} - ${errorText.substring(0, 100)}`);
+        }
+        return res;
     }
-    return res;
+    
+    // Should not reach here, but just in case
+    throw new Error(`[${label}] Unexpected state: proxy attempted but no response returned`);
 }
 
 async function updateLastUpdated(db, collectionRegistry, cid, userType, dataType, logger) {

package/functions/task-engine/helpers/social_helpers.js

@@ -90,13 +90,59 @@ async function handleSocialFetch(taskData, config, dependencies, batchCounterRef
 
             logger.log('INFO', `[SocialFetch/${taskId}] Requesting URL: ${url}`);
 
+            // Use circuit breaker logic: always try proxy first, fallback to direct only after proxy fails
+            const { shouldTryProxy, recordProxyOutcome, getFailureCount, getMaxFailures } = require('../utils/proxy_circuit_breaker');
+            
             let response;
-            try {
-                response = await proxyManager.fetch(url, { headers: requestHeaders });
-                if (!response.ok) throw new Error(`Status ${response.status}`);
-                headerManager.updatePerformance(selectedHeader.id, true);
-            } catch (err) {
-                logger.log('WARN', `[SocialFetch/${taskId}] Fetch failed for ${url}: ${err.message}. Retrying direct.`);
+            let proxyAttempted = false;
+            let proxyFailed = false;
+            
+            // Always try proxy first (unless circuit breaker is open and we know it will fail)
+            if (shouldTryProxy()) {
+                proxyAttempted = true;
+                try {
+                    response = await proxyManager.fetch(url, { headers: requestHeaders });
+                    if (response.ok) {
+                        recordProxyOutcome(true);
+                        headerManager.updatePerformance(selectedHeader.id, true);
+                        logger.log('TRACE', `[SocialFetch/${taskId}] Proxy fetch succeeded`);
+                    } else {
+                        recordProxyOutcome(false);
+                        proxyFailed = true;
+                        headerManager.updatePerformance(selectedHeader.id, false);
+                        logger.log('WARN', `[SocialFetch/${taskId}] Proxy returned status ${response.status} for ${url}. Failures: ${getFailureCount()}/${getMaxFailures()}`);
+                    }
+                } catch (err) {
+                    recordProxyOutcome(false);
+                    proxyFailed = true;
+                    headerManager.updatePerformance(selectedHeader.id, false);
+                    logger.log('WARN', `[SocialFetch/${taskId}] Proxy failed for ${url}: ${err.message}. Failures: ${getFailureCount()}/${getMaxFailures()}`);
+                }
+            } else {
+                // Circuit breaker is open, but try proxy once more to check if it recovered
+                logger.log('INFO', `[SocialFetch/${taskId}] Circuit breaker open (${getFailureCount()}/${getMaxFailures()} failures), attempting proxy once more`);
+                proxyAttempted = true;
+                try {
+                    response = await proxyManager.fetch(url, { headers: requestHeaders });
+                    if (response.ok) {
+                        recordProxyOutcome(true); // Reset circuit breaker on success
+                        headerManager.updatePerformance(selectedHeader.id, true);
+                        logger.log('INFO', `[SocialFetch/${taskId}] Proxy succeeded despite circuit breaker. Resetting circuit breaker.`);
+                    } else {
+                        proxyFailed = true;
+                        headerManager.updatePerformance(selectedHeader.id, false);
+                        logger.log('WARN', `[SocialFetch/${taskId}] Proxy failed (circuit breaker open): Status ${response.status}`);
+                    }
+                } catch (err) {
+                    proxyFailed = true;
+                    headerManager.updatePerformance(selectedHeader.id, false);
+                    logger.log('WARN', `[SocialFetch/${taskId}] Proxy failed (circuit breaker open): ${err.message}`);
+                }
+            }
+            
+            // Fallback to direct fetch only after proxy fails
+            if (proxyFailed || !proxyAttempted) {
+                logger.log('INFO', `[SocialFetch/${taskId}] Falling back to direct fetch for ${url}${proxyFailed ? ' (proxy failed)' : ''}`);
                 try {
                     const directFetch = typeof fetch !== 'undefined' ? fetch : require('node-fetch');
                     response = await directFetch(url, { headers: requestHeaders });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.543",
+  "version": "1.0.544",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [