bulltrackers-module 1.0.676 → 1.0.678

package/functions/api-v2/helpers/data-fetchers/firestore.js

@@ -2061,7 +2061,12 @@ const fetchTrendingPopularInvestors = async (db) => {
             console.log(`[Trending] Got result for ${dateKey}:`, JSON.stringify(result).substring(0, 500));
             // If we got a result, return it (even if trending array is empty)
             if (result) {
-                const trending = result.trending || [];
+                // Handle case where data is wrapped under "undefined" key
+                let trending = result.trending;
+                if (!trending && result.undefined && result.undefined.trending) {
+                    trending = result.undefined.trending;
+                }
+                trending = trending || [];
                 console.log(`[Trending] Returning trending array with ${trending.length} items`);
                 return trending;
             }
@@ -2089,7 +2094,12 @@ const fetchPopularInvestorCategories = async (db) => {
             console.log(`[Categories] Got result for ${dateKey}:`, JSON.stringify(result).substring(0, 500));
             // If we got a result, return it (even if categories object is empty)
             if (result) {
-                const categories = result.categories || {};
+                // Handle case where data is wrapped under "undefined" key
+                let categories = result.categories;
+                if (!categories && result.undefined && result.undefined.categories) {
+                    categories = result.undefined.categories;
+                }
+                categories = categories || {};
                 console.log(`[Categories] Returning categories object with ${Object.keys(categories).length} keys`);
                 return categories;
             }

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -8,6 +8,64 @@ const { IntelligentProxyManager } = require('../../core/utils/intelligent_proxy_
 const { IntelligentHeaderManager } = require('../../core/utils/intelligent_header_manager');
 const zlib = require('zlib');
 
+/**
+ * Fetches individual user rankings data by CID
+ * @param {string} cid - Customer ID
+ * @param {object} headers - Request headers to use
+ * @param {object} proxyManager - ProxyManager instance
+ * @param {object} logger - Logger instance
+ * @returns {object|null} - User rankings data or null if failed
+ */
+async function fetchIndividualUserRankings(cid, headers, proxyManager, logger) {
+    const individualUrl = `https://www.etoro.com/sapi/rankings/cid/${cid}/rankings/?Period=OneYearAgo`;
+    
+    try {
+        logger.log('INFO', `[PopularInvestorFetch] Fetching individual rankings for CID: ${cid}`);
+        
+        // Try with proxy first
+        try {
+            const response = await proxyManager.fetch(individualUrl, {
+                method: 'GET',
+                headers
+            });
+
+            if (response.ok) {
+                const data = await response.json();
+                if (data && data.Data) {
+                    logger.log('SUCCESS', `[PopularInvestorFetch] Successfully fetched individual rankings for CID: ${cid} via proxy`);
+                    return data.Data; // Return the Data object which matches the Items schema
+                }
+            }
+        } catch (proxyError) {
+            logger.log('WARN', `[PopularInvestorFetch] Proxy fetch failed for CID ${cid}: ${proxyError.message}`);
+        }
+        
+        // Fallback to direct fetch
+        try {
+            const directResponse = await fetch(individualUrl, {
+                method: 'GET',
+                headers
+            });
+
+            if (directResponse.ok) {
+                const data = await directResponse.json();
+                if (data && data.Data) {
+                    logger.log('SUCCESS', `[PopularInvestorFetch] Successfully fetched individual rankings for CID: ${cid} via direct fetch`);
+                    return data.Data;
+                }
+            }
+        } catch (directError) {
+            logger.log('WARN', `[PopularInvestorFetch] Direct fetch failed for CID ${cid}: ${directError.message}`);
+        }
+        
+        logger.log('ERROR', `[PopularInvestorFetch] Failed to fetch individual rankings for CID: ${cid} from all sources`);
+        return null;
+    } catch (error) {
+        logger.log('ERROR', `[PopularInvestorFetch] Error fetching individual rankings for CID ${cid}`, { errorMessage: error.message });
+        return null;
+    }
+}
+
 /**
  * Fetches the top Popular Investors and stores the raw result in Firestore.
  * @param {object} dependencies - Contains db, logger.
@@ -117,6 +175,90 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
         await headerManager.flushPerformanceUpdates();
     }
 
+    // 5.5. Check for missing users from master list and fetch them individually
+    if (data && data.Items && Array.isArray(data.Items)) {
+        try {
+            logger.log('INFO', '[PopularInvestorFetch] Checking for missing users from master list...');
+            
+            // Get master list path
+            let masterListPath = 'system_state/popular_investor_master_list';
+            if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                try {
+                    const registryPath = collectionRegistry.getCollectionPath('system', 'popularInvestorMasterList', {});
+                    masterListPath = registryPath;
+                } catch (e) {
+                    logger.log('WARN', `[PopularInvestorFetch] Failed to get master list path from registry, using default: ${e.message}`);
+                }
+            }
+            
+            const masterListRef = db.doc(masterListPath);
+            const masterListDoc = await masterListRef.get();
+            
+            if (masterListDoc.exists) {
+                const masterListData = masterListDoc.data();
+                const masterInvestors = masterListData.investors || {};
+                
+                // Build a Set of CIDs from the fetched data for fast lookup
+                const fetchedCids = new Set(data.Items.map(item => String(item.CustomerId)));
+                
+                // Identify missing CIDs
+                const masterCids = Object.keys(masterInvestors);
+                const missingCids = masterCids.filter(cid => !fetchedCids.has(cid));
+                
+                if (missingCids.length > 0) {
+                    logger.log('INFO', `[PopularInvestorFetch] Found ${missingCids.length} missing users from master list. Fetching individually...`);
+                    
+                    // Prepare headers for individual fetches
+                    const requestHeaders = {
+                        'Accept': 'application/json',
+                        'Referer': 'https://www.etoro.com/',
+                        ...(await headerManager.selectHeader()).header
+                    };
+                    
+                    // Fetch missing users with rate limiting
+                    const missingUserData = [];
+                    let successCount = 0;
+                    let failureCount = 0;
+                    
+                    for (const cid of missingCids) {
+                        const userData = await fetchIndividualUserRankings(cid, requestHeaders, proxyManager, logger);
+                        
+                        if (userData) {
+                            missingUserData.push(userData);
+                            successCount++;
+                        } else {
+                            failureCount++;
+                            logger.log('WARN', `[PopularInvestorFetch] Failed to fetch data for missing user CID: ${cid} (${masterInvestors[cid].username})`);
+                        }
+                        
+                        // Add small delay between requests to avoid rate limiting
+                        if (missingCids.length > 10 && missingCids.indexOf(cid) < missingCids.length - 1) {
+                            await new Promise(resolve => setTimeout(resolve, 200)); // 200ms delay
+                        }
+                    }
+                    
+                    // Append successfully fetched missing users to the main data
+                    if (missingUserData.length > 0) {
+                        data.Items.push(...missingUserData);
+                        data.TotalRows += missingUserData.length;
+                        logger.log('SUCCESS', `[PopularInvestorFetch] Successfully fetched ${successCount}/${missingCids.length} missing users. Updated Items array from ${data.Items.length - missingUserData.length} to ${data.Items.length} users.`);
+                    }
+                    
+                    if (failureCount > 0) {
+                        logger.log('WARN', `[PopularInvestorFetch] Failed to fetch ${failureCount}/${missingCids.length} missing users.`);
+                    }
+                } else {
+                    logger.log('INFO', '[PopularInvestorFetch] All users from master list are present in the main fetch. No missing users to fetch individually.');
+                }
+            } else {
+                logger.log('INFO', '[PopularInvestorFetch] Master list document does not exist yet. Skipping missing user check.');
+            }
+        } catch (missingUserError) {
+            logger.log('WARN', `[PopularInvestorFetch] Error while checking/fetching missing users: ${missingUserError.message}. Continuing with main fetch data.`);
+            // Non-critical error, continue with whatever data we have
+        }
+    }
+
     // 6. Final Validation & Storage
     if (data && data.Items && Array.isArray(data.Items)) {
         try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.676",
+  "version": "1.0.678",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [