bulltrackers-module 1.0.351 → 1.0.352

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -0,0 +1,132 @@
+/**
+ * @fileoverview Logic for fetching and storing Popular Investor rankings.
+ * Integrates IntelligentProxyManager for IP rotation and IntelligentHeaderManager for 
+ * user-agent/header rotation and performance tracking.
+ */
+const { IntelligentProxyManager } = require('../../core/utils/intelligent_proxy_manager');
+const { IntelligentHeaderManager } = require('../../core/utils/intelligent_header_manager');
+
+/**
+ * Fetches the top Popular Investors and stores the raw result in Firestore.
+ * @param {object} dependencies - Contains db, logger.
+ * @param {object} config - Configuration object.
+ * @param {string} config.rankingsApiUrl - The eToro Rankings API URL.
+ * @param {string} config.rankingsCollectionName - e.g., 'popular_investor_rankings'.
+ * @param {object} config.proxyConfig - Configuration for the IntelligentProxyManager.
+ * @param {object} config.headerConfig - Configuration for the IntelligentHeaderManager.
+ */
+async function fetchAndStorePopularInvestors(dependencies, config) {
+    const { db, logger } = dependencies;
+    const { rankingsApiUrl, rankingsCollectionName, proxyConfig, headerConfig } = config;
+
+    if (!rankingsApiUrl || !rankingsCollectionName || !proxyConfig || !headerConfig) {
+        throw new Error("[PopularInvestorFetch] Missing required config (rankingsApiUrl, rankingsCollectionName, proxyConfig, headerConfig).");
+    }
+
+    const today = new Date().toISOString().split('T')[0]; // YYYY-MM-DD
+    logger.log('INFO', `[PopularInvestorFetch] Starting fetch for date: ${today}`);
+
+    // 1. Initialize Managers
+    const headerManager = new IntelligentHeaderManager(db, logger, headerConfig);
+    const proxyManager = new IntelligentProxyManager(db, logger, proxyConfig);
+
+    let data = null;
+    let fetchSuccess = false;
+    let selectedHeaderId = 'fallback';
+
+    try {
+        // 2. Select the best performing header
+        const { id, header } = await headerManager.selectHeader();
+        selectedHeaderId = id;
+        
+        logger.log('INFO', `[PopularInvestorFetch] Selected header configuration: ${id}`);
+
+        // Merge selected headers with specific API requirements
+        // We prioritize the dynamic headers but ensure critical fields like Accept exist
+        const requestHeaders = {
+            'Accept': 'application/json',
+            'Referer': 'https://www.etoro.com/',
+            ...header
+        };
+
+        // 3. Attempt Fetch via Proxy Manager
+        try {
+            logger.log('INFO', '[PopularInvestorFetch] Attempting fetch via Proxy Manager...');
+            
+            const response = await proxyManager.fetch(rankingsApiUrl, {
+                method: 'GET',
+                headers: requestHeaders
+            });
+
+            if (response.ok) {
+                data = await response.json();
+                fetchSuccess = true;
+                logger.log('SUCCESS', '[PopularInvestorFetch] Successfully fetched data via Proxy.');
+            } else {
+                logger.log('WARN', `[PopularInvestorFetch] Proxy fetch failed with status ${response.status}.`);
+            }
+        } catch (proxyError) {
+            logger.log('ERROR', '[PopularInvestorFetch] Error during Proxy Manager execution.', { errorMessage: proxyError.message });
+        }
+
+        // 4. Fallback: Direct Fetch (if Proxy failed)
+        if (!data) {
+            logger.log('WARN', '[PopularInvestorFetch] Falling back to Direct Node Fetch...');
+            try {
+                const directResponse = await fetch(rankingsApiUrl, {
+                    method: 'GET',
+                    headers: requestHeaders
+                });
+
+                if (directResponse.ok) {
+                    data = await directResponse.json();
+                    fetchSuccess = true;
+                    logger.log('SUCCESS', '[PopularInvestorFetch] Successfully fetched data via Direct Fetch.');
+                } else {
+                    throw new Error(`Direct fetch failed with status: ${directResponse.status}`);
+                }
+            } catch (directError) {
+                logger.log('ERROR', '[PopularInvestorFetch] Direct fetch failed.', { errorMessage: directError.message });
+            }
+        }
+
+    } catch (error) {
+        logger.log('ERROR', '[PopularInvestorFetch] Critical error in fetch orchestration.', { errorMessage: error.message });
+    } finally {
+        // 5. Update Header Performance (Reinforcement Learning)
+        // This ensures the system "learns" which headers are getting blocked vs accepted
+        logger.log('INFO', `[PopularInvestorFetch] Updating header performance for ${selectedHeaderId}: Success=${fetchSuccess}`);
+        headerManager.updatePerformance(selectedHeaderId, fetchSuccess);
+        await headerManager.flushPerformanceUpdates();
+    }
+
+    // 6. Final Validation & Storage
+    if (data && data.Items && Array.isArray(data.Items)) {
+        try {
+            const docRef = db.collection(rankingsCollectionName).doc(today);
+            
+            await docRef.set({
+                fetchedAt: new Date(),
+                totalRows: data.TotalRows,
+                itemsCount: data.Items.length,
+                ...data
+            });
+
+            logger.log('SUCCESS', `[PopularInvestorFetch] Stored ${data.TotalRows} rankings into ${rankingsCollectionName}/${today}`);
+            return { success: true, count: data.TotalRows };
+
+        } catch (dbError) {
+            logger.log('ERROR', '[PopularInvestorFetch] Failed to write to Firestore.', { errorMessage: dbError.message });
+            throw dbError;
+        }
+    } else {
+        const errorMsg = fetchSuccess 
+            ? '[PopularInvestorFetch] Fetched data format is invalid (missing Items array).' 
+            : '[PopularInvestorFetch] Failed to fetch data from all sources.';
+            
+        logger.log('ERROR', errorMsg);
+        throw new Error(errorMsg);
+    }
+}
+
+module.exports = { fetchAndStorePopularInvestors };

package/functions/fetch-popular-investors/index.js

@@ -0,0 +1,8 @@
+/**
+ * @fileoverview Entry point for the Popular Investor Fetcher module.
+ */
+const { fetchAndStorePopularInvestors } = require('./helpers/fetch_helpers');
+
+module.exports = {
+    runPopularInvestorFetch: fetchAndStorePopularInvestors
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.351",
+  "version": "1.0.352",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -19,7 +19,8 @@
     "functions/social-orchestrator/",
     "functions/social-task-handler/",
     "functions/price-backfill/",
-    "functions/root-data-indexer/"
+    "functions/root-data-indexer/",
+    "functions/fetch-popular-investors"
   ],
   "keywords": [
     "bulltrackers",