bulltrackers-module 1.0.721 → 1.0.723

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -1,40 +1,30 @@
 /**
  * @fileoverview Logic for fetching and storing Popular Investor rankings.
- * Integrates IntelligentProxyManager for IP rotation and IntelligentHeaderManager for 
- * user-agent/header rotation and performance tracking.
- * UPDATED: Added compression support for large rankings data to avoid Firestore index limits.
+ * REFACTORED: Fully migrated to BigQuery. Removed Firestore storage logic.
+ * NOW: Uses BigQuery for both 'Rankings' (Historical) and 'Master List' (State).
+ * DEDUPLICATION: Uses 'insertRowsWithMerge' to ensure idempotency.
  */
 const { IntelligentProxyManager } = require('../../core/utils/intelligent_proxy_manager');
 const { IntelligentHeaderManager } = require('../../core/utils/intelligent_header_manager');
-const zlib = require('zlib');
+const { 
+    insertRowsWithMerge, 
+    ensurePIRankingsTable, 
+    ensurePIMasterListTable, 
+    queryPIMasterList 
+} = require('../../core/utils/bigquery_utils');
 
 /**
  * Fetches individual user rankings data by CID
- * @param {string} cid - Customer ID
- * @param {object} headers - Request headers to use
- * @param {object} proxyManager - ProxyManager instance
- * @param {object} logger - Logger instance
- * @returns {object|null} - User rankings data or null if failed
  */
 async function fetchIndividualUserRankings(cid, headers, proxyManager, logger) {
     const individualUrl = `https://www.etoro.com/sapi/rankings/cid/${cid}/rankings/?Period=OneYearAgo`;
-    
     try {
-        logger.log('INFO', `[PopularInvestorFetch] Fetching individual rankings for CID: ${cid}`);
-        
         // Try with proxy first
         try {
-            const response = await proxyManager.fetch(individualUrl, {
-                method: 'GET',
-                headers
-            });
-
+            const response = await proxyManager.fetch(individualUrl, { method: 'GET', headers });
             if (response.ok) {
                 const data = await response.json();
-                if (data && data.Data) {
-                    logger.log('SUCCESS', `[PopularInvestorFetch] Successfully fetched individual rankings for CID: ${cid} via proxy`);
-                    return data.Data; // Return the Data object which matches the Items schema
-                }
+                if (data && data.Data) return data.Data;
             }
         } catch (proxyError) {
             logger.log('WARN', `[PopularInvestorFetch] Proxy fetch failed for CID ${cid}: ${proxyError.message}`);
@@ -42,23 +32,14 @@ async function fetchIndividualUserRankings(cid, headers, proxyManager, logger) {
         
         // Fallback to direct fetch
         try {
-            const directResponse = await fetch(individualUrl, {
-                method: 'GET',
-                headers
-            });
-
+            const directResponse = await fetch(individualUrl, { method: 'GET', headers });
             if (directResponse.ok) {
                 const data = await directResponse.json();
-                if (data && data.Data) {
-                    logger.log('SUCCESS', `[PopularInvestorFetch] Successfully fetched individual rankings for CID: ${cid} via direct fetch`);
-                    return data.Data;
-                }
+                if (data && data.Data) return data.Data;
             }
         } catch (directError) {
             logger.log('WARN', `[PopularInvestorFetch] Direct fetch failed for CID ${cid}: ${directError.message}`);
         }
-        
-        logger.log('ERROR', `[PopularInvestorFetch] Failed to fetch individual rankings for CID: ${cid} from all sources`);
         return null;
     } catch (error) {
         logger.log('ERROR', `[PopularInvestorFetch] Error fetching individual rankings for CID ${cid}`, { errorMessage: error.message });
@@ -67,39 +48,22 @@ async function fetchIndividualUserRankings(cid, headers, proxyManager, logger) {
 }
 
 /**
- * Fetches the top Popular Investors and stores the raw result in Firestore.
- * @param {object} dependencies - Contains db, logger.
- * @param {object} config - Configuration object.
- * @param {string} config.rankingsApiUrl - The eToro Rankings API URL.
- * @param {string} config.rankingsCollectionName - e.g., 'popular_investor_rankings'.
- * @param {object} config.proxyConfig - Configuration for the IntelligentProxyManager.
- * @param {object} config.headerConfig - Configuration for the IntelligentHeaderManager.
+ * Fetches the top Popular Investors and stores them directly to BigQuery.
  */
 async function fetchAndStorePopularInvestors(config, dependencies) {
-    const { db, logger, collectionRegistry } = dependencies;
-    const { rankingsApiUrl, rankingsCollectionName, proxyConfig, headerConfig } = config;
+    const { db, logger } = dependencies;
+    const { rankingsApiUrl, proxyConfig, headerConfig } = config;
 
-    // Get collection name from registry if available, fallback to config
-    const { getCollectionPath } = collectionRegistry || {};
-    let finalRankingsCollectionName = rankingsCollectionName;
-    
-    if (getCollectionPath) {
-        try {
-            // Extract collection name from registry path: popular_investor_rankings/{date}
-            const basePath = getCollectionPath('rootData', 'popularInvestorRankings', { date: '2025-01-01' });
-            // Path is like "popular_investor_rankings/2025-01-01", extract collection name
-            finalRankingsCollectionName = basePath.split('/')[0];
-        } catch (e) {
-            logger.log('WARN', `[PopularInvestorFetch] Failed to get collection from registry, using config: ${e.message}`);
-        }
+    // Validation
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        throw new Error("[PopularInvestorFetch] BIGQUERY_ENABLED is false. This module now strictly requires BigQuery.");
     }
-
-    if (!rankingsApiUrl || !finalRankingsCollectionName || !proxyConfig || !headerConfig) {
-        throw new Error("[PopularInvestorFetch] Missing required config (rankingsApiUrl, rankingsCollectionName, proxyConfig, headerConfig).");
+    if (!rankingsApiUrl || !proxyConfig || !headerConfig) {
+        throw new Error("[PopularInvestorFetch] Missing required config.");
     }
 
-    const today = new Date().toISOString().split('T')[0]; // YYYY-MM-DD
-    logger.log('INFO', `[PopularInvestorFetch] Starting fetch for date: ${today}`);
+    const today = new Date().toISOString().split('T')[0];
+    logger.log('INFO', `[PopularInvestorFetch] Starting BigQuery-native fetch for date: ${today}`);
 
     // 1. Initialize Managers
     const headerManager = new IntelligentHeaderManager(db, logger, headerConfig);
@@ -110,444 +74,165 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
     let selectedHeaderId = 'fallback';
 
     try {
-        // 2. Select the best performing header
+        // 2. Select Header & Fetch
         const { id, header } = await headerManager.selectHeader();
         selectedHeaderId = id;
         
-        logger.log('INFO', `[PopularInvestorFetch] Selected header configuration: ${id}`);
-
-        // Merge selected headers with specific API requirements
-        // We prioritize the dynamic headers but ensure critical fields like Accept exist
         const requestHeaders = {
             'Accept': 'application/json',
             'Referer': 'https://www.etoro.com/',
             ...header
         };
 
-        // 3. Attempt Fetch via Proxy Manager
+        // Proxy Fetch
         try {
-            logger.log('INFO', '[PopularInvestorFetch] Attempting fetch via Proxy Manager...');
-            
-            const response = await proxyManager.fetch(rankingsApiUrl, {
-                method: 'GET',
-                headers: requestHeaders
-            });
-
+            const response = await proxyManager.fetch(rankingsApiUrl, { method: 'GET', headers: requestHeaders });
             if (response.ok) {
                 data = await response.json();
                 fetchSuccess = true;
-                logger.log('SUCCESS', '[PopularInvestorFetch] Successfully fetched data via Proxy.');
-            } else {
-                logger.log('WARN', `[PopularInvestorFetch] Proxy fetch failed with status ${response.status}.`);
             }
-        } catch (proxyError) {
-            logger.log('ERROR', '[PopularInvestorFetch] Error during Proxy Manager execution.', { errorMessage: proxyError.message });
-        }
+        } catch (e) { logger.log('WARN', `[PopularInvestorFetch] Proxy failed: ${e.message}`); }
 
-        // 4. Fallback: Direct Fetch (if Proxy failed)
+        // Fallback Fetch
         if (!data) {
-            logger.log('WARN', '[PopularInvestorFetch] Falling back to Direct Node Fetch...');
             try {
-                const directResponse = await fetch(rankingsApiUrl, {
-                    method: 'GET',
-                    headers: requestHeaders
-                });
-
+                const directResponse = await fetch(rankingsApiUrl, { method: 'GET', headers: requestHeaders });
                 if (directResponse.ok) {
                     data = await directResponse.json();
                     fetchSuccess = true;
-                    logger.log('SUCCESS', '[PopularInvestorFetch] Successfully fetched data via Direct Fetch.');
-                } else {
-                    throw new Error(`Direct fetch failed with status: ${directResponse.status}`);
                 }
-            } catch (directError) {
-                logger.log('ERROR', '[PopularInvestorFetch] Direct fetch failed.', { errorMessage: directError.message });
-            }
+            } catch (e) { logger.log('ERROR', `[PopularInvestorFetch] Direct fetch failed: ${e.message}`); }
         }
 
     } catch (error) {
-        logger.log('ERROR', '[PopularInvestorFetch] Critical error in fetch orchestration.', { errorMessage: error.message });
+        logger.log('ERROR', '[PopularInvestorFetch] Fetch error', error);
     } finally {
-        // 5. Update Header Performance (Reinforcement Learning)
-        // This ensures the system "learns" which headers are getting blocked vs accepted
-        logger.log('INFO', `[PopularInvestorFetch] Updating header performance for ${selectedHeaderId}: Success=${fetchSuccess}`);
         headerManager.updatePerformance(selectedHeaderId, fetchSuccess);
-        await headerManager.flushPerformanceUpdates();
     }
 
-    // 5.5. Check for missing users from master list and fetch them individually
-    if (data && data.Items && Array.isArray(data.Items)) {
-        try {
-            logger.log('INFO', '[PopularInvestorFetch] Checking for missing users from master list...');
-            
-            // Get master list path
-            let masterListPath = 'system_state/popular_investor_master_list';
-            if (collectionRegistry && collectionRegistry.getCollectionPath) {
-                try {
-                    const registryPath = collectionRegistry.getCollectionPath('system', 'popularInvestorMasterList', {});
-                    masterListPath = registryPath;
-                } catch (e) {
-                    logger.log('WARN', `[PopularInvestorFetch] Failed to get master list path from registry, using default: ${e.message}`);
-                }
-            }
-            
-            const masterListRef = db.doc(masterListPath);
-            const masterListDoc = await masterListRef.get();
+    if (!data || !data.Items || !Array.isArray(data.Items)) {
+        throw new Error('[PopularInvestorFetch] Failed to fetch valid data.');
+    }
+
+    // 3. Check for Missing Users (Using BigQuery Master List)
+    // We now fetch the state from BigQuery instead of Firestore
+    let bqMasterList = {};
+    try {
+        logger.log('INFO', '[PopularInvestorFetch] Querying BigQuery Master List for missing user check...');
+        bqMasterList = await queryPIMasterList(logger) || {};
+        
+        const fetchedCids = new Set(data.Items.map(item => String(item.CustomerId)));
+        const knownCids = Object.keys(bqMasterList);
+        const missingCids = knownCids.filter(cid => !fetchedCids.has(cid));
+
+        if (missingCids.length > 0) {
+            logger.log('INFO', `[PopularInvestorFetch] Found ${missingCids.length} missing users. Fetching individually...`);
+            const { header } = await headerManager.selectHeader();
             
-            if (masterListDoc.exists) {
-                const masterListData = masterListDoc.data();
-                const masterInvestors = masterListData.investors || {};
-                
-                // Build a Set of CIDs from the fetched data for fast lookup
-                const fetchedCids = new Set(data.Items.map(item => String(item.CustomerId)));
-                
-                // Identify missing CIDs
-                const masterCids = Object.keys(masterInvestors);
-                const missingCids = masterCids.filter(cid => !fetchedCids.has(cid));
-                
-                if (missingCids.length > 0) {
-                    logger.log('INFO', `[PopularInvestorFetch] Found ${missingCids.length} missing users from master list. Fetching individually...`);
-                    
-                    // Prepare headers for individual fetches
-                    const requestHeaders = {
-                        'Accept': 'application/json',
-                        'Referer': 'https://www.etoro.com/',
-                        ...(await headerManager.selectHeader()).header
-                    };
-                    
-                    // Fetch missing users with rate limiting
-                    const missingUserData = [];
-                    let successCount = 0;
-                    let failureCount = 0;
-                    
-                    for (const cid of missingCids) {
-                        const userData = await fetchIndividualUserRankings(cid, requestHeaders, proxyManager, logger);
-                        
-                        if (userData) {
-                            missingUserData.push(userData);
-                            successCount++;
-                        } else {
-                            failureCount++;
-                            logger.log('WARN', `[PopularInvestorFetch] Failed to fetch data for missing user CID: ${cid} (${masterInvestors[cid].username})`);
-                        }
-                        
-                        // Add small delay between requests to avoid rate limiting
-                        if (missingCids.length > 10 && missingCids.indexOf(cid) < missingCids.length - 1) {
-                            await new Promise(resolve => setTimeout(resolve, 200)); // 200ms delay
-                        }
-                    }
-                    
-                    // Append successfully fetched missing users to the main data
-                    if (missingUserData.length > 0) {
-                        data.Items.push(...missingUserData);
-                        data.TotalRows += missingUserData.length;
-                        logger.log('SUCCESS', `[PopularInvestorFetch] Successfully fetched ${successCount}/${missingCids.length} missing users. Updated Items array from ${data.Items.length - missingUserData.length} to ${data.Items.length} users.`);
-                    }
-                    
-                    if (failureCount > 0) {
-                        logger.log('WARN', `[PopularInvestorFetch] Failed to fetch ${failureCount}/${missingCids.length} missing users.`);
-                    }
-                } else {
-                    logger.log('INFO', '[PopularInvestorFetch] All users from master list are present in the main fetch. No missing users to fetch individually.');
+            for (const cid of missingCids) {
+                const userData = await fetchIndividualUserRankings(cid, { ...header, 'Referer': 'https://www.etoro.com/' }, proxyManager, logger);
+                if (userData) {
+                    data.Items.push(userData);
+                    data.TotalRows++;
                 }
-            } else {
-                logger.log('INFO', '[PopularInvestorFetch] Master list document does not exist yet. Skipping missing user check.');
+                await new Promise(r => setTimeout(r, 200)); // Rate limit
             }
-        } catch (missingUserError) {
-            logger.log('WARN', `[PopularInvestorFetch] Error while checking/fetching missing users: ${missingUserError.message}. Continuing with main fetch data.`);
-            // Non-critical error, continue with whatever data we have
         }
+    } catch (e) {
+        logger.log('WARN', `[PopularInvestorFetch] Failed to check master list: ${e.message}`);
     }
 
-    // 6. Final Validation & Storage
-    if (data && data.Items && Array.isArray(data.Items)) {
-        try {
-            const docRef = db.collection(finalRankingsCollectionName).doc(today);
-            
-            // Prepare data for storage
-            const dataToStore = {
-                fetchedAt: new Date(),
-                totalRows: data.TotalRows,
-                itemsCount: data.Items.length,
-                ...data
-            };
-            
-            // Apply compression if data is large enough (similar to ResultCommitter pattern)
-            let firestorePayload;
-            try {
-                const jsonString = JSON.stringify(dataToStore);
-                const rawBuffer = Buffer.from(jsonString);
-                const SIZE_THRESHOLD = 50 * 1024; // 50KB threshold (same as ResultCommitter)
-                const MAX_COMPRESSED_SIZE = 900 * 1024; // 900KB max (Firestore limit is 1MB)
-                
-                if (rawBuffer.length > SIZE_THRESHOLD) {
-                    logger.log('INFO', `[PopularInvestorFetch] Rankings data size ${(rawBuffer.length / 1024).toFixed(2)}KB exceeds threshold. Compressing...`);
-                    
-                    // Gzip the JSON string
-                    const compressedBuffer = zlib.gzipSync(rawBuffer);
-                    
-                    if (compressedBuffer.length < MAX_COMPRESSED_SIZE) {
-                        // Create the compressed wrapper structure
-                        firestorePayload = {
-                            fetchedAt: new Date(), // Keep outer timestamp for indexing
-                            totalRows: data.TotalRows, // Keep outer metadata for indexing
-                            itemsCount: data.Items.length, // Keep outer metadata for indexing
-                            _compressed: true,
-                            payload: compressedBuffer
-                        };
-                        
-                        logger.log('INFO', `[PopularInvestorFetch] Compressed rankings data: ${(rawBuffer.length / 1024).toFixed(2)}KB -> ${(compressedBuffer.length / 1024).toFixed(2)}KB`);
-                    } else {
-                        // Compressed size still too large, store uncompressed (will need sharding in future)
-                        logger.log('WARN', `[PopularInvestorFetch] Compressed size ${(compressedBuffer.length / 1024).toFixed(2)}KB still exceeds limit. Storing uncompressed (may hit index limits).`);
-                        firestorePayload = dataToStore;
-                    }
-                } else {
-                    // Data is small enough, store uncompressed
-                    firestorePayload = dataToStore;
-                }
-            } catch (compressionError) {
-                logger.log('WARN', `[PopularInvestorFetch] Compression failed, storing uncompressed: ${compressionError.message}`);
-                firestorePayload = dataToStore;
-            }
-            
-            await docRef.set(firestorePayload);
+    // 4. Store Rankings to BigQuery (Deduplicated)
+    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+    
+    try {
+        await ensurePIRankingsTable(logger);
+        
+        const fetchedAt = new Date().toISOString();
+        const rankingsRows = data.Items.map((item, index) => ({
+            date: today,
+            pi_id: parseInt(item.CustomerId, 10),
+            username: item.UserName || null,
+            rank: index + 1,
+            category: item.Category || null,
+            rankings_data: item, // Store full JSON
+            fetched_at: fetchedAt
+        }));
+
+        // MERGE: Overwrite if (date, pi_id) exists. 
+        // This effectively handles "duplicate data" by updating the record to the latest fetch.
+        await insertRowsWithMerge(datasetId, 'pi_rankings', rankingsRows, ['date', 'pi_id'], logger);
+        logger.log('SUCCESS', `[PopularInvestorFetch] Merged ${rankingsRows.length} ranking records into BigQuery.`);
+
+    } catch (e) {
+        logger.log('ERROR', `[PopularInvestorFetch] Failed to write rankings to BigQuery: ${e.message}`);
+        throw e;
+    }
 
-            logger.log('SUCCESS', `[PopularInvestorFetch] Stored ${data.TotalRows} rankings into ${finalRankingsCollectionName}/${today}${firestorePayload._compressed ? ' (compressed)' : ''}`);
-            
-            // Write rankings to BigQuery (one row per PI)
-            if (process.env.BIGQUERY_ENABLED !== 'false') {
-                try {
-                    const { insertRows, ensurePIRankingsTable } = require('../../core/utils/bigquery_utils');
-                    await ensurePIRankingsTable(logger);
-                    
-                    const fetchedAt = new Date().toISOString();
-                    const bigqueryRows = data.Items.map((item, index) => {
-                        return {
-                            date: today,
-                            pi_id: parseInt(item.CustomerId, 10),
-                            username: item.UserName || null,
-                            rank: index + 1, // Rank is position in array (1-indexed)
-                            category: item.Category || null,
-                            rankings_data: item, // Store full item data as JSON
-                            fetched_at: fetchedAt
-                        };
-                    });
-                    
-                    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
-                    await insertRows(datasetId, 'pi_rankings', bigqueryRows, logger);
-                    
-                    logger.log('INFO', `[PopularInvestorFetch] Successfully stored ${bigqueryRows.length} ranking records to BigQuery`);
-                } catch (bqError) {
-                    logger.log('WARN', `[PopularInvestorFetch] BigQuery rankings write failed: ${bqError.message}`);
-                    // Continue - Firestore write succeeded
-                }
-            }
-            
-            // Update the master list of Popular Investors
-            // Use batched writes to avoid 500 field transform limit
-            try {
-                const { FieldValue } = require('@google-cloud/firestore');
-                let masterListPath = 'system_state/popular_investor_master_list';
-                
-                if (collectionRegistry && collectionRegistry.getCollectionPath) {
-                    try {
-                        // Get the path from registry
-                        const registryPath = collectionRegistry.getCollectionPath('system', 'popularInvestorMasterList', {});
-                        masterListPath = registryPath;
-                    } catch (e) {
-                        logger.log('WARN', `[PopularInvestorFetch] Failed to get master list path from registry, using default: ${e.message}`);
-                    }
-                }
-                
-                const masterListRef = db.doc(masterListPath);
-                const masterListDoc = await masterListRef.get();
-                
-                const now = new Date();
-                const existingInvestors = masterListDoc.exists ? (masterListDoc.data().investors || {}) : {};
-                const investorsToUpdate = {};
-                let newInvestorsCount = 0;
-                let updatedInvestorsCount = 0;
-                
-                // Identify which investors need updating (only those in current fetch)
-                for (const item of data.Items) {
-                    const cid = String(item.CustomerId);
-                    const username = item.UserName;
-                    
-                    if (!cid || !username) continue;
-                    
-                    if (!existingInvestors[cid]) {
-                        // New PI discovered
-                        investorsToUpdate[cid] = {
-                            cid: cid,
-                            username: username,
-                            firstSeenAt: now,
-                            lastSeenAt: now
-                        };
-                        newInvestorsCount++;
-                    } else {
-                        // Existing PI - check if username changed or needs lastSeenAt update
-                        const needsUpdate = existingInvestors[cid].username !== username;
-                        if (needsUpdate) {
-                            investorsToUpdate[cid] = {
-                                ...existingInvestors[cid],
-                                username: username,
-                                lastSeenAt: now
-                            };
-                            updatedInvestorsCount++;
-                        } else {
-                            // Just update lastSeenAt timestamp
-                            investorsToUpdate[cid] = {
-                                ...existingInvestors[cid],
-                                lastSeenAt: now
-                            };
-                            updatedInvestorsCount++;
-                        }
-                    }
-                }
-                
-                // Use batched writes to update only changed investors
-                // Since we're using regular Date objects (not serverTimestamp()), we avoid field transform limits
-                // But we still batch to handle large numbers of investors efficiently
-                const BATCH_SIZE = 450; // Firestore allows 500 operations per batch, leave room for metadata
-                const investorsToUpdateEntries = Object.entries(investorsToUpdate);
-                const totalBatches = Math.ceil(investorsToUpdateEntries.length / BATCH_SIZE);
-                
-                // If document doesn't exist, create it with all investors in first batch
-                if (!masterListDoc.exists && investorsToUpdateEntries.length > 0) {
-                    const batch = db.batch();
-                    const finalInvestorsMap = { ...existingInvestors, ...investorsToUpdate };
-                    batch.set(masterListRef, {
-                        investors: finalInvestorsMap,
-                        lastUpdated: FieldValue.serverTimestamp(),
-                        totalInvestors: Object.keys(finalInvestorsMap).length
-                    }, { merge: true });
-                    await batch.commit();
-                    logger.log('INFO', `[PopularInvestorFetch] Created master list with ${Object.keys(finalInvestorsMap).length} investors`);
-                } else if (investorsToUpdateEntries.length > 0) {
-                    // Document exists - update only changed investors in batches
-                    for (let batchIdx = 0; batchIdx < totalBatches; batchIdx++) {
-                        const batch = db.batch();
-                        const startIdx = batchIdx * BATCH_SIZE;
-                        const endIdx = Math.min(startIdx + BATCH_SIZE, investorsToUpdateEntries.length);
-                        const batchEntries = investorsToUpdateEntries.slice(startIdx, endIdx);
-                        
-                        // Build update object with all investors in this batch
-                        const batchUpdateData = {};
-                        for (const [cid, investorData] of batchEntries) {
-                            batchUpdateData[`investors.${cid}`] = investorData;
-                        }
-                        
-                        // Update metadata (lastUpdated, totalInvestors) only in the last batch
-                        if (batchIdx === totalBatches - 1) {
-                            const finalInvestorsMap = { ...existingInvestors, ...investorsToUpdate };
-                            batchUpdateData.lastUpdated = FieldValue.serverTimestamp();
-                            batchUpdateData.totalInvestors = Object.keys(finalInvestorsMap).length;
-                        }
-                        
-                        batch.update(masterListRef, batchUpdateData);
-                        await batch.commit();
-                        
-                        if (totalBatches > 1) {
-                            logger.log('INFO', `[PopularInvestorFetch] Updated master list batch ${batchIdx + 1}/${totalBatches} (${batchEntries.length} investors)`);
-                        }
-                    }
-                }
-                
-                logger.log('SUCCESS', `[PopularInvestorFetch] Updated master list: ${newInvestorsCount} new, ${updatedInvestorsCount} updated. Total unique PIs: ${Object.keys({ ...existingInvestors, ...investorsToUpdate }).length}`);
-                
-                // Write master list updates to BigQuery
-                if (process.env.BIGQUERY_ENABLED !== 'false' && Object.keys(investorsToUpdate).length > 0) {
-                    try {
-                        const { insertRowsWithMerge, ensurePIMasterListTable } = require('../../core/utils/bigquery_utils');
-                        await ensurePIMasterListTable(logger);
-                        
-                        const now = new Date().toISOString();
-                        const bigqueryRows = Object.entries(investorsToUpdate).map(([cid, investorData]) => {
-                            // Handle Firestore Timestamp objects
-                            const convertTimestamp = (ts) => {
-                                if (!ts) return now;
-                                if (ts instanceof Date) return ts.toISOString();
-                                if (ts.toDate && typeof ts.toDate === 'function') return ts.toDate().toISOString();
-                                if (typeof ts === 'string') return ts;
-                                return now;
-                            };
-                            
-                            return {
-                                cid: parseInt(cid, 10),
-                                username: investorData.username,
-                                first_seen_at: convertTimestamp(investorData.firstSeenAt),
-                                last_seen_at: convertTimestamp(investorData.lastSeenAt),
-                                last_updated: now
-                            };
-                        });
-                        
-                        const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
-                        // Use MERGE to update existing records or insert new ones
-                        await insertRowsWithMerge(datasetId, 'pi_master_list', bigqueryRows, ['cid'], logger);
-                        
-                        logger.log('INFO', `[PopularInvestorFetch] Successfully stored ${bigqueryRows.length} master list records to BigQuery`);
-                    } catch (bqError) {
-                        logger.log('WARN', `[PopularInvestorFetch] BigQuery master list write failed: ${bqError.message}`);
-                        // Continue - Firestore write succeeded
-                    }
-                }
-            } catch (masterListError) {
-                logger.log('WARN', `[PopularInvestorFetch] Failed to update master list: ${masterListError.message}`);
-                // Non-critical, continue
-            }
-            
-            // Update root data indexer for today's date after rankings data is stored
-            try {
-                const { runRootDataIndexer } = require('../../root-data-indexer/index');
-                
-                // Access rootDataIndexer from config (passed from index.js) or use defaults
-                // Using bracket notation to avoid TypeScript errors
-                const rootDataIndexerConfig = (config && typeof config === 'object' && config['rootDataIndexer']) 
-                    ? config['rootDataIndexer'] 
-                    : null;
-                
-                if (!rootDataIndexerConfig) {
-                    logger.log('WARN', `[PopularInvestorFetch] Root data indexer config not provided. Skipping index update.`);
-                } else {
-                    // Merge rankings collection name into the full config
-                    const indexerConfig = {
-                        ...rootDataIndexerConfig,
-                        collections: {
-                            ...rootDataIndexerConfig.collections,
-                            piRankings: finalRankingsCollectionName // Override with actual collection name used
-                        },
-                        targetDate: today // Index only today's date for speed
-                    };
-                    
-                    logger.log('INFO', `[PopularInvestorFetch] Triggering root data indexer for date ${today} after rankings data storage...`);
-                    const result = await runRootDataIndexer(indexerConfig, dependencies);
-                    
-                    if (result.success && result.count > 0) {
-                        logger.log('INFO', `[PopularInvestorFetch] Root data indexer completed successfully for date ${today} (updated ${result.count} dates)`);
-                    } else {
-                        logger.log('WARN', `[PopularInvestorFetch] Root data indexer completed but no dates were updated for ${today}`);
-                    }
-                }
-            } catch (indexerError) {
-                logger.log('ERROR', `[PopularInvestorFetch] Failed to run root data indexer for ${today}`, indexerError);
-                // Continue - rankings data is stored, indexer failure is non-critical
+    // 5. Update Master List in BigQuery (State Management)
+    try {
+        await ensurePIMasterListTable(logger);
+        
+        const now = new Date().toISOString();
+        const masterListUpdates = [];
+
+        // Calculate updates based on what we just fetched vs what we know from BQ
+        for (const item of data.Items) {
+            const cid = String(item.CustomerId);
+            const existing = bqMasterList[cid];
+
+            if (!existing) {
+                // New Discovery
+                masterListUpdates.push({
+                    cid: parseInt(cid, 10),
+                    username: item.UserName,
+                    first_seen_at: now,
+                    last_seen_at: now,
+                    last_updated: now
+                });
+            } else {
+                // Existing - Update timestamps and username if changed
+                masterListUpdates.push({
+                    cid: parseInt(cid, 10),
+                    username: item.UserName, // Update username in case of rebrand
+                    first_seen_at: existing.firstSeenAt ? (existing.firstSeenAt.value || existing.firstSeenAt) : now, // Preserve original discovery time
+                    last_seen_at: now,
+                    last_updated: now
+                });
             }
-            
-            return { success: true, count: data.TotalRows };
+        }
 
-        } catch (dbError) {
-            logger.log('ERROR', '[PopularInvestorFetch] Failed to write to Firestore.', { errorMessage: dbError.message });
-            throw dbError;
+        if (masterListUpdates.length > 0) {
+            // MERGE: Overwrite based on 'cid'
+            await insertRowsWithMerge(datasetId, 'pi_master_list', masterListUpdates, ['cid'], logger);
+            logger.log('SUCCESS', `[PopularInvestorFetch] Updated Master List for ${masterListUpdates.length} investors in BigQuery.`);
         }
-    } else {
-        const errorMsg = fetchSuccess 
-            ? '[PopularInvestorFetch] Fetched data format is invalid (missing Items array).' 
-            : '[PopularInvestorFetch] Failed to fetch data from all sources.';
-            
-        logger.log('ERROR', errorMsg);
-        throw new Error(errorMsg);
+
+    } catch (e) {
+        logger.log('WARN', `[PopularInvestorFetch] Failed to update Master List: ${e.message}`);
+    }
+
+    // 6. Trigger Root Data Indexer
+    try {
+        const { runRootDataIndexer } = require('../../root-data-indexer/index');
+        const rootDataIndexerConfig = config.rootDataIndexer || {};
+        
+        // Pass the BQ table name as the collection hint, though the indexer should auto-detect BQ availability
+        const indexerConfig = {
+            ...rootDataIndexerConfig,
+            collections: { 
+                ...rootDataIndexerConfig.collections, 
+                piRankings: 'pi_rankings' 
+            },
+            targetDate: today
+        };
+        
+        await runRootDataIndexer(indexerConfig, dependencies);
+    } catch (e) {
+        logger.log('WARN', `[PopularInvestorFetch] Indexer trigger failed: ${e.message}`);
     }
+
+    return { success: true, count: data.TotalRows, storage: 'BIGQUERY_ONLY' };
 }
 
 module.exports = { fetchAndStorePopularInvestors };