bulltrackers-module 1.0.709 → 1.0.712

package/functions/computation-system/persistence/ResultCommitter.js

@@ -16,6 +16,7 @@ const validationOverrides                  = require('../config/validation_overr
 const pLimit                               = require('p-limit'); 
 const zlib                                 = require('zlib'); 
 const { Storage }                          = require('@google-cloud/storage');
+const { ensureComputationResultsTable, insertRows } = require('../../core/utils/bigquery_utils');
 
 const storage = new Storage(); // Singleton GCS Client
 const NON_RETRYABLE_ERRORS = [ 'PERMISSION_DENIED', 'DATA_LOSS', 'FAILED_PRECONDITION' ];
@@ -210,7 +211,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     const historicalDocRef = db.collection(config.resultsCollection).doc(historicalDate).collection(config.resultsSubcollection).doc(calc.manifest.category).collection(config.computationsSubcollection).doc(name);
                     
                     // Recursive call allows GCS logic to apply per-day
-                    const stats = await writeSingleResult(dailyData, historicalDocRef, name, historicalDate, calc.manifest.category, logger, config, deps, 0, 'STANDARD', false, dailyExpireAt);
+                    const stats = await writeSingleResult(dailyData, historicalDocRef, name, historicalDate, calc.manifest.category, logger, config, deps, 0, 'STANDARD', false, dailyExpireAt, isAlertComputation, isPageComputation);
                     runMetrics.io.writes += stats.opCounts.writes;
                     runMetrics.io.deletes += stats.opCounts.deletes;
 
@@ -225,7 +226,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 const runExpireAt = calculateExpirationDate(dStr, ttlDays);
 
                 const mainDocRef = db.collection(config.resultsCollection).doc(dStr).collection(config.resultsSubcollection).doc(calc.manifest.category).collection(config.computationsSubcollection).doc(name);
-                const writeStats = await writeSingleResult(result, mainDocRef, name, dStr, calc.manifest.category, logger, config, deps, currentShardIndex, flushMode, isInitialWrite, runExpireAt);
+                const writeStats = await writeSingleResult(result, mainDocRef, name, dStr, calc.manifest.category, logger, config, deps, currentShardIndex, flushMode, isInitialWrite, runExpireAt, isAlertComputation, isPageComputation);
                 
                 runMetrics.storage.sizeBytes  = writeStats.totalSize;
                 runMetrics.storage.isSharded  = writeStats.isSharded;
@@ -294,7 +295,7 @@ async function fetchContracts(db, calcNames) {
     return map;
 }
 
-async function writeSingleResult(result, docRef, name, dateContext, category, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD', isInitialWrite = false, expireAt = null) {
+async function writeSingleResult(result, docRef, name, dateContext, category, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD', isInitialWrite = false, expireAt = null, isAlertComputation = false) {
     const opCounts = { writes: 0, deletes: 0 };
     
     // Check if previously sharded (so we can clean up if moving to GCS or Compressed)
@@ -358,6 +359,13 @@ async function writeSingleResult(result, docRef, name, dateContext, category, lo
 
             logger.log('INFO', `[GCS] ${name}: Offloaded ${(totalSize/1024).toFixed(0)}KB to ${fileName}`);
             
+            // Write to BigQuery (await to ensure completion before function returns)
+            // Errors are caught and logged but don't fail the operation
+            // Pass isAlertComputation flag to use streaming for alerts, load jobs for others
+            await writeToBigQuery(result, name, dateContext, category, logger, isAlertComputation).catch(err => {
+                logger.log('WARN', `[BigQuery] Failed to write ${name} for ${dateContext}: ${err.message}`);
+            });
+            
             return { totalSize, isSharded: false, shardCount: 1, nextShardIndex: startShardIndex, opCounts, location: 'GCS' };
 
         } catch (gcsErr) {
@@ -394,6 +402,12 @@ async function writeSingleResult(result, docRef, name, dateContext, category, lo
                 opCounts.writes += 1;
                 logger.log('INFO', `[Compression] ${name}: Compressed ${(totalSize/1024).toFixed(0)}KB -> ${(compressedBuffer.length/1024).toFixed(0)}KB.`);
                 
+                // Write to BigQuery (await to ensure completion before function returns)
+                // Errors are caught and logged but don't fail the operation
+                await writeToBigQuery(result, name, dateContext, category, logger).catch(err => {
+                    logger.log('WARN', `[BigQuery] Failed to write ${name} for ${dateContext}: ${err.message}`);
+                });
+                
                 return { totalSize: compressedBuffer.length, isSharded: false, shardCount: 1, nextShardIndex: startShardIndex, opCounts, location: 'FIRESTORE' };
             }
         } catch (compErr) {
@@ -475,6 +489,13 @@ async function writeSingleResult(result, docRef, name, dateContext, category, lo
         shardingError.stage = 'SHARDING_LIMIT_EXCEEDED';
         throw shardingError; 
     }
+    
+    // Write to BigQuery (await to ensure completion before function returns)
+    // Errors are caught and logged but don't fail the operation
+    await writeToBigQuery(result, name, dateContext, category, logger).catch(err => {
+        logger.log('WARN', `[BigQuery] Failed to write ${name} for ${dateContext}: ${err.message}`);
+    });
+    
     finalStats.opCounts = opCounts;
     return finalStats;
 }
@@ -483,6 +504,76 @@ async function writeSingleResult(result, docRef, name, dateContext, category, lo
 // HELPERS
 // =============================================================================
 
+/**
+ * Write computation result to BigQuery (errors are logged but don't fail Firestore writes)
+ * @param {object} result - Computation result data
+ * @param {string} name - Computation name
+ * @param {string} dateContext - Date string (YYYY-MM-DD)
+ * @param {string} category - Category (e.g., 'popular-investor', 'alerts')
+ * @param {object} logger - Logger instance
+ * @param {boolean} isAlertComputation - If true, uses streaming inserts (immediate). If false, uses load jobs (batched, free).
+ */
+async function writeToBigQuery(result, name, dateContext, category, logger, isAlertComputation = false) {
+    // Skip if BigQuery is disabled via environment variable
+    if (process.env.BIGQUERY_ENABLED === 'false') {
+        return;
+    }
+    
+    try {
+        // Size check: BigQuery streaming inserts have a 10MB limit per row
+        // Estimate size by stringifying the result
+        const estimatedSize = JSON.stringify(result).length;
+        const MAX_BIGQUERY_ROW_SIZE = 9 * 1024 * 1024; // 9MB safety limit (10MB is hard limit)
+        
+        if (estimatedSize > MAX_BIGQUERY_ROW_SIZE) {
+            if (logger) {
+                logger.log('WARN', `[BigQuery] Skipping ${name} (${dateContext}): Result too large for streaming (${(estimatedSize/1024/1024).toFixed(2)}MB). Data is in GCS/Firestore.`);
+            }
+            // Return early - don't attempt insert that will fail
+            // The data is still available in Firestore/GCS, so this is acceptable
+            return;
+        }
+        
+        // Ensure table exists
+        await ensureComputationResultsTable(logger);
+        
+        // Extract metadata (cids if present)
+        const metadata = {};
+        if (result.cids && Array.isArray(result.cids)) {
+            metadata.cids = result.cids;
+        }
+        
+        // Prepare row for BigQuery
+        const row = {
+            date: dateContext,
+            computation_name: name,
+            category: category,
+            result_data: result, // Full result as JSON
+            metadata: Object.keys(metadata).length > 0 ? metadata : null,
+            created_at: new Date().toISOString()
+        };
+        
+        const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+        
+        // Use streaming inserts for alert computations (immediate, time-sensitive)
+        // Use load jobs for non-alert computations (batched, free)
+        const { insertRows: insertRowsLoadJob, insertRowsStreaming } = require('../../core/utils/bigquery_utils');
+        
+        if (isAlertComputation) {
+            await insertRowsStreaming(datasetId, 'computation_results', [row], logger);
+        } else {
+            await insertRowsLoadJob(datasetId, 'computation_results', [row], logger);
+        }
+        
+    } catch (error) {
+        // Log but don't throw - BigQuery write failure shouldn't break Firestore writes
+        if (logger) {
+            logger.log('WARN', `[BigQuery] Write failed for ${name} (${dateContext}): ${error.message}`);
+        }
+        // Don't re-throw - ensure Firestore writes always succeed
+    }
+}
+
 async function cleanupOldShards(docRef, name, config, deps, metrics) {
     const shardCol = docRef.collection('_shards');
     const shardDocs = await shardCol.listDocuments();

package/functions/computation-system/utils/data_loader.js

@@ -68,7 +68,42 @@ async function getPortfolioPartRefs(config, deps, dateString, requiredUserTypes
     const types = requiredUserTypes ? new Set(requiredUserTypes.map(t => t.toUpperCase())) : null;
     const fetchAll = !types || types.size === 0 || types.has('ALL');
 
-    logger.log('INFO', `Getting portfolio part references for date: ${dateString}. Filter: ${fetchAll ? 'ALL' : Array.from(types).join(',')}`);
+    // =========================================================================
+    // BIGQUERY FIRST: Try BigQuery before Firestore
+    // =========================================================================
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryPortfolioData } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryPortfolioData(dateString, null, requiredUserTypes, logger);
+            
+            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for portfolio data (${dateString}): ${Object.keys(bigqueryData).length} users`);
+                
+                // Transform BigQuery data into "ref-like" objects for compatibility
+                const bigqueryRefs = Object.keys(bigqueryData).map(userId => {
+                    const dataObj = bigqueryData[userId];
+                    return {
+                        ref: null, // No Firestore ref needed
+                        type: dataObj.user_type || 'UNKNOWN',
+                        cid: userId,
+                        collectionType: 'BIGQUERY',
+                        bigqueryData: dataObj.portfolio_data || {} // Extract the actual portfolio data
+                    };
+                });
+                
+                return bigqueryRefs;
+            } else {
+                logger.log('INFO', `[DataLoader] ⚠️ BigQuery returned no portfolio data for ${dateString}, falling back to Firestore`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery portfolio query failed for ${dateString}, falling back to Firestore: ${bqError.message}`);
+        }
+    }
+    
+    // =========================================================================
+    // FIRESTORE FALLBACK: Original logic (backwards compatibility)
+    // =========================================================================
+    logger.log('INFO', `[DataLoader] 📂 Using Firestore for portfolio data (${dateString}). Filter: ${fetchAll ? 'ALL' : Array.from(types || []).join(',')}`);
     
     const allPartRefs = [];
     
@@ -143,12 +178,62 @@ async function loadDataByRefs(config, deps, refObjects) {
     
     if (!refObjects || !refObjects.length) return {};
     
+    // =========================================================================
+    // CHECK FOR PRICE DATA (BigQuery marker)
+    // =========================================================================
+    const priceBigQueryMarker = refObjects.find(r => r._bigquery === true);
+    if (priceBigQueryMarker) {
+        // This is a price data load request - use BigQuery
+        try {
+            const { queryAssetPrices } = require('../../core/utils/bigquery_utils');
+            const priceData = await queryAssetPrices(null, null, null, logger);
+            
+            if (priceData && Object.keys(priceData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Loaded ${Object.keys(priceData).length} instruments from BigQuery for price data`);
+                return priceData;
+            }
+            
+            // If BigQuery returns empty, fallback to Firestore
+            logger.log('WARN', `[DataLoader] BigQuery returned no price data, falling back to Firestore`);
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery price load failed, falling back to Firestore: ${bqError.message}`);
+            // Fall through to Firestore
+        }
+    }
+    
+    // =========================================================================
+    // SEPARATE BIGQUERY AND FIRESTORE REFS (for portfolio/history data)
+    // =========================================================================
+    const bigqueryRefs = refObjects.filter(r => r.collectionType === 'BIGQUERY');
+    const firestoreRefs = refObjects.filter(r => r.collectionType !== 'BIGQUERY' && !r._bigquery);
+    
     const mergedPortfolios = {};
+    
+    // Load from BigQuery cache (data already fetched in getPortfolioPartRefs/getHistoryPartRefs)
+    if (bigqueryRefs.length > 0) {
+        logger.log('INFO', `[DataLoader] 📊 Loading ${bigqueryRefs.length} records from BigQuery cache`);
+        bigqueryRefs.forEach(ref => {
+            if (ref.bigqueryData) {
+                // Extract the actual data (portfolio_data or history_data)
+                // The bigqueryData is the JSON object from BigQuery
+                mergedPortfolios[ref.cid] = ref.bigqueryData;
+            }
+        });
+    }
+    
+    // Load from Firestore (existing logic)
+    if (firestoreRefs.length === 0) {
+        return mergedPortfolios; // All data came from BigQuery
+    }
+    
+    logger.log('INFO', `[DataLoader] 📂 Loading ${firestoreRefs.length} records from Firestore`);
     const batchSize = config.partRefBatchSize || 10;
 
-    for (let i = 0; i < refObjects.length; i += batchSize) { 
-        const batch = refObjects.slice(i, i + batchSize); 
-        const refs = batch.map(b => b.ref);
+    for (let i = 0; i < firestoreRefs.length; i += batchSize) {
+        const batch = firestoreRefs.slice(i, i + batchSize);
+        const refs = batch.map(b => b.ref).filter(ref => ref !== null); // Filter out null refs (BigQuery refs)
+        
+        if (refs.length === 0) continue; // Skip if all refs are BigQuery refs
         
         const snapshots = await withRetry(() => db.getAll(...refs), `getAll(batch ${Math.floor(i/batchSize)})`);
         
@@ -250,9 +335,26 @@ async function loadDailyInsights(config, deps, dateString) {
     const cached = await tryLoadFromGCS(config, dateString, 'insights', logger);
     if (cached) return cached;
 
-    // 2. FIRESTORE FALLBACK
+    // 2. BIGQUERY FIRST (if enabled)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryInstrumentInsights } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryInstrumentInsights(dateString, logger);
+            
+            if (bigqueryData && Array.isArray(bigqueryData) && bigqueryData.length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for instrument insights (${dateString}): ${bigqueryData.length} instruments`);
+                // Return in same format as Firestore: { insights: [...] }
+                return { insights: bigqueryData };
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery insights query failed, falling back to Firestore: ${bqError.message}`);
+            // Fall through to Firestore
+        }
+    }
+
+    // 3. FIRESTORE FALLBACK
     const insightsCollectionName = config.insightsCollectionName || 'daily_instrument_insights';
-    logger.log('INFO', `Loading daily insights for ${dateString} from ${insightsCollectionName}`);
+    logger.log('INFO', `Loading daily insights for ${dateString} from ${insightsCollectionName} (Firestore)`);
     try { 
         const docRef = db.collection(insightsCollectionName).doc(dateString);
         const docSnap = await withRetry(() => docRef.get(), `getInsights(${dateString})`);
@@ -379,7 +481,42 @@ async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes =
     const types = requiredUserTypes ? new Set(requiredUserTypes.map(t => t.toUpperCase())) : null;
     const fetchAll = !types || types.size === 0 || types.has('ALL');
 
-    logger.log('INFO', `Getting history part references for ${dateString}. Filter: ${fetchAll ? 'ALL' : Array.from(types).join(',')}`);
+    // =========================================================================
+    // BIGQUERY FIRST: Try BigQuery before Firestore
+    // =========================================================================
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryHistoryData } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryHistoryData(dateString, null, requiredUserTypes, logger);
+            
+            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for trade history data (${dateString}): ${Object.keys(bigqueryData).length} users`);
+                
+                // Transform BigQuery data into "ref-like" objects for compatibility
+                const bigqueryRefs = Object.keys(bigqueryData).map(userId => {
+                    const dataObj = bigqueryData[userId];
+                    return {
+                        ref: null, // No Firestore ref needed
+                        type: dataObj.user_type || 'UNKNOWN',
+                        cid: userId,
+                        collectionType: 'BIGQUERY',
+                        bigqueryData: dataObj.history_data || {} // Extract the actual history data
+                    };
+                });
+                
+                return bigqueryRefs;
+            } else {
+                logger.log('INFO', `[DataLoader] ⚠️ BigQuery returned no history data for ${dateString}, falling back to Firestore`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery history query failed for ${dateString}, falling back to Firestore: ${bqError.message}`);
+        }
+    }
+    
+    // =========================================================================
+    // FIRESTORE FALLBACK: Original logic (backwards compatibility)
+    // =========================================================================
+    logger.log('INFO', `[DataLoader] 📂 Using Firestore for trade history data (${dateString}). Filter: ${fetchAll ? 'ALL' : Array.from(types || []).join(',')}`);
     
     const allPartRefs = [];
     
@@ -447,6 +584,31 @@ async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes =
 async function* streamPortfolioData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
     const { logger } = deps;
 
+    // =========================================================================
+    // BIGQUERY FIRST: Try BigQuery before GCS/Firestore
+    // =========================================================================
+    if (!providedRefs && process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryPortfolioData } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryPortfolioData(dateString, null, requiredUserTypes, logger);
+            
+            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Streaming portfolio data from BigQuery (${dateString}): ${Object.keys(bigqueryData).length} users`);
+                
+                // Transform to expected format and yield
+                const transformedData = {};
+                Object.keys(bigqueryData).forEach(userId => {
+                    transformedData[userId] = bigqueryData[userId].portfolio_data || {};
+                });
+                
+                yield transformedData;
+                return; // Exit early, data loaded from BigQuery
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery portfolio stream failed for ${dateString}, falling back: ${bqError.message}`);
+        }
+    }
+
     // 1. GCS FAST PATH (Snapshot) - Only if full run (no providedRefs)
     if (!providedRefs) { 
         try {
@@ -498,6 +660,31 @@ async function* streamPortfolioData(config, deps, dateString, providedRefs = nul
 async function* streamHistoryData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
     const { logger } = deps;
 
+    // =========================================================================
+    // BIGQUERY FIRST: Try BigQuery before Firestore
+    // =========================================================================
+    if (!providedRefs && process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryHistoryData } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryHistoryData(dateString, null, requiredUserTypes, logger);
+            
+            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Streaming history data from BigQuery (${dateString}): ${Object.keys(bigqueryData).length} users`);
+                
+                // Transform to expected format and yield
+                const transformedData = {};
+                Object.keys(bigqueryData).forEach(userId => {
+                    transformedData[userId] = bigqueryData[userId].history_data || {};
+                });
+                
+                yield transformedData;
+                return; // Exit early, data loaded from BigQuery
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery history stream failed for ${dateString}, falling back: ${bqError.message}`);
+        }
+    }
+
     // 1. GCS FAST PATH (JSONL Streaming) - Only if full run
     if (!providedRefs) {
         try {
@@ -555,9 +742,21 @@ async function* streamHistoryData(config, deps, dateString, providedRefs = null,
     logger.log('INFO', `[streamHistoryData] Finished streaming for ${dateString}.`);
 }
 
-/** Stage 9: Get all price shard references (Basic) */
+/** Stage 9: Get all price shard references (Basic) 
+ * [UPDATED] Returns special marker for BigQuery mode, or Firestore refs for fallback
+ */
 async function getPriceShardRefs(config, deps) {
-    const { db, logger, calculationUtils } = deps;
+    const { logger } = deps;
+    
+    // Try BigQuery first if enabled
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        // Return a special marker object to indicate BigQuery mode
+        // The loader will detect this and load from BigQuery instead
+        return [{ _bigquery: true }];
+    }
+    
+    // Fallback to Firestore
+    const { db, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     const collection = config.priceCollection || 'asset_prices';
     try {
@@ -590,9 +789,25 @@ async function loadPopularInvestorRankings(config, deps, dateString) {
     const cached = await tryLoadFromGCS(config, dateString, 'rankings', logger);
     if (cached) return cached;
 
-    // 2. FIRESTORE FALLBACK
+    // 2. BIGQUERY FIRST (if enabled)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryPIRankings } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryPIRankings(dateString, logger);
+            
+            if (bigqueryData && bigqueryData.Items && bigqueryData.Items.length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for PI rankings (${dateString}): ${bigqueryData.Items.length} items`);
+                return bigqueryData.Items;
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery rankings query failed, falling back to Firestore: ${bqError.message}`);
+            // Fall through to Firestore
+        }
+    }
+
+    // 3. FIRESTORE FALLBACK
     const collectionName = config.popularInvestorRankingsCollection || 'popular_investor_rankings';
-    logger.log('INFO', `Loading Popular Investor Rankings for ${dateString}`);
+    logger.log('INFO', `Loading Popular Investor Rankings for ${dateString} from Firestore`);
     try {
         const docRef = db.collection(collectionName).doc(dateString);
         const docSnap = await withRetry(() => docRef.get(), `getRankings(${dateString})`);
@@ -794,10 +1009,26 @@ async function loadPopularInvestorMasterList(config, deps, dateString = null) {
         if (cached) return cached;
     }
 
-    // 2. FIRESTORE FALLBACK
+    // 2. BIGQUERY FIRST (if enabled)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryPIMasterList } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryPIMasterList(logger);
+            
+            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for PI master list: ${Object.keys(bigqueryData).length} PIs`);
+                return bigqueryData;
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery master list query failed, falling back to Firestore: ${bqError.message}`);
+            // Fall through to Firestore
+        }
+    }
+
+    // 3. FIRESTORE FALLBACK
     const collectionName = config.piMasterListCollection || 'system_state';
     const docId = config.piMasterListDocId || 'popular_investor_master_list';
-    logger.log('INFO', `Loading Popular Investor Master List from ${collectionName}/${docId}`);
+    logger.log('INFO', `Loading Popular Investor Master List from ${collectionName}/${docId} (Firestore)`);
     try {
         const docRef = db.collection(collectionName).doc(docId);
         const docSnap = await withRetry(() => docRef.get(), 'getPIMasterList');