bulltrackers-module 1.0.709 → 1.0.712

package/functions/core/utils/firestore_utils.js

@@ -281,8 +281,8 @@ function needsUpdate(lastUpdated, todayStr) {
 
 /**
  * [NEW] Fetches Popular Investors from the master list and filters by last updated times.
- * UPDATED: Uses the master list as single source of truth, then checks last updated timestamps
- * for each data type (portfolio, tradeHistory, socialPosts) against TODAY to determine who needs updating.
+ * UPDATED: Uses BigQuery first for master list, then checks BigQuery for today's data to filter users.
+ * Falls back to Firestore if BigQuery is disabled or fails.
  */
 async function getPopularInvestorsToUpdate(dependencies, config) {
     const { db, logger, collectionRegistry } = dependencies;
@@ -291,53 +291,122 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
     logger.log('INFO', `[Core Utils] Getting Popular Investors to update (Checking against date: ${todayStr})...`);
     
     try {
-        // Get the master list of Popular Investors
-        let masterListPath = 'system_state/popular_investor_master_list';
+        let investors = {};
+        let masterListSource = 'UNKNOWN';
         
-        if (collectionRegistry && collectionRegistry.getCollectionPath) {
+        // =========================================================================
+        // BIGQUERY FIRST: Try BigQuery for master list
+        // =========================================================================
+        if (process.env.BIGQUERY_ENABLED !== 'false') {
             try {
-                masterListPath = collectionRegistry.getCollectionPath('system', 'popularInvestorMasterList', {});
-            } catch (err) {
-                logger.log('WARN', `[Core Utils] Failed to get master list path from registry, using default: ${err.message}`);
+                const { queryPIMasterList } = require('../../core/utils/bigquery_utils');
+                const bigqueryMasterList = await queryPIMasterList(logger);
+                
+                if (bigqueryMasterList && Object.keys(bigqueryMasterList).length > 0) {
+                    investors = bigqueryMasterList;
+                    masterListSource = 'BIGQUERY';
+                    logger.log('INFO', `[Core Utils] ✅ Loaded PI master list from BigQuery: ${Object.keys(investors).length} investors`);
+                }
+            } catch (bqError) {
+                logger.log('WARN', `[Core Utils] BigQuery master list query failed, falling back to Firestore: ${bqError.message}`);
             }
         }
         
-        const masterListRef = db.doc(masterListPath);
-        const masterListDoc = await masterListRef.get();
-        
-        if (!masterListDoc.exists) {
-            logger.log('WARN', `[Core Utils] Master list not found. Falling back to legacy method.`);
-            return await getPopularInvestorsToUpdateLegacy(dependencies, config);
+        // =========================================================================
+        // FIRESTORE FALLBACK: If BigQuery didn't return data
+        // =========================================================================
+        if (Object.keys(investors).length === 0) {
+            let masterListPath = 'system_state/popular_investor_master_list';
+            
+            if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                try {
+                    masterListPath = collectionRegistry.getCollectionPath('system', 'popularInvestorMasterList', {});
+                } catch (err) {
+                    logger.log('WARN', `[Core Utils] Failed to get master list path from registry, using default: ${err.message}`);
+                }
+            }
+            
+            const masterListRef = db.doc(masterListPath);
+            const masterListDoc = await masterListRef.get();
+            
+            if (!masterListDoc.exists) {
+                logger.log('WARN', `[Core Utils] Master list not found in Firestore. Falling back to legacy method.`);
+                return await getPopularInvestorsToUpdateLegacy(dependencies, config);
+            }
+            
+            const masterListData = masterListDoc.data();
+            investors = masterListData.investors || {};
+            masterListSource = 'FIRESTORE';
+            logger.log('INFO', `[Core Utils] Loaded PI master list from Firestore: ${Object.keys(investors).length} investors`);
         }
         
-        const masterListData = masterListDoc.data();
-        const investors = masterListData.investors || {};
-        
         if (Object.keys(investors).length === 0) {
             logger.log('WARN', `[Core Utils] Master list is empty. Returning empty array.`);
             return [];
         }
         
+        // =========================================================================
+        // FILTER: Check BigQuery for today's data to determine who needs updating
+        // =========================================================================
         const targets = [];
         let skippedCount = 0;
         
+        // Get today's data from BigQuery to check who's already updated
+        let todayPortfolioUsers = new Set();
+        let todayHistoryUsers = new Set();
+        let todaySocialUsers = new Set();
+        
+        if (process.env.BIGQUERY_ENABLED !== 'false') {
+            try {
+                const { queryPortfolioData, queryHistoryData, querySocialData } = require('../../core/utils/bigquery_utils');
+                
+                // Query BigQuery for today's data (only for POPULAR_INVESTOR user type)
+                const [portfolioData, historyData, socialData] = await Promise.all([
+                    queryPortfolioData(todayStr, null, ['POPULAR_INVESTOR'], logger).catch(() => null),
+                    queryHistoryData(todayStr, null, ['POPULAR_INVESTOR'], logger).catch(() => null),
+                    querySocialData(todayStr, null, ['POPULAR_INVESTOR'], logger).catch(() => null)
+                ]);
+                
+                if (portfolioData) {
+                    todayPortfolioUsers = new Set(Object.keys(portfolioData));
+                }
+                if (historyData) {
+                    todayHistoryUsers = new Set(Object.keys(historyData));
+                }
+                if (socialData) {
+                    todaySocialUsers = new Set(Object.keys(socialData));
+                }
+                
+                logger.log('INFO', `[Core Utils] BigQuery filter: ${todayPortfolioUsers.size} portfolios, ${todayHistoryUsers.size} histories, ${todaySocialUsers.size} social posts for ${todayStr}`);
+            } catch (bqError) {
+                logger.log('WARN', `[Core Utils] BigQuery filter query failed, using Firestore fallback: ${bqError.message}`);
+            }
+        }
+        
+        // Process each investor from master list
         for (const [cid, piData] of Object.entries(investors)) {
             const username = piData.username || String(cid);
             
-            // Construct path manually to avoid registry lookup errors on "profile"
-            const piDocPath = `PopularInvestors/${cid}`;
-            const piDoc = await db.doc(piDocPath).get();
-            
-            // Default: All need update
-            let components = { portfolio: true, tradeHistory: true, socialPosts: true };
+            // Check if already updated today (BigQuery first, then Firestore)
+            let components = {
+                portfolio: !todayPortfolioUsers.has(cid),
+                tradeHistory: !todayHistoryUsers.has(cid),
+                socialPosts: !todaySocialUsers.has(cid)
+            };
             
-            if (piDoc.exists) {
-                const data = piDoc.data();
-                const lastUpdated = data.lastUpdated || {};
+            // If BigQuery didn't have data, check Firestore
+            if (todayPortfolioUsers.size === 0 && todayHistoryUsers.size === 0 && todaySocialUsers.size === 0) {
+                const piDocPath = `PopularInvestors/${cid}`;
+                const piDoc = await db.doc(piDocPath).get();
                 
-                components.portfolio = needsUpdate(lastUpdated.portfolio, todayStr);
-                components.tradeHistory = needsUpdate(lastUpdated.tradeHistory, todayStr);
-                components.socialPosts = needsUpdate(lastUpdated.socialPosts, todayStr);
+                if (piDoc.exists) {
+                    const data = piDoc.data();
+                    const lastUpdated = data.lastUpdated || {};
+                    
+                    components.portfolio = needsUpdate(lastUpdated.portfolio, todayStr);
+                    components.tradeHistory = needsUpdate(lastUpdated.tradeHistory, todayStr);
+                    components.socialPosts = needsUpdate(lastUpdated.socialPosts, todayStr);
+                }
             }
             
             // Only add if at least one component needs updating
@@ -348,7 +417,7 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
             }
         }
         
-        logger.log('INFO', `[Core Utils] Found ${Object.keys(investors).length} Popular Investors in master list. Skipped ${skippedCount} (updated today). ${targets.length} queued for update.`);
+        logger.log('INFO', `[Core Utils] Found ${Object.keys(investors).length} Popular Investors in master list (${masterListSource}). Skipped ${skippedCount} (updated today). ${targets.length} queued for update.`);
         return targets;
         
     } catch (error) {

package/functions/etoro-price-fetcher/helpers/handler_helpers.js

@@ -16,11 +16,24 @@ const SHARD_SIZE = 40;
  * @returns {Promise<{success: boolean, message: string, instrumentsProcessed?: number}>}
  */
 exports.fetchAndStorePrices = async (config, dependencies) => {
-    const { db, logger, headerManager, proxyManager, collectionRegistry } = dependencies;
+    const { db, logger, headerManager, proxyManager, collectionRegistry, calculationUtils } = dependencies;
     logger.log('INFO', '[PriceFetcherHelpers] Starting Daily Closing Price Update...');
     let selectedHeader = null;
     let wasSuccessful = false;
     
+    // Load instrument mappings for ticker information (needed for BigQuery)
+    let instrumentMappings = null;
+    if (process.env.BIGQUERY_ENABLED !== 'false' && calculationUtils?.loadInstrumentMappings) {
+        try {
+            instrumentMappings = await calculationUtils.loadInstrumentMappings();
+            if (instrumentMappings?.instrumentToTicker) {
+                logger.log('INFO', `[PriceFetcherHelpers] Loaded ${Object.keys(instrumentMappings.instrumentToTicker).length} instrument mappings for ticker lookup`);
+            }
+        } catch (mappingError) {
+            logger.log('WARN', `[PriceFetcherHelpers] Failed to load instrument mappings: ${mappingError.message}. Ticker will be set to 'unknown_{instrumentId}'`);
+        }
+    }
+    
     // Get collection names from registry if available, fallback to hardcoded
     const { getCollectionPath } = collectionRegistry || {};
     let priceCollectionName = 'asset_prices';
@@ -50,23 +63,82 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
         wasSuccessful = true;
         const results = await response.json();
         if (!Array.isArray(results)) { throw new Error('Invalid response format from API. Expected an array.'); }        
-        logger.log('INFO', `[PriceFetcherHelpers] Received ${results.length} instrument prices. Sharding...`);
-        const shardUpdates = {}; 
+        logger.log('INFO', `[PriceFetcherHelpers] Received ${results.length} instrument prices. Processing for BigQuery...`);
+        
+        // Transform daily prices to BigQuery rows
+        const fetchedAt = new Date().toISOString();
+        const bigqueryRows = [];
+        const shardUpdates = {}; // Keep for Firestore backward compatibility
+        
         for (const instrumentData of results) {
             const dailyData = instrumentData?.ClosingPrices?.Daily;
             const instrumentId = instrumentData.InstrumentId;
+            
             if (instrumentId && dailyData?.Price && dailyData?.Date) {
                 const instrumentIdStr = String(instrumentId);
-                const dateKey = dailyData.Date.substring(0, 10);
-                const shardId = `shard_${parseInt(instrumentIdStr, 10) % SHARD_SIZE}`;
-                if (!shardUpdates[shardId]) { shardUpdates[shardId] = {}; }
-                const pricePath = `${instrumentIdStr}.prices.${dateKey}`;
-                const updatePath = `${instrumentIdStr}.lastUpdated`;
-                shardUpdates[shardId][pricePath] = dailyData.Price;
-                shardUpdates[shardId][updatePath] = FieldValue.serverTimestamp(); } }
-        const batchPromises = [];
-        for (const shardId in shardUpdates) { const docRef = db.collection(priceCollectionName).doc(shardId); const payload = shardUpdates[shardId]; batchPromises.push(docRef.update(payload)); }
-        await Promise.all(batchPromises);
+                const dateKey = dailyData.Date.substring(0, 10); // Extract YYYY-MM-DD
+                
+                // Get ticker from mappings if available
+                let ticker = `unknown_${instrumentId}`;
+                if (instrumentMappings?.instrumentToTicker?.[instrumentIdStr]) {
+                    ticker = instrumentMappings.instrumentToTicker[instrumentIdStr];
+                } else if (instrumentData.Ticker) {
+                    ticker = instrumentData.Ticker;
+                }
+                
+                // Prepare BigQuery row
+                bigqueryRows.push({
+                    date: dateKey,
+                    instrument_id: parseInt(instrumentId, 10),
+                    ticker: ticker,
+                    price: dailyData.Price,
+                    open: null, // Daily API doesn't provide OHLC, only closing price
+                    high: null,
+                    low: null,
+                    close: dailyData.Price,
+                    volume: null,
+                    fetched_at: fetchedAt
+                });
+                
+                // Also prepare Firestore update for backward compatibility
+                if (process.env.FIRESTORE_PRICE_FETCH !== 'false') {
+                    const shardId = `shard_${parseInt(instrumentIdStr, 10) % SHARD_SIZE}`;
+                    if (!shardUpdates[shardId]) { shardUpdates[shardId] = {}; }
+                    const pricePath = `${instrumentIdStr}.prices.${dateKey}`;
+                    const updatePath = `${instrumentIdStr}.lastUpdated`;
+                    shardUpdates[shardId][pricePath] = dailyData.Price;
+                    shardUpdates[shardId][updatePath] = FieldValue.serverTimestamp();
+                }
+            }
+        }
+        
+        // Write to BigQuery using load jobs (free, batched)
+        if (process.env.BIGQUERY_ENABLED !== 'false' && bigqueryRows.length > 0) {
+            try {
+                const { insertRows, ensureAssetPricesTable } = require('../../core/utils/bigquery_utils');
+                await ensureAssetPricesTable(logger);
+                
+                const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                await insertRows(datasetId, 'asset_prices', bigqueryRows, logger);
+                
+                logger.log('INFO', `[PriceFetcherHelpers] Successfully stored ${bigqueryRows.length} daily price records to BigQuery`);
+            } catch (bqError) {
+                logger.log('ERROR', `[PriceFetcherHelpers] BigQuery write failed: ${bqError.message}`);
+                // Continue - don't fail the entire fetch for BigQuery errors
+            }
+        }
+        
+        // Also write to Firestore for backward compatibility (if needed)
+        if (process.env.FIRESTORE_PRICE_FETCH !== 'false' && Object.keys(shardUpdates).length > 0) {
+            const batchPromises = [];
+            for (const shardId in shardUpdates) { 
+                const docRef = db.collection(priceCollectionName).doc(shardId); 
+                const payload = shardUpdates[shardId]; 
+                batchPromises.push(docRef.update(payload)); 
+            }
+            await Promise.all(batchPromises);
+            logger.log('INFO', `[PriceFetcherHelpers] Also stored prices to ${batchPromises.length} Firestore shards`);
+        }
         
         // Extract all dates from the price data and create a date tracking document
         const priceDatesSet = new Set();

package/functions/fetch-insights/helpers/handler_helpers.js

@@ -133,6 +133,32 @@ exports.fetchAndStoreInsights = async (config, dependencies) => {
 
         await docRef.set(firestorePayload);
         
+        // Write insights to BigQuery (one row per instrument)
+        if (process.env.BIGQUERY_ENABLED !== 'false') {
+            try {
+                const { insertRows, ensureInstrumentInsightsTable } = require('../../core/utils/bigquery_utils');
+                await ensureInstrumentInsightsTable(logger);
+                
+                const fetchedAt = new Date().toISOString();
+                const bigqueryRows = insightsData.map(insight => {
+                    return {
+                        date: today,
+                        instrument_id: parseInt(insight.instrumentId, 10),
+                        insights_data: insight, // Store full insight object as JSON
+                        fetched_at: fetchedAt
+                    };
+                });
+                
+                const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                await insertRows(datasetId, 'instrument_insights', bigqueryRows, logger);
+                
+                logger.log('INFO', `[FetchInsightsHelpers] Successfully stored ${bigqueryRows.length} insight records to BigQuery`);
+            } catch (bqError) {
+                logger.log('WARN', `[FetchInsightsHelpers] BigQuery insights write failed: ${bqError.message}`);
+                // Continue - Firestore write succeeded
+            }
+        }
+        
         // Update root data indexer for today's date after insights data is stored
         try {
             const { runRootDataIndexer } = require('../../root-data-indexer/index');

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -315,6 +315,35 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
 
             logger.log('SUCCESS', `[PopularInvestorFetch] Stored ${data.TotalRows} rankings into ${finalRankingsCollectionName}/${today}${firestorePayload._compressed ? ' (compressed)' : ''}`);
             
+            // Write rankings to BigQuery (one row per PI)
+            if (process.env.BIGQUERY_ENABLED !== 'false') {
+                try {
+                    const { insertRows, ensurePIRankingsTable } = require('../../core/utils/bigquery_utils');
+                    await ensurePIRankingsTable(logger);
+                    
+                    const fetchedAt = new Date().toISOString();
+                    const bigqueryRows = data.Items.map((item, index) => {
+                        return {
+                            date: today,
+                            pi_id: parseInt(item.CustomerId, 10),
+                            username: item.UserName || null,
+                            rank: index + 1, // Rank is position in array (1-indexed)
+                            category: item.Category || null,
+                            rankings_data: item, // Store full item data as JSON
+                            fetched_at: fetchedAt
+                        };
+                    });
+                    
+                    const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                    await insertRows(datasetId, 'pi_rankings', bigqueryRows, logger);
+                    
+                    logger.log('INFO', `[PopularInvestorFetch] Successfully stored ${bigqueryRows.length} ranking records to BigQuery`);
+                } catch (bqError) {
+                    logger.log('WARN', `[PopularInvestorFetch] BigQuery rankings write failed: ${bqError.message}`);
+                    // Continue - Firestore write succeeded
+                }
+            }
+            
             // Update the master list of Popular Investors
             // Use batched writes to avoid 500 field transform limit
             try {
@@ -426,6 +455,43 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
                 }
                 
                 logger.log('SUCCESS', `[PopularInvestorFetch] Updated master list: ${newInvestorsCount} new, ${updatedInvestorsCount} updated. Total unique PIs: ${Object.keys({ ...existingInvestors, ...investorsToUpdate }).length}`);
+                
+                // Write master list updates to BigQuery
+                if (process.env.BIGQUERY_ENABLED !== 'false' && Object.keys(investorsToUpdate).length > 0) {
+                    try {
+                        const { insertRowsWithMerge, ensurePIMasterListTable } = require('../../core/utils/bigquery_utils');
+                        await ensurePIMasterListTable(logger);
+                        
+                        const now = new Date().toISOString();
+                        const bigqueryRows = Object.entries(investorsToUpdate).map(([cid, investorData]) => {
+                            // Handle Firestore Timestamp objects
+                            const convertTimestamp = (ts) => {
+                                if (!ts) return now;
+                                if (ts instanceof Date) return ts.toISOString();
+                                if (ts.toDate && typeof ts.toDate === 'function') return ts.toDate().toISOString();
+                                if (typeof ts === 'string') return ts;
+                                return now;
+                            };
+                            
+                            return {
+                                cid: parseInt(cid, 10),
+                                username: investorData.username,
+                                first_seen_at: convertTimestamp(investorData.firstSeenAt),
+                                last_seen_at: convertTimestamp(investorData.lastSeenAt),
+                                last_updated: now
+                            };
+                        });
+                        
+                        const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                        // Use MERGE to update existing records or insert new ones
+                        await insertRowsWithMerge(datasetId, 'pi_master_list', bigqueryRows, ['cid'], logger);
+                        
+                        logger.log('INFO', `[PopularInvestorFetch] Successfully stored ${bigqueryRows.length} master list records to BigQuery`);
+                    } catch (bqError) {
+                        logger.log('WARN', `[PopularInvestorFetch] BigQuery master list write failed: ${bqError.message}`);
+                        // Continue - Firestore write succeeded
+                    }
+                }
             } catch (masterListError) {
                 logger.log('WARN', `[PopularInvestorFetch] Failed to update master list: ${masterListError.message}`);
                 // Non-critical, continue

package/functions/price-backfill/helpers/handler_helpers.js

@@ -1,14 +1,17 @@
 /**
  * @fileoverview Main pipe: pipe.maintenance.runBackfillAssetPrices
  * A one-time function to backfill historical price data from eToro's
- * candle API into the new sharded `asset_prices` collection.
+ * candle API into BigQuery `asset_prices` table.
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
 const pLimit = require('p-limit'); 
 const CONCURRENT_REQUESTS = 10;
 const DAYS_TO_FETCH = 365;
-const SHARD_SIZE = 40;
+const SHARD_SIZE = 40; // Legacy, kept for Firestore compatibility if needed
+
+// BigQuery batch size for load jobs (100MB limit, but we batch smaller for efficiency)
+const BIGQUERY_BATCH_SIZE = 10000; // rows per batch
 
 /**
  * Main pipe: pipe.maintenance.runBackfillAssetPrices
@@ -42,14 +45,60 @@ exports.runBackfillAssetPrices = async (config, dependencies) => {
                 headerManager.updatePerformance(selectedHeader.id, wasSuccess);
                 const data = await response.json();
                 const candles = data?.Candles?.[0]?.Candles;
-                if (!Array.isArray(candles) || candles.length === 0) { logger.log('WARN', `[PriceBackfill] No candle data returned for ${ticker} (${instrumentId})`); return; }
-                const prices = {};
-                for (const candle of candles) { const dateKey = candle.FromDate.substring(0, 10); prices[dateKey] = candle.Close; }
-                const shardId = `shard_${parseInt(instrumentId, 10) % SHARD_SIZE}`;
-                const docRef = db.collection('asset_prices').doc(shardId); 
-                const payload = { [instrumentId]: { ticker: ticker, prices: prices, lastUpdated: FieldValue.serverTimestamp() } };
-                await docRef.set(payload, { merge: true });
-                logger.log('TRACE', `[PriceBackfill] Successfully stored data for ${ticker} (${instrumentId}) in ${shardId}`);
+                if (!Array.isArray(candles) || candles.length === 0) { 
+                    logger.log('WARN', `[PriceBackfill] No candle data returned for ${ticker} (${instrumentId})`); 
+                    return; 
+                }
+                
+                // Transform candles to BigQuery rows
+                const fetchedAt = new Date().toISOString();
+                const bigqueryRows = candles.map(candle => {
+                    const dateStr = candle.FromDate.substring(0, 10); // Extract YYYY-MM-DD
+                    return {
+                        date: dateStr,
+                        instrument_id: parseInt(instrumentId, 10),
+                        ticker: ticker,
+                        price: candle.Close || null,
+                        open: candle.Open || null,
+                        high: candle.High || null,
+                        low: candle.Low || null,
+                        close: candle.Close || null,
+                        volume: candle.Volume || null,
+                        fetched_at: fetchedAt
+                    };
+                });
+                
+                // Write to BigQuery using load jobs (free, batched)
+                if (process.env.BIGQUERY_ENABLED !== 'false') {
+                    try {
+                        const { insertRows, ensureAssetPricesTable } = require('../../core/utils/bigquery_utils');
+                        await ensureAssetPricesTable(logger);
+                        
+                        const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+                        await insertRows(datasetId, 'asset_prices', bigqueryRows, logger);
+                        
+                        logger.log('TRACE', `[PriceBackfill] Successfully stored ${bigqueryRows.length} price records for ${ticker} (${instrumentId}) to BigQuery`);
+                    } catch (bqError) {
+                        logger.log('ERROR', `[PriceBackfill] BigQuery write failed for ${ticker} (${instrumentId}): ${bqError.message}`);
+                        // Continue - don't fail the entire backfill for one instrument
+                    }
+                }
+                
+                // Also write to Firestore for backward compatibility (if needed)
+                // This can be removed once all readers are migrated to BigQuery
+                if (process.env.FIRESTORE_PRICE_BACKFILL !== 'false') {
+                    const prices = {};
+                    for (const candle of candles) { 
+                        const dateKey = candle.FromDate.substring(0, 10); 
+                        prices[dateKey] = candle.Close; 
+                    }
+                    const shardId = `shard_${parseInt(instrumentId, 10) % SHARD_SIZE}`;
+                    const docRef = db.collection('asset_prices').doc(shardId); 
+                    const payload = { [instrumentId]: { ticker: ticker, prices: prices, lastUpdated: FieldValue.serverTimestamp() } };
+                    await docRef.set(payload, { merge: true });
+                    logger.log('TRACE', `[PriceBackfill] Also stored data for ${ticker} (${instrumentId}) in Firestore ${shardId}`);
+                }
+                
                 successCount++;
             } catch (err) {
                 logger.log('ERROR', `[PriceBackfill] Failed to process instrument ${instrumentId}`, { err: err.message });

package/functions/root-data-indexer/index.js

@@ -245,6 +245,56 @@ exports.runRootDataIndexer = async (config, dependencies) => {
 
             // --- Define Refs & Check Paths ---
             
+            // =========================================================================
+            // BIGQUERY FIRST: Check BigQuery tables for data availability
+            // =========================================================================
+            let bigqueryHasPortfolio = false;
+            let bigqueryHasHistory = false;
+            let bigqueryHasSocial = false;
+            let bigqueryHasInsights = false;
+            let bigqueryHasPrices = false;
+            let bigqueryHasPIRankings = false;
+            let bigqueryHasPIMasterList = false;
+            
+            if (process.env.BIGQUERY_ENABLED !== 'false') {
+                try {
+                    const { 
+                        queryPortfolioData, 
+                        queryHistoryData, 
+                        querySocialData,
+                        queryInstrumentInsights,
+                        queryAssetPrices,
+                        queryPIRankings,
+                        queryPIMasterList
+                    } = require('../../core/utils/bigquery_utils');
+                    
+                    // Check BigQuery for today's data (or target date)
+                    const [portfolioData, historyData, socialData, insightsData, pricesData, rankingsData, masterListData] = await Promise.all([
+                        queryPortfolioData(dateStr, null, null, logger).catch(() => null),
+                        queryHistoryData(dateStr, null, null, logger).catch(() => null),
+                        querySocialData(dateStr, null, null, logger).catch(() => null),
+                        queryInstrumentInsights(dateStr, logger).catch(() => null),
+                        queryAssetPrices(dateStr, dateStr, null, logger).catch(() => null), // Check for specific date
+                        queryPIRankings(dateStr, logger).catch(() => null),
+                        queryPIMasterList(logger).catch(() => null)
+                    ]);
+                    
+                    bigqueryHasPortfolio = portfolioData && Object.keys(portfolioData).length > 0;
+                    bigqueryHasHistory = historyData && Object.keys(historyData).length > 0;
+                    bigqueryHasSocial = socialData && Object.keys(socialData).length > 0;
+                    bigqueryHasInsights = insightsData && Array.isArray(insightsData) && insightsData.length > 0;
+                    bigqueryHasPrices = pricesData && Object.keys(pricesData).length > 0;
+                    bigqueryHasPIRankings = rankingsData && Array.isArray(rankingsData) && rankingsData.length > 0;
+                    bigqueryHasPIMasterList = masterListData && Object.keys(masterListData).length > 0;
+                    
+                    if (bigqueryHasPortfolio || bigqueryHasHistory || bigqueryHasSocial || bigqueryHasInsights || bigqueryHasPrices || bigqueryHasPIRankings) {
+                        logger.log('INFO', `[RootDataIndexer/${dateStr}] ✅ Found data in BigQuery: portfolio=${bigqueryHasPortfolio}, history=${bigqueryHasHistory}, social=${bigqueryHasSocial}, insights=${bigqueryHasInsights}, prices=${bigqueryHasPrices}, rankings=${bigqueryHasPIRankings}`);
+                    }
+                } catch (bqError) {
+                    logger.log('WARN', `[RootDataIndexer/${dateStr}] BigQuery check failed, using Firestore fallback: ${bqError.message}`);
+                }
+            }
+            
             // 1. Standard Retail
             // Path: {normalPortfolios}/19M/snapshots/{YYYY-MM-DD}/parts/part_*
             const normPortPartsRef = db.collection(safeCollections.normalPortfolios).doc(CANARY_BLOCK_ID).collection('snapshots').doc(dateStr).collection('parts');
@@ -435,25 +485,29 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             logger.log('INFO', `[RootDataIndexer/${dateStr}] Social check results - PI: ${foundPISocial}, Signed-in: ${foundSignedInSocial}`);
 
             // --- Assign to Availability ---
-            availability.details.normalPortfolio     = normPortExists;
-            availability.details.speculatorPortfolio = specPortExists;
-            availability.details.normalHistory       = normHistExists;
-            availability.details.speculatorHistory   = specHistExists;
-            availability.details.piRankings          = piRankingsSnap.exists;
+            // Use BigQuery results if available, otherwise fall back to Firestore
+            availability.details.normalPortfolio     = bigqueryHasPortfolio || normPortExists;
+            availability.details.speculatorPortfolio = bigqueryHasPortfolio || specPortExists;
+            availability.details.normalHistory       = bigqueryHasHistory || normHistExists;
+            availability.details.speculatorHistory   = bigqueryHasHistory || specHistExists;
+            availability.details.piRankings          = bigqueryHasPIRankings || piRankingsSnap.exists;
             
-            availability.details.piPortfolios        = piPortExists; 
-            availability.details.piDeepPortfolios    = piDeepExists; 
-            availability.details.piHistory           = piHistExists;
+            availability.details.piPortfolios        = bigqueryHasPortfolio || piPortExists; 
+            availability.details.piDeepPortfolios    = piDeepExists; // Legacy only, no BigQuery equivalent
+            availability.details.piHistory           = bigqueryHasHistory || piHistExists;
             
             // PI & Signed-In Social Flags (Strict)
-            availability.details.piSocial            = foundPISocial;
-            availability.details.hasPISocial         = foundPISocial; 
-            availability.details.signedInSocial      = foundSignedInSocial;
-            availability.details.hasSignedInSocial   = foundSignedInSocial; 
-
+            // Use BigQuery if available, otherwise use Firestore tracking documents
+            const finalPISocial = bigqueryHasSocial || foundPISocial;
+            const finalSignedInSocial = bigqueryHasSocial || foundSignedInSocial;
+            availability.details.piSocial            = finalPISocial;
+            availability.details.hasPISocial         = finalPISocial; 
+            availability.details.signedInSocial      = finalSignedInSocial;
+            availability.details.hasSignedInSocial   = finalSignedInSocial; 
+            
             // Signed-In Flags
-            availability.details.signedInUserPortfolio = signedInPortExists;
-            availability.details.signedInUserHistory   = signedInHistExists;
+            availability.details.signedInUserPortfolio = bigqueryHasPortfolio || signedInPortExists;
+            availability.details.signedInUserHistory   = bigqueryHasHistory || signedInHistExists;
             availability.details.signedInUserVerification = !verificationsQuery.empty;
             
             // New Root Data Types for Profile Metrics
@@ -461,20 +515,18 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             availability.details.piPageViews = piPageViewsSnap.exists;
             availability.details.watchlistMembership = watchlistMembershipSnap.exists;
             availability.details.piAlertHistory = piAlertHistorySnap.exists;
+            
+            // Aggregates (use BigQuery if available, otherwise Firestore)
+            availability.hasPortfolio = bigqueryHasPortfolio || normPortExists || specPortExists || piPortExists || signedInPortExists;
+            availability.hasHistory   = bigqueryHasHistory || normHistExists || specHistExists || piHistExists || signedInHistExists;
+            availability.hasInsights  = bigqueryHasInsights || insightsSnap.exists;
+            availability.hasSocial    = bigqueryHasSocial || finalPISocial || finalSignedInSocial || genericSocialExists;
 
-            // Aggregates
-            availability.hasPortfolio = normPortExists || specPortExists || piPortExists || signedInPortExists;
-            availability.hasHistory   = normHistExists || specHistExists || piHistExists || signedInHistExists;
-            availability.hasInsights  = insightsSnap.exists;
-            availability.hasSocial    = foundPISocial || foundSignedInSocial || genericSocialExists; 
-
-            // [NEW] Assign Master List Availability
-            availability.details.piMasterList = piMasterListSnap.exists;
+            // [NEW] Assign Master List Availability (BigQuery first, then Firestore)
+            availability.details.piMasterList = bigqueryHasPIMasterList || piMasterListSnap.exists;
             
-            // Price Check
-            // Check if the target date exists in the price availability set
-            // The set is populated from the price tracking document's datesAvailable array
-            const hasPriceForDate = priceAvailabilitySet.has(dateStr);
+            // Price Check (BigQuery first, then Firestore tracking document)
+            const hasPriceForDate = bigqueryHasPrices || priceAvailabilitySet.has(dateStr);
             availability.hasPrices = hasPriceForDate;
             
             if (targetDate && !hasPriceForDate) {