bulltrackers-module 1.0.710 → 1.0.713

package/functions/computation-system/utils/data_loader.js

@@ -68,7 +68,42 @@ async function getPortfolioPartRefs(config, deps, dateString, requiredUserTypes
     const types = requiredUserTypes ? new Set(requiredUserTypes.map(t => t.toUpperCase())) : null;
     const fetchAll = !types || types.size === 0 || types.has('ALL');
 
-    logger.log('INFO', `Getting portfolio part references for date: ${dateString}. Filter: ${fetchAll ? 'ALL' : Array.from(types).join(',')}`);
+    // =========================================================================
+    // BIGQUERY FIRST: Try BigQuery before Firestore
+    // =========================================================================
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryPortfolioData } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryPortfolioData(dateString, null, requiredUserTypes, logger);
+            
+            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for portfolio data (${dateString}): ${Object.keys(bigqueryData).length} users`);
+                
+                // Transform BigQuery data into "ref-like" objects for compatibility
+                const bigqueryRefs = Object.keys(bigqueryData).map(userId => {
+                    const dataObj = bigqueryData[userId];
+                    return {
+                        ref: null, // No Firestore ref needed
+                        type: dataObj.user_type || 'UNKNOWN',
+                        cid: userId,
+                        collectionType: 'BIGQUERY',
+                        bigqueryData: dataObj.portfolio_data || {} // Extract the actual portfolio data
+                    };
+                });
+                
+                return bigqueryRefs;
+            } else {
+                logger.log('INFO', `[DataLoader] ⚠️ BigQuery returned no portfolio data for ${dateString}, falling back to Firestore`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery portfolio query failed for ${dateString}, falling back to Firestore: ${bqError.message}`);
+        }
+    }
+    
+    // =========================================================================
+    // FIRESTORE FALLBACK: Original logic (backwards compatibility)
+    // =========================================================================
+    logger.log('INFO', `[DataLoader] 📂 Using Firestore for portfolio data (${dateString}). Filter: ${fetchAll ? 'ALL' : Array.from(types || []).join(',')}`);
     
     const allPartRefs = [];
     
@@ -143,12 +178,62 @@ async function loadDataByRefs(config, deps, refObjects) {
     
     if (!refObjects || !refObjects.length) return {};
     
+    // =========================================================================
+    // CHECK FOR PRICE DATA (BigQuery marker)
+    // =========================================================================
+    const priceBigQueryMarker = refObjects.find(r => r._bigquery === true);
+    if (priceBigQueryMarker) {
+        // This is a price data load request - use BigQuery
+        try {
+            const { queryAssetPrices } = require('../../core/utils/bigquery_utils');
+            const priceData = await queryAssetPrices(null, null, null, logger);
+            
+            if (priceData && Object.keys(priceData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Loaded ${Object.keys(priceData).length} instruments from BigQuery for price data`);
+                return priceData;
+            }
+            
+            // If BigQuery returns empty, fallback to Firestore
+            logger.log('WARN', `[DataLoader] BigQuery returned no price data, falling back to Firestore`);
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery price load failed, falling back to Firestore: ${bqError.message}`);
+            // Fall through to Firestore
+        }
+    }
+    
+    // =========================================================================
+    // SEPARATE BIGQUERY AND FIRESTORE REFS (for portfolio/history data)
+    // =========================================================================
+    const bigqueryRefs = refObjects.filter(r => r.collectionType === 'BIGQUERY');
+    const firestoreRefs = refObjects.filter(r => r.collectionType !== 'BIGQUERY' && !r._bigquery);
+    
     const mergedPortfolios = {};
+    
+    // Load from BigQuery cache (data already fetched in getPortfolioPartRefs/getHistoryPartRefs)
+    if (bigqueryRefs.length > 0) {
+        logger.log('INFO', `[DataLoader] 📊 Loading ${bigqueryRefs.length} records from BigQuery cache`);
+        bigqueryRefs.forEach(ref => {
+            if (ref.bigqueryData) {
+                // Extract the actual data (portfolio_data or history_data)
+                // The bigqueryData is the JSON object from BigQuery
+                mergedPortfolios[ref.cid] = ref.bigqueryData;
+            }
+        });
+    }
+    
+    // Load from Firestore (existing logic)
+    if (firestoreRefs.length === 0) {
+        return mergedPortfolios; // All data came from BigQuery
+    }
+    
+    logger.log('INFO', `[DataLoader] 📂 Loading ${firestoreRefs.length} records from Firestore`);
     const batchSize = config.partRefBatchSize || 10;
 
-    for (let i = 0; i < refObjects.length; i += batchSize) { 
-        const batch = refObjects.slice(i, i + batchSize); 
-        const refs = batch.map(b => b.ref);
+    for (let i = 0; i < firestoreRefs.length; i += batchSize) {
+        const batch = firestoreRefs.slice(i, i + batchSize);
+        const refs = batch.map(b => b.ref).filter(ref => ref !== null); // Filter out null refs (BigQuery refs)
+        
+        if (refs.length === 0) continue; // Skip if all refs are BigQuery refs
         
         const snapshots = await withRetry(() => db.getAll(...refs), `getAll(batch ${Math.floor(i/batchSize)})`);
         
@@ -250,9 +335,26 @@ async function loadDailyInsights(config, deps, dateString) {
     const cached = await tryLoadFromGCS(config, dateString, 'insights', logger);
     if (cached) return cached;
 
-    // 2. FIRESTORE FALLBACK
+    // 2. BIGQUERY FIRST (if enabled)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryInstrumentInsights } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryInstrumentInsights(dateString, logger);
+            
+            if (bigqueryData && Array.isArray(bigqueryData) && bigqueryData.length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for instrument insights (${dateString}): ${bigqueryData.length} instruments`);
+                // Return in same format as Firestore: { insights: [...] }
+                return { insights: bigqueryData };
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery insights query failed, falling back to Firestore: ${bqError.message}`);
+            // Fall through to Firestore
+        }
+    }
+
+    // 3. FIRESTORE FALLBACK
     const insightsCollectionName = config.insightsCollectionName || 'daily_instrument_insights';
-    logger.log('INFO', `Loading daily insights for ${dateString} from ${insightsCollectionName}`);
+    logger.log('INFO', `Loading daily insights for ${dateString} from ${insightsCollectionName} (Firestore)`);
     try { 
         const docRef = db.collection(insightsCollectionName).doc(dateString);
         const docSnap = await withRetry(() => docRef.get(), `getInsights(${dateString})`);
@@ -379,7 +481,42 @@ async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes =
     const types = requiredUserTypes ? new Set(requiredUserTypes.map(t => t.toUpperCase())) : null;
     const fetchAll = !types || types.size === 0 || types.has('ALL');
 
-    logger.log('INFO', `Getting history part references for ${dateString}. Filter: ${fetchAll ? 'ALL' : Array.from(types).join(',')}`);
+    // =========================================================================
+    // BIGQUERY FIRST: Try BigQuery before Firestore
+    // =========================================================================
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryHistoryData } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryHistoryData(dateString, null, requiredUserTypes, logger);
+            
+            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for trade history data (${dateString}): ${Object.keys(bigqueryData).length} users`);
+                
+                // Transform BigQuery data into "ref-like" objects for compatibility
+                const bigqueryRefs = Object.keys(bigqueryData).map(userId => {
+                    const dataObj = bigqueryData[userId];
+                    return {
+                        ref: null, // No Firestore ref needed
+                        type: dataObj.user_type || 'UNKNOWN',
+                        cid: userId,
+                        collectionType: 'BIGQUERY',
+                        bigqueryData: dataObj.history_data || {} // Extract the actual history data
+                    };
+                });
+                
+                return bigqueryRefs;
+            } else {
+                logger.log('INFO', `[DataLoader] ⚠️ BigQuery returned no history data for ${dateString}, falling back to Firestore`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery history query failed for ${dateString}, falling back to Firestore: ${bqError.message}`);
+        }
+    }
+    
+    // =========================================================================
+    // FIRESTORE FALLBACK: Original logic (backwards compatibility)
+    // =========================================================================
+    logger.log('INFO', `[DataLoader] 📂 Using Firestore for trade history data (${dateString}). Filter: ${fetchAll ? 'ALL' : Array.from(types || []).join(',')}`);
     
     const allPartRefs = [];
     
@@ -447,6 +584,31 @@ async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes =
 async function* streamPortfolioData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
     const { logger } = deps;
 
+    // =========================================================================
+    // BIGQUERY FIRST: Try BigQuery before GCS/Firestore
+    // =========================================================================
+    if (!providedRefs && process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryPortfolioData } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryPortfolioData(dateString, null, requiredUserTypes, logger);
+            
+            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Streaming portfolio data from BigQuery (${dateString}): ${Object.keys(bigqueryData).length} users`);
+                
+                // Transform to expected format and yield
+                const transformedData = {};
+                Object.keys(bigqueryData).forEach(userId => {
+                    transformedData[userId] = bigqueryData[userId].portfolio_data || {};
+                });
+                
+                yield transformedData;
+                return; // Exit early, data loaded from BigQuery
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery portfolio stream failed for ${dateString}, falling back: ${bqError.message}`);
+        }
+    }
+
     // 1. GCS FAST PATH (Snapshot) - Only if full run (no providedRefs)
     if (!providedRefs) { 
         try {
@@ -498,6 +660,31 @@ async function* streamPortfolioData(config, deps, dateString, providedRefs = nul
 async function* streamHistoryData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
     const { logger } = deps;
 
+    // =========================================================================
+    // BIGQUERY FIRST: Try BigQuery before Firestore
+    // =========================================================================
+    if (!providedRefs && process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryHistoryData } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryHistoryData(dateString, null, requiredUserTypes, logger);
+            
+            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Streaming history data from BigQuery (${dateString}): ${Object.keys(bigqueryData).length} users`);
+                
+                // Transform to expected format and yield
+                const transformedData = {};
+                Object.keys(bigqueryData).forEach(userId => {
+                    transformedData[userId] = bigqueryData[userId].history_data || {};
+                });
+                
+                yield transformedData;
+                return; // Exit early, data loaded from BigQuery
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery history stream failed for ${dateString}, falling back: ${bqError.message}`);
+        }
+    }
+
     // 1. GCS FAST PATH (JSONL Streaming) - Only if full run
     if (!providedRefs) {
         try {
@@ -555,9 +742,21 @@ async function* streamHistoryData(config, deps, dateString, providedRefs = null,
     logger.log('INFO', `[streamHistoryData] Finished streaming for ${dateString}.`);
 }
 
-/** Stage 9: Get all price shard references (Basic) */
+/** Stage 9: Get all price shard references (Basic) 
+ * [UPDATED] Returns special marker for BigQuery mode, or Firestore refs for fallback
+ */
 async function getPriceShardRefs(config, deps) {
-    const { db, logger, calculationUtils } = deps;
+    const { logger } = deps;
+    
+    // Try BigQuery first if enabled
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        // Return a special marker object to indicate BigQuery mode
+        // The loader will detect this and load from BigQuery instead
+        return [{ _bigquery: true }];
+    }
+    
+    // Fallback to Firestore
+    const { db, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     const collection = config.priceCollection || 'asset_prices';
     try {
@@ -590,9 +789,25 @@ async function loadPopularInvestorRankings(config, deps, dateString) {
     const cached = await tryLoadFromGCS(config, dateString, 'rankings', logger);
     if (cached) return cached;
 
-    // 2. FIRESTORE FALLBACK
+    // 2. BIGQUERY FIRST (if enabled)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryPIRankings } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryPIRankings(dateString, logger);
+            
+            if (bigqueryData && bigqueryData.Items && bigqueryData.Items.length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for PI rankings (${dateString}): ${bigqueryData.Items.length} items`);
+                return bigqueryData.Items;
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery rankings query failed, falling back to Firestore: ${bqError.message}`);
+            // Fall through to Firestore
+        }
+    }
+
+    // 3. FIRESTORE FALLBACK
     const collectionName = config.popularInvestorRankingsCollection || 'popular_investor_rankings';
-    logger.log('INFO', `Loading Popular Investor Rankings for ${dateString}`);
+    logger.log('INFO', `Loading Popular Investor Rankings for ${dateString} from Firestore`);
     try {
         const docRef = db.collection(collectionName).doc(dateString);
         const docSnap = await withRetry(() => docRef.get(), `getRankings(${dateString})`);
@@ -794,10 +1009,26 @@ async function loadPopularInvestorMasterList(config, deps, dateString = null) {
         if (cached) return cached;
     }
 
-    // 2. FIRESTORE FALLBACK
+    // 2. BIGQUERY FIRST (if enabled)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryPIMasterList } = require('../../core/utils/bigquery_utils');
+            const bigqueryData = await queryPIMasterList(logger);
+            
+            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for PI master list: ${Object.keys(bigqueryData).length} PIs`);
+                return bigqueryData;
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DataLoader] BigQuery master list query failed, falling back to Firestore: ${bqError.message}`);
+            // Fall through to Firestore
+        }
+    }
+
+    // 3. FIRESTORE FALLBACK
     const collectionName = config.piMasterListCollection || 'system_state';
     const docId = config.piMasterListDocId || 'popular_investor_master_list';
-    logger.log('INFO', `Loading Popular Investor Master List from ${collectionName}/${docId}`);
+    logger.log('INFO', `Loading Popular Investor Master List from ${collectionName}/${docId} (Firestore)`);
     try {
         const docRef = db.collection(collectionName).doc(docId);
         const docSnap = await withRetry(() => docRef.get(), 'getPIMasterList');