bulltrackers-module 1.0.731 → 1.0.733

package/functions/computation-system/utils/data_loader.js

@@ -1,597 +0,0 @@
-/**
- * @fileoverview Data Loading Layer.
- * REFACTORED:
- * 1. Routes 90% of data fetching to BigQuery (Portfolios, History, Prices, Insights, Rankings, etc.).
- * 2. Retains Firestore logic ONLY for:
- * - Verifications (user_verifications)
- * - Retail Users (Normal/Speculator Portfolios/History)
- * - Generic Social Feed (Legacy compatibility)
- */
-const { 
-    queryPortfolioData, 
-    queryHistoryData, 
-    querySocialData, 
-    queryAssetPrices,
-    queryAllPricesForDate,
-    queryPricesForTickers,
-    queryInstrumentInsights, 
-    queryPIRankings,
-    queryTickerMappings,
-    queryPIMasterList,
-    queryPIRatings,
-    queryPIPageViews,
-    queryWatchlistMembership,
-    queryPIAlertHistory
-} = require('../../core/utils/bigquery_utils');
-
-const { normalizeName } = require('./utils');
-
-// =============================================================================
-// 1. PORTFOLIOS
-// =============================================================================
-exports.loadDailyPortfolios = async (config, deps, dateStr, userTypes = []) => {
-    const { db, logger } = deps;
-    
-    // Normalize user types
-    const types = Array.isArray(userTypes) ? userTypes : [userTypes];
-    const isRetail = types.some(t => ['NORMAL', 'SPECULATOR'].includes(t.toUpperCase()));
-    const isMigrated = types.some(t => ['POPULAR_INVESTOR', 'SIGNED_IN_USER'].includes(t.toUpperCase()));
-
-    let results = {};
-
-    // A. BigQuery (PIs & SignedIn)
-    if (isMigrated && process.env.BIGQUERY_ENABLED !== 'false') {
-        const bqData = await queryPortfolioData(dateStr, null, types, logger);
-        if (bqData) Object.assign(results, bqData);
-    }
-
-    // B. Firestore (Retail / Fallback)
-    // Note: If we need Retail data, we MUST check Firestore as it wasn't migrated.
-    if (isRetail) {
-        if (types.includes('NORMAL')) {
-            const normalData = await loadRetailFirestore(db, 'NormalUserPortfolios', dateStr);
-            Object.assign(results, normalData);
-        }
-        if (types.includes('SPECULATOR')) {
-            const specData = await loadRetailFirestore(db, 'SpeculatorPortfolios', dateStr);
-            Object.assign(results, specData);
-        }
-    }
-
-    return results;
-};
-
-// =============================================================================
-// 2. TRADE HISTORY
-// =============================================================================
-exports.loadDailyHistory = async (config, deps, dateStr, userTypes = []) => {
-    const { db, logger } = deps;
-    const types = Array.isArray(userTypes) ? userTypes : [userTypes];
-    const isRetail = types.some(t => ['NORMAL', 'SPECULATOR'].includes(t.toUpperCase()));
-    const isMigrated = types.some(t => ['POPULAR_INVESTOR', 'SIGNED_IN_USER'].includes(t.toUpperCase()));
-
-    let results = {};
-
-    if (isMigrated && process.env.BIGQUERY_ENABLED !== 'false') {
-        const bqData = await queryHistoryData(dateStr, null, types, logger);
-        if (bqData) Object.assign(results, bqData);
-    }
-
-    if (isRetail) {
-        if (types.includes('NORMAL')) {
-            const normalData = await loadRetailFirestore(db, 'NormalUserTradeHistory', dateStr);
-            Object.assign(results, normalData);
-        }
-        if (types.includes('SPECULATOR')) {
-            const specData = await loadRetailFirestore(db, 'SpeculatorTradeHistory', dateStr);
-            Object.assign(results, specData);
-        }
-    }
-    return results;
-};
-
-// =============================================================================
-// 3. SOCIAL
-// =============================================================================
-exports.loadDailySocialPostInsights = async (config, deps, dateStr, userTypes = []) => {
-    const { db, logger } = deps;
-    const types = Array.isArray(userTypes) ? userTypes : (userTypes ? [userTypes] : []);
-
-    // A. BigQuery (User-Specific Social)
-    if (types.length > 0 && process.env.BIGQUERY_ENABLED !== 'false') {
-        return querySocialData(dateStr, null, types, logger);
-    }
-
-    // B. Firestore (Generic Feed - Legacy)
-    // If no user types specified, assume generic feed fetch
-    const collection = config.socialInsightsCollection || 'daily_social_insights';
-    try {
-        const snap = await db.collection(collection).doc(dateStr).collection('posts').get();
-        if (snap.empty) return {};
-        const data = {};
-        snap.forEach(doc => data[doc.id] = doc.data());
-        return data;
-    } catch (e) {
-        logger.log('WARN', `[DataLoader] Failed to load generic social for ${dateStr}: ${e.message}`);
-        return {};
-    }
-};
-
-// =============================================================================
-// 4. MARKET DATA (Prices)
-// =============================================================================
-exports.getPriceShardRefs = async (config, deps) => {
-    // Legacy Shard Helper - In BQ world, we don't use shards but CachedDataLoader expects this structure.
-    // We return a "virtual" shard array that signals CachedDataLoader to load from BQ.
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        return [ { _bigquery: true } ];
-    }
-    // Fallback to Firestore Logic - return array of doc refs
-    const { db } = deps;
-    const collection = config.assetPricesCollection || 'asset_prices';
-    const snapshot = await db.collection(collection).listDocuments();
-    const refs = [];
-    snapshot.forEach(doc => refs.push(doc));
-    return refs;
-};
-
-exports.getRelevantShardRefs = async (config, deps, targetIds) => {
-    // In BQ mode, we don't shard by instrument; return single virtual shard
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        return [ { _bigquery: true, targetIds: targetIds || [] } ];
-    }
-    // Firestore behavior - return array of doc refs (same as getPriceShardRefs for now)
-    return exports.getPriceShardRefs(config, deps);
-};
-
-// =============================================================================
-// 5. ROOT DATA TYPES (Simple Mappings)
-// =============================================================================
-
-exports.loadDailyInsights = async (config, deps, dateStr) => {
-    const { logger } = deps;
-
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const rows = await queryInstrumentInsights(dateStr, logger);
-            if (Array.isArray(rows) && rows.length > 0) {
-                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for instrument insights (${dateStr}): ${rows.length} instruments`);
-                // Wrap in Firestore-shaped document format for InsightsExtractor compatibility
-                return { insights: rows };
-            }
-        } catch (e) {
-            logger.log('WARN', `[DataLoader] BigQuery insights query failed for ${dateStr}: ${e.message}`);
-        }
-    }
-
-    // No Firestore fallback by design – return empty but correctly shaped
-    return { insights: [] };
-};
-
-exports.loadPopularInvestorRankings = async (config, deps, dateStr) => {
-    const data = await queryPIRankings(dateStr, deps.logger);
-    return data ? data.Items : [];
-};
-
-exports.loadPIRatings = async (config, deps, dateStr) => {
-    return queryPIRatings(dateStr, deps.logger);
-};
-
-exports.loadPIPageViews = async (config, deps, dateStr) => {
-    return queryPIPageViews(dateStr, deps.logger);
-};
-
-exports.loadWatchlistMembership = async (config, deps, dateStr) => {
-    return queryWatchlistMembership(dateStr, deps.logger);
-};
-
-exports.loadPIAlertHistory = async (config, deps, dateStr) => {
-    return queryPIAlertHistory(dateStr, deps.logger);
-};
-
-exports.loadPopularInvestorMasterList = async (config, deps) => {
-    return queryPIMasterList(deps.logger);
-};
-
-exports.loadPIWatchlistData = async (config, deps, piCid) => {
-    // Watchlist data is time-series in BQ. For "Current State" (ID based),
-    // we query the most recent date available for this PI.
-    // This is a specialized query not in standard utils, so we implement it here or assume caller passes date.
-    // However, CachedDataLoader expects (cid) -> Data.
-    // We'll return null here as WatchlistMembership (by date) is the preferred method now.
-    deps.logger.log('WARN', '[DataLoader] loadPIWatchlistData (by CID) is deprecated in favor of loadWatchlistMembership (by Date).');
-    return null;
-};
-
-// =============================================================================
-// 6. EXCEPTIONS (Firestore Only)
-// =============================================================================
-
-exports.loadVerificationProfiles = async (config, deps, dateStr) => {
-    const { db, logger } = deps;
-    try {
-        // Verifications are a single collection, not date-partitioned snapshots
-        const snap = await db.collection('user_verifications').get();
-        const verifications = {};
-        snap.forEach(doc => verifications[doc.id] = doc.data());
-        return verifications;
-    } catch (e) {
-        logger.log('ERROR', `[DataLoader] Failed to load verifications: ${e.message}`);
-        return {};
-    }
-};
-
-// =============================================================================
-// HELPERS
-// =============================================================================
-
-// =============================================================================
-// 7. PRICE DATA BY REFS (For PriceBatchExecutor)
-// =============================================================================
-
-/**
- * Load price data from an array of shard references (virtual or Firestore doc refs).
- * Used by PriceBatchExecutor for batch price computations.
- * @param {object} config - Configuration object
- * @param {object} deps - Dependencies (db, logger, etc.)
- * @param {Array} shardRefs - Array of shard references (virtual BigQuery objects or Firestore doc refs)
- * @returns {Promise<object>} Combined price data object keyed by instrument ID
- */
-exports.loadDataByRefs = async (config, deps, shardRefs) => {
-    const { logger } = deps;
-    
-    if (!Array.isArray(shardRefs) || shardRefs.length === 0) {
-        return {};
-    }
-
-    // Check if we're in BigQuery mode (virtual shards)
-    const isBigQuery = shardRefs.some(ref => ref && ref._bigquery === true);
-    
-    if (isBigQuery && process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            // Extract targetIds from virtual shards if present
-            const targetIds = shardRefs
-                .filter(ref => ref._bigquery && ref.targetIds && ref.targetIds.length > 0)
-                .flatMap(ref => ref.targetIds);
-            
-            // Query BigQuery for prices
-            // queryAssetPrices signature: (startDateStr, endDateStr, instrumentIds, logger)
-            const { queryAssetPrices } = require('../../core/utils/bigquery_utils');
-            const pricesData = await queryAssetPrices(null, null, targetIds.length > 0 ? targetIds : null, logger);
-            
-            // Filter by targetIds if specified
-            if (targetIds.length > 0 && pricesData) {
-                const targetSet = new Set(targetIds.map(id => String(id)));
-                const filtered = {};
-                for (const [instrumentId, priceData] of Object.entries(pricesData)) {
-                    if (targetSet.has(String(instrumentId))) {
-                        filtered[instrumentId] = priceData;
-                    }
-                }
-                return filtered;
-            }
-            
-            return pricesData || {};
-        } catch (e) {
-            logger.log('ERROR', `[DataLoader] BigQuery price load failed: ${e.message}`);
-            return {};
-        }
-    }
-
-    // Firestore fallback - load from doc refs
-    const combined = {};
-    try {
-        const loadPromises = shardRefs.map(async (docRef) => {
-            try {
-                const snap = await docRef.get();
-                if (snap.exists) {
-                    const data = snap.data();
-                    // Firestore price shards are nested: { instrumentId: { prices: {...} } }
-                    Object.assign(combined, data);
-                }
-            } catch (e) {
-                logger.log('WARN', `[DataLoader] Failed to load price shard: ${e.message}`);
-            }
-        });
-        
-        await Promise.all(loadPromises);
-    } catch (e) {
-        logger.log('ERROR', `[DataLoader] Failed to load price data from refs: ${e.message}`);
-    }
-    
-    return combined;
-};
-
-// =============================================================================
-// HELPERS
-// =============================================================================
-
-async function loadRetailFirestore(db, collectionName, dateStr) {
-    const CANARY_ID = '19M'; // Legacy Block
-    try {
-        const partsRef = db.collection(collectionName).doc(CANARY_ID)
-            .collection('snapshots').doc(dateStr).collection('parts');
-        
-        const snap = await partsRef.get();
-        if (snap.empty) return {};
-        
-        const combined = {};
-        snap.forEach(doc => Object.assign(combined, doc.data()));
-        return combined;
-    } catch (e) {
-        return {};
-    }
-}
-
-// =============================================================================
-// 8. STREAMING DATA (For StandardExecutor)
-// =============================================================================
-
-/**
- * Get portfolio part references for streaming.
- * In BigQuery mode, returns virtual refs that signal to load from BQ.
- * @param {object} config - Configuration object
- * @param {object} deps - Dependencies (db, logger)
- * @param {string} dateStr - Date string (YYYY-MM-DD)
- * @param {Array<string>|null} userTypes - User types to filter (null = all)
- * @returns {Promise<Array>} Array of part references
- */
-exports.getPortfolioPartRefs = async (config, deps, dateStr, userTypes = null) => {
-    const { logger } = deps;
-    
-    // In BigQuery mode, return virtual refs with metadata
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        logger.log('INFO', `[DataLoader] Using BigQuery virtual refs for portfolios (${dateStr})`);
-        return [{ 
-            _bigquery: true, 
-            type: 'portfolio', 
-            date: dateStr, 
-            userTypes: userTypes 
-        }];
-    }
-    
-    // Firestore fallback - return actual document references
-    const { db } = deps;
-    const refs = [];
-    
-    // Check both PI and SignedIn collections
-    const collections = [
-        { name: 'PopularInvestorPortfolios', userType: 'POPULAR_INVESTOR' },
-        { name: 'SignedInUserPortfolios', userType: 'SIGNED_IN_USER' }
-    ];
-    
-    for (const col of collections) {
-        if (userTypes && !userTypes.includes(col.userType)) continue;
-        
-        try {
-            const partsSnap = await db.collection(col.name)
-                .doc('latest')
-                .collection('snapshots')
-                .doc(dateStr)
-                .collection('parts')
-                .listDocuments();
-            
-            for (const docRef of partsSnap) {
-                refs.push({ ref: docRef, userType: col.userType, collection: col.name });
-            }
-        } catch (e) {
-            logger.log('WARN', `[DataLoader] Failed to get portfolio refs from ${col.name}: ${e.message}`);
-        }
-    }
-    
-    return refs;
-};
-
-/**
- * Get history part references for streaming.
- * @param {object} config - Configuration object
- * @param {object} deps - Dependencies (db, logger)
- * @param {string} dateStr - Date string (YYYY-MM-DD)
- * @param {Array<string>|null} userTypes - User types to filter (null = all)
- * @returns {Promise<Array>} Array of part references
- */
-exports.getHistoryPartRefs = async (config, deps, dateStr, userTypes = null) => {
-    const { logger } = deps;
-    
-    // In BigQuery mode, return virtual refs with metadata
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        logger.log('INFO', `[DataLoader] Using BigQuery virtual refs for history (${dateStr})`);
-        return [{ 
-            _bigquery: true, 
-            type: 'history', 
-            date: dateStr, 
-            userTypes: userTypes 
-        }];
-    }
-    
-    // Firestore fallback - return actual document references
-    const { db } = deps;
-    const refs = [];
-    
-    const collections = [
-        { name: 'PopularInvestorTradeHistory', userType: 'POPULAR_INVESTOR' },
-        { name: 'SignedInUserTradeHistory', userType: 'SIGNED_IN_USER' }
-    ];
-    
-    for (const col of collections) {
-        if (userTypes && !userTypes.includes(col.userType)) continue;
-        
-        try {
-            const partsSnap = await db.collection(col.name)
-                .doc('latest')
-                .collection('snapshots')
-                .doc(dateStr)
-                .collection('parts')
-                .listDocuments();
-            
-            for (const docRef of partsSnap) {
-                refs.push({ ref: docRef, userType: col.userType, collection: col.name });
-            }
-        } catch (e) {
-            logger.log('WARN', `[DataLoader] Failed to get history refs from ${col.name}: ${e.message}`);
-        }
-    }
-    
-    return refs;
-};
-
-/**
- * Stream portfolio data in chunks (async generator).
- * Yields objects of { cid: portfolioData } for memory efficiency.
- * @param {object} config - Configuration object
- * @param {object} deps - Dependencies (db, logger)
- * @param {string} dateStr - Date string (YYYY-MM-DD)
- * @param {Array} refs - Part references from getPortfolioPartRefs
- * @param {Array<string>|null} userTypes - User types to filter
- * @yields {object} Chunk of portfolio data { cid: data }
- */
-exports.streamPortfolioData = async function* (config, deps, dateStr, refs, userTypes = null) {
-    const { logger } = deps;
-    
-    if (!refs || refs.length === 0) {
-        logger.log('WARN', `[DataLoader] No portfolio refs provided for streaming (${dateStr})`);
-        return;
-    }
-    
-    // BigQuery mode - load all at once and yield in chunks
-    const isBigQuery = refs.some(r => r && r._bigquery === true);
-    
-    if (isBigQuery && process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const types = userTypes || ['POPULAR_INVESTOR', 'SIGNED_IN_USER'];
-            const rawData = await queryPortfolioData(dateStr, null, types, logger);
-            
-            if (rawData && Object.keys(rawData).length > 0) {
-                // Transform BigQuery format to computation system format
-                // BigQuery returns: { cid: { portfolio_data: {...}, user_type: '...', fetched_at: '...' } }
-                // Computation system expects: { cid: { ...portfolioFields..., _userType: '...' } }
-                const data = {};
-                for (const [cid, record] of Object.entries(rawData)) {
-                    // Unwrap portfolio_data and add _userType metadata
-                    const portfolioData = record.portfolio_data || {};
-                    data[cid] = {
-                        ...portfolioData,
-                        _userType: record.user_type
-                    };
-                }
-                
-                logger.log('INFO', `[DataLoader] ✅ Streaming ${Object.keys(data).length} portfolios from BigQuery (${dateStr})`);
-                
-                // Yield in chunks of 100 users for memory efficiency
-                const CHUNK_SIZE = 100;
-                const entries = Object.entries(data);
-                
-                for (let i = 0; i < entries.length; i += CHUNK_SIZE) {
-                    const chunk = {};
-                    entries.slice(i, i + CHUNK_SIZE).forEach(([cid, portfolio]) => {
-                        chunk[cid] = portfolio;
-                    });
-                    yield chunk;
-                }
-                return;
-            }
-        } catch (e) {
-            logger.log('ERROR', `[DataLoader] BigQuery portfolio stream failed: ${e.message}`);
-        }
-    }
-    
-    // Firestore fallback - stream from refs
-    for (const refInfo of refs) {
-        if (!refInfo.ref) continue;
-        
-        try {
-            const snap = await refInfo.ref.get();
-            if (snap.exists) {
-                const data = snap.data();
-                // Add user type metadata
-                const enriched = {};
-                for (const [cid, portfolio] of Object.entries(data)) {
-                    enriched[cid] = { ...portfolio, _userType: refInfo.userType };
-                }
-                yield enriched;
-            }
-        } catch (e) {
-            logger.log('WARN', `[DataLoader] Failed to stream portfolio part: ${e.message}`);
-        }
-    }
-};
-
-/**
- * Stream history data in chunks (async generator).
- * @param {object} config - Configuration object
- * @param {object} deps - Dependencies (db, logger)
- * @param {string} dateStr - Date string (YYYY-MM-DD)
- * @param {Array} refs - Part references from getHistoryPartRefs
- * @param {Array<string>|null} userTypes - User types to filter
- * @yields {object} Chunk of history data { cid: data }
- */
-exports.streamHistoryData = async function* (config, deps, dateStr, refs, userTypes = null) {
-    const { logger } = deps;
-    
-    if (!refs || refs.length === 0) {
-        logger.log('WARN', `[DataLoader] No history refs provided for streaming (${dateStr})`);
-        return;
-    }
-    
-    // BigQuery mode - load all at once and yield in chunks
-    const isBigQuery = refs.some(r => r && r._bigquery === true);
-    
-    if (isBigQuery && process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const types = userTypes || ['POPULAR_INVESTOR', 'SIGNED_IN_USER'];
-            const rawData = await queryHistoryData(dateStr, null, types, logger);
-            
-            if (rawData && Object.keys(rawData).length > 0) {
-                // Transform BigQuery format to computation system format
-                // BigQuery returns: { cid: { history_data: {...}, user_type: '...', fetched_at: '...' } }
-                // Computation system expects: { cid: { ...historyFields..., _userType: '...' } }
-                const data = {};
-                for (const [cid, record] of Object.entries(rawData)) {
-                    // Unwrap history_data and add _userType metadata
-                    const historyData = record.history_data || {};
-                    data[cid] = {
-                        ...historyData,
-                        _userType: record.user_type
-                    };
-                }
-                
-                logger.log('INFO', `[DataLoader] ✅ Streaming ${Object.keys(data).length} history records from BigQuery (${dateStr})`);
-                
-                // Yield in chunks of 100 users
-                const CHUNK_SIZE = 100;
-                const entries = Object.entries(data);
-                
-                for (let i = 0; i < entries.length; i += CHUNK_SIZE) {
-                    const chunk = {};
-                    entries.slice(i, i + CHUNK_SIZE).forEach(([cid, history]) => {
-                        chunk[cid] = history;
-                    });
-                    yield chunk;
-                }
-                return;
-            }
-        } catch (e) {
-            logger.log('ERROR', `[DataLoader] BigQuery history stream failed: ${e.message}`);
-        }
-    }
-    
-    // Firestore fallback - stream from refs
-    for (const refInfo of refs) {
-        if (!refInfo.ref) continue;
-        
-        try {
-            const snap = await refInfo.ref.get();
-            if (snap.exists) {
-                const data = snap.data();
-                // Add user type metadata
-                const enriched = {};
-                for (const [cid, history] of Object.entries(data)) {
-                    enriched[cid] = { ...history, _userType: refInfo.userType };
-                }
-                yield enriched;
-            }
-        } catch (e) {
-            logger.log('WARN', `[DataLoader] Failed to stream history part: ${e.message}`);
-        }
-    }
-};