bulltrackers-module 1.0.721 → 1.0.722

package/functions/computation-system/utils/data_loader.js

@@ -1,1159 +1,324 @@
 /**
- * @fileoverview Data loader sub-pipes for the Computation System.
- * REFACTORED: Now stateless and receive dependencies.
- * UPDATED: Integrated GCS Snapshot "Fast Path" for massive read reduction across ALL data types.
- * FIXED: Added strict userType filtering to prevent fetching unnecessary data.
- * UPDATED: Verification now uses CollectionGroup query due to per-user storage.
- * UPDATED: Ratings now correctly handles flattened top-level schema.
+ * @fileoverview Data Loading Layer.
+ * REFACTORED:
+ * 1. Routes 90% of data fetching to BigQuery (Portfolios, History, Prices, Insights, Rankings, etc.).
+ * 2. Retains Firestore logic ONLY for:
+ * - Verifications (user_verifications)
+ * - Retail Users (Normal/Speculator Portfolios/History)
+ * - Generic Social Feed (Legacy compatibility)
  */
-const zlib = require('zlib'); 
-const { Storage } = require('@google-cloud/storage');
-const readline = require('readline');
-
-// Singleton Storage Client
-const storage = new Storage(); 
-
-// Helper for decompressing any doc if needed (Firestore Legacy)
-function tryDecompress(data) {
-    if (data && data._compressed === true && data.payload) {
-        try {
-            return JSON.parse(zlib.gunzipSync(data.payload).toString());
-        } catch (e) {
-            console.error('[DataLoader] Decompression failed', e);
-            return {};
-        }
-    }
-    return data;
-}
-
-// --- GCS FAST PATH HELPER ---
-async function tryLoadFromGCS(config, dateString, snapshotName, logger) {
-    if (!dateString) return null;
-    const bucketName = config.gcsBucketName || 'bulltrackers';
-    try {
-        const bucket = storage.bucket(bucketName);
-        const file = bucket.file(`${dateString}/snapshots/${snapshotName}.json.gz`);
-        const [exists] = await file.exists();
-        
-        if (exists) {
-            logger.log('INFO', `[DataLoader] ⚡️ GCS HIT: ${snapshotName} for ${dateString}`);
-            const [content] = await file.download();
-            
-            // FIX: GCS client auto-decompresses if Content-Encoding is gzip.
-            // We try gunzip first; if it fails with header check, it's likely already JSON.
-            try {
-                return JSON.parse(zlib.gunzipSync(content).toString());
-            } catch (zipError) {
-                if (zipError.message && zipError.message.includes('incorrect header check')) {
-                    // Content was already decompressed by the client
-                    return JSON.parse(content.toString());
-                }
-                throw zipError;
-            }
-        }
-    } catch (e) {
-        logger.log('WARN', `[DataLoader] GCS Check Failed (${snapshotName}): ${e.message}`);
-    }
-    return null;
-}
-
-/** --- Data Loader Sub-Pipes (Stateless, Dependency-Injection) --- */
-
-/** Stage 1: Get portfolio part document references for a given date */
-async function getPortfolioPartRefs(config, deps, dateString, requiredUserTypes = null) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
+const { 
+    queryPortfolioData, 
+    queryHistoryData, 
+    querySocialData, 
+    queryAssetPrices,
+    queryAllPricesForDate,
+    queryPricesForTickers,
+    queryInstrumentInsights, 
+    queryPIRankings,
+    queryTickerMappings,
+    queryPIMasterList,
+    queryPIRatings,
+    queryPIPageViews,
+    queryWatchlistMembership,
+    queryPIAlertHistory
+} = require('../../core/utils/bigquery_utils');
+
+const { normalizeName } = require('./utils');
+
+// =============================================================================
+// 1. PORTFOLIOS
+// =============================================================================
+exports.loadDailyPortfolios = async (config, deps, dateStr, userTypes = []) => {
+    const { db, logger } = deps;
     
-    // Normalize required types. If null/empty or contains 'ALL', fetch everything.
-    const types = requiredUserTypes ? new Set(requiredUserTypes.map(t => t.toUpperCase())) : null;
-    const fetchAll = !types || types.size === 0 || types.has('ALL');
+    // Normalize user types
+    const types = Array.isArray(userTypes) ? userTypes : [userTypes];
+    const isRetail = types.some(t => ['NORMAL', 'SPECULATOR'].includes(t.toUpperCase()));
+    const isMigrated = types.some(t => ['POPULAR_INVESTOR', 'SIGNED_IN_USER'].includes(t.toUpperCase()));
 
-    // =========================================================================
-    // BIGQUERY FIRST: Try BigQuery before Firestore
-    // =========================================================================
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const { queryPortfolioData } = require('../../core/utils/bigquery_utils');
-            const bigqueryData = await queryPortfolioData(dateString, null, requiredUserTypes, logger);
-            
-            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
-                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for portfolio data (${dateString}): ${Object.keys(bigqueryData).length} users`);
-                
-                // Transform BigQuery data into "ref-like" objects for compatibility
-                const bigqueryRefs = Object.keys(bigqueryData).map(userId => {
-                    const dataObj = bigqueryData[userId];
-                    return {
-                        ref: null, // No Firestore ref needed
-                        type: dataObj.user_type || 'UNKNOWN',
-                        cid: userId,
-                        collectionType: 'BIGQUERY',
-                        bigqueryData: dataObj.portfolio_data || {} // Extract the actual portfolio data
-                    };
-                });
-                
-                return bigqueryRefs;
-            } else {
-                logger.log('INFO', `[DataLoader] ⚠️ BigQuery returned no portfolio data for ${dateString}, falling back to Firestore`);
-            }
-        } catch (bqError) {
-            logger.log('WARN', `[DataLoader] BigQuery portfolio query failed for ${dateString}, falling back to Firestore: ${bqError.message}`);
-        }
-    }
-    
-    // =========================================================================
-    // FIRESTORE FALLBACK: Original logic (backwards compatibility)
-    // =========================================================================
-    logger.log('INFO', `[DataLoader] 📂 Using Firestore for portfolio data (${dateString}). Filter: ${fetchAll ? 'ALL' : Array.from(types || []).join(',')}`);
-    
-    const allPartRefs = [];
-    
-    // NEW STRUCTURE: Read from date-based collections (per-user documents)
-    try {
-        if (fetchAll || types.has('SIGNED_IN_USER')) {
-            const signedInPortCollectionName = 'SignedInUserPortfolioData';
-            const signedInPortDateDoc = db.collection(signedInPortCollectionName).doc(dateString);
-            const signedInPortSubcollections = await withRetry(
-                () => signedInPortDateDoc.listCollections(),
-                `listSignedInPortfolios(${dateString})`
-            );
-            
-            signedInPortSubcollections.forEach(subcol => {
-                const cid = subcol.id;
-                allPartRefs.push({ ref: subcol.doc(cid), type: 'SIGNED_IN_USER', cid: cid, collectionType: 'NEW_STRUCTURE' });
-            });
-        }
-        
-        if (fetchAll || types.has('POPULAR_INVESTOR')) {
-            const piPortCollectionName = 'PopularInvestorPortfolioData';
-            const piPortDateDoc = db.collection(piPortCollectionName).doc(dateString);
-            const piPortSubcollections = await withRetry(
-                () => piPortDateDoc.listCollections(),
-                `listPIPortfolios(${dateString})`
-            );
-            
-            piPortSubcollections.forEach(subcol => {
-                const cid = subcol.id;
-                allPartRefs.push({ ref: subcol.doc(cid), type: 'POPULAR_INVESTOR', cid: cid, collectionType: 'NEW_STRUCTURE' });
-            });
-        }
-    } catch (newStructError) {
-        logger.log('WARN', `Failed to load from new structure, falling back to legacy: ${newStructError.message}`);
-    }
-    
-    // LEGACY STRUCTURE: Read from block-based collections
-    const collectionsToQuery = [];
-    if ((fetchAll || types.has('NORMAL')) && config.normalUserPortfolioCollection) collectionsToQuery.push({ name: config.normalUserPortfolioCollection, type: 'NORMAL' });
-    if ((fetchAll || types.has('SPECULATOR')) && config.speculatorPortfolioCollection) collectionsToQuery.push({ name: config.speculatorPortfolioCollection, type: 'SPECULATOR' });
-    if ((fetchAll || types.has('POPULAR_INVESTOR')) && config.piPortfolioCollection) collectionsToQuery.push({ name: config.piPortfolioCollection, type: 'POPULAR_INVESTOR' });
-    if ((fetchAll || types.has('SIGNED_IN_USER')) && config.signedInUsersCollection) collectionsToQuery.push({ name: config.signedInUsersCollection, type: 'SIGNED_IN_USER' });
-
-    for (const { name: collectionName, type: collectionType } of collectionsToQuery) {
-        try {
-            const blockDocsQuery = db.collection(collectionName);
-            const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
-            if (!blockDocRefs.length) continue; 
+    let results = {};
 
-            const partsPromises = blockDocRefs.map(blockDocRef => { 
-                const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection || 'snapshots').doc(dateString).collection(config.partsSubcollection || 'parts'); 
-                return withRetry(() => partsCollectionRef.listDocuments(), `listParts(${partsCollectionRef.path})`); 
-            });
-
-            const partDocArrays = await Promise.all(partsPromises);
-            partDocArrays.forEach(partDocs => { 
-                allPartRefs.push(...partDocs.map(ref => ({ ref, type: collectionType, collectionType: 'LEGACY' }))); 
-            });
-        } catch (legacyError) {
-            logger.log('WARN', `Failed to load legacy collection ${collectionName}: ${legacyError.message}`);
-        }
+    // A. BigQuery (PIs & SignedIn)
+    if (isMigrated && process.env.BIGQUERY_ENABLED !== 'false') {
+        const bqData = await queryPortfolioData(dateStr, null, types, logger);
+        if (bqData) Object.assign(results, bqData);
     }
-    
-    logger.log('INFO', `Found ${allPartRefs.length} total portfolio refs for ${dateString} for types: ${fetchAll ? 'ALL' : Array.from(types).join(',')}`);
-    return allPartRefs;
-}
 
-/** Stage 2: Load data from an array of document references */
-async function loadDataByRefs(config, deps, refObjects) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
-    
-    if (!refObjects || !refObjects.length) return {};
-    
-    // =========================================================================
-    // CHECK FOR PRICE DATA (BigQuery marker)
-    // =========================================================================
-    const priceBigQueryMarker = refObjects.find(r => r._bigquery === true);
-    if (priceBigQueryMarker) {
-        // This is a price data load request - use BigQuery
-        try {
-            const { queryAssetPrices } = require('../../core/utils/bigquery_utils');
-            const priceData = await queryAssetPrices(null, null, null, logger);
-            
-            if (priceData && Object.keys(priceData).length > 0) {
-                logger.log('INFO', `[DataLoader] ✅ Loaded ${Object.keys(priceData).length} instruments from BigQuery for price data`);
-                return priceData;
-            }
-            
-            // If BigQuery returns empty, fallback to Firestore
-            logger.log('WARN', `[DataLoader] BigQuery returned no price data, falling back to Firestore`);
-        } catch (bqError) {
-            logger.log('WARN', `[DataLoader] BigQuery price load failed, falling back to Firestore: ${bqError.message}`);
-            // Fall through to Firestore
+    // B. Firestore (Retail / Fallback)
+    // Note: If we need Retail data, we MUST check Firestore as it wasn't migrated.
+    if (isRetail) {
+        if (types.includes('NORMAL')) {
+            const normalData = await loadRetailFirestore(db, 'NormalUserPortfolios', dateStr);
+            Object.assign(results, normalData);
         }
-    }
-    
-    // =========================================================================
-    // SEPARATE BIGQUERY AND FIRESTORE REFS (for portfolio/history data)
-    // =========================================================================
-    const bigqueryRefs = refObjects.filter(r => r.collectionType === 'BIGQUERY');
-    const firestoreRefs = refObjects.filter(r => r.collectionType !== 'BIGQUERY' && !r._bigquery);
-    
-    const mergedPortfolios = {};
-    
-    // Load from BigQuery cache (data already fetched in getPortfolioPartRefs/getHistoryPartRefs)
-    if (bigqueryRefs.length > 0) {
-        logger.log('INFO', `[DataLoader] 📊 Loading ${bigqueryRefs.length} records from BigQuery cache`);
-        bigqueryRefs.forEach(ref => {
-            if (ref.bigqueryData) {
-                // Extract the actual data (portfolio_data or history_data)
-                // The bigqueryData is the JSON object from BigQuery
-                mergedPortfolios[ref.cid] = ref.bigqueryData;
-            }
-        });
-    }
-    
-    // Load from Firestore (existing logic)
-    if (firestoreRefs.length === 0) {
-        return mergedPortfolios; // All data came from BigQuery
-    }
-    
-    logger.log('INFO', `[DataLoader] 📂 Loading ${firestoreRefs.length} records from Firestore`);
-    const batchSize = config.partRefBatchSize || 10;
-
-    for (let i = 0; i < firestoreRefs.length; i += batchSize) {
-        const batch = firestoreRefs.slice(i, i + batchSize);
-        const refs = batch.map(b => b.ref).filter(ref => ref !== null); // Filter out null refs (BigQuery refs)
-        
-        if (refs.length === 0) continue; // Skip if all refs are BigQuery refs
-        
-        const snapshots = await withRetry(() => db.getAll(...refs), `getAll(batch ${Math.floor(i/batchSize)})`);
-        
-        const deepFetchPromises = [];
-
-        for (let j = 0; j < snapshots.length; j++) {
-            const doc = snapshots[j];
-            const meta = batch[j];
-            
-            if (!doc.exists) continue;
-
-            const rawData = doc.data();
-            let chunkData;
-            
-            if (meta.collectionType === 'NEW_STRUCTURE') {
-                const cid = meta.cid || doc.id;
-                const userData = tryDecompress(rawData);
-                chunkData = { [cid]: userData };
-                
-                if (meta.type === 'POPULAR_INVESTOR') {
-                    chunkData[cid]._userType = 'POPULAR_INVESTOR';
-                    if (chunkData[cid].deepPositions) chunkData[cid].DeepPositions = chunkData[cid].deepPositions;
-                } else if (meta.type === 'SIGNED_IN_USER') {
-                    chunkData[cid]._userType = 'SIGNED_IN_USER';
-                }
-                
-                deepFetchPromises.push(Promise.resolve(chunkData));
-            } else {
-                chunkData = tryDecompress(rawData); 
-
-                if (meta.type === 'POPULAR_INVESTOR' && config.piDeepPortfolioCollection) {
-                    const pathSegments = doc.ref.path.split('/'); 
-                    const deepCollection = config.piDeepPortfolioCollection;
-                    const deepPath = `${deepCollection}/${pathSegments[1]}/${pathSegments[2]}/${pathSegments[3]}/${pathSegments[4]}/${pathSegments[5]}`;
-                    
-                    deepFetchPromises.push(
-                        db.doc(deepPath).get().then(deepSnap => {
-                            if (deepSnap.exists) {
-                                const deepChunk = tryDecompress(deepSnap.data());
-                                for (const [uid, pData] of Object.entries(chunkData)) {
-                                    if (deepChunk[uid] && deepChunk[uid].positions) pData.DeepPositions = deepChunk[uid].positions;
-                                }
-                            }
-                            for (const pData of Object.values(chunkData)) pData._userType = 'POPULAR_INVESTOR';
-                            return chunkData;
-                        }).catch(() => chunkData)
-                    );
-                } else if (meta.type === 'SIGNED_IN_USER') {
-                    for (const pData of Object.values(chunkData)) pData._userType = 'SIGNED_IN_USER';
-                    deepFetchPromises.push(Promise.resolve(chunkData));
-                } else {
-                    deepFetchPromises.push(Promise.resolve(chunkData));
-                }
-            }
+        if (types.includes('SPECULATOR')) {
+            const specData = await loadRetailFirestore(db, 'SpeculatorPortfolios', dateStr);
+            Object.assign(results, specData);
         }
-
-        const resolvedChunks = await Promise.all(deepFetchPromises);
-        resolvedChunks.forEach(chunk => {
-             if (chunk && typeof chunk === 'object') Object.assign(mergedPortfolios, chunk);
-        });
     }
-    return mergedPortfolios;
-}
-
-/** Stage 3: Load a full day map by delegating to loadDataByRefs */
-async function loadFullDayMap(config, deps, partRefs, dateString) { 
-    // 1. GCS FAST PATH
-    const cached = await tryLoadFromGCS(config, dateString, 'portfolios', deps.logger);
-    if (cached) return cached;
-
-    // 2. FIRESTORE FALLBACK
-    const { logger } = deps;
-    if (!partRefs.length) return {};
-    logger.log('TRACE', `Loading full day map from ${partRefs.length} references...`);
-    const fullMap = await loadDataByRefs(config, deps, partRefs);
-    logger.log('TRACE', `Full day map loaded with ${Object.keys(fullMap).length} users`);
-    return fullMap;
-}
 
-/** Stage 3.5: Load Daily Portfolios (Wrapper for Series Loading) */
-async function loadDailyPortfolios(config, deps, dateString, requiredUserTypes = null) {
-    // 1. GCS FAST PATH
-    const cached = await tryLoadFromGCS(config, dateString, 'portfolios', deps.logger);
-    if (cached) return cached;
+    return results;
+};
 
-    // 2. FIRESTORE FALLBACK
-    // [FIX] Now passing requiredUserTypes to prevent fetching all users (e.g. NormalUserPortfolios)
-    const partRefs = await getPortfolioPartRefs(config, deps, dateString, requiredUserTypes);
-    if (partRefs.length === 0) return {};
-    return loadDataByRefs(config, deps, partRefs);
-}
+// =============================================================================
+// 2. TRADE HISTORY
+// =============================================================================
+exports.loadDailyHistory = async (config, deps, dateStr, userTypes = []) => {
+    const { db, logger } = deps;
+    const types = Array.isArray(userTypes) ? userTypes : [userTypes];
+    const isRetail = types.some(t => ['NORMAL', 'SPECULATOR'].includes(t.toUpperCase()));
+    const isMigrated = types.some(t => ['POPULAR_INVESTOR', 'SIGNED_IN_USER'].includes(t.toUpperCase()));
 
-/** Stage 4: Load daily instrument insights */
-async function loadDailyInsights(config, deps, dateString) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
-
-    // 1. GCS FAST PATH
-    const cached = await tryLoadFromGCS(config, dateString, 'insights', logger);
-    if (cached) return cached;
-
-    // 2. BIGQUERY FIRST (if enabled)
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const { queryInstrumentInsights } = require('../../core/utils/bigquery_utils');
-            const bigqueryData = await queryInstrumentInsights(dateString, logger);
-            
-            if (bigqueryData && Array.isArray(bigqueryData) && bigqueryData.length > 0) {
-                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for instrument insights (${dateString}): ${bigqueryData.length} instruments`);
-                // Return in same format as Firestore: { insights: [...] }
-                return { insights: bigqueryData };
-            }
-        } catch (bqError) {
-            logger.log('WARN', `[DataLoader] BigQuery insights query failed, falling back to Firestore: ${bqError.message}`);
-            // Fall through to Firestore
-        }
-    }
+    let results = {};
 
-    // 3. FIRESTORE FALLBACK
-    const insightsCollectionName = config.insightsCollectionName || 'daily_instrument_insights';
-    logger.log('INFO', `Loading daily insights for ${dateString} from ${insightsCollectionName} (Firestore)`);
-    try { 
-        const docRef = db.collection(insightsCollectionName).doc(dateString);
-        const docSnap = await withRetry(() => docRef.get(), `getInsights(${dateString})`);
-        if (!docSnap.exists) { logger.log('WARN', `Insights not found for ${dateString}`); return null; }
-        logger.log('TRACE', `Successfully loaded insights for ${dateString}`);
-        return tryDecompress(docSnap.data());
-    } catch (error) { 
-        logger.log('ERROR', `Failed to load daily insights for ${dateString}`, { errorMessage: error.message });
-        return null;
+    if (isMigrated && process.env.BIGQUERY_ENABLED !== 'false') {
+        const bqData = await queryHistoryData(dateStr, null, types, logger);
+        if (bqData) Object.assign(results, bqData);
     }
-}
 
-/** Stage 5: Load and Partition Social Data */
-async function loadDailySocialPostInsights(config, deps, dateString) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
-    
-    // 1. GCS FAST PATH
-    const cached = await tryLoadFromGCS(config, dateString, 'social', logger);
-    if (cached) return cached;
-
-    // 2. BIGQUERY FIRST (if enabled)
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const { querySocialData } = require('../../core/utils/bigquery_utils');
-            const bigqueryData = await querySocialData(dateString, null, null, logger);
-            
-            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
-                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for social data (${dateString}): ${Object.keys(bigqueryData).length} users`);
-                
-                // Transform BigQuery data to expected format: { generic: {}, pi: {}, signedIn: {} }
-                // BigQuery returns: { userId: { posts_data: { posts: {...}, postCount: N }, user_type: '...' } }
-                const result = { generic: {}, pi: {}, signedIn: {} };
-                
-                for (const [userId, userData] of Object.entries(bigqueryData)) {
-                    const userType = userData.user_type || 'UNKNOWN';
-                    
-                    // Handle posts_data - may be object (parsed JSON) or string (needs parsing)
-                    let postsData = userData.posts_data || {};
-                    if (typeof postsData === 'string') {
-                        try {
-                            postsData = JSON.parse(postsData);
-                        } catch (e) {
-                            logger.log('WARN', `[DataLoader] Failed to parse posts_data for user ${userId}: ${e.message}`);
-                            continue;
-                        }
-                    }
-                    
-                    // Extract posts map from posts_data structure: { posts: {...}, postCount: N }
-                    const posts = postsData.posts || {};
-                    
-                    // Partition by user type
-                    if (userType === 'POPULAR_INVESTOR') {
-                        result.pi[userId] = posts;
-                    } else if (userType === 'SIGNED_IN_USER') {
-                        result.signedIn[userId] = posts;
-                    } else {
-                        // Generic/unknown user types go to generic
-                        result.generic[userId] = posts;
-                    }
-                }
-                
-                logger.log('INFO', `[DataLoader] ✅ Loaded Social Data from BigQuery: ${Object.keys(result.generic).length} Generic, ${Object.keys(result.pi).length} PIs, ${Object.keys(result.signedIn).length} Signed-In`);
-                return result;
-            }
-        } catch (bqError) {
-            logger.log('WARN', `[DataLoader] BigQuery social query failed for ${dateString}, falling back to Firestore: ${bqError.message}`);
-            // Fall through to Firestore
+    if (isRetail) {
+        if (types.includes('NORMAL')) {
+            const normalData = await loadRetailFirestore(db, 'NormalUserTradeHistory', dateStr);
+            Object.assign(results, normalData);
+        }
+        if (types.includes('SPECULATOR')) {
+            const specData = await loadRetailFirestore(db, 'SpeculatorTradeHistory', dateStr);
+            Object.assign(results, specData);
         }
     }
+    return results;
+};
 
-    // 3. FIRESTORE FALLBACK
-    logger.log('INFO', `Loading and partitioning social data for ${dateString} (Firestore)`);
-    
-    const result = { generic: {}, pi: {}, signedIn: {} };
+// =============================================================================
+// 3. SOCIAL
+// =============================================================================
+exports.loadDailySocialPostInsights = async (config, deps, dateStr, userTypes = []) => {
+    const { db, logger } = deps;
+    const types = Array.isArray(userTypes) ? userTypes : (userTypes ? [userTypes] : []);
 
-    // NEW STRUCTURE
-    try {
-        const signedInSocialCollectionName = 'SignedInUserSocialPostData';
-        const signedInSocialDateDoc = db.collection(signedInSocialCollectionName).doc(dateString);
-        const signedInSocialSubcollections = await withRetry(() => signedInSocialDateDoc.listCollections(), `listSignedInSocial(${dateString})`);
-        
-        for (const subcol of signedInSocialSubcollections) {
-            const cid = subcol.id;
-            const cidDoc = await subcol.doc(cid).get();
-            if (cidDoc.exists) {
-                const cidData = tryDecompress(cidDoc.data());
-                if (cidData.posts && typeof cidData.posts === 'object') {
-                    if (!result.signedIn[cid]) result.signedIn[cid] = {};
-                    Object.assign(result.signedIn[cid], cidData.posts);
-                }
-            }
-        }
-        
-        const piSocialCollectionName = 'PopularInvestorSocialPostData';
-        const piSocialDateDoc = db.collection(piSocialCollectionName).doc(dateString);
-        const piSocialSubcollections = await withRetry(() => piSocialDateDoc.listCollections(), `listPISocial(${dateString})`);
-        
-        for (const subcol of piSocialSubcollections) {
-            const cid = subcol.id;
-            const cidDoc = await subcol.doc(cid).get();
-            if (cidDoc.exists) {
-                const cidData = tryDecompress(cidDoc.data());
-                if (cidData.posts && typeof cidData.posts === 'object') {
-                    if (!result.pi[cid]) result.pi[cid] = {};
-                    Object.assign(result.pi[cid], cidData.posts);
-                }
-            }
-        }
-        
-        const instrumentSocialCollectionName = 'InstrumentFeedSocialPostData';
-        const instrumentSocialDateDoc = db.collection(instrumentSocialCollectionName).doc(dateString);
-        const instrumentSocialPostsCol = instrumentSocialDateDoc.collection('posts');
-        const instrumentSocialSnapshot = await withRetry(() => instrumentSocialPostsCol.limit(1000).get(), `getInstrumentSocial(${dateString})`);
-        
-        instrumentSocialSnapshot.forEach(doc => {
-            const data = tryDecompress(doc.data());
-            result.generic[doc.id] = data;
-        });
-        
-        logger.log('INFO', `Loaded Social Data (NEW): ${Object.keys(result.generic).length} Generic, ${Object.keys(result.pi).length} PIs.`);
-    } catch (newStructError) {
-        logger.log('WARN', `Failed to load from new structure: ${newStructError.message}`);
+    // A. BigQuery (User-Specific Social)
+    if (types.length > 0 && process.env.BIGQUERY_ENABLED !== 'false') {
+        return querySocialData(dateStr, null, types, logger);
     }
 
-    // LEGACY STRUCTURE
-    const PI_COL_NAME = config.piSocialCollectionName || config.piSocialCollection || 'pi_social_posts';
-    const SIGNED_IN_COL_NAME = config.signedInUserSocialCollection || 'signed_in_users_social';
-    const startDate = new Date(dateString + 'T00:00:00Z');
-    const endDate = new Date(dateString + 'T23:59:59Z');
-
+    // B. Firestore (Generic Feed - Legacy)
+    // If no user types specified, assume generic feed fetch
+    const collection = config.socialInsightsCollection || 'daily_social_insights';
     try {
-        const postsQuery = db.collectionGroup('posts').where('fetchedAt', '>=', startDate).where('fetchedAt', '<=', endDate);
-        const querySnapshot = await withRetry(() => postsQuery.get(), `getSocialPosts(${dateString})`);
-        
-        if (!querySnapshot.empty) {
-            querySnapshot.forEach(doc => { 
-                const data = tryDecompress(doc.data());
-                const path = doc.ref.path; 
-                if (path.includes(PI_COL_NAME)) {
-                    const parts = path.split('/');
-                    const colIndex = parts.indexOf(PI_COL_NAME);
-                    if (colIndex !== -1 && parts[colIndex + 1]) {
-                        const userId = parts[colIndex + 1];
-                        if (!result.pi[userId]) result.pi[userId] = {};
-                        result.pi[userId][doc.id] = data;
-                    }
-                } else if (path.includes(SIGNED_IN_COL_NAME)) {
-                    const parts = path.split('/');
-                    const colIndex = parts.indexOf(SIGNED_IN_COL_NAME);
-                    if (colIndex !== -1 && parts[colIndex + 1]) {
-                        const userId = parts[colIndex + 1];
-                        if (!result.signedIn[userId]) result.signedIn[userId] = {};
-                        result.signedIn[userId][doc.id] = data;
-                    }
-                } else {
-                    result.generic[doc.id] = data;
-                }
-            });
-        }
-    } catch (error) {
-        logger.log('ERROR', `Failed to load social posts (legacy): ${error.message}`);
+        const snap = await db.collection(collection).doc(dateStr).collection('posts').get();
+        if (snap.empty) return {};
+        const data = {};
+        snap.forEach(doc => data[doc.id] = doc.data());
+        return data;
+    } catch (e) {
+        logger.log('WARN', `[DataLoader] Failed to load generic social for ${dateStr}: ${e.message}`);
+        return {};
     }
+};
 
-    return result;
-}
-
-/** Stage 6: Get history part references for a given date */
-async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes = null) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
-    
-    // Normalize required types
-    const types = requiredUserTypes ? new Set(requiredUserTypes.map(t => t.toUpperCase())) : null;
-    const fetchAll = !types || types.size === 0 || types.has('ALL');
-
-    // =========================================================================
-    // BIGQUERY FIRST: Try BigQuery before Firestore
-    // =========================================================================
+// =============================================================================
+// 4. MARKET DATA (Prices)
+// =============================================================================
+exports.getPriceShardRefs = async (config, deps) => {
+    // Legacy Shard Helper - In BQ world, we don't use shards but CachedDataLoader expects this structure.
+    // We return a "virtual" shard array that signals CachedDataLoader to load from BQ.
     if (process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const { queryHistoryData } = require('../../core/utils/bigquery_utils');
-            const bigqueryData = await queryHistoryData(dateString, null, requiredUserTypes, logger);
-            
-            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
-                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for trade history data (${dateString}): ${Object.keys(bigqueryData).length} users`);
-                
-                // Transform BigQuery data into "ref-like" objects for compatibility
-                const bigqueryRefs = Object.keys(bigqueryData).map(userId => {
-                    const dataObj = bigqueryData[userId];
-                    return {
-                        ref: null, // No Firestore ref needed
-                        type: dataObj.user_type || 'UNKNOWN',
-                        cid: userId,
-                        collectionType: 'BIGQUERY',
-                        bigqueryData: dataObj.history_data || {} // Extract the actual history data
-                    };
-                });
-                
-                return bigqueryRefs;
-            } else {
-                logger.log('INFO', `[DataLoader] ⚠️ BigQuery returned no history data for ${dateString}, falling back to Firestore`);
-            }
-        } catch (bqError) {
-            logger.log('WARN', `[DataLoader] BigQuery history query failed for ${dateString}, falling back to Firestore: ${bqError.message}`);
-        }
-    }
-    
-    // =========================================================================
-    // FIRESTORE FALLBACK: Original logic (backwards compatibility)
-    // =========================================================================
-    logger.log('INFO', `[DataLoader] 📂 Using Firestore for trade history data (${dateString}). Filter: ${fetchAll ? 'ALL' : Array.from(types || []).join(',')}`);
-    
-    const allPartRefs = [];
-    
-    // NEW STRUCTURE
-    try {
-        if (fetchAll || types.has('SIGNED_IN_USER')) {
-            const signedInHistCollectionName = 'SignedInUserTradeHistoryData';
-            const signedInHistDateDoc = db.collection(signedInHistCollectionName).doc(dateString);
-            const signedInHistSubcollections = await withRetry(
-                () => signedInHistDateDoc.listCollections(),
-                `listSignedInHistory(${dateString})`
-            );
-            signedInHistSubcollections.forEach(subcol => {
-                allPartRefs.push({ ref: subcol.doc(subcol.id), type: 'SIGNED_IN_USER', cid: subcol.id, collectionType: 'NEW_STRUCTURE' });
-            });
-        }
-        
-        if (fetchAll || types.has('POPULAR_INVESTOR')) {
-            const piHistCollectionName = 'PopularInvestorTradeHistoryData';
-            const piHistDateDoc = db.collection(piHistCollectionName).doc(dateString);
-            const piHistSubcollections = await withRetry(
-                () => piHistDateDoc.listCollections(),
-                `listPIHistory(${dateString})`
-            );
-            piHistSubcollections.forEach(subcol => {
-                allPartRefs.push({ ref: subcol.doc(subcol.id), type: 'POPULAR_INVESTOR', cid: subcol.id, collectionType: 'NEW_STRUCTURE' });
-            });
-        }
-    } catch (newStructError) {
-        logger.log('WARN', `Failed to load from new structure: ${newStructError.message}`);
+        return [ { _bigquery: true } ];
+    }
+    // Fallback to Firestore Logic - return array of doc refs
+    const { db } = deps;
+    const collection = config.assetPricesCollection || 'asset_prices';
+    const snapshot = await db.collection(collection).listDocuments();
+    const refs = [];
+    snapshot.forEach(doc => refs.push(doc));
+    return refs;
+};
+
+exports.getRelevantShardRefs = async (config, deps, targetIds) => {
+    // In BQ mode, we don't shard by instrument; return single virtual shard
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        return [ { _bigquery: true, targetIds: targetIds || [] } ];
     }
-    
-    // LEGACY STRUCTURE
-    const collectionsToQuery = [];
-    if ((fetchAll || types.has('NORMAL')) && config.normalUserHistoryCollection) collectionsToQuery.push(config.normalUserHistoryCollection);
-    if ((fetchAll || types.has('SPECULATOR')) && config.speculatorHistoryCollection) collectionsToQuery.push(config.speculatorHistoryCollection);
-    if ((fetchAll || types.has('POPULAR_INVESTOR')) && config.piHistoryCollection) collectionsToQuery.push(config.piHistoryCollection);
-    if ((fetchAll || types.has('SIGNED_IN_USER')) && config.signedInHistoryCollection) collectionsToQuery.push(config.signedInHistoryCollection);
+    // Firestore behavior - return array of doc refs (same as getPriceShardRefs for now)
+    return exports.getPriceShardRefs(config, deps);
+};
 
-    for (const collectionName of collectionsToQuery) {
-        try {
-            const blockDocsQuery = db.collection(collectionName);
-            const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
-            if (!blockDocRefs.length) continue;
-            
-            const partsPromises = blockDocRefs.map(blockDocRef => { 
-                const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection || 'snapshots').doc(dateString).collection(config.partsSubcollection || 'parts');
-                return withRetry(() => partsCollectionRef.listDocuments(), `listParts(${partsCollectionRef.path})`); 
-            });
-            
-            const partDocArrays = await Promise.all(partsPromises);
-            partDocArrays.forEach(partDocs => { 
-                allPartRefs.push(...partDocs.map(ref => ({ ref, type: 'PART', collectionType: 'LEGACY' }))); 
-            });
-        } catch (legacyError) {
-            logger.log('WARN', `Failed to load legacy history collection ${collectionName}: ${legacyError.message}`);
-        }
-    }
-    
-    logger.log('INFO', `Found ${allPartRefs.length} total history refs for ${dateString}`);
-    return allPartRefs;
-}
+// =============================================================================
+// 5. ROOT DATA TYPES (Simple Mappings)
+// =============================================================================
 
-/** Stage 7: Stream portfolio data in chunks */
-async function* streamPortfolioData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
+exports.loadDailyInsights = async (config, deps, dateStr) => {
     const { logger } = deps;
 
-    // =========================================================================
-    // BIGQUERY FIRST: Try BigQuery before GCS/Firestore
-    // =========================================================================
-    if (!providedRefs && process.env.BIGQUERY_ENABLED !== 'false') {
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
         try {
-            const { queryPortfolioData } = require('../../core/utils/bigquery_utils');
-            const bigqueryData = await queryPortfolioData(dateString, null, requiredUserTypes, logger);
-            
-            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
-                logger.log('INFO', `[DataLoader] ✅ Streaming portfolio data from BigQuery (${dateString}): ${Object.keys(bigqueryData).length} users`);
-                
-                // Transform to expected format and yield
-                const transformedData = {};
-                Object.keys(bigqueryData).forEach(userId => {
-                    transformedData[userId] = bigqueryData[userId].portfolio_data || {};
-                });
-                
-                yield transformedData;
-                return; // Exit early, data loaded from BigQuery
-            }
-        } catch (bqError) {
-            logger.log('WARN', `[DataLoader] BigQuery portfolio stream failed for ${dateString}, falling back: ${bqError.message}`);
-        }
-    }
-
-    // 1. GCS FAST PATH (Snapshot) - Only if full run (no providedRefs)
-    if (!providedRefs) { 
-        try {
-            const bucketName = config.gcsBucketName || 'bulltrackers';
-            const bucket = storage.bucket(bucketName);
-            const file = bucket.file(`${dateString}/snapshots/portfolios.json.gz`);
-            const [exists] = await file.exists();
-            
-            if (exists) {
-                logger.log('INFO', `[DataLoader] ⚡️ STREAMING: Hydrating Portfolios from GCS Snapshot`);
-                const [content] = await file.download();
-                
-                // FIX: Handle Double Decompression
-                let fullData;
-                try {
-                    fullData = JSON.parse(zlib.gunzipSync(content).toString());
-                } catch (zipError) {
-                     if (zipError.message && zipError.message.includes('incorrect header check')) {
-                        fullData = JSON.parse(content.toString());
-                    } else {
-                        throw zipError;
-                    }
-                }
-
-                yield fullData; // Yield all in one chunk as it fits in memory
-                return;
+            const rows = await queryInstrumentInsights(dateStr, logger);
+            if (Array.isArray(rows) && rows.length > 0) {
+                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for instrument insights (${dateStr}): ${rows.length} instruments`);
+                // Wrap in Firestore-shaped document format for InsightsExtractor compatibility
+                return { insights: rows };
             }
         } catch (e) {
-            logger.log('WARN', `[DataLoader] GCS Portfolio Stream failed: ${e.message}. Falling back.`);
+            logger.log('WARN', `[DataLoader] BigQuery insights query failed for ${dateStr}: ${e.message}`);
         }
     }
 
-    // 2. FIRESTORE FALLBACK
-    const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString, requiredUserTypes));
-    if (refs.length === 0) { logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`); return; }
-    
-    const batchSize = config.partRefBatchSize || 10; 
-    logger.log('INFO', `[streamPortfolioData] Streaming ${refs.length} portfolio parts in chunks of ${batchSize}...`);
-    
-    for (let i = 0; i < refs.length; i += batchSize) { 
-        const batchRefs = refs.slice(i, i + batchSize); 
-        const data = await loadDataByRefs(config, deps, batchRefs);
-        yield data; 
-    }
-    logger.log('INFO', `[streamPortfolioData] Finished streaming for ${dateString}.`);
-}
+    // No Firestore fallback by design – return empty but correctly shaped
+    return { insights: [] };
+};
 
-/** Stage 8: Stream history data in chunks */
-async function* streamHistoryData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
-    const { logger } = deps;
+exports.loadPopularInvestorRankings = async (config, deps, dateStr) => {
+    const data = await queryPIRankings(dateStr, deps.logger);
+    return data ? data.Items : [];
+};
 
-    // =========================================================================
-    // BIGQUERY FIRST: Try BigQuery before Firestore
-    // =========================================================================
-    if (!providedRefs && process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const { queryHistoryData } = require('../../core/utils/bigquery_utils');
-            const bigqueryData = await queryHistoryData(dateString, null, requiredUserTypes, logger);
-            
-            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
-                logger.log('INFO', `[DataLoader] ✅ Streaming history data from BigQuery (${dateString}): ${Object.keys(bigqueryData).length} users`);
-                
-                // Transform to expected format and yield
-                const transformedData = {};
-                Object.keys(bigqueryData).forEach(userId => {
-                    transformedData[userId] = bigqueryData[userId].history_data || {};
-                });
-                
-                yield transformedData;
-                return; // Exit early, data loaded from BigQuery
-            }
-        } catch (bqError) {
-            logger.log('WARN', `[DataLoader] BigQuery history stream failed for ${dateString}, falling back: ${bqError.message}`);
-        }
-    }
+exports.loadPIRatings = async (config, deps, dateStr) => {
+    return queryPIRatings(dateStr, deps.logger);
+};
 
-    // 1. GCS FAST PATH (JSONL Streaming) - Only if full run
-    if (!providedRefs) {
-        try {
-            const bucketName = config.gcsBucketName || 'bulltrackers';
-            const bucket = storage.bucket(bucketName);
-            const file = bucket.file(`${dateString}/snapshots/history.jsonl.gz`);
-            const [exists] = await file.exists();
+exports.loadPIPageViews = async (config, deps, dateStr) => {
+    return queryPIPageViews(dateStr, deps.logger);
+};
 
-            if (exists) {
-                logger.log('INFO', `[DataLoader] ⚡️ STREAMING: Hydrating History from GCS (JSONL)`);
-                
-                const fileStream = file.createReadStream();
-                const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
+exports.loadWatchlistMembership = async (config, deps, dateStr) => {
+    return queryWatchlistMembership(dateStr, deps.logger);
+};
 
-                let currentBatch = {};
-                let count = 0;
-                const BATCH_SIZE = 50; 
+exports.loadPIAlertHistory = async (config, deps, dateStr) => {
+    return queryPIAlertHistory(dateStr, deps.logger);
+};
 
-                for await (const line of rl) {
-                    if (!line.trim()) continue;
-                    try {
-                        const userEntry = JSON.parse(line);
-                        Object.assign(currentBatch, userEntry);
-                        count++;
-                        if (count >= BATCH_SIZE) {
-                            yield currentBatch;
-                            currentBatch = {};
-                            count = 0;
-                        }
-                    } catch (parseErr) {
-                        logger.log('ERROR', `[DataLoader] JSONL Parse Error: ${parseErr.message}`);
-                    }
-                }
-                if (Object.keys(currentBatch).length > 0) yield currentBatch;
-                logger.log('INFO', `[DataLoader] Finished streaming History from GCS.`);
-                return;
-            }
-        } catch (e) {
-            logger.log('WARN', `[DataLoader] GCS History Stream failed: ${e.message}. Falling back.`);
-        }
-    }
+exports.loadPopularInvestorMasterList = async (config, deps) => {
+    return queryPIMasterList(deps.logger);
+};
 
-    // 2. FIRESTORE FALLBACK
-    const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString, requiredUserTypes));
-    if (refs.length === 0) { logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`); return; }
-    
-    const batchSize = config.partRefBatchSize || 10; 
-    logger.log('INFO', `[streamHistoryData] Streaming ${refs.length} history parts in chunks of ${batchSize}...`);
-    
-    for (let i = 0; i < refs.length; i += batchSize) { 
-        const batchRefs = refs.slice(i, i + batchSize); 
-        const data = await loadDataByRefs(config, deps, batchRefs); 
-        yield data;  
-    }
-    logger.log('INFO', `[streamHistoryData] Finished streaming for ${dateString}.`);
-}
+exports.loadPIWatchlistData = async (config, deps, piCid) => {
+    // Watchlist data is time-series in BQ. For "Current State" (ID based),
+    // we query the most recent date available for this PI.
+    // This is a specialized query not in standard utils, so we implement it here or assume caller passes date.
+    // However, CachedDataLoader expects (cid) -> Data.
+    // We'll return null here as WatchlistMembership (by date) is the preferred method now.
+    deps.logger.log('WARN', '[DataLoader] loadPIWatchlistData (by CID) is deprecated in favor of loadWatchlistMembership (by Date).');
+    return null;
+};
 
-/** Stage 9: Get all price shard references (Basic) 
- * [UPDATED] Returns special marker for BigQuery mode, or Firestore refs for fallback
- */
-async function getPriceShardRefs(config, deps) {
-    const { logger } = deps;
-    
-    // Try BigQuery first if enabled
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        // Return a special marker object to indicate BigQuery mode
-        // The loader will detect this and load from BigQuery instead
-        return [{ _bigquery: true }];
-    }
-    
-    // Fallback to Firestore
-    const { db, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
-    const collection = config.priceCollection || 'asset_prices';
+// =============================================================================
+// 6. EXCEPTIONS (Firestore Only)
+// =============================================================================
+
+exports.loadVerificationProfiles = async (config, deps, dateStr) => {
+    const { db, logger } = deps;
     try {
-        const collectionRef = db.collection(collection);
-        const refs = await withRetry(() => collectionRef.listDocuments(), `listDocuments(${collection})`);
-        return refs;
+        // Verifications are a single collection, not date-partitioned snapshots
+        const snap = await db.collection('user_verifications').get();
+        const verifications = {};
+        snap.forEach(doc => verifications[doc.id] = doc.data());
+        return verifications;
     } catch (e) {
-        logger.log('ERROR', `Failed to list price shards: ${e.message}`);
-        return [];
+        logger.log('ERROR', `[DataLoader] Failed to load verifications: ${e.message}`);
+        return {};
     }
-}
+};
 
-/** Stage 10: Smart Shard Lookup System (DEPRECATED/SIMPLIFIED) */
-async function ensurePriceShardIndex(config, deps) {
-   return {}; // Deprecated
-}
+// =============================================================================
+// HELPERS
+// =============================================================================
 
-async function getRelevantShardRefs(config, deps, targetInstrumentIds) {
-    const { logger } = deps;
-    logger.log('INFO', `[ShardLookup] Smart indexing disabled. Fetching all price shards.`);
-    return getPriceShardRefs(config, deps);
-}
-
-/** Stage 11: Load Popular Investor Rankings */
-async function loadPopularInvestorRankings(config, deps, dateString) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
+// =============================================================================
+// 7. PRICE DATA BY REFS (For PriceBatchExecutor)
+// =============================================================================
 
-    // 1. GCS FAST PATH
-    const cached = await tryLoadFromGCS(config, dateString, 'rankings', logger);
-    if (cached) return cached;
-
-    // 2. BIGQUERY FIRST (if enabled)
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const { queryPIRankings } = require('../../core/utils/bigquery_utils');
-            const bigqueryData = await queryPIRankings(dateString, logger);
-            
-            if (bigqueryData && bigqueryData.Items && bigqueryData.Items.length > 0) {
-                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for PI rankings (${dateString}): ${bigqueryData.Items.length} items`);
-                return bigqueryData.Items;
-            }
-        } catch (bqError) {
-            logger.log('WARN', `[DataLoader] BigQuery rankings query failed, falling back to Firestore: ${bqError.message}`);
-            // Fall through to Firestore
-        }
-    }
-
-    // 3. FIRESTORE FALLBACK
-    const collectionName = config.popularInvestorRankingsCollection || 'popular_investor_rankings';
-    logger.log('INFO', `Loading Popular Investor Rankings for ${dateString} from Firestore`);
-    try {
-        const docRef = db.collection(collectionName).doc(dateString);
-        const docSnap = await withRetry(() => docRef.get(), `getRankings(${dateString})`);
-        if (!docSnap.exists) { logger.log('WARN', `Rankings not found for ${dateString}`); return null; }
-        const data = tryDecompress(docSnap.data());
-        return data.Items || []; 
-    } catch (error) {
-        logger.log('ERROR', `Failed to load Rankings for ${dateString}: ${error.message}`);
-        return null;
-    }
-}
-
-/** Stage 12: Load User Verification Profiles 
- * [UPDATED] Scans global verification data via CollectionGroup since it's now stored per-user.
- * [UPDATED] Added optional dateString param to support GCS snapshot checks.
+/**
+ * Load price data from an array of shard references (virtual or Firestore doc refs).
+ * Used by PriceBatchExecutor for batch price computations.
+ * @param {object} config - Configuration object
+ * @param {object} deps - Dependencies (db, logger, etc.)
+ * @param {Array} shardRefs - Array of shard references (virtual BigQuery objects or Firestore doc refs)
+ * @returns {Promise<object>} Combined price data object keyed by instrument ID
  */
-async function loadVerificationProfiles(config, deps, dateString = null) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
-
-    // 1. GCS FAST PATH (If date provided)
-    if (dateString) {
-        const cached = await tryLoadFromGCS(config, dateString, 'verification', logger);
-        if (cached) return cached;
-    }
+exports.loadDataByRefs = async (config, deps, shardRefs) => {
+    const { logger } = deps;
     
-    // 2. FIRESTORE FALLBACK (Global Scan)
-    logger.log('INFO', `Loading Verification Profiles (CollectionGroup: verification/data)`);
-    try {
-        const snapshot = await withRetry(() => db.collectionGroup('verification').get(), 'getVerificationsGroup');
-        if (snapshot.empty) return {};
-        
-        const profiles = {};
-        let count = 0;
-        snapshot.forEach(doc => {
-            if (doc.id !== 'data') return;
-            const raw = tryDecompress(doc.data());
-            if (raw.etoroCID) {
-                profiles[raw.etoroCID] = {
-                    cid: raw.etoroCID,
-                    username: raw.etoroUsername,
-                    aboutMe: "", 
-                    aboutMeShort: "",
-                    isVerified: !!(raw.verifiedAt),
-                    restrictions: []
-                };
-                count++;
-            }
-        });
-        logger.log('INFO', `Loaded ${count} verification profiles.`);
-        return profiles;
-    } catch (error) {
-        logger.log('ERROR', `Failed to load Verification Profiles: ${error.message}`);
+    if (!Array.isArray(shardRefs) || shardRefs.length === 0) {
         return {};
     }
-}
-
-/** Stage 13: Load PI Ratings Data */
-async function loadPIRatings(config, deps, dateString) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
-
-    // 1. GCS FAST PATH
-    const cached = await tryLoadFromGCS(config, dateString, 'ratings', logger);
-    if (cached) return cached;
-
-    // 2. BIGQUERY FIRST (if enabled)
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const { queryPIRatings } = require('../../core/utils/bigquery_utils');
-            const bigqueryData = await queryPIRatings(dateString, logger);
-            if (bigqueryData) {
-                logger.log('INFO', `[DataLoader] ✅ Loaded PI Ratings from BigQuery for ${dateString}`);
-                return bigqueryData;
-            }
-        } catch (error) {
-            logger.log('WARN', `[DataLoader] BigQuery PI Ratings query failed, falling back to Firestore: ${error.message}`);
-        }
-    }
 
-    // 3. FIRESTORE FALLBACK
-    const collectionName = config.piRatingsCollection || 'PIRatingsData';
-    logger.log('INFO', `Loading PI Ratings from Firestore for ${dateString}`);
-    try {
-        const docRef = db.collection(collectionName).doc(dateString);
-        const docSnap = await withRetry(() => docRef.get(), `getPIRatings(${dateString})`);
-        if (!docSnap.exists) { 
-            logger.log('WARN', `PI Ratings not found for ${dateString}`); 
-            return {}; 
-        }
-        const data = tryDecompress(docSnap.data());
-        const { date, lastUpdated, ...piRatings } = data;
-        return piRatings; 
-    } catch (error) {
-        logger.log('ERROR', `Failed to load PI Ratings: ${error.message}`);
-        return {};
-    }
-}
-
-/** Stage 14: Load PI Page Views Data */
-async function loadPIPageViews(config, deps, dateString) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
-
-    // 1. GCS FAST PATH
-    const cached = await tryLoadFromGCS(config, dateString, 'page_views', logger);
-    if (cached) return cached;
-
-    // 2. BIGQUERY FIRST (if enabled)
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const { queryPIPageViews } = require('../../core/utils/bigquery_utils');
-            const bigqueryData = await queryPIPageViews(dateString, logger);
-            if (bigqueryData) {
-                logger.log('INFO', `[DataLoader] ✅ Loaded PI Page Views from BigQuery for ${dateString}`);
-                return bigqueryData;
-            }
-        } catch (error) {
-            logger.log('WARN', `[DataLoader] BigQuery PI Page Views query failed, falling back to Firestore: ${error.message}`);
-        }
-    }
-
-    // 3. FIRESTORE FALLBACK
-    const collectionName = config.piPageViewsCollection || 'PIPageViewsData';
-    logger.log('INFO', `Loading PI Page Views from Firestore for ${dateString}`);
-    try {
-        const docRef = db.collection(collectionName).doc(dateString);
-        const docSnap = await withRetry(() => docRef.get(), `getPIPageViews(${dateString})`);
-        if (!docSnap.exists) { logger.log('WARN', `PI Page Views not found`); return null; }
-        const data = tryDecompress(docSnap.data());
-        const { date, lastUpdated, ...piPageViews } = data;
-        return piPageViews; 
-    } catch (error) {
-        logger.log('ERROR', `Failed to load PI Page Views: ${error.message}`);
-        return null;
-    }
-}
-
-/** Stage 15: Load Watchlist Membership Data */
-async function loadWatchlistMembership(config, deps, dateString) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
+    // Check if we're in BigQuery mode (virtual shards)
+    const isBigQuery = shardRefs.some(ref => ref && ref._bigquery === true);
     
-    // 1. GCS FAST PATH
-    const cached = await tryLoadFromGCS(config, dateString, 'watchlist', logger);
-    if (cached) return cached;
-
-    // 2. BIGQUERY FIRST (if enabled)
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
+    if (isBigQuery && process.env.BIGQUERY_ENABLED !== 'false') {
         try {
-            const { queryWatchlistMembership } = require('../../core/utils/bigquery_utils');
-            const bigqueryData = await queryWatchlistMembership(dateString, logger);
-            if (bigqueryData) {
-                logger.log('INFO', `[DataLoader] ✅ Loaded Watchlist Membership from BigQuery for ${dateString}`);
-                return bigqueryData;
+            // Extract targetIds from virtual shards if present
+            const targetIds = shardRefs
+                .filter(ref => ref._bigquery && ref.targetIds && ref.targetIds.length > 0)
+                .flatMap(ref => ref.targetIds);
+            
+            // Query BigQuery for prices
+            // queryAssetPrices signature: (startDateStr, endDateStr, instrumentIds, logger)
+            const { queryAssetPrices } = require('../../core/utils/bigquery_utils');
+            const pricesData = await queryAssetPrices(null, null, targetIds.length > 0 ? targetIds : null, logger);
+            
+            // Filter by targetIds if specified
+            if (targetIds.length > 0 && pricesData) {
+                const targetSet = new Set(targetIds.map(id => String(id)));
+                const filtered = {};
+                for (const [instrumentId, priceData] of Object.entries(pricesData)) {
+                    if (targetSet.has(String(instrumentId))) {
+                        filtered[instrumentId] = priceData;
+                    }
+                }
+                return filtered;
             }
-        } catch (error) {
-            logger.log('WARN', `[DataLoader] BigQuery Watchlist Membership query failed, falling back to Firestore: ${error.message}`);
+            
+            return pricesData || {};
+        } catch (e) {
+            logger.log('ERROR', `[DataLoader] BigQuery price load failed: ${e.message}`);
+            return {};
         }
     }
 
-    // 3. FIRESTORE FALLBACK
-    const collectionName = config.watchlistMembershipCollection || 'WatchlistMembershipData';
-    logger.log('INFO', `Loading Watchlist Membership from Firestore for ${dateString}`);
+    // Firestore fallback - load from doc refs
+    const combined = {};
     try {
-        const docRef = db.collection(collectionName).doc(dateString);
-        const docSnap = await withRetry(() => docRef.get(), `getWatchlistMembership(${dateString})`);
-        if (!docSnap.exists) { logger.log('WARN', `Watchlist Membership not found`); return null; }
-        const data = tryDecompress(docSnap.data());
-        const { date, lastUpdated, ...watchlistMembership } = data;
-        return watchlistMembership; 
-    } catch (error) {
-        logger.log('ERROR', `Failed to load Watchlist Membership: ${error.message}`);
-        return null;
-    }
-}
-
-/** Stage 16: Load PI Alert History Data */
-async function loadPIAlertHistory(config, deps, dateString) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
-
-    // 1. GCS FAST PATH
-    const cached = await tryLoadFromGCS(config, dateString, 'alerts', logger);
-    if (cached) return cached;
-
-    // 2. BIGQUERY FIRST (if enabled)
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const { queryPIAlertHistory } = require('../../core/utils/bigquery_utils');
-            const bigqueryData = await queryPIAlertHistory(dateString, logger);
-            if (bigqueryData) {
-                logger.log('INFO', `[DataLoader] ✅ Loaded PI Alert History from BigQuery for ${dateString}`);
-                return bigqueryData;
+        const loadPromises = shardRefs.map(async (docRef) => {
+            try {
+                const snap = await docRef.get();
+                if (snap.exists) {
+                    const data = snap.data();
+                    // Firestore price shards are nested: { instrumentId: { prices: {...} } }
+                    Object.assign(combined, data);
+                }
+            } catch (e) {
+                logger.log('WARN', `[DataLoader] Failed to load price shard: ${e.message}`);
             }
-        } catch (error) {
-            logger.log('WARN', `[DataLoader] BigQuery PI Alert History query failed, falling back to Firestore: ${error.message}`);
-        }
-    }
-
-    // 3. FIRESTORE FALLBACK
-    const collectionName = config.piAlertHistoryCollection || 'PIAlertHistoryData';
-    logger.log('INFO', `Loading PI Alert History from Firestore for ${dateString}`);
-    try {
-        const docRef = db.collection(collectionName).doc(dateString);
-        const docSnap = await withRetry(() => docRef.get(), `getPIAlertHistory(${dateString})`);
-        if (!docSnap.exists) { logger.log('WARN', `PI Alert History not found`); return null; }
-        const data = tryDecompress(docSnap.data());
-        const { date, lastUpdated, ...piAlertHistory } = data;
-        return piAlertHistory; 
-    } catch (error) {
-        logger.log('ERROR', `Failed to load PI Alert History: ${error.message}`);
-        return null;
-    }
-}
-
-/** Stage 17: Load PI-Centric Watchlist Data (Targeted - Keep as Firestore) */
-async function loadPIWatchlistData(config, deps, piCid) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
-    const piCidStr = String(piCid);
-    
-    logger.log('INFO', `Loading PI Watchlist Data for PI ${piCid}`);
-    try {
-        const docRef = db.collection('PopularInvestors').doc(piCidStr).collection('watchlistData').doc('current');
-        const docSnap = await withRetry(() => docRef.get(), `getPIWatchlistData(${piCidStr})`);
-        if (!docSnap.exists) { logger.log('WARN', `PI Watchlist Data not found for PI ${piCidStr}`); return null; }
-        return tryDecompress(docSnap.data());
-    } catch (error) {
-        logger.log('ERROR', `Failed to load PI Watchlist Data for PI ${piCidStr}: ${error.message}`);
-        return null;
+        });
+        
+        await Promise.all(loadPromises);
+    } catch (e) {
+        logger.log('ERROR', `[DataLoader] Failed to load price data from refs: ${e.message}`);
     }
-}
-
-// Load Popular Investor Master List
-async function loadPopularInvestorMasterList(config, deps, dateString = null) {
-    const { db, logger, calculationUtils } = deps;
-    const { withRetry } = calculationUtils;
     
-    // 1. GCS FAST PATH (If date context exists)
-    if (dateString) {
-        const cached = await tryLoadFromGCS(config, dateString, 'master_list', logger);
-        if (cached) return cached;
-    }
+    return combined;
+};
 
-    // 2. BIGQUERY FIRST (if enabled)
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const { queryPIMasterList } = require('../../core/utils/bigquery_utils');
-            const bigqueryData = await queryPIMasterList(logger);
-            
-            if (bigqueryData && Object.keys(bigqueryData).length > 0) {
-                logger.log('INFO', `[DataLoader] ✅ Using BigQuery for PI master list: ${Object.keys(bigqueryData).length} PIs`);
-                return bigqueryData;
-            }
-        } catch (bqError) {
-            logger.log('WARN', `[DataLoader] BigQuery master list query failed, falling back to Firestore: ${bqError.message}`);
-            // Fall through to Firestore
-        }
-    }
+// =============================================================================
+// HELPERS
+// =============================================================================
 
-    // 3. FIRESTORE FALLBACK
-    const collectionName = config.piMasterListCollection || 'system_state';
-    const docId = config.piMasterListDocId || 'popular_investor_master_list';
-    logger.log('INFO', `Loading Popular Investor Master List from ${collectionName}/${docId} (Firestore)`);
+async function loadRetailFirestore(db, collectionName, dateStr) {
+    const CANARY_ID = '19M'; // Legacy Block
     try {
-        const docRef = db.collection(collectionName).doc(docId);
-        const docSnap = await withRetry(() => docRef.get(), 'getPIMasterList');
-        if (!docSnap.exists) { logger.log('WARN', 'Popular Investor Master List not found.'); return {}; }
-        const data = tryDecompress(docSnap.data());
-        return data.investors || data; 
-    } catch (error) {
-        logger.log('ERROR', `Failed to load PI Master List: ${error.message}`);
+        const partsRef = db.collection(collectionName).doc(CANARY_ID)
+            .collection('snapshots').doc(dateStr).collection('parts');
+        
+        const snap = await partsRef.get();
+        if (snap.empty) return {};
+        
+        const combined = {};
+        snap.forEach(doc => Object.assign(combined, doc.data()));
+        return combined;
+    } catch (e) {
         return {};
     }
-}
-
-module.exports = {
-    getPortfolioPartRefs, 
-    loadDataByRefs, 
-    loadFullDayMap,
-    loadDailyPortfolios, 
-    loadDailyInsights, 
-    loadDailySocialPostInsights, 
-    getHistoryPartRefs, 
-    streamPortfolioData, 
-    streamHistoryData,
-    getPriceShardRefs,
-    ensurePriceShardIndex,
-    getRelevantShardRefs,
-    loadPopularInvestorRankings,
-    loadVerificationProfiles,
-    loadPIRatings,
-    loadPIPageViews,
-    loadWatchlistMembership,
-    loadPIAlertHistory,
-    loadPopularInvestorMasterList,
-    loadPIWatchlistData,
-};
+}