bulltrackers-module 1.0.657 → 1.0.659

package/functions/computation-system/utils/data_loader.js

@@ -1,14 +1,19 @@
 /**
  * @fileoverview Data loader sub-pipes for the Computation System.
  * REFACTORED: Now stateless and receive dependencies.
+ * UPDATED: Integrated GCS Snapshot "Fast Path" for massive read reduction across ALL data types.
  * FIXED: Added strict userType filtering to prevent fetching unnecessary data.
  * UPDATED: Verification now uses CollectionGroup query due to per-user storage.
- * UPDATED: Ratings now correctly handles flattened top-level schema (keys like "reviews.ID").
- * REMOVED: Redundant Price Shard Indexing logic.
+ * UPDATED: Ratings now correctly handles flattened top-level schema.
  */
 const zlib = require('zlib'); 
+const { Storage } = require('@google-cloud/storage');
+const readline = require('readline');
 
-// Helper for decompressing any doc if needed
+// Singleton Storage Client
+const storage = new Storage(); 
+
+// Helper for decompressing any doc if needed (Firestore Legacy)
 function tryDecompress(data) {
     if (data && data._compressed === true && data.payload) {
         try {
@@ -21,6 +26,26 @@ function tryDecompress(data) {
     return data;
 }
 
+// --- GCS FAST PATH HELPER ---
+async function tryLoadFromGCS(config, dateString, snapshotName, logger) {
+    if (!dateString) return null;
+    const bucketName = config.gcsBucketName || 'bulltrackers';
+    try {
+        const bucket = storage.bucket(bucketName);
+        const file = bucket.file(`${dateString}/snapshots/${snapshotName}.json.gz`);
+        const [exists] = await file.exists();
+        
+        if (exists) {
+            logger.log('INFO', `[DataLoader] ⚡️ GCS HIT: ${snapshotName} for ${dateString}`);
+            const [content] = await file.download();
+            return JSON.parse(zlib.gunzipSync(content).toString());
+        }
+    } catch (e) {
+        logger.log('WARN', `[DataLoader] GCS Check Failed (${snapshotName}): ${e.message}`);
+    }
+    return null;
+}
+
 /** --- Data Loader Sub-Pipes (Stateless, Dependency-Injection) --- */
 
 /** Stage 1: Get portfolio part document references for a given date */
@@ -38,7 +63,6 @@ async function getPortfolioPartRefs(config, deps, dateString, requiredUserTypes
     
     // NEW STRUCTURE: Read from date-based collections (per-user documents)
     try {
-        // Signed-In User Portfolios
         if (fetchAll || types.has('SIGNED_IN_USER')) {
             const signedInPortCollectionName = 'SignedInUserPortfolioData';
             const signedInPortDateDoc = db.collection(signedInPortCollectionName).doc(dateString);
@@ -49,16 +73,10 @@ async function getPortfolioPartRefs(config, deps, dateString, requiredUserTypes
             
             signedInPortSubcollections.forEach(subcol => {
                 const cid = subcol.id;
-                allPartRefs.push({ 
-                    ref: subcol.doc(cid), 
-                    type: 'SIGNED_IN_USER',
-                    cid: cid,
-                    collectionType: 'NEW_STRUCTURE'
-                });
+                allPartRefs.push({ ref: subcol.doc(cid), type: 'SIGNED_IN_USER', cid: cid, collectionType: 'NEW_STRUCTURE' });
             });
         }
         
-        // Popular Investor Portfolios
         if (fetchAll || types.has('POPULAR_INVESTOR')) {
             const piPortCollectionName = 'PopularInvestorPortfolioData';
             const piPortDateDoc = db.collection(piPortCollectionName).doc(dateString);
@@ -69,12 +87,7 @@ async function getPortfolioPartRefs(config, deps, dateString, requiredUserTypes
             
             piPortSubcollections.forEach(subcol => {
                 const cid = subcol.id;
-                allPartRefs.push({ 
-                    ref: subcol.doc(cid), 
-                    type: 'POPULAR_INVESTOR',
-                    cid: cid,
-                    collectionType: 'NEW_STRUCTURE'
-                });
+                allPartRefs.push({ ref: subcol.doc(cid), type: 'POPULAR_INVESTOR', cid: cid, collectionType: 'NEW_STRUCTURE' });
             });
         }
     } catch (newStructError) {
@@ -83,42 +96,25 @@ async function getPortfolioPartRefs(config, deps, dateString, requiredUserTypes
     
     // LEGACY STRUCTURE: Read from block-based collections
     const collectionsToQuery = [];
-    
-    if ((fetchAll || types.has('NORMAL')) && config.normalUserPortfolioCollection) 
-        collectionsToQuery.push({ name: config.normalUserPortfolioCollection, type: 'NORMAL' });
-        
-    if ((fetchAll || types.has('SPECULATOR')) && config.speculatorPortfolioCollection) 
-        collectionsToQuery.push({ name: config.speculatorPortfolioCollection, type: 'SPECULATOR' });
-        
-    if ((fetchAll || types.has('POPULAR_INVESTOR')) && config.piPortfolioCollection) 
-        collectionsToQuery.push({ name: config.piPortfolioCollection, type: 'POPULAR_INVESTOR' });
-        
-    if ((fetchAll || types.has('SIGNED_IN_USER')) && config.signedInUsersCollection) 
-        collectionsToQuery.push({ name: config.signedInUsersCollection, type: 'SIGNED_IN_USER' });
+    if ((fetchAll || types.has('NORMAL')) && config.normalUserPortfolioCollection) collectionsToQuery.push({ name: config.normalUserPortfolioCollection, type: 'NORMAL' });
+    if ((fetchAll || types.has('SPECULATOR')) && config.speculatorPortfolioCollection) collectionsToQuery.push({ name: config.speculatorPortfolioCollection, type: 'SPECULATOR' });
+    if ((fetchAll || types.has('POPULAR_INVESTOR')) && config.piPortfolioCollection) collectionsToQuery.push({ name: config.piPortfolioCollection, type: 'POPULAR_INVESTOR' });
+    if ((fetchAll || types.has('SIGNED_IN_USER')) && config.signedInUsersCollection) collectionsToQuery.push({ name: config.signedInUsersCollection, type: 'SIGNED_IN_USER' });
 
     for (const { name: collectionName, type: collectionType } of collectionsToQuery) {
         try {
             const blockDocsQuery = db.collection(collectionName);
             const blockDocRefs = await withRetry(() => blockDocsQuery.listDocuments(), `listDocuments(${collectionName})`);
-            
             if (!blockDocRefs.length) continue; 
 
             const partsPromises = blockDocRefs.map(blockDocRef => { 
-                const partsCollectionRef = blockDocRef
-                    .collection(config.snapshotsSubcollection || 'snapshots')
-                    .doc(dateString)
-                    .collection(config.partsSubcollection || 'parts'); 
+                const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection || 'snapshots').doc(dateString).collection(config.partsSubcollection || 'parts'); 
                 return withRetry(() => partsCollectionRef.listDocuments(), `listParts(${partsCollectionRef.path})`); 
             });
 
             const partDocArrays = await Promise.all(partsPromises);
-            
             partDocArrays.forEach(partDocs => { 
-                allPartRefs.push(...partDocs.map(ref => ({ 
-                    ref, 
-                    type: collectionType,
-                    collectionType: 'LEGACY'
-                }))); 
+                allPartRefs.push(...partDocs.map(ref => ({ ref, type: collectionType, collectionType: 'LEGACY' }))); 
             });
         } catch (legacyError) {
             logger.log('WARN', `Failed to load legacy collection ${collectionName}: ${legacyError.message}`);
@@ -163,9 +159,7 @@ async function loadDataByRefs(config, deps, refObjects) {
                 
                 if (meta.type === 'POPULAR_INVESTOR') {
                     chunkData[cid]._userType = 'POPULAR_INVESTOR';
-                    if (chunkData[cid].deepPositions) {
-                        chunkData[cid].DeepPositions = chunkData[cid].deepPositions;
-                    }
+                    if (chunkData[cid].deepPositions) chunkData[cid].DeepPositions = chunkData[cid].deepPositions;
                 } else if (meta.type === 'SIGNED_IN_USER') {
                     chunkData[cid]._userType = 'SIGNED_IN_USER';
                 }
@@ -184,21 +178,15 @@ async function loadDataByRefs(config, deps, refObjects) {
                             if (deepSnap.exists) {
                                 const deepChunk = tryDecompress(deepSnap.data());
                                 for (const [uid, pData] of Object.entries(chunkData)) {
-                                    if (deepChunk[uid] && deepChunk[uid].positions) {
-                                        pData.DeepPositions = deepChunk[uid].positions;
-                                    }
+                                    if (deepChunk[uid] && deepChunk[uid].positions) pData.DeepPositions = deepChunk[uid].positions;
                                 }
                             }
-                            for (const pData of Object.values(chunkData)) {
-                                pData._userType = 'POPULAR_INVESTOR';
-                            }
+                            for (const pData of Object.values(chunkData)) pData._userType = 'POPULAR_INVESTOR';
                             return chunkData;
                         }).catch(() => chunkData)
                     );
                 } else if (meta.type === 'SIGNED_IN_USER') {
-                    for (const pData of Object.values(chunkData)) {
-                        pData._userType = 'SIGNED_IN_USER';
-                    }
+                    for (const pData of Object.values(chunkData)) pData._userType = 'SIGNED_IN_USER';
                     deepFetchPromises.push(Promise.resolve(chunkData));
                 } else {
                     deepFetchPromises.push(Promise.resolve(chunkData));
@@ -208,16 +196,19 @@ async function loadDataByRefs(config, deps, refObjects) {
 
         const resolvedChunks = await Promise.all(deepFetchPromises);
         resolvedChunks.forEach(chunk => {
-             if (chunk && typeof chunk === 'object') {
-                Object.assign(mergedPortfolios, chunk);
-             }
+             if (chunk && typeof chunk === 'object') Object.assign(mergedPortfolios, chunk);
         });
     }
     return mergedPortfolios;
 }
 
 /** Stage 3: Load a full day map by delegating to loadDataByRefs */
-async function loadFullDayMap(config, deps, partRefs) {
+async function loadFullDayMap(config, deps, partRefs, dateString) { 
+    // 1. GCS FAST PATH
+    const cached = await tryLoadFromGCS(config, dateString, 'portfolios', deps.logger);
+    if (cached) return cached;
+
+    // 2. FIRESTORE FALLBACK
     const { logger } = deps;
     if (!partRefs.length) return {};
     logger.log('TRACE', `Loading full day map from ${partRefs.length} references...`);
@@ -230,6 +221,12 @@ async function loadFullDayMap(config, deps, partRefs) {
 async function loadDailyInsights(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
+
+    // 1. GCS FAST PATH
+    const cached = await tryLoadFromGCS(config, dateString, 'insights', logger);
+    if (cached) return cached;
+
+    // 2. FIRESTORE FALLBACK
     const insightsCollectionName = config.insightsCollectionName || 'daily_instrument_insights';
     logger.log('INFO', `Loading daily insights for ${dateString} from ${insightsCollectionName}`);
     try { 
@@ -246,27 +243,23 @@ async function loadDailyInsights(config, deps, dateString) {
 
 /** Stage 5: Load and Partition Social Data */
 async function loadDailySocialPostInsights(config, deps, dateString) {
-    const { db, logger, calculationUtils, collectionRegistry } = deps;
+    const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     
+    // 1. GCS FAST PATH
+    const cached = await tryLoadFromGCS(config, dateString, 'social', logger);
+    if (cached) return cached;
+
+    // 2. FIRESTORE FALLBACK
     logger.log('INFO', `Loading and partitioning social data for ${dateString}`);
     
-    // 1. Initialize Buckets
-    const result = {
-        generic: {},            // Map<PostId, Data> - For Normal/Speculator
-        pi: {},                 // Map<UserId, Map<PostId, Data>> - For Popular Investors
-        signedIn: {}            // Map<UserId, Map<PostId, Data>> - For Signed-In Users
-    };
-
-    // NEW STRUCTURE: Read from date-based collections
+    const result = { generic: {}, pi: {}, signedIn: {} };
+
+    // NEW STRUCTURE
     try {
-        // Signed-In User Social: SignedInUserSocialPostData/{date}/{cid}/{cid}
         const signedInSocialCollectionName = 'SignedInUserSocialPostData';
         const signedInSocialDateDoc = db.collection(signedInSocialCollectionName).doc(dateString);
-        const signedInSocialSubcollections = await withRetry(
-            () => signedInSocialDateDoc.listCollections(),
-            `listSignedInSocial(${dateString})`
-        );
+        const signedInSocialSubcollections = await withRetry(() => signedInSocialDateDoc.listCollections(), `listSignedInSocial(${dateString})`);
         
         for (const subcol of signedInSocialSubcollections) {
             const cid = subcol.id;
@@ -280,13 +273,9 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
             }
         }
         
-        // Popular Investor Social: PopularInvestorSocialPostData/{date}/{cid}/{cid}
         const piSocialCollectionName = 'PopularInvestorSocialPostData';
         const piSocialDateDoc = db.collection(piSocialCollectionName).doc(dateString);
-        const piSocialSubcollections = await withRetry(
-            () => piSocialDateDoc.listCollections(),
-            `listPISocial(${dateString})`
-        );
+        const piSocialSubcollections = await withRetry(() => piSocialDateDoc.listCollections(), `listPISocial(${dateString})`);
         
         for (const subcol of piSocialSubcollections) {
             const cid = subcol.id;
@@ -300,44 +289,35 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
             }
         }
         
-        // Instrument Social: InstrumentFeedSocialPostData/{date}/posts/{postId}
         const instrumentSocialCollectionName = 'InstrumentFeedSocialPostData';
         const instrumentSocialDateDoc = db.collection(instrumentSocialCollectionName).doc(dateString);
         const instrumentSocialPostsCol = instrumentSocialDateDoc.collection('posts');
-        const instrumentSocialSnapshot = await withRetry(
-            () => instrumentSocialPostsCol.limit(1000).get(),
-            `getInstrumentSocial(${dateString})`
-        );
+        const instrumentSocialSnapshot = await withRetry(() => instrumentSocialPostsCol.limit(1000).get(), `getInstrumentSocial(${dateString})`);
         
         instrumentSocialSnapshot.forEach(doc => {
             const data = tryDecompress(doc.data());
             result.generic[doc.id] = data;
         });
         
-        logger.log('INFO', `Loaded Social Data (NEW): ${Object.keys(result.generic).length} Generic, ${Object.keys(result.pi).length} PIs, ${Object.keys(result.signedIn).length} SignedIn.`);
+        logger.log('INFO', `Loaded Social Data (NEW): ${Object.keys(result.generic).length} Generic, ${Object.keys(result.pi).length} PIs.`);
     } catch (newStructError) {
-        logger.log('WARN', `Failed to load from new structure, falling back to legacy: ${newStructError.message}`);
+        logger.log('WARN', `Failed to load from new structure: ${newStructError.message}`);
     }
 
-    // LEGACY STRUCTURE: CollectionGroup query (for backward compatibility)
+    // LEGACY STRUCTURE
     const PI_COL_NAME = config.piSocialCollectionName || config.piSocialCollection || 'pi_social_posts';
     const SIGNED_IN_COL_NAME = config.signedInUserSocialCollection || 'signed_in_users_social';
-
     const startDate = new Date(dateString + 'T00:00:00Z');
     const endDate = new Date(dateString + 'T23:59:59Z');
 
     try {
-        const postsQuery = db.collectionGroup('posts')
-            .where('fetchedAt', '>=', startDate)
-            .where('fetchedAt', '<=', endDate);
-
+        const postsQuery = db.collectionGroup('posts').where('fetchedAt', '>=', startDate).where('fetchedAt', '<=', endDate);
         const querySnapshot = await withRetry(() => postsQuery.get(), `getSocialPosts(${dateString})`);
         
         if (!querySnapshot.empty) {
             querySnapshot.forEach(doc => { 
                 const data = tryDecompress(doc.data());
                 const path = doc.ref.path; 
-                
                 if (path.includes(PI_COL_NAME)) {
                     const parts = path.split('/');
                     const colIndex = parts.indexOf(PI_COL_NAME);
@@ -346,8 +326,7 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
                         if (!result.pi[userId]) result.pi[userId] = {};
                         result.pi[userId][doc.id] = data;
                     }
-                } 
-                else if (path.includes(SIGNED_IN_COL_NAME)) {
+                } else if (path.includes(SIGNED_IN_COL_NAME)) {
                     const parts = path.split('/');
                     const colIndex = parts.indexOf(SIGNED_IN_COL_NAME);
                     if (colIndex !== -1 && parts[colIndex + 1]) {
@@ -355,18 +334,13 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
                         if (!result.signedIn[userId]) result.signedIn[userId] = {};
                         result.signedIn[userId][doc.id] = data;
                     }
-                } 
-                else {
+                } else {
                     result.generic[doc.id] = data;
                 }
             });
-            logger.log('INFO', `Loaded Social Data (LEGACY): ${Object.keys(result.generic).length} Generic, ${Object.keys(result.pi).length} PIs, ${Object.keys(result.signedIn).length} SignedIn.`);
-        } else {
-            logger.log('WARN', `No social posts found for ${dateString} via CollectionGroup.`);
         }
-
     } catch (error) {
-        logger.log('ERROR', `Failed to load social posts: ${error.message}`);
+        logger.log('ERROR', `Failed to load social posts (legacy): ${error.message}`);
     }
 
     return result;
@@ -395,12 +369,7 @@ async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes =
                 `listSignedInHistory(${dateString})`
             );
             signedInHistSubcollections.forEach(subcol => {
-                allPartRefs.push({ 
-                    ref: subcol.doc(subcol.id), 
-                    type: 'SIGNED_IN_USER',
-                    cid: subcol.id,
-                    collectionType: 'NEW_STRUCTURE'
-                });
+                allPartRefs.push({ ref: subcol.doc(subcol.id), type: 'SIGNED_IN_USER', cid: subcol.id, collectionType: 'NEW_STRUCTURE' });
             });
         }
         
@@ -412,12 +381,7 @@ async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes =
                 `listPIHistory(${dateString})`
             );
             piHistSubcollections.forEach(subcol => {
-                allPartRefs.push({ 
-                    ref: subcol.doc(subcol.id), 
-                    type: 'POPULAR_INVESTOR',
-                    cid: subcol.id,
-                    collectionType: 'NEW_STRUCTURE'
-                });
+                allPartRefs.push({ ref: subcol.doc(subcol.id), type: 'POPULAR_INVESTOR', cid: subcol.id, collectionType: 'NEW_STRUCTURE' });
             });
         }
     } catch (newStructError) {
@@ -426,14 +390,10 @@ async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes =
     
     // LEGACY STRUCTURE
     const collectionsToQuery = [];
-    if ((fetchAll || types.has('NORMAL')) && config.normalUserHistoryCollection) 
-        collectionsToQuery.push(config.normalUserHistoryCollection);
-    if ((fetchAll || types.has('SPECULATOR')) && config.speculatorHistoryCollection) 
-        collectionsToQuery.push(config.speculatorHistoryCollection);
-    if ((fetchAll || types.has('POPULAR_INVESTOR')) && config.piHistoryCollection) 
-        collectionsToQuery.push(config.piHistoryCollection);
-    if ((fetchAll || types.has('SIGNED_IN_USER')) && config.signedInHistoryCollection) 
-        collectionsToQuery.push(config.signedInHistoryCollection);
+    if ((fetchAll || types.has('NORMAL')) && config.normalUserHistoryCollection) collectionsToQuery.push(config.normalUserHistoryCollection);
+    if ((fetchAll || types.has('SPECULATOR')) && config.speculatorHistoryCollection) collectionsToQuery.push(config.speculatorHistoryCollection);
+    if ((fetchAll || types.has('POPULAR_INVESTOR')) && config.piHistoryCollection) collectionsToQuery.push(config.piHistoryCollection);
+    if ((fetchAll || types.has('SIGNED_IN_USER')) && config.signedInHistoryCollection) collectionsToQuery.push(config.signedInHistoryCollection);
 
     for (const collectionName of collectionsToQuery) {
         try {
@@ -442,18 +402,13 @@ async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes =
             if (!blockDocRefs.length) continue;
             
             const partsPromises = blockDocRefs.map(blockDocRef => { 
-                const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection || 'snapshots')
-                    .doc(dateString).collection(config.partsSubcollection || 'parts');
+                const partsCollectionRef = blockDocRef.collection(config.snapshotsSubcollection || 'snapshots').doc(dateString).collection(config.partsSubcollection || 'parts');
                 return withRetry(() => partsCollectionRef.listDocuments(), `listParts(${partsCollectionRef.path})`); 
             });
             
             const partDocArrays = await Promise.all(partsPromises);
             partDocArrays.forEach(partDocs => { 
-                allPartRefs.push(...partDocs.map(ref => ({ 
-                    ref, 
-                    type: 'PART',
-                    collectionType: 'LEGACY'
-                }))); 
+                allPartRefs.push(...partDocs.map(ref => ({ ref, type: 'PART', collectionType: 'LEGACY' }))); 
             });
         } catch (legacyError) {
             logger.log('WARN', `Failed to load legacy history collection ${collectionName}: ${legacyError.message}`);
@@ -467,6 +422,28 @@ async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes =
 /** Stage 7: Stream portfolio data in chunks */
 async function* streamPortfolioData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
     const { logger } = deps;
+
+    // 1. GCS FAST PATH (Snapshot) - Only if full run (no providedRefs)
+    if (!providedRefs) { 
+        try {
+            const bucketName = config.gcsBucketName || 'bulltrackers';
+            const bucket = storage.bucket(bucketName);
+            const file = bucket.file(`${dateString}/snapshots/portfolios.json.gz`);
+            const [exists] = await file.exists();
+            
+            if (exists) {
+                logger.log('INFO', `[DataLoader] ⚡️ STREAMING: Hydrating Portfolios from GCS Snapshot`);
+                const [content] = await file.download();
+                const fullData = JSON.parse(zlib.gunzipSync(content).toString());
+                yield fullData; // Yield all in one chunk as it fits in memory
+                return;
+            }
+        } catch (e) {
+            logger.log('WARN', `[DataLoader] GCS Portfolio Stream failed: ${e.message}. Falling back.`);
+        }
+    }
+
+    // 2. FIRESTORE FALLBACK
     const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString, requiredUserTypes));
     if (refs.length === 0) { logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`); return; }
     
@@ -484,6 +461,50 @@ async function* streamPortfolioData(config, deps, dateString, providedRefs = nul
 /** Stage 8: Stream history data in chunks */
 async function* streamHistoryData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
     const { logger } = deps;
+
+    // 1. GCS FAST PATH (JSONL Streaming) - Only if full run
+    if (!providedRefs) {
+        try {
+            const bucketName = config.gcsBucketName || 'bulltrackers';
+            const bucket = storage.bucket(bucketName);
+            const file = bucket.file(`${dateString}/snapshots/history.jsonl.gz`);
+            const [exists] = await file.exists();
+
+            if (exists) {
+                logger.log('INFO', `[DataLoader] ⚡️ STREAMING: Hydrating History from GCS (JSONL)`);
+                
+                const fileStream = file.createReadStream().pipe(zlib.createGunzip());
+                const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
+
+                let currentBatch = {};
+                let count = 0;
+                const BATCH_SIZE = 50; 
+
+                for await (const line of rl) {
+                    if (!line.trim()) continue;
+                    try {
+                        const userEntry = JSON.parse(line);
+                        Object.assign(currentBatch, userEntry);
+                        count++;
+                        if (count >= BATCH_SIZE) {
+                            yield currentBatch;
+                            currentBatch = {};
+                            count = 0;
+                        }
+                    } catch (parseErr) {
+                        logger.log('ERROR', `[DataLoader] JSONL Parse Error: ${parseErr.message}`);
+                    }
+                }
+                if (Object.keys(currentBatch).length > 0) yield currentBatch;
+                logger.log('INFO', `[DataLoader] Finished streaming History from GCS.`);
+                return;
+            }
+        } catch (e) {
+            logger.log('WARN', `[DataLoader] GCS History Stream failed: ${e.message}. Falling back.`);
+        }
+    }
+
+    // 2. FIRESTORE FALLBACK
     const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString, requiredUserTypes));
     if (refs.length === 0) { logger.log('WARN', `[streamHistoryData] No history refs found for ${dateString}. Stream is empty.`); return; }
     
@@ -515,19 +536,12 @@ async function getPriceShardRefs(config, deps) {
 
 /** Stage 10: Smart Shard Lookup System (DEPRECATED/SIMPLIFIED) */
 async function ensurePriceShardIndex(config, deps) {
-   // [DEPRECATED] This function previously built an index in 'system_metadata/price_shard_index'.
-   // It has been removed to avoid performing computation/indexing in the data loader.
-   // Use 'Fetch All' strategy in Stage 9 instead.
-   return {};
+   return {}; // Deprecated
 }
 
 async function getRelevantShardRefs(config, deps, targetInstrumentIds) {
     const { logger } = deps;
-    
-    // [UPDATED] Smart shard lookup is disabled due to missing index infrastructure
-    // and to avoid computing indexes during load time.
-    // Falling back to Stage 9 (Fetch All Shards).
-    logger.log('INFO', `[ShardLookup] Smart indexing disabled. Fetching all price shards for ${targetInstrumentIds ? targetInstrumentIds.length : 'all'} instruments.`);
+    logger.log('INFO', `[ShardLookup] Smart indexing disabled. Fetching all price shards.`);
     return getPriceShardRefs(config, deps);
 }
 
@@ -535,19 +549,18 @@ async function getRelevantShardRefs(config, deps, targetInstrumentIds) {
 async function loadPopularInvestorRankings(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
+
+    // 1. GCS FAST PATH
+    const cached = await tryLoadFromGCS(config, dateString, 'rankings', logger);
+    if (cached) return cached;
+
+    // 2. FIRESTORE FALLBACK
     const collectionName = config.popularInvestorRankingsCollection || 'popular_investor_rankings';
-    
     logger.log('INFO', `Loading Popular Investor Rankings for ${dateString}`);
-    
     try {
         const docRef = db.collection(collectionName).doc(dateString);
         const docSnap = await withRetry(() => docRef.get(), `getRankings(${dateString})`);
-        
-        if (!docSnap.exists) { 
-            logger.log('WARN', `Rankings not found for ${dateString}`); 
-            return null; 
-        }
-        
+        if (!docSnap.exists) { logger.log('WARN', `Rankings not found for ${dateString}`); return null; }
         const data = tryDecompress(docSnap.data());
         return data.Items || []; 
     } catch (error) {
@@ -558,48 +571,41 @@ async function loadPopularInvestorRankings(config, deps, dateString) {
 
 /** Stage 12: Load User Verification Profiles 
  * [UPDATED] Scans global verification data via CollectionGroup since it's now stored per-user.
+ * [UPDATED] Added optional dateString param to support GCS snapshot checks.
  */
-async function loadVerificationProfiles(config, deps) {
+async function loadVerificationProfiles(config, deps, dateString = null) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
+
+    // 1. GCS FAST PATH (If date provided)
+    if (dateString) {
+        const cached = await tryLoadFromGCS(config, dateString, 'verification', logger);
+        if (cached) return cached;
+    }
     
-    // Verification is now stored at /SignedInUsers/{cid}/verification/data
-    // To fetch globally, we must use a CollectionGroup query on 'verification'
-    // and filter for the document ID 'data'.
-    
+    // 2. FIRESTORE FALLBACK (Global Scan)
     logger.log('INFO', `Loading Verification Profiles (CollectionGroup: verification/data)`);
-
     try {
-        // Warning: This requires a Firestore Index if used with complex filters, but basic get() usually works.
         const snapshot = await withRetry(() => db.collectionGroup('verification').get(), 'getVerificationsGroup');
-        
         if (snapshot.empty) return {};
         
         const profiles = {};
         let count = 0;
-
         snapshot.forEach(doc => {
-            if (doc.id !== 'data') return; // Enforce specific document ID from schema
-
+            if (doc.id !== 'data') return;
             const raw = tryDecompress(doc.data());
-            
-            // Map new schema fields to internal profile structure
-            // New Schema: { etoroCID, etoroUsername, verifiedAt, setupCompletedAt ... }
             if (raw.etoroCID) {
                 profiles[raw.etoroCID] = {
                     cid: raw.etoroCID,
                     username: raw.etoroUsername,
-                    // 'aboutMe' and 'restrictions' are NOT present in the new schema.
-                    // Defaulting to empty values to preserve downstream compatibility.
                     aboutMe: "", 
                     aboutMeShort: "",
-                    isVerified: !!(raw.verifiedAt), // Using existence of verifiedAt as flag
+                    isVerified: !!(raw.verifiedAt),
                     restrictions: []
                 };
                 count++;
             }
         });
-        
         logger.log('INFO', `Loaded ${count} verification profiles.`);
         return profiles;
     } catch (error) {
@@ -608,59 +614,42 @@ async function loadVerificationProfiles(config, deps) {
     }
 }
 
-/** Stage 13: Load PI Ratings Data 
- * [UPDATED] Reads from /PiReviews/{date}/shards/daily_log.
- * [FIXED] Handles FLATTENED schema where keys like "reviews.ID" are at the top level.
- * Returns RAW logs grouped by PI. NO COMPUTATION.
- */
+/** Stage 13: Load PI Ratings Data */
 async function loadPIRatings(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    
-    // New Path: /PiReviews/{date}/shards/daily_log
-    
+
+    // 1. GCS FAST PATH
+    const cached = await tryLoadFromGCS(config, dateString, 'ratings', logger);
+    if (cached) return cached;
+
+    // 2. FIRESTORE FALLBACK
     logger.log('INFO', `Loading PI Ratings (Raw Logs) for ${dateString}`);
-    
     try {
         const shardsColRef = db.collection('PiReviews').doc(dateString).collection('shards');
         const shardDocs = await withRetry(() => shardsColRef.listDocuments(), `listRatingShards(${dateString})`);
 
         if (!shardDocs || shardDocs.length === 0) {
-            logger.log('WARN', `No rating shards found for ${dateString} at ${shardsColRef.path}`);
+            logger.log('WARN', `No rating shards found for ${dateString}`);
             return {};
         }
 
         const rawReviewsByPi = {}; 
-
         for (const docRef of shardDocs) {
             const docSnap = await docRef.get();
             if (!docSnap.exists) continue;
-
             const rawData = tryDecompress(docSnap.data());
-            
-            // SCHEMA HANDLING:
-            // Keys at the root of the document are the review IDs (e.g. "reviews.29312236_31075566").
-            // We iterate over all values and check if they look like review objects.
-            
             Object.values(rawData).forEach(entry => {
-                // Check for valid review object structure
                 if (entry && typeof entry === 'object' && entry.piCid && entry.rating !== undefined) {
-                    
-                    if (!rawReviewsByPi[entry.piCid]) {
-                        rawReviewsByPi[entry.piCid] = [];
-                    }
-                    
-                    // Store the raw entry directly.
+                    if (!rawReviewsByPi[entry.piCid]) rawReviewsByPi[entry.piCid] = [];
                     rawReviewsByPi[entry.piCid].push(entry);
                 }
             });
         }
-
         logger.log('INFO', `Loaded raw reviews for ${Object.keys(rawReviewsByPi).length} PIs.`);
         return rawReviewsByPi; 
-
     } catch (error) {
-        logger.log('ERROR', `Failed to load PI Ratings for ${dateString}: ${error.message}`);
+        logger.log('ERROR', `Failed to load PI Ratings: ${error.message}`);
         return {};
     }
 }
@@ -669,24 +658,23 @@ async function loadPIRatings(config, deps, dateString) {
 async function loadPIPageViews(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
+
+    // 1. GCS FAST PATH
+    const cached = await tryLoadFromGCS(config, dateString, 'page_views', logger);
+    if (cached) return cached;
+
+    // 2. FIRESTORE FALLBACK
     const collectionName = config.piPageViewsCollection || 'PIPageViewsData';
-    
     logger.log('INFO', `Loading PI Page Views for ${dateString}`);
-    
     try {
         const docRef = db.collection(collectionName).doc(dateString);
         const docSnap = await withRetry(() => docRef.get(), `getPIPageViews(${dateString})`);
-        
-        if (!docSnap.exists) { 
-            logger.log('WARN', `PI Page Views not found for ${dateString}`); 
-            return null; 
-        }
-        
+        if (!docSnap.exists) { logger.log('WARN', `PI Page Views not found`); return null; }
         const data = tryDecompress(docSnap.data());
         const { date, lastUpdated, ...piPageViews } = data;
         return piPageViews; 
     } catch (error) {
-        logger.log('ERROR', `Failed to load PI Page Views for ${dateString}: ${error.message}`);
+        logger.log('ERROR', `Failed to load PI Page Views: ${error.message}`);
         return null;
     }
 }
@@ -695,24 +683,23 @@ async function loadPIPageViews(config, deps, dateString) {
 async function loadWatchlistMembership(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    const collectionName = config.watchlistMembershipCollection || 'WatchlistMembershipData';
     
+    // 1. GCS FAST PATH
+    const cached = await tryLoadFromGCS(config, dateString, 'watchlist', logger);
+    if (cached) return cached;
+
+    // 2. FIRESTORE FALLBACK
+    const collectionName = config.watchlistMembershipCollection || 'WatchlistMembershipData';
     logger.log('INFO', `Loading Watchlist Membership for ${dateString}`);
-    
     try {
         const docRef = db.collection(collectionName).doc(dateString);
         const docSnap = await withRetry(() => docRef.get(), `getWatchlistMembership(${dateString})`);
-        
-        if (!docSnap.exists) { 
-            logger.log('WARN', `Watchlist Membership not found for ${dateString}`); 
-            return null; 
-        }
-        
+        if (!docSnap.exists) { logger.log('WARN', `Watchlist Membership not found`); return null; }
         const data = tryDecompress(docSnap.data());
         const { date, lastUpdated, ...watchlistMembership } = data;
         return watchlistMembership; 
     } catch (error) {
-        logger.log('ERROR', `Failed to load Watchlist Membership for ${dateString}: ${error.message}`);
+        logger.log('ERROR', `Failed to load Watchlist Membership: ${error.message}`);
         return null;
     }
 }
@@ -721,51 +708,39 @@ async function loadWatchlistMembership(config, deps, dateString) {
 async function loadPIAlertHistory(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
+
+    // 1. GCS FAST PATH
+    const cached = await tryLoadFromGCS(config, dateString, 'alerts', logger);
+    if (cached) return cached;
+
+    // 2. FIRESTORE FALLBACK
     const collectionName = config.piAlertHistoryCollection || 'PIAlertHistoryData';
-    
     logger.log('INFO', `Loading PI Alert History for ${dateString}`);
-    
     try {
         const docRef = db.collection(collectionName).doc(dateString);
         const docSnap = await withRetry(() => docRef.get(), `getPIAlertHistory(${dateString})`);
-        
-        if (!docSnap.exists) { 
-            logger.log('WARN', `PI Alert History not found for ${dateString}`); 
-            return null; 
-        }
-        
+        if (!docSnap.exists) { logger.log('WARN', `PI Alert History not found`); return null; }
         const data = tryDecompress(docSnap.data());
         const { date, lastUpdated, ...piAlertHistory } = data;
         return piAlertHistory; 
     } catch (error) {
-        logger.log('ERROR', `Failed to load PI Alert History for ${dateString}: ${error.message}`);
+        logger.log('ERROR', `Failed to load PI Alert History: ${error.message}`);
         return null;
     }
 }
 
-/** Stage 17: Load PI-Centric Watchlist Data */
+/** Stage 17: Load PI-Centric Watchlist Data (Targeted - Keep as Firestore) */
 async function loadPIWatchlistData(config, deps, piCid) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     const piCidStr = String(piCid);
     
     logger.log('INFO', `Loading PI Watchlist Data for PI ${piCid}`);
-    
     try {
-        const docRef = db.collection('PopularInvestors')
-            .doc(piCidStr)
-            .collection('watchlistData')
-            .doc('current');
-        
+        const docRef = db.collection('PopularInvestors').doc(piCidStr).collection('watchlistData').doc('current');
         const docSnap = await withRetry(() => docRef.get(), `getPIWatchlistData(${piCidStr})`);
-        
-        if (!docSnap.exists) { 
-            logger.log('WARN', `PI Watchlist Data not found for PI ${piCidStr}`); 
-            return null; 
-        }
-        
-        const data = tryDecompress(docSnap.data());
-        return data; 
+        if (!docSnap.exists) { logger.log('WARN', `PI Watchlist Data not found for PI ${piCidStr}`); return null; }
+        return tryDecompress(docSnap.data());
     } catch (error) {
         logger.log('ERROR', `Failed to load PI Watchlist Data for PI ${piCidStr}: ${error.message}`);
         return null;
@@ -773,24 +748,24 @@ async function loadPIWatchlistData(config, deps, piCid) {
 }
 
 // Load Popular Investor Master List
-async function loadPopularInvestorMasterList(config, deps) {
+async function loadPopularInvestorMasterList(config, deps, dateString = null) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     
+    // 1. GCS FAST PATH (If date context exists)
+    if (dateString) {
+        const cached = await tryLoadFromGCS(config, dateString, 'master_list', logger);
+        if (cached) return cached;
+    }
+
+    // 2. FIRESTORE FALLBACK
     const collectionName = config.piMasterListCollection || 'system_state';
     const docId = config.piMasterListDocId || 'popular_investor_master_list';
-
     logger.log('INFO', `Loading Popular Investor Master List from ${collectionName}/${docId}`);
-
     try {
         const docRef = db.collection(collectionName).doc(docId);
         const docSnap = await withRetry(() => docRef.get(), 'getPIMasterList');
-
-        if (!docSnap.exists) {
-            logger.log('WARN', 'Popular Investor Master List not found.');
-            return {};
-        }
-
+        if (!docSnap.exists) { logger.log('WARN', 'Popular Investor Master List not found.'); return {}; }
         const data = tryDecompress(docSnap.data());
         return data.investors || data; 
     } catch (error) {