bulltrackers-module 1.0.631 → 1.0.632

package/functions/computation-system/context/ContextFactory.js

@@ -20,17 +20,32 @@ class ContextFactory {
 
     static buildPerUserContext(options) {
         const { 
-            todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory, 
-            userId, userType, dateStr, metadata, mappings, insights, socialData, 
-            computedDependencies, previousComputedDependencies, config, deps,
+            todayPortfolio, 
+            yesterdayPortfolio, 
+            todayHistory, 
+            yesterdayHistory, 
+            userId, 
+            userType, 
+            dateStr, 
+            metadata, 
+            mappings, 
+            insights, 
+            socialData, 
+            computedDependencies, 
+            previousComputedDependencies, 
+            config, 
+            deps,
             verification, 
-            rankings, yesterdayRankings,         // User-specific rank entries
-            allRankings, allRankingsYesterday,   // Global rank lists
+            rankings,
+            yesterdayRankings,         
+            allRankings,                
+            allRankingsYesterday,   
             allVerifications,
-            // [NEW] New Root Data Types for Profile Metrics
-            ratings, pageViews, watchlistMembership, alertHistory,
+            ratings, 
+            pageViews, 
+            watchlistMembership, 
+            alertHistory,
             piMasterList,
-            // [NEW] Series Data (Lookback for Root Data or Computation Results)
             seriesData
         } = options;
         
@@ -56,14 +71,11 @@ class ContextFactory {
                 rankings: allRankings || [],
                 rankingsYesterday: allRankingsYesterday || [],
                 verifications: allVerifications || {},
-                // [NEW] New Root Data Types for Profile Metrics
                 ratings: ratings || {},
                 pageViews: pageViews || {},
                 watchlistMembership: watchlistMembership || {},
                 alertHistory: alertHistory || {},
                 piMasterList: piMasterList || {},
-                // [NEW] Expose Series Data
-                // Structure: { root: { [type]: { [date]: data } }, results: { [date]: { [calcName]: data } } }
                 series: seriesData || {} 
             }
         };
@@ -71,13 +83,23 @@ class ContextFactory {
 
     static buildMetaContext(options) {
         const { 
-            dateStr, metadata, mappings, insights, socialData, prices, 
-            computedDependencies, previousComputedDependencies, config, deps,
-            allRankings, allRankingsYesterday,
+            dateStr, 
+            metadata, 
+            mappings, 
+            insights, 
+            socialData, 
+            prices, 
+            computedDependencies, 
+            previousComputedDependencies, 
+            config, 
+            deps,
+            allRankings, 
+            allRankingsYesterday,
             allVerifications,
-            // [NEW] New Root Data Types
-            ratings, pageViews, watchlistMembership, alertHistory,
-            // [NEW] Series Data
+            ratings, 
+            pageViews, 
+            watchlistMembership, 
+            alertHistory,
             seriesData
         } = options;
         
@@ -99,7 +121,6 @@ class ContextFactory {
                 pageViews: pageViews || {},
                 watchlistMembership: watchlistMembership || {},
                 alertHistory: alertHistory || {},
-                // [NEW] Expose Series Data
                 series: seriesData || {}
             }
         };

package/functions/computation-system/data/AvailabilityChecker.js

@@ -30,11 +30,11 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
         let isAvailable = false;
         
         if (dep === 'portfolio') {
-            if (userType === 'speculator' && rootDataStatus.speculatorPortfolio) isAvailable = true;
-            else if (userType === 'normal' && rootDataStatus.normalPortfolio) isAvailable = true;
+            if (userType === 'speculator'            && rootDataStatus.speculatorPortfolio) isAvailable = true;
+            else if (userType === 'normal'           && rootDataStatus.normalPortfolio) isAvailable = true;
             else if (userType === 'popular_investor' && rootDataStatus.piPortfolios) isAvailable = true;
-            else if (userType === 'signed_in_user' && rootDataStatus.signedInUserPortfolio) isAvailable = true;
-            else if (userType === 'all' && rootDataStatus.hasPortfolio) isAvailable = true;
+            else if (userType === 'signed_in_user'   && rootDataStatus.signedInUserPortfolio) isAvailable = true;
+            else if (userType === 'all'              && rootDataStatus.hasPortfolio) isAvailable = true;
             
             if (!isAvailable) {
                 // [OPTIMIZATION] Optimistic Series Check
@@ -284,12 +284,28 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
             logger.log('WARN', `[Availability] Index not found for ${dateStr}. Assuming NO data.`);
             return {
                 status: { 
-                    hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false,
-                    speculatorPortfolio: false, normalPortfolio: false, speculatorHistory: false, normalHistory: false,
-                    piRankings: false, piPortfolios: false, piDeepPortfolios: false, piHistory: false,
-                    signedInUserPortfolio: false, signedInUserHistory: false, signedInUserVerification: false,
-                    hasPISocial: false, hasSignedInSocial: false,
-                    piRatings: false, piPageViews: false, watchlistMembership: false, piAlertHistory: false
+                    hasPortfolio: false, 
+                    hasHistory: false, 
+                    hasSocial: false, 
+                    hasInsights: false, 
+                    hasPrices: false,
+                    speculatorPortfolio: false, 
+                    normalPortfolio: false, 
+                    speculatorHistory: false, 
+                    normalHistory: false,
+                    piRankings: false, 
+                    piPortfolios: false, 
+                    piDeepPortfolios: false, 
+                    piHistory: false,
+                    signedInUserPortfolio: false, 
+                    signedInUserHistory: false, 
+                    signedInUserVerification: false,
+                    hasPISocial: false, 
+                    hasSignedInSocial: false,
+                    piRatings: false, 
+                    piPageViews: false, 
+                    watchlistMembership: false, 
+                    piAlertHistory: false
                 }
             };
         }
@@ -361,5 +377,5 @@ module.exports = {
     checkRootDependencies, 
     checkRootDataAvailability, 
     getViableCalculations,
-    getAvailabilityWindow // [NEW] Exported
+    getAvailabilityWindow 
 };

package/functions/computation-system/executors/MetaExecutor.js

@@ -10,7 +10,7 @@ const { normalizeName }     = require('../utils/utils');
 const { CachedDataLoader }  = require('../data/CachedDataLoader');
 const { ContextFactory }    = require('../context/ContextFactory');
 const { commitResults }     = require('../persistence/ResultCommitter');
-const { fetchResultSeries } = require('../data/DependencyFetcher'); // [NEW] Import series fetcher
+const { fetchResultSeries } = require('../data/DependencyFetcher');
 
 class MetaExecutor {
     static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
@@ -47,6 +47,8 @@ class MetaExecutor {
         let ratings = null, pageViews = null, watchlistMembership = null, alertHistory = null;
         if (needsNewRootData) {
             const loadPromises = [];
+            
+            // Ratings
             if (calcs.some(c => c.rootDataDependencies?.includes('ratings'))) {
                 loadPromises.push(loader.loadRatings(dStr).then(r => { ratings = r; }).catch(e => {
                     // Only catch if ALL calcs allow missing roots
@@ -61,6 +63,8 @@ class MetaExecutor {
                     }
                 }));
             }
+            
+            // PageViews
             if (calcs.some(c => c.rootDataDependencies?.includes('pageViews'))) {
                 loadPromises.push(loader.loadPageViews(dStr).then(pv => { pageViews = pv; }).catch(e => {
                     const allAllowMissing = calcs.every(c => {
@@ -74,6 +78,8 @@ class MetaExecutor {
                     }
                 }));
             }
+            
+            // Watchlist
             if (calcs.some(c => c.rootDataDependencies?.includes('watchlist'))) {
                 loadPromises.push(loader.loadWatchlistMembership(dStr).then(w => { watchlistMembership = w; }).catch(e => {
                     const allAllowMissing = calcs.every(c => {
@@ -87,6 +93,8 @@ class MetaExecutor {
                     }
                 }));
             }
+            
+            // Alerts
             if (calcs.some(c => c.rootDataDependencies?.includes('alerts'))) {
                 loadPromises.push(loader.loadAlertHistory(dStr).then(a => { alertHistory = a; }).catch(e => {
                     const allAllowMissing = calcs.every(c => {
@@ -100,6 +108,7 @@ class MetaExecutor {
                     }
                 }));
             }
+            
             await Promise.all(loadPromises);
             
             // [FIX] Enforce canHaveMissingRoots - validate after loading
@@ -163,7 +172,8 @@ class MetaExecutor {
             else if (type === 'insights') loaderMethod = 'loadInsights';
             else if (type === 'ratings') loaderMethod = 'loadRatings';
             else if (type === 'watchlist') loaderMethod = 'loadWatchlistMembership';
-            // Add other root types if needed
+            // [CRITICAL UPDATE] Add rankings support for Meta lookbacks
+            else if (type === 'rankings') loaderMethod = 'loadRankings';
 
             if (loaderMethod) {
                 logger.log('INFO', `[MetaExecutor] Loading ${days}-day series for Root Data '${type}'...`);
@@ -222,7 +232,8 @@ class MetaExecutor {
             }
         }
 
-        return await commitResults(state, dStr, passName, config, deps);
+        // CRITICAL FIX: Pass 'isInitialWrite: true' to ensure proper cleanup of old meta data
+        return await commitResults(state, dStr, passName, config, deps, false, { isInitialWrite: true });
     }
 
     static async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps, config, deps, loader) {

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,15 +1,8 @@
-/**
- * {
- * type: uploaded file
- * fileName: computation-system/executors/StandardExecutor.js
- * }
- */
 const { normalizeName, getEarliestDataDates }                          = require('../utils/utils');
 const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
 const { CachedDataLoader }                                             = require('../data/CachedDataLoader');
 const { ContextFactory }                                               = require('../context/ContextFactory');
 const { commitResults }                                                = require('../persistence/ResultCommitter');
-// [NEW] Import series fetcher for computation results
 const { fetchResultSeries }                                            = require('../data/DependencyFetcher');
 const mathLayer                                                        = require('../layers/index');
 const { performance }                                                  = require('perf_hooks');
@@ -167,7 +160,8 @@ class StandardExecutor {
             else if (type === 'insights') loaderMethod = 'loadInsights';
             else if (type === 'ratings') loaderMethod = 'loadRatings';
             else if (type === 'watchlist') loaderMethod = 'loadWatchlistMembership';
-            // Add other root types as needed...
+            // [CRITICAL UPDATE] Add rankings support for AUM lookback
+            else if (type === 'rankings') loaderMethod = 'loadRankings';
 
             if (loaderMethod) {
                 logger.log('INFO', `[StandardExecutor] Loading ${days}-day series for Root Data '${type}'...`);
@@ -487,4 +481,4 @@ class StandardExecutor {
     }
 }
 
-module.exports = { StandardExecutor };
+module.exports = { StandardExecutor };

package/functions/computation-system/persistence/ResultCommitter.js

@@ -4,6 +4,7 @@
  * UPDATED: Added support for 'isPage' mode to store per-user data in subcollections.
  * UPDATED: Implemented TTL retention policy. Defaults to 90 days from the computation date.
  * UPDATED: Fixed issue where switching to 'isPage' mode didn't clean up old sharded/raw data.
+ * CRITICAL FIX: Fixed sharding logic to prevent wiping existing shards during INTERMEDIATE flushes.
  */
 const { commitBatchInChunks, generateDataHash, FieldValue } = require('../utils/utils')
 const { updateComputationStatus }          = require('./StatusRepository');
@@ -138,8 +139,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 continue; 
             }
 
-            // [NEW] Page Computation Logic (Fan-Out) with TTL
-            // Bypasses standard compression/sharding to write per-user documents
             // [NEW] Page Computation Logic (Fan-Out) with TTL
             // Bypasses standard compression/sharding to write per-user documents
             if (isPageComputation && !isEmpty) {
@@ -429,7 +428,9 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
     let finalStats = { totalSize: 0, isSharded: false, shardCount: 1, nextShardIndex: startShardIndex };
     let rootMergeOption = !isInitialWrite; 
 
-    let shouldWipeShards = wasSharded; 
+    // CRITICAL FIX: Only wipe existing shards if this is the INITIAL write for this batch run.
+    // If we are flushing intermediate chunks, we should NOT wipe the shards created by previous chunks!
+    let shouldWipeShards = wasSharded && isInitialWrite; 
 
     for (let attempt = 0; attempt < strategies.length; attempt++) {
         if (committed) break;

package/functions/computation-system/utils/data_loader.js

@@ -1,10 +1,10 @@
 /**
  * @fileoverview Data loader sub-pipes for the Computation System.
  * REFACTORED: Now stateless and receive dependencies.
- * FIXED: Added strict userType filtering to prevent fetching unnecessary data (e.g. Normal users for PI calcs).
- * --- NEW: Updated to read PI/Signed-In data from SHARDS (Parts) instead of individual docs. ---
- * --- NEW: Logic to merge Overall and Deep PI data from corresponding shards. ---
- * --- UPDATED: Added loaders for Rankings and Verification data. ---
+ * FIXED: Added strict userType filtering to prevent fetching unnecessary data.
+ * UPDATED: Verification now uses CollectionGroup query due to per-user storage.
+ * UPDATED: Ratings now correctly handles flattened top-level schema (keys like "reviews.ID").
+ * REMOVED: Redundant Price Shard Indexing logic.
  */
 const zlib = require('zlib'); 
 
@@ -23,11 +23,9 @@ function tryDecompress(data) {
 
 /** --- Data Loader Sub-Pipes (Stateless, Dependency-Injection) --- */
 
-/** * Stage 1: Get portfolio part document references for a given date 
- * [UPDATED] Accepts requiredUserTypes to filter collections.
- */
+/** Stage 1: Get portfolio part document references for a given date */
 async function getPortfolioPartRefs(config, deps, dateString, requiredUserTypes = null) {
-    const { db, logger, calculationUtils, collectionRegistry } = deps;
+    const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     
     // Normalize required types. If null/empty or contains 'ALL', fetch everything.
@@ -250,7 +248,6 @@ async function loadDailyInsights(config, deps, dateString) {
 async function loadDailySocialPostInsights(config, deps, dateString) {
     const { db, logger, calculationUtils, collectionRegistry } = deps;
     const { withRetry } = calculationUtils;
-    const { getCollectionPath } = collectionRegistry || {};
     
     logger.log('INFO', `Loading and partitioning social data for ${dateString}`);
     
@@ -262,7 +259,6 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
     };
 
     // NEW STRUCTURE: Read from date-based collections
-    // Structure: Collection/{date}/{cid}/{cid} for user social, Collection/{date}/posts/{postId} for instrument
     try {
         // Signed-In User Social: SignedInUserSocialPostData/{date}/{cid}/{cid}
         const signedInSocialCollectionName = 'SignedInUserSocialPostData';
@@ -279,7 +275,6 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
                 const cidData = tryDecompress(cidDoc.data());
                 if (cidData.posts && typeof cidData.posts === 'object') {
                     if (!result.signedIn[cid]) result.signedIn[cid] = {};
-                    // Posts are stored as a map in the document
                     Object.assign(result.signedIn[cid], cidData.posts);
                 }
             }
@@ -328,13 +323,10 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
     const PI_COL_NAME = config.piSocialCollectionName || config.piSocialCollection || 'pi_social_posts';
     const SIGNED_IN_COL_NAME = config.signedInUserSocialCollection || 'signed_in_users_social';
 
-    // 2. Define Time Range (UTC Day)
     const startDate = new Date(dateString + 'T00:00:00Z');
     const endDate = new Date(dateString + 'T23:59:59Z');
 
     try {
-        // 3. Fetch ALL with CollectionGroup
-        // NOTE: Requires Firestore Index: CollectionId 'posts', Field 'fetchedAt' (ASC/DESC)
         const postsQuery = db.collectionGroup('posts')
             .where('fetchedAt', '>=', startDate)
             .where('fetchedAt', '<=', endDate);
@@ -346,9 +338,7 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
                 const data = tryDecompress(doc.data());
                 const path = doc.ref.path; 
                 
-                // 4. Partition Logic based on Path
                 if (path.includes(PI_COL_NAME)) {
-                    // Path format: .../pi_social_posts/{userId}/posts/{postId}
                     const parts = path.split('/');
                     const colIndex = parts.indexOf(PI_COL_NAME);
                     if (colIndex !== -1 && parts[colIndex + 1]) {
@@ -358,7 +348,6 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
                     }
                 } 
                 else if (path.includes(SIGNED_IN_COL_NAME)) {
-                    // Path format: .../signed_in_users_social/{userId}/posts/{postId}
                     const parts = path.split('/');
                     const colIndex = parts.indexOf(SIGNED_IN_COL_NAME);
                     if (colIndex !== -1 && parts[colIndex + 1]) {
@@ -368,7 +357,6 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
                     }
                 } 
                 else {
-                    // Default: Generic Instrument Posts
                     result.generic[doc.id] = data;
                 }
             });
@@ -384,11 +372,9 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
     return result;
 }
 
-/** * Stage 6: Get history part references for a given date 
- * [UPDATED] Accepts requiredUserTypes to filter collections.
- */
+/** Stage 6: Get history part references for a given date */
 async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes = null) {
-    const { db, logger, calculationUtils, collectionRegistry } = deps;
+    const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     
     // Normalize required types
@@ -478,9 +464,7 @@ async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes =
     return allPartRefs;
 }
 
-/** * Stage 7: Stream portfolio data in chunks 
- * [UPDATED] Passes requiredUserTypes to getPortfolioPartRefs
- */
+/** Stage 7: Stream portfolio data in chunks */
 async function* streamPortfolioData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
     const { logger } = deps;
     const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString, requiredUserTypes));
@@ -497,9 +481,7 @@ async function* streamPortfolioData(config, deps, dateString, providedRefs = nul
     logger.log('INFO', `[streamPortfolioData] Finished streaming for ${dateString}.`);
 }
 
-/** * Stage 8: Stream history data in chunks 
- * [UPDATED] Passes requiredUserTypes to getHistoryPartRefs
- */
+/** Stage 8: Stream history data in chunks */
 async function* streamHistoryData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
     const { logger } = deps;
     const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString, requiredUserTypes));
@@ -531,80 +513,22 @@ async function getPriceShardRefs(config, deps) {
     }
 }
 
-/** Stage 10: Smart Shard Lookup System */
+/** Stage 10: Smart Shard Lookup System (DEPRECATED/SIMPLIFIED) */
 async function ensurePriceShardIndex(config, deps) {
-    const { db, logger } = deps;
-    const metadataCol = config.metadataCollection || 'system_metadata';
-    const indexDocRef = db.collection(metadataCol).doc('price_shard_index');
-    
-    // 1. Try to fetch existing index
-    const snap = await indexDocRef.get();
-    if (snap.exists) {
-        const data = snap.data();
-        const lastUpdated = data.lastUpdated ? new Date(data.lastUpdated).getTime() : 0;
-        const now = Date.now();
-        const oneDayMs = 24 * 60 * 60 * 1000;
-        if ((now - lastUpdated) < oneDayMs) { return data.index || {}; }
-        logger.log('INFO', '[ShardIndex] Index is stale (>24h). Rebuilding...');
-    } else {
-        logger.log('INFO', '[ShardIndex] Index not found. Building new Price Shard Index...');
-    }
-    
-    // 2. Build Index
-    const collection = config.priceCollection || 'asset_prices';
-    const snapshot = await db.collection(collection).get();
-    
-    const index = {};
-    let shardCount = 0;
-    
-    snapshot.forEach(doc => {
-        shardCount++;
-        const rawData = doc.data(); 
-        const data = tryDecompress(rawData);
-
-        if (data.history) {
-            Object.keys(data.history).forEach(instId => {
-                index[instId] = doc.id;
-            });
-        }
-    });
-
-    // 3. Save Index
-    await indexDocRef.set({
-        index: index,
-        lastUpdated: new Date().toISOString(),
-        shardCount: shardCount
-    });
-    
-    logger.log('INFO', `[ShardIndex] Built index for ${Object.keys(index).length} instruments across ${shardCount} shards.`);
-    return index;
+   // [DEPRECATED] This function previously built an index in 'system_metadata/price_shard_index'.
+   // It has been removed to avoid performing computation/indexing in the data loader.
+   // Use 'Fetch All' strategy in Stage 9 instead.
+   return {};
 }
 
 async function getRelevantShardRefs(config, deps, targetInstrumentIds) {
-    const { db, logger } = deps;
-    
-    if (!targetInstrumentIds || targetInstrumentIds.length === 0) {
-        return getPriceShardRefs(config, deps);
-    }
-
-    logger.log('INFO', `[ShardLookup] Resolving shards for ${targetInstrumentIds.length} specific instruments...`);
-    
-    const index = await ensurePriceShardIndex(config, deps);
-    const uniqueShardIds = new Set();
-    const collection = config.priceCollection || 'asset_prices';
-
-    let foundCount = 0;
-    for (const id of targetInstrumentIds) {
-        const shardId = index[id];
-        if (shardId) {
-            uniqueShardIds.add(shardId);
-            foundCount++;
-        }
-    }
-    
-    logger.log('INFO', `[ShardLookup] Mapped ${foundCount}/${targetInstrumentIds.length} instruments to ${uniqueShardIds.size} unique shards.`);
+    const { logger } = deps;
     
-    return Array.from(uniqueShardIds).map(id => db.collection(collection).doc(id));
+    // [UPDATED] Smart shard lookup is disabled due to missing index infrastructure
+    // and to avoid computing indexes during load time.
+    // Falling back to Stage 9 (Fetch All Shards).
+    logger.log('INFO', `[ShardLookup] Smart indexing disabled. Fetching all price shards for ${targetInstrumentIds ? targetInstrumentIds.length : 'all'} instruments.`);
+    return getPriceShardRefs(config, deps);
 }
 
 /** Stage 11: Load Popular Investor Rankings */
@@ -625,40 +549,58 @@ async function loadPopularInvestorRankings(config, deps, dateString) {
         }
         
         const data = tryDecompress(docSnap.data());
-        return data.Items || []; // Returns the array of PI objects
+        return data.Items || []; 
     } catch (error) {
         logger.log('ERROR', `Failed to load Rankings for ${dateString}: ${error.message}`);
         return null;
     }
 }
 
-/** Stage 12: Load User Verification Profiles */
+/** Stage 12: Load User Verification Profiles 
+ * [UPDATED] Scans global verification data via CollectionGroup since it's now stored per-user.
+ */
 async function loadVerificationProfiles(config, deps) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    const collectionName = config.verificationCollection || 'verified_users';
     
-    logger.log('INFO', `Loading Verification Profiles`);
+    // Verification is now stored at /SignedInUsers/{cid}/verification/data
+    // To fetch globally, we must use a CollectionGroup query on 'verification'
+    // and filter for the document ID 'data'.
+    
+    logger.log('INFO', `Loading Verification Profiles (CollectionGroup: verification/data)`);
 
     try {
-        const snapshot = await withRetry(() => db.collection(collectionName).get(), 'getVerifications');
+        // Warning: This requires a Firestore Index if used with complex filters, but basic get() usually works.
+        const snapshot = await withRetry(() => db.collectionGroup('verification').get(), 'getVerificationsGroup');
         
         if (snapshot.empty) return {};
         
         const profiles = {};
+        let count = 0;
+
         snapshot.forEach(doc => {
+            if (doc.id !== 'data') return; // Enforce specific document ID from schema
+
             const raw = tryDecompress(doc.data());
-            // [FIX] Normalize Verification Data Structure
-            profiles[doc.id] = {
-                cid: raw.realCID || raw.cid,
-                username: raw.username,
-                aboutMe: raw.userBio?.aboutMe || raw.aboutMe || "",
-                aboutMeShort: raw.userBio?.aboutMeShort || raw.aboutMeShort || "",
-                isVerified: raw.isVerified === true,
-                restrictions: raw.CustomerRestrictions || []
-            };
+            
+            // Map new schema fields to internal profile structure
+            // New Schema: { etoroCID, etoroUsername, verifiedAt, setupCompletedAt ... }
+            if (raw.etoroCID) {
+                profiles[raw.etoroCID] = {
+                    cid: raw.etoroCID,
+                    username: raw.etoroUsername,
+                    // 'aboutMe' and 'restrictions' are NOT present in the new schema.
+                    // Defaulting to empty values to preserve downstream compatibility.
+                    aboutMe: "", 
+                    aboutMeShort: "",
+                    isVerified: !!(raw.verifiedAt), // Using existence of verifiedAt as flag
+                    restrictions: []
+                };
+                count++;
+            }
         });
         
+        logger.log('INFO', `Loaded ${count} verification profiles.`);
         return profiles;
     } catch (error) {
         logger.log('ERROR', `Failed to load Verification Profiles: ${error.message}`);
@@ -666,30 +608,60 @@ async function loadVerificationProfiles(config, deps) {
     }
 }
 
-/** Stage 13: Load PI Ratings Data */
+/** Stage 13: Load PI Ratings Data 
+ * [UPDATED] Reads from /PiReviews/{date}/shards/daily_log.
+ * [FIXED] Handles FLATTENED schema where keys like "reviews.ID" are at the top level.
+ * Returns RAW logs grouped by PI. NO COMPUTATION.
+ */
 async function loadPIRatings(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    const collectionName = config.piRatingsCollection || 'PIRatingsData';
     
-    logger.log('INFO', `Loading PI Ratings for ${dateString}`);
+    // New Path: /PiReviews/{date}/shards/daily_log
+    
+    logger.log('INFO', `Loading PI Ratings (Raw Logs) for ${dateString}`);
     
     try {
-        const docRef = db.collection(collectionName).doc(dateString);
-        const docSnap = await withRetry(() => docRef.get(), `getPIRatings(${dateString})`);
-        
-        if (!docSnap.exists) { 
-            logger.log('WARN', `PI Ratings not found for ${dateString}`); 
-            return null; 
+        const shardsColRef = db.collection('PiReviews').doc(dateString).collection('shards');
+        const shardDocs = await withRetry(() => shardsColRef.listDocuments(), `listRatingShards(${dateString})`);
+
+        if (!shardDocs || shardDocs.length === 0) {
+            logger.log('WARN', `No rating shards found for ${dateString} at ${shardsColRef.path}`);
+            return {};
         }
-        
-        const data = tryDecompress(docSnap.data());
-        // Remove the date key and lastUpdated, return just the PI data
-        const { date, lastUpdated, ...piRatings } = data;
-        return piRatings; // Returns { piCid: { averageRating, totalRatings, ratingsByUser, ... } }
+
+        const rawReviewsByPi = {}; 
+
+        for (const docRef of shardDocs) {
+            const docSnap = await docRef.get();
+            if (!docSnap.exists) continue;
+
+            const rawData = tryDecompress(docSnap.data());
+            
+            // SCHEMA HANDLING:
+            // Keys at the root of the document are the review IDs (e.g. "reviews.29312236_31075566").
+            // We iterate over all values and check if they look like review objects.
+            
+            Object.values(rawData).forEach(entry => {
+                // Check for valid review object structure
+                if (entry && typeof entry === 'object' && entry.piCid && entry.rating !== undefined) {
+                    
+                    if (!rawReviewsByPi[entry.piCid]) {
+                        rawReviewsByPi[entry.piCid] = [];
+                    }
+                    
+                    // Store the raw entry directly.
+                    rawReviewsByPi[entry.piCid].push(entry);
+                }
+            });
+        }
+
+        logger.log('INFO', `Loaded raw reviews for ${Object.keys(rawReviewsByPi).length} PIs.`);
+        return rawReviewsByPi; 
+
     } catch (error) {
         logger.log('ERROR', `Failed to load PI Ratings for ${dateString}: ${error.message}`);
-        return null;
+        return {};
     }
 }
 
@@ -711,9 +683,8 @@ async function loadPIPageViews(config, deps, dateString) {
         }
         
         const data = tryDecompress(docSnap.data());
-        // Remove the date key and lastUpdated, return just the PI data
         const { date, lastUpdated, ...piPageViews } = data;
-        return piPageViews; // Returns { piCid: { totalViews, uniqueViewers, viewsByUser, ... } }
+        return piPageViews; 
     } catch (error) {
         logger.log('ERROR', `Failed to load PI Page Views for ${dateString}: ${error.message}`);
         return null;
@@ -738,9 +709,8 @@ async function loadWatchlistMembership(config, deps, dateString) {
         }
         
         const data = tryDecompress(docSnap.data());
-        // Remove the date key and lastUpdated, return just the PI data
         const { date, lastUpdated, ...watchlistMembership } = data;
-        return watchlistMembership; // Returns { piCid: { totalUsers, users, publicWatchlistCount, ... } }
+        return watchlistMembership; 
     } catch (error) {
         logger.log('ERROR', `Failed to load Watchlist Membership for ${dateString}: ${error.message}`);
         return null;
@@ -765,19 +735,15 @@ async function loadPIAlertHistory(config, deps, dateString) {
         }
         
         const data = tryDecompress(docSnap.data());
-        // Remove the date key and lastUpdated, return just the PI data
         const { date, lastUpdated, ...piAlertHistory } = data;
-        return piAlertHistory; // Returns { piCid: { alertType: { triggered, count, triggeredFor, ... } } }
+        return piAlertHistory; 
     } catch (error) {
         logger.log('ERROR', `Failed to load PI Alert History for ${dateString}: ${error.message}`);
         return null;
     }
 }
 
-/** Stage 17: Load PI-Centric Watchlist Data
- * Loads watchlist data from PopularInvestors/{piCid}/watchlistData/current
- * This provides time-series data of watchlist additions per PI
- */
+/** Stage 17: Load PI-Centric Watchlist Data */
 async function loadPIWatchlistData(config, deps, piCid) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
@@ -799,19 +765,18 @@ async function loadPIWatchlistData(config, deps, piCid) {
         }
         
         const data = tryDecompress(docSnap.data());
-        return data; // Returns { totalUsers, userCids: [], dailyAdditions: { date: { count, userCids, timestamp } }, lastUpdated }
+        return data; 
     } catch (error) {
         logger.log('ERROR', `Failed to load PI Watchlist Data for PI ${piCidStr}: ${error.message}`);
         return null;
     }
 }
 
-// [NEW] Load Popular Investor Master List
+// Load Popular Investor Master List
 async function loadPopularInvestorMasterList(config, deps) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     
-    // Default to 'system_state' collection, 'popular_investor_master_list' doc
     const collectionName = config.piMasterListCollection || 'system_state';
     const docId = config.piMasterListDocId || 'popular_investor_master_list';
 
@@ -827,9 +792,6 @@ async function loadPopularInvestorMasterList(config, deps) {
         }
 
         const data = tryDecompress(docSnap.data());
-        // Structure is { investors: { cid: { username, ... } } } or direct map { cid: { ... } }
-        // Based on user input, it looks like a direct map of CIDs or a field holding the map.
-        // We return the raw object which acts as the map.
         return data.investors || data; 
     } catch (error) {
         logger.log('ERROR', `Failed to load PI Master List: ${error.message}`);
@@ -855,6 +817,6 @@ module.exports = {
     loadPIPageViews,
     loadWatchlistMembership,
     loadPIAlertHistory,
-    loadPopularInvestorMasterList, // [NEW]
+    loadPopularInvestorMasterList,
     loadPIWatchlistData,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.631",
+  "version": "1.0.632",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [