bulltrackers-module 1.0.630 → 1.0.632

package/functions/alert-system/helpers/alert_helpers.js

@@ -6,7 +6,7 @@
 const { FieldValue } = require('@google-cloud/firestore');
 const zlib = require('zlib');
 const { getAlertTypeByComputation, generateAlertMessage } = require('./alert_type_registry');
-const { writeWithMigration } = require('../../old-generic-api/user-api/helpers/core/path_resolution_helpers');
+// Migration helpers removed - write directly to new path
 
 /**
  * Process alerts for a specific PI from computation results
@@ -65,39 +65,30 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
       };
       
       // Write to alerts collection (not notifications) - alerts are separate from system notifications
-      // Use writeWithMigration to write to new path (with legacy fallback)
-      const writePromise = writeWithMigration(
-        db,
-        'signedInUsers',
-        'alerts', // Write to alerts collection, not notifications
-        { cid: String(userCid) },
-        {
-          // Alert-specific format (simplified from notification format)
-          alertId: notificationId,
-          piCid: Number(piCid),
-          piUsername: piUsername,
-          alertType: alertType.id,
-          alertTypeName: alertType.name,
-          message: alertMessage,
-          severity: alertType.severity,
-          watchlistId: subscription.watchlistId,
-          watchlistName: subscription.watchlistName,
-          read: false,
-          createdAt: FieldValue.serverTimestamp(),
-          computationDate: computationDate,
-          computationName: alertType.computationName,
-          ...(computationMetadata || {})
-        },
-        {
-          isCollection: true,
-          merge: false,
-          dataType: 'alerts',
-          documentId: notificationId,
-          dualWrite: false, // Disable dual write - we're fully migrated to new path
-          config,
-          collectionRegistry
-        }
-      ).catch(err => {
+      // Write directly to new path: SignedInUsers/{userCid}/alerts/{alertId}
+      const alertData = {
+        alertId: notificationId,
+        piCid: Number(piCid),
+        piUsername: piUsername,
+        alertType: alertType.id,
+        alertTypeName: alertType.name,
+        message: alertMessage,
+        severity: alertType.severity,
+        watchlistId: subscription.watchlistId,
+        watchlistName: subscription.watchlistName,
+        read: false,
+        createdAt: FieldValue.serverTimestamp(),
+        computationDate: computationDate,
+        computationName: alertType.computationName,
+        ...(computationMetadata || {})
+      };
+      
+      const writePromise = db.collection('SignedInUsers')
+        .doc(String(userCid))
+        .collection('alerts')
+        .doc(notificationId)
+        .set(alertData)
+        .catch(err => {
         logger.log('ERROR', `[processAlertForPI] Failed to write alert for CID ${userCid}: ${err.message}`, err);
         throw err; // Re-throw so we know if writes are failing
       });
@@ -114,9 +105,21 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
     // 6. Update global rootdata collection for computation system
     // (Wrap in try-catch to prevent crashing the alert if metrics fail)
     try {
-        const { updateAlertHistoryRootData } = require('../../old-generic-api/user-api/helpers/rootdata/rootdata_aggregation_helpers');
+        const { runRootDataIndexer } = require('../../root-data-indexer/index');
         const triggeredUserCids = subscriptions.map(s => s.userCid);
-        await updateAlertHistoryRootData(db, logger, piCid, alertType, computationMetadata, computationDate, triggeredUserCids);
+        
+        // Update alert history root data by running root data indexer for the specific date
+        // The indexer will detect and update PIAlertHistoryData availability
+        const indexerConfig = {
+            availabilityCollection: 'root_data_availability',
+            targetDate: computationDate,
+            collections: {
+                piAlertHistory: 'PIAlertHistoryData'
+            }
+        };
+        
+        await runRootDataIndexer(indexerConfig, { db, logger });
+        logger.log('INFO', `[processAlertForPI] Updated root data indexer for date ${computationDate}`);
     } catch (e) {
         logger.log('WARN', `[processAlertForPI] Failed to update history rootdata: ${e.message}`);
     }
@@ -158,14 +161,18 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
   // Check for developer accounts with pretendSubscribedToAllAlerts flag enabled
   // This allows developers to test the alert system without manually configuring subscriptions
   try {
-    const { getDevOverride, isDeveloperAccount } = require('../../old-generic-api/user-api/helpers/dev/dev_helpers');
-    const { getPIMasterList } = require('../../old-generic-api/user-api/helpers/core/user_status_helpers');
+    const { isDeveloper } = require('../../api-v2/helpers/data-fetchers/firestore.js');
+    const { fetchPopularInvestorMasterList } = require('../../api-v2/helpers/data-fetchers/firestore.js');
     const config = dependencies.config || {};
-    const collectionRegistry = dependencies.collectionRegistry || null;
     
-    // Get all popular investors from master list for username lookup
-    const allInvestors = await getPIMasterList(db, collectionRegistry, logger);
-    const piUsername = allInvestors[String(piCid)]?.username || `PI-${piCid}`;
+    // Get PI username from master list
+    let piUsername = `PI-${piCid}`;
+    try {
+      const piData = await fetchPopularInvestorMasterList(db, String(piCid));
+      piUsername = piData.username || piUsername;
+    } catch (e) {
+      // PI not in master list, use fallback
+    }
     
     // Default alert config with all alert types enabled (for dev override)
     const allAlertsEnabledConfig = {
@@ -186,8 +193,9 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
     for (const devOverrideDoc of devOverridesSnapshot.docs) {
       const devUserCid = Number(devOverrideDoc.id);
       
-      // Verify this is actually a developer account (security check)
-      if (!isDeveloperAccount(devUserCid)) {
+      // Verify this is actually a developer account (security check) - using api-v2 helper
+      const isDev = await isDeveloper(db, String(devUserCid));
+      if (!isDev) {
         continue;
       }
       
@@ -244,44 +252,26 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
   }
   
   // Step 2: For each user, read their watchlists from SignedInUsers/{cid}/watchlists
-  const { readWithMigration } = require('../../old-generic-api/user-api/helpers/core/path_resolution_helpers');
-  // const { collectionRegistry } = dependencies; // Already destructured above
-  // const config = dependencies.config || {}; // Already defined
-  
+  // Read directly from new path (no migration needed)
   for (const userCidStr of userCids) {
     try {
       const userCid = Number(userCidStr);
       
       // Read all watchlists for this user from new path
-      const watchlistsResult = await readWithMigration(
-        db,
-        'signedInUsers',
-        'watchlists',
-        { cid: userCid },
-        {
-          isCollection: true,
-          dataType: 'watchlists',
-          config,
-          logger,
-          collectionRegistry: collectionRegistry
-        }
-      );
+      const watchlistsSnapshot = await db.collection('SignedInUsers')
+        .doc(String(userCid))
+        .collection('watchlists')
+        .get();
       
-      if (!watchlistsResult) {
+      if (watchlistsSnapshot.empty) {
         continue;
       }
       
-      // Get watchlists from snapshot or data
-      let watchlists = [];
-      if (watchlistsResult.snapshot && !watchlistsResult.snapshot.empty) {
-        watchlists = watchlistsResult.snapshot.docs.map(doc => ({
-          id: doc.id,
-          ...doc.data()
-        }));
-      } else if (watchlistsResult.data) {
-        // If it's a single document, wrap it
-        watchlists = [watchlistsResult.data];
-      }
+      // Get watchlists from snapshot
+      const watchlists = watchlistsSnapshot.docs.map(doc => ({
+        id: doc.id,
+        ...doc.data()
+      }));
       
       // Step 3: Check each watchlist for the PI and alert config
       for (const watchlistData of watchlists) {
@@ -337,11 +327,13 @@ function shouldTriggerAlert(subscription, alertTypeId) {
  */
 async function getPIUsername(db, piCid) {
   try {
-    // Try to get from master list first (single source of truth)
-    const { getPIUsernameFromMasterList } = require('../../old-generic-api/user-api/helpers/core/user_status_helpers');
-    const username = await getPIUsernameFromMasterList(db, piCid, null, null);
+    // Try to get from master list first (single source of truth) - using api-v2 helper
+    const { fetchPopularInvestorMasterList } = require('../../api-v2/helpers/data-fetchers/firestore.js');
+    const piData = await fetchPopularInvestorMasterList(db, String(piCid));
     
-    if (username) return username;
+    if (piData && piData.username) {
+      return piData.username;
+    }
     
     // Fallback: try to get from any subscription
     const subscriptionsSnapshot = await db.collection('watchlist_subscriptions')

package/functions/alert-system/index.js

@@ -427,35 +427,25 @@ async function sendAllClearNotification(db, logger, userCid, piCid, piUsername,
     };
     
     // Write to alerts collection (not notifications) - alerts are separate from system notifications
-    const { writeWithMigration } = require('../old-generic-api/user-api/helpers/core/path_resolution_helpers');
-    await writeWithMigration(
-      db,
-      'signedInUsers',
-      'alerts', // Write to alerts collection, not notifications
-      { cid: String(userCid) },
-      {
-        // Alert-specific format
-        alertId: notificationId,
-        piCid: Number(piCid),
-        piUsername: piUsername,
-        alertType: 'all_clear',
-        alertTypeName: 'All Clear',
-        message: `${piUsername} was processed, all clear today!`,
-        severity: 'info',
-        read: false,
-        createdAt: FieldValue.serverTimestamp(),
-        computationDate: date
-      },
-      {
-        isCollection: true,
-        merge: false,
-        dataType: 'alerts',
-        documentId: notificationId,
-        dualWrite: false, // Disable dual write - we're fully migrated to new path
-        config,
-        collectionRegistry
-      }
-    );
+    // Write directly to new path: SignedInUsers/{userCid}/alerts/{alertId}
+    const alertData = {
+      alertId: notificationId,
+      piCid: Number(piCid),
+      piUsername: piUsername,
+      alertType: 'all_clear',
+      alertTypeName: 'All Clear',
+      message: `${piUsername} was processed, all clear today!`,
+      severity: 'info',
+      read: false,
+      createdAt: FieldValue.serverTimestamp(),
+      computationDate: date
+    };
+    
+    await db.collection('SignedInUsers')
+      .doc(String(userCid))
+      .collection('alerts')
+      .doc(notificationId)
+      .set(alertData);
     
     logger.log('INFO', `[sendAllClearNotification] Sent all-clear notification to user ${userCid} for PI ${piCid}`);
     

package/functions/api-v2/helpers/notification_helpers.js

@@ -0,0 +1,187 @@
+/**
+ * Notification helpers for task engine and computation system
+ * Replaces old-generic-api notification helpers
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+
+/**
+ * Notify task engine completion
+ * @param {Firestore} db - Firestore instance
+ * @param {Object} logger - Logger instance
+ * @param {string|number} userCid - User CID
+ * @param {string} requestId - Request ID
+ * @param {string} username - Username
+ * @param {boolean} success - Whether the task succeeded
+ * @param {string|null} errorMessage - Error message if failed
+ * @param {Object} options - Additional options (collectionRegistry, config, etc.)
+ */
+async function notifyTaskEngineComplete(db, logger, userCid, requestId, username, success, errorMessage, options = {}) {
+    try {
+        const notificationData = {
+            type: 'sync',
+            subType: 'complete',
+            title: success ? 'Sync Complete' : 'Sync Failed',
+            message: success 
+                ? `Data sync for ${username} completed successfully`
+                : `Data sync for ${username} failed: ${errorMessage || 'Unknown error'}`,
+            read: false,
+            timestamp: FieldValue.serverTimestamp(),
+            createdAt: FieldValue.serverTimestamp(),
+            metadata: {
+                requestId,
+                username,
+                success,
+                errorMessage: errorMessage || null
+            }
+        };
+
+        await db.collection('SignedInUsers')
+            .doc(String(userCid))
+            .collection('notifications')
+            .doc(requestId)
+            .set(notificationData);
+
+        logger?.log('INFO', `[notifyTaskEngineComplete] Notification sent for user ${userCid}, request ${requestId}`);
+    } catch (error) {
+        logger?.log('WARN', `[notifyTaskEngineComplete] Failed to send notification: ${error.message}`);
+    }
+}
+
+/**
+ * Notify task engine progress
+ * @param {Firestore} db - Firestore instance
+ * @param {Object} logger - Logger instance
+ * @param {string|number} userCid - User CID
+ * @param {string} requestId - Request ID
+ * @param {string} username - Username
+ * @param {string} stage - Progress stage (e.g., 'started', 'portfolio_complete')
+ * @param {string|null} dataType - Data type (e.g., 'portfolio', 'tradeHistory')
+ * @param {Object} options - Additional options
+ */
+async function notifyTaskEngineProgress(db, logger, userCid, requestId, username, stage, dataType, options = {}) {
+    try {
+        const stageMessages = {
+            'started': 'Data sync started',
+            'portfolio_complete': 'Portfolio data fetched',
+            'history_complete': 'Trade history fetched',
+            'social_complete': 'Social posts fetched',
+            'indexing': 'Indexing data',
+            'computing': 'Running computations'
+        };
+
+        const notificationData = {
+            type: 'sync',
+            subType: 'progress',
+            title: 'Sync Progress',
+            message: stageMessages[stage] || `Sync ${stage}`,
+            read: false,
+            timestamp: FieldValue.serverTimestamp(),
+            createdAt: FieldValue.serverTimestamp(),
+            metadata: {
+                requestId,
+                username,
+                stage,
+                dataType: dataType || null
+            }
+        };
+
+        await db.collection('SignedInUsers')
+            .doc(String(userCid))
+            .collection('notifications')
+            .doc(`${requestId}_${stage}`)
+            .set(notificationData);
+
+        logger?.log('INFO', `[notifyTaskEngineProgress] Progress notification sent for user ${userCid}, stage ${stage}`);
+    } catch (error) {
+        logger?.log('WARN', `[notifyTaskEngineProgress] Failed to send notification: ${error.message}`);
+    }
+}
+
+/**
+ * Notify that PI data was refreshed
+ * @param {Firestore} db - Firestore instance
+ * @param {Object} logger - Logger instance
+ * @param {Object} collectionRegistry - Collection registry
+ * @param {string|number} piCid - PI CID
+ * @param {string} username - Username
+ * @param {Object} config - Config object
+ */
+async function notifyPIDataRefreshed(db, logger, collectionRegistry, piCid, username, config) {
+    try {
+        // This is a non-critical notification - just log it
+        logger?.log('INFO', `[notifyPIDataRefreshed] PI ${piCid} (${username}) data refreshed`);
+    } catch (error) {
+        logger?.log('WARN', `[notifyPIDataRefreshed] Failed: ${error.message}`);
+    }
+}
+
+/**
+ * Notify computation completion
+ * @param {Firestore} db - Firestore instance
+ * @param {Object} logger - Logger instance
+ * @param {string|number} userCid - User CID
+ * @param {string} requestId - Request ID
+ * @param {string} computation - Computation name
+ * @param {string} displayName - Display name for computation
+ * @param {boolean} success - Whether computation succeeded
+ * @param {string|null} errorMessage - Error message if failed
+ * @param {Object} options - Additional options
+ */
+async function notifyComputationComplete(db, logger, userCid, requestId, computation, displayName, success, errorMessage, options = {}) {
+    try {
+        const notificationData = {
+            type: 'computation',
+            subType: 'complete',
+            title: success ? 'Computation Complete' : 'Computation Failed',
+            message: success
+                ? `${displayName} completed successfully`
+                : `${displayName} failed: ${errorMessage || 'Unknown error'}`,
+            read: false,
+            timestamp: FieldValue.serverTimestamp(),
+            createdAt: FieldValue.serverTimestamp(),
+            metadata: {
+                requestId,
+                computation,
+                displayName,
+                success,
+                errorMessage: errorMessage || null
+            }
+        };
+
+        await db.collection('SignedInUsers')
+            .doc(String(userCid))
+            .collection('notifications')
+            .doc(`${requestId}_${computation}`)
+            .set(notificationData);
+
+        logger?.log('INFO', `[notifyComputationComplete] Notification sent for user ${userCid}, computation ${computation}`);
+    } catch (error) {
+        logger?.log('WARN', `[notifyComputationComplete] Failed to send notification: ${error.message}`);
+    }
+}
+
+/**
+ * Get computation display name
+ * @param {string} computation - Computation name
+ * @returns {string} Display name
+ */
+function getComputationDisplayName(computation) {
+    const displayNames = {
+        'SignedInUserProfileMetrics': 'Profile Metrics',
+        'PopularInvestorProfileMetrics': 'PI Profile Metrics',
+        'SignedInUserPIPersonalizedMetrics': 'Personalized PI Metrics',
+        'SimilarInvestors': 'Similar Investors',
+        'RecommendedHedges': 'Recommended Hedges',
+        'PopularInvestorRankings': 'PI Rankings'
+    };
+    return displayNames[computation] || computation;
+}
+
+module.exports = {
+    notifyTaskEngineComplete,
+    notifyTaskEngineProgress,
+    notifyPIDataRefreshed,
+    notifyComputationComplete,
+    getComputationDisplayName
+};

package/functions/computation-system/context/ContextFactory.js

@@ -20,17 +20,32 @@ class ContextFactory {
 
     static buildPerUserContext(options) {
         const { 
-            todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory, 
-            userId, userType, dateStr, metadata, mappings, insights, socialData, 
-            computedDependencies, previousComputedDependencies, config, deps,
+            todayPortfolio, 
+            yesterdayPortfolio, 
+            todayHistory, 
+            yesterdayHistory, 
+            userId, 
+            userType, 
+            dateStr, 
+            metadata, 
+            mappings, 
+            insights, 
+            socialData, 
+            computedDependencies, 
+            previousComputedDependencies, 
+            config, 
+            deps,
             verification, 
-            rankings, yesterdayRankings,         // User-specific rank entries
-            allRankings, allRankingsYesterday,   // Global rank lists
+            rankings,
+            yesterdayRankings,         
+            allRankings,                
+            allRankingsYesterday,   
             allVerifications,
-            // [NEW] New Root Data Types for Profile Metrics
-            ratings, pageViews, watchlistMembership, alertHistory,
+            ratings, 
+            pageViews, 
+            watchlistMembership, 
+            alertHistory,
             piMasterList,
-            // [NEW] Series Data (Lookback for Root Data or Computation Results)
             seriesData
         } = options;
         
@@ -56,14 +71,11 @@ class ContextFactory {
                 rankings: allRankings || [],
                 rankingsYesterday: allRankingsYesterday || [],
                 verifications: allVerifications || {},
-                // [NEW] New Root Data Types for Profile Metrics
                 ratings: ratings || {},
                 pageViews: pageViews || {},
                 watchlistMembership: watchlistMembership || {},
                 alertHistory: alertHistory || {},
                 piMasterList: piMasterList || {},
-                // [NEW] Expose Series Data
-                // Structure: { root: { [type]: { [date]: data } }, results: { [date]: { [calcName]: data } } }
                 series: seriesData || {} 
             }
         };
@@ -71,13 +83,23 @@ class ContextFactory {
 
     static buildMetaContext(options) {
         const { 
-            dateStr, metadata, mappings, insights, socialData, prices, 
-            computedDependencies, previousComputedDependencies, config, deps,
-            allRankings, allRankingsYesterday,
+            dateStr, 
+            metadata, 
+            mappings, 
+            insights, 
+            socialData, 
+            prices, 
+            computedDependencies, 
+            previousComputedDependencies, 
+            config, 
+            deps,
+            allRankings, 
+            allRankingsYesterday,
             allVerifications,
-            // [NEW] New Root Data Types
-            ratings, pageViews, watchlistMembership, alertHistory,
-            // [NEW] Series Data
+            ratings, 
+            pageViews, 
+            watchlistMembership, 
+            alertHistory,
             seriesData
         } = options;
         
@@ -99,7 +121,6 @@ class ContextFactory {
                 pageViews: pageViews || {},
                 watchlistMembership: watchlistMembership || {},
                 alertHistory: alertHistory || {},
-                // [NEW] Expose Series Data
                 series: seriesData || {}
             }
         };

package/functions/computation-system/data/AvailabilityChecker.js

@@ -30,11 +30,11 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
         let isAvailable = false;
         
         if (dep === 'portfolio') {
-            if (userType === 'speculator' && rootDataStatus.speculatorPortfolio) isAvailable = true;
-            else if (userType === 'normal' && rootDataStatus.normalPortfolio) isAvailable = true;
+            if (userType === 'speculator'            && rootDataStatus.speculatorPortfolio) isAvailable = true;
+            else if (userType === 'normal'           && rootDataStatus.normalPortfolio) isAvailable = true;
             else if (userType === 'popular_investor' && rootDataStatus.piPortfolios) isAvailable = true;
-            else if (userType === 'signed_in_user' && rootDataStatus.signedInUserPortfolio) isAvailable = true;
-            else if (userType === 'all' && rootDataStatus.hasPortfolio) isAvailable = true;
+            else if (userType === 'signed_in_user'   && rootDataStatus.signedInUserPortfolio) isAvailable = true;
+            else if (userType === 'all'              && rootDataStatus.hasPortfolio) isAvailable = true;
             
             if (!isAvailable) {
                 // [OPTIMIZATION] Optimistic Series Check
@@ -284,12 +284,28 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
             logger.log('WARN', `[Availability] Index not found for ${dateStr}. Assuming NO data.`);
             return {
                 status: { 
-                    hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false,
-                    speculatorPortfolio: false, normalPortfolio: false, speculatorHistory: false, normalHistory: false,
-                    piRankings: false, piPortfolios: false, piDeepPortfolios: false, piHistory: false,
-                    signedInUserPortfolio: false, signedInUserHistory: false, signedInUserVerification: false,
-                    hasPISocial: false, hasSignedInSocial: false,
-                    piRatings: false, piPageViews: false, watchlistMembership: false, piAlertHistory: false
+                    hasPortfolio: false, 
+                    hasHistory: false, 
+                    hasSocial: false, 
+                    hasInsights: false, 
+                    hasPrices: false,
+                    speculatorPortfolio: false, 
+                    normalPortfolio: false, 
+                    speculatorHistory: false, 
+                    normalHistory: false,
+                    piRankings: false, 
+                    piPortfolios: false, 
+                    piDeepPortfolios: false, 
+                    piHistory: false,
+                    signedInUserPortfolio: false, 
+                    signedInUserHistory: false, 
+                    signedInUserVerification: false,
+                    hasPISocial: false, 
+                    hasSignedInSocial: false,
+                    piRatings: false, 
+                    piPageViews: false, 
+                    watchlistMembership: false, 
+                    piAlertHistory: false
                 }
             };
         }
@@ -361,5 +377,5 @@ module.exports = {
     checkRootDependencies, 
     checkRootDataAvailability, 
     getViableCalculations,
-    getAvailabilityWindow // [NEW] Exported
+    getAvailabilityWindow 
 };

package/functions/computation-system/executors/MetaExecutor.js

@@ -10,7 +10,7 @@ const { normalizeName }     = require('../utils/utils');
 const { CachedDataLoader }  = require('../data/CachedDataLoader');
 const { ContextFactory }    = require('../context/ContextFactory');
 const { commitResults }     = require('../persistence/ResultCommitter');
-const { fetchResultSeries } = require('../data/DependencyFetcher'); // [NEW] Import series fetcher
+const { fetchResultSeries } = require('../data/DependencyFetcher');
 
 class MetaExecutor {
     static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
@@ -47,6 +47,8 @@ class MetaExecutor {
         let ratings = null, pageViews = null, watchlistMembership = null, alertHistory = null;
         if (needsNewRootData) {
             const loadPromises = [];
+            
+            // Ratings
             if (calcs.some(c => c.rootDataDependencies?.includes('ratings'))) {
                 loadPromises.push(loader.loadRatings(dStr).then(r => { ratings = r; }).catch(e => {
                     // Only catch if ALL calcs allow missing roots
@@ -61,6 +63,8 @@ class MetaExecutor {
                     }
                 }));
             }
+            
+            // PageViews
             if (calcs.some(c => c.rootDataDependencies?.includes('pageViews'))) {
                 loadPromises.push(loader.loadPageViews(dStr).then(pv => { pageViews = pv; }).catch(e => {
                     const allAllowMissing = calcs.every(c => {
@@ -74,6 +78,8 @@ class MetaExecutor {
                     }
                 }));
             }
+            
+            // Watchlist
             if (calcs.some(c => c.rootDataDependencies?.includes('watchlist'))) {
                 loadPromises.push(loader.loadWatchlistMembership(dStr).then(w => { watchlistMembership = w; }).catch(e => {
                     const allAllowMissing = calcs.every(c => {
@@ -87,6 +93,8 @@ class MetaExecutor {
                     }
                 }));
             }
+            
+            // Alerts
             if (calcs.some(c => c.rootDataDependencies?.includes('alerts'))) {
                 loadPromises.push(loader.loadAlertHistory(dStr).then(a => { alertHistory = a; }).catch(e => {
                     const allAllowMissing = calcs.every(c => {
@@ -100,6 +108,7 @@ class MetaExecutor {
                     }
                 }));
             }
+            
             await Promise.all(loadPromises);
             
             // [FIX] Enforce canHaveMissingRoots - validate after loading
@@ -163,7 +172,8 @@ class MetaExecutor {
             else if (type === 'insights') loaderMethod = 'loadInsights';
             else if (type === 'ratings') loaderMethod = 'loadRatings';
             else if (type === 'watchlist') loaderMethod = 'loadWatchlistMembership';
-            // Add other root types if needed
+            // [CRITICAL UPDATE] Add rankings support for Meta lookbacks
+            else if (type === 'rankings') loaderMethod = 'loadRankings';
 
             if (loaderMethod) {
                 logger.log('INFO', `[MetaExecutor] Loading ${days}-day series for Root Data '${type}'...`);
@@ -222,7 +232,8 @@ class MetaExecutor {
             }
         }
 
-        return await commitResults(state, dStr, passName, config, deps);
+        // CRITICAL FIX: Pass 'isInitialWrite: true' to ensure proper cleanup of old meta data
+        return await commitResults(state, dStr, passName, config, deps, false, { isInitialWrite: true });
     }
 
     static async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps, config, deps, loader) {

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,15 +1,8 @@
-/**
- * {
- * type: uploaded file
- * fileName: computation-system/executors/StandardExecutor.js
- * }
- */
 const { normalizeName, getEarliestDataDates }                          = require('../utils/utils');
 const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
 const { CachedDataLoader }                                             = require('../data/CachedDataLoader');
 const { ContextFactory }                                               = require('../context/ContextFactory');
 const { commitResults }                                                = require('../persistence/ResultCommitter');
-// [NEW] Import series fetcher for computation results
 const { fetchResultSeries }                                            = require('../data/DependencyFetcher');
 const mathLayer                                                        = require('../layers/index');
 const { performance }                                                  = require('perf_hooks');
@@ -167,7 +160,8 @@ class StandardExecutor {
             else if (type === 'insights') loaderMethod = 'loadInsights';
             else if (type === 'ratings') loaderMethod = 'loadRatings';
             else if (type === 'watchlist') loaderMethod = 'loadWatchlistMembership';
-            // Add other root types as needed...
+            // [CRITICAL UPDATE] Add rankings support for AUM lookback
+            else if (type === 'rankings') loaderMethod = 'loadRankings';
 
             if (loaderMethod) {
                 logger.log('INFO', `[StandardExecutor] Loading ${days}-day series for Root Data '${type}'...`);
@@ -487,4 +481,4 @@ class StandardExecutor {
     }
 }
 
-module.exports = { StandardExecutor };
+module.exports = { StandardExecutor };

package/functions/computation-system/helpers/computation_worker.js

@@ -232,7 +232,7 @@ async function handleComputationTask(message, config, dependencies) {
         await db.doc(ledgerPath).update({ status: 'COMPLETED', completedAt: new Date() });
         await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, metrics, triggerReason, resourceTier);
         
-        const { notifyComputationComplete, getComputationDisplayName } = require('../../old-generic-api/user-api/helpers/notifications/notification_helpers.js');
+        const { notifyComputationComplete, getComputationDisplayName } = require('../../api-v2/helpers/notification_helpers.js');
         // Send notification if this was an on-demand computation
         if (metadata?.onDemand && metadata?.requestId && metadata?.requestingUserCid) {
             try {

package/functions/computation-system/persistence/ResultCommitter.js

@@ -4,6 +4,7 @@
  * UPDATED: Added support for 'isPage' mode to store per-user data in subcollections.
  * UPDATED: Implemented TTL retention policy. Defaults to 90 days from the computation date.
  * UPDATED: Fixed issue where switching to 'isPage' mode didn't clean up old sharded/raw data.
+ * CRITICAL FIX: Fixed sharding logic to prevent wiping existing shards during INTERMEDIATE flushes.
  */
 const { commitBatchInChunks, generateDataHash, FieldValue } = require('../utils/utils')
 const { updateComputationStatus }          = require('./StatusRepository');
@@ -138,8 +139,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 continue; 
             }
 
-            // [NEW] Page Computation Logic (Fan-Out) with TTL
-            // Bypasses standard compression/sharding to write per-user documents
             // [NEW] Page Computation Logic (Fan-Out) with TTL
             // Bypasses standard compression/sharding to write per-user documents
             if (isPageComputation && !isEmpty) {
@@ -429,7 +428,9 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
     let finalStats = { totalSize: 0, isSharded: false, shardCount: 1, nextShardIndex: startShardIndex };
     let rootMergeOption = !isInitialWrite; 
 
-    let shouldWipeShards = wasSharded; 
+    // CRITICAL FIX: Only wipe existing shards if this is the INITIAL write for this batch run.
+    // If we are flushing intermediate chunks, we should NOT wipe the shards created by previous chunks!
+    let shouldWipeShards = wasSharded && isInitialWrite; 
 
     for (let attempt = 0; attempt < strategies.length; attempt++) {
         if (committed) break;

package/functions/computation-system/utils/data_loader.js

@@ -1,10 +1,10 @@
 /**
  * @fileoverview Data loader sub-pipes for the Computation System.
  * REFACTORED: Now stateless and receive dependencies.
- * FIXED: Added strict userType filtering to prevent fetching unnecessary data (e.g. Normal users for PI calcs).
- * --- NEW: Updated to read PI/Signed-In data from SHARDS (Parts) instead of individual docs. ---
- * --- NEW: Logic to merge Overall and Deep PI data from corresponding shards. ---
- * --- UPDATED: Added loaders for Rankings and Verification data. ---
+ * FIXED: Added strict userType filtering to prevent fetching unnecessary data.
+ * UPDATED: Verification now uses CollectionGroup query due to per-user storage.
+ * UPDATED: Ratings now correctly handles flattened top-level schema (keys like "reviews.ID").
+ * REMOVED: Redundant Price Shard Indexing logic.
  */
 const zlib = require('zlib'); 
 
@@ -23,11 +23,9 @@ function tryDecompress(data) {
 
 /** --- Data Loader Sub-Pipes (Stateless, Dependency-Injection) --- */
 
-/** * Stage 1: Get portfolio part document references for a given date 
- * [UPDATED] Accepts requiredUserTypes to filter collections.
- */
+/** Stage 1: Get portfolio part document references for a given date */
 async function getPortfolioPartRefs(config, deps, dateString, requiredUserTypes = null) {
-    const { db, logger, calculationUtils, collectionRegistry } = deps;
+    const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     
     // Normalize required types. If null/empty or contains 'ALL', fetch everything.
@@ -250,7 +248,6 @@ async function loadDailyInsights(config, deps, dateString) {
 async function loadDailySocialPostInsights(config, deps, dateString) {
     const { db, logger, calculationUtils, collectionRegistry } = deps;
     const { withRetry } = calculationUtils;
-    const { getCollectionPath } = collectionRegistry || {};
     
     logger.log('INFO', `Loading and partitioning social data for ${dateString}`);
     
@@ -262,7 +259,6 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
     };
 
     // NEW STRUCTURE: Read from date-based collections
-    // Structure: Collection/{date}/{cid}/{cid} for user social, Collection/{date}/posts/{postId} for instrument
     try {
         // Signed-In User Social: SignedInUserSocialPostData/{date}/{cid}/{cid}
         const signedInSocialCollectionName = 'SignedInUserSocialPostData';
@@ -279,7 +275,6 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
                 const cidData = tryDecompress(cidDoc.data());
                 if (cidData.posts && typeof cidData.posts === 'object') {
                     if (!result.signedIn[cid]) result.signedIn[cid] = {};
-                    // Posts are stored as a map in the document
                     Object.assign(result.signedIn[cid], cidData.posts);
                 }
             }
@@ -328,13 +323,10 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
     const PI_COL_NAME = config.piSocialCollectionName || config.piSocialCollection || 'pi_social_posts';
     const SIGNED_IN_COL_NAME = config.signedInUserSocialCollection || 'signed_in_users_social';
 
-    // 2. Define Time Range (UTC Day)
     const startDate = new Date(dateString + 'T00:00:00Z');
     const endDate = new Date(dateString + 'T23:59:59Z');
 
     try {
-        // 3. Fetch ALL with CollectionGroup
-        // NOTE: Requires Firestore Index: CollectionId 'posts', Field 'fetchedAt' (ASC/DESC)
         const postsQuery = db.collectionGroup('posts')
             .where('fetchedAt', '>=', startDate)
             .where('fetchedAt', '<=', endDate);
@@ -346,9 +338,7 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
                 const data = tryDecompress(doc.data());
                 const path = doc.ref.path; 
                 
-                // 4. Partition Logic based on Path
                 if (path.includes(PI_COL_NAME)) {
-                    // Path format: .../pi_social_posts/{userId}/posts/{postId}
                     const parts = path.split('/');
                     const colIndex = parts.indexOf(PI_COL_NAME);
                     if (colIndex !== -1 && parts[colIndex + 1]) {
@@ -358,7 +348,6 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
                     }
                 } 
                 else if (path.includes(SIGNED_IN_COL_NAME)) {
-                    // Path format: .../signed_in_users_social/{userId}/posts/{postId}
                     const parts = path.split('/');
                     const colIndex = parts.indexOf(SIGNED_IN_COL_NAME);
                     if (colIndex !== -1 && parts[colIndex + 1]) {
@@ -368,7 +357,6 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
                     }
                 } 
                 else {
-                    // Default: Generic Instrument Posts
                     result.generic[doc.id] = data;
                 }
             });
@@ -384,11 +372,9 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
     return result;
 }
 
-/** * Stage 6: Get history part references for a given date 
- * [UPDATED] Accepts requiredUserTypes to filter collections.
- */
+/** Stage 6: Get history part references for a given date */
 async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes = null) {
-    const { db, logger, calculationUtils, collectionRegistry } = deps;
+    const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     
     // Normalize required types
@@ -478,9 +464,7 @@ async function getHistoryPartRefs(config, deps, dateString, requiredUserTypes =
     return allPartRefs;
 }
 
-/** * Stage 7: Stream portfolio data in chunks 
- * [UPDATED] Passes requiredUserTypes to getPortfolioPartRefs
- */
+/** Stage 7: Stream portfolio data in chunks */
 async function* streamPortfolioData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
     const { logger } = deps;
     const refs = providedRefs || (await getPortfolioPartRefs(config, deps, dateString, requiredUserTypes));
@@ -497,9 +481,7 @@ async function* streamPortfolioData(config, deps, dateString, providedRefs = nul
     logger.log('INFO', `[streamPortfolioData] Finished streaming for ${dateString}.`);
 }
 
-/** * Stage 8: Stream history data in chunks 
- * [UPDATED] Passes requiredUserTypes to getHistoryPartRefs
- */
+/** Stage 8: Stream history data in chunks */
 async function* streamHistoryData(config, deps, dateString, providedRefs = null, requiredUserTypes = null) {
     const { logger } = deps;
     const refs = providedRefs || (await getHistoryPartRefs(config, deps, dateString, requiredUserTypes));
@@ -531,80 +513,22 @@ async function getPriceShardRefs(config, deps) {
     }
 }
 
-/** Stage 10: Smart Shard Lookup System */
+/** Stage 10: Smart Shard Lookup System (DEPRECATED/SIMPLIFIED) */
 async function ensurePriceShardIndex(config, deps) {
-    const { db, logger } = deps;
-    const metadataCol = config.metadataCollection || 'system_metadata';
-    const indexDocRef = db.collection(metadataCol).doc('price_shard_index');
-    
-    // 1. Try to fetch existing index
-    const snap = await indexDocRef.get();
-    if (snap.exists) {
-        const data = snap.data();
-        const lastUpdated = data.lastUpdated ? new Date(data.lastUpdated).getTime() : 0;
-        const now = Date.now();
-        const oneDayMs = 24 * 60 * 60 * 1000;
-        if ((now - lastUpdated) < oneDayMs) { return data.index || {}; }
-        logger.log('INFO', '[ShardIndex] Index is stale (>24h). Rebuilding...');
-    } else {
-        logger.log('INFO', '[ShardIndex] Index not found. Building new Price Shard Index...');
-    }
-    
-    // 2. Build Index
-    const collection = config.priceCollection || 'asset_prices';
-    const snapshot = await db.collection(collection).get();
-    
-    const index = {};
-    let shardCount = 0;
-    
-    snapshot.forEach(doc => {
-        shardCount++;
-        const rawData = doc.data(); 
-        const data = tryDecompress(rawData);
-
-        if (data.history) {
-            Object.keys(data.history).forEach(instId => {
-                index[instId] = doc.id;
-            });
-        }
-    });
-
-    // 3. Save Index
-    await indexDocRef.set({
-        index: index,
-        lastUpdated: new Date().toISOString(),
-        shardCount: shardCount
-    });
-    
-    logger.log('INFO', `[ShardIndex] Built index for ${Object.keys(index).length} instruments across ${shardCount} shards.`);
-    return index;
+   // [DEPRECATED] This function previously built an index in 'system_metadata/price_shard_index'.
+   // It has been removed to avoid performing computation/indexing in the data loader.
+   // Use 'Fetch All' strategy in Stage 9 instead.
+   return {};
 }
 
 async function getRelevantShardRefs(config, deps, targetInstrumentIds) {
-    const { db, logger } = deps;
-    
-    if (!targetInstrumentIds || targetInstrumentIds.length === 0) {
-        return getPriceShardRefs(config, deps);
-    }
-
-    logger.log('INFO', `[ShardLookup] Resolving shards for ${targetInstrumentIds.length} specific instruments...`);
-    
-    const index = await ensurePriceShardIndex(config, deps);
-    const uniqueShardIds = new Set();
-    const collection = config.priceCollection || 'asset_prices';
-
-    let foundCount = 0;
-    for (const id of targetInstrumentIds) {
-        const shardId = index[id];
-        if (shardId) {
-            uniqueShardIds.add(shardId);
-            foundCount++;
-        }
-    }
-    
-    logger.log('INFO', `[ShardLookup] Mapped ${foundCount}/${targetInstrumentIds.length} instruments to ${uniqueShardIds.size} unique shards.`);
+    const { logger } = deps;
     
-    return Array.from(uniqueShardIds).map(id => db.collection(collection).doc(id));
+    // [UPDATED] Smart shard lookup is disabled due to missing index infrastructure
+    // and to avoid computing indexes during load time.
+    // Falling back to Stage 9 (Fetch All Shards).
+    logger.log('INFO', `[ShardLookup] Smart indexing disabled. Fetching all price shards for ${targetInstrumentIds ? targetInstrumentIds.length : 'all'} instruments.`);
+    return getPriceShardRefs(config, deps);
 }
 
 /** Stage 11: Load Popular Investor Rankings */
@@ -625,40 +549,58 @@ async function loadPopularInvestorRankings(config, deps, dateString) {
         }
         
         const data = tryDecompress(docSnap.data());
-        return data.Items || []; // Returns the array of PI objects
+        return data.Items || []; 
     } catch (error) {
         logger.log('ERROR', `Failed to load Rankings for ${dateString}: ${error.message}`);
         return null;
     }
 }
 
-/** Stage 12: Load User Verification Profiles */
+/** Stage 12: Load User Verification Profiles 
+ * [UPDATED] Scans global verification data via CollectionGroup since it's now stored per-user.
+ */
 async function loadVerificationProfiles(config, deps) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    const collectionName = config.verificationCollection || 'verified_users';
     
-    logger.log('INFO', `Loading Verification Profiles`);
+    // Verification is now stored at /SignedInUsers/{cid}/verification/data
+    // To fetch globally, we must use a CollectionGroup query on 'verification'
+    // and filter for the document ID 'data'.
+    
+    logger.log('INFO', `Loading Verification Profiles (CollectionGroup: verification/data)`);
 
     try {
-        const snapshot = await withRetry(() => db.collection(collectionName).get(), 'getVerifications');
+        // Warning: This requires a Firestore Index if used with complex filters, but basic get() usually works.
+        const snapshot = await withRetry(() => db.collectionGroup('verification').get(), 'getVerificationsGroup');
         
         if (snapshot.empty) return {};
         
         const profiles = {};
+        let count = 0;
+
         snapshot.forEach(doc => {
+            if (doc.id !== 'data') return; // Enforce specific document ID from schema
+
             const raw = tryDecompress(doc.data());
-            // [FIX] Normalize Verification Data Structure
-            profiles[doc.id] = {
-                cid: raw.realCID || raw.cid,
-                username: raw.username,
-                aboutMe: raw.userBio?.aboutMe || raw.aboutMe || "",
-                aboutMeShort: raw.userBio?.aboutMeShort || raw.aboutMeShort || "",
-                isVerified: raw.isVerified === true,
-                restrictions: raw.CustomerRestrictions || []
-            };
+            
+            // Map new schema fields to internal profile structure
+            // New Schema: { etoroCID, etoroUsername, verifiedAt, setupCompletedAt ... }
+            if (raw.etoroCID) {
+                profiles[raw.etoroCID] = {
+                    cid: raw.etoroCID,
+                    username: raw.etoroUsername,
+                    // 'aboutMe' and 'restrictions' are NOT present in the new schema.
+                    // Defaulting to empty values to preserve downstream compatibility.
+                    aboutMe: "", 
+                    aboutMeShort: "",
+                    isVerified: !!(raw.verifiedAt), // Using existence of verifiedAt as flag
+                    restrictions: []
+                };
+                count++;
+            }
         });
         
+        logger.log('INFO', `Loaded ${count} verification profiles.`);
         return profiles;
     } catch (error) {
         logger.log('ERROR', `Failed to load Verification Profiles: ${error.message}`);
@@ -666,30 +608,60 @@ async function loadVerificationProfiles(config, deps) {
     }
 }
 
-/** Stage 13: Load PI Ratings Data */
+/** Stage 13: Load PI Ratings Data 
+ * [UPDATED] Reads from /PiReviews/{date}/shards/daily_log.
+ * [FIXED] Handles FLATTENED schema where keys like "reviews.ID" are at the top level.
+ * Returns RAW logs grouped by PI. NO COMPUTATION.
+ */
 async function loadPIRatings(config, deps, dateString) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    const collectionName = config.piRatingsCollection || 'PIRatingsData';
     
-    logger.log('INFO', `Loading PI Ratings for ${dateString}`);
+    // New Path: /PiReviews/{date}/shards/daily_log
+    
+    logger.log('INFO', `Loading PI Ratings (Raw Logs) for ${dateString}`);
     
     try {
-        const docRef = db.collection(collectionName).doc(dateString);
-        const docSnap = await withRetry(() => docRef.get(), `getPIRatings(${dateString})`);
-        
-        if (!docSnap.exists) { 
-            logger.log('WARN', `PI Ratings not found for ${dateString}`); 
-            return null; 
+        const shardsColRef = db.collection('PiReviews').doc(dateString).collection('shards');
+        const shardDocs = await withRetry(() => shardsColRef.listDocuments(), `listRatingShards(${dateString})`);
+
+        if (!shardDocs || shardDocs.length === 0) {
+            logger.log('WARN', `No rating shards found for ${dateString} at ${shardsColRef.path}`);
+            return {};
         }
-        
-        const data = tryDecompress(docSnap.data());
-        // Remove the date key and lastUpdated, return just the PI data
-        const { date, lastUpdated, ...piRatings } = data;
-        return piRatings; // Returns { piCid: { averageRating, totalRatings, ratingsByUser, ... } }
+
+        const rawReviewsByPi = {}; 
+
+        for (const docRef of shardDocs) {
+            const docSnap = await docRef.get();
+            if (!docSnap.exists) continue;
+
+            const rawData = tryDecompress(docSnap.data());
+            
+            // SCHEMA HANDLING:
+            // Keys at the root of the document are the review IDs (e.g. "reviews.29312236_31075566").
+            // We iterate over all values and check if they look like review objects.
+            
+            Object.values(rawData).forEach(entry => {
+                // Check for valid review object structure
+                if (entry && typeof entry === 'object' && entry.piCid && entry.rating !== undefined) {
+                    
+                    if (!rawReviewsByPi[entry.piCid]) {
+                        rawReviewsByPi[entry.piCid] = [];
+                    }
+                    
+                    // Store the raw entry directly.
+                    rawReviewsByPi[entry.piCid].push(entry);
+                }
+            });
+        }
+
+        logger.log('INFO', `Loaded raw reviews for ${Object.keys(rawReviewsByPi).length} PIs.`);
+        return rawReviewsByPi; 
+
     } catch (error) {
         logger.log('ERROR', `Failed to load PI Ratings for ${dateString}: ${error.message}`);
-        return null;
+        return {};
     }
 }
 
@@ -711,9 +683,8 @@ async function loadPIPageViews(config, deps, dateString) {
         }
         
         const data = tryDecompress(docSnap.data());
-        // Remove the date key and lastUpdated, return just the PI data
         const { date, lastUpdated, ...piPageViews } = data;
-        return piPageViews; // Returns { piCid: { totalViews, uniqueViewers, viewsByUser, ... } }
+        return piPageViews; 
     } catch (error) {
         logger.log('ERROR', `Failed to load PI Page Views for ${dateString}: ${error.message}`);
         return null;
@@ -738,9 +709,8 @@ async function loadWatchlistMembership(config, deps, dateString) {
         }
         
         const data = tryDecompress(docSnap.data());
-        // Remove the date key and lastUpdated, return just the PI data
         const { date, lastUpdated, ...watchlistMembership } = data;
-        return watchlistMembership; // Returns { piCid: { totalUsers, users, publicWatchlistCount, ... } }
+        return watchlistMembership; 
     } catch (error) {
         logger.log('ERROR', `Failed to load Watchlist Membership for ${dateString}: ${error.message}`);
         return null;
@@ -765,19 +735,15 @@ async function loadPIAlertHistory(config, deps, dateString) {
         }
         
         const data = tryDecompress(docSnap.data());
-        // Remove the date key and lastUpdated, return just the PI data
         const { date, lastUpdated, ...piAlertHistory } = data;
-        return piAlertHistory; // Returns { piCid: { alertType: { triggered, count, triggeredFor, ... } } }
+        return piAlertHistory; 
     } catch (error) {
         logger.log('ERROR', `Failed to load PI Alert History for ${dateString}: ${error.message}`);
         return null;
     }
 }
 
-/** Stage 17: Load PI-Centric Watchlist Data
- * Loads watchlist data from PopularInvestors/{piCid}/watchlistData/current
- * This provides time-series data of watchlist additions per PI
- */
+/** Stage 17: Load PI-Centric Watchlist Data */
 async function loadPIWatchlistData(config, deps, piCid) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
@@ -799,19 +765,18 @@ async function loadPIWatchlistData(config, deps, piCid) {
         }
         
         const data = tryDecompress(docSnap.data());
-        return data; // Returns { totalUsers, userCids: [], dailyAdditions: { date: { count, userCids, timestamp } }, lastUpdated }
+        return data; 
     } catch (error) {
         logger.log('ERROR', `Failed to load PI Watchlist Data for PI ${piCidStr}: ${error.message}`);
         return null;
     }
 }
 
-// [NEW] Load Popular Investor Master List
+// Load Popular Investor Master List
 async function loadPopularInvestorMasterList(config, deps) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     
-    // Default to 'system_state' collection, 'popular_investor_master_list' doc
     const collectionName = config.piMasterListCollection || 'system_state';
     const docId = config.piMasterListDocId || 'popular_investor_master_list';
 
@@ -827,9 +792,6 @@ async function loadPopularInvestorMasterList(config, deps) {
         }
 
         const data = tryDecompress(docSnap.data());
-        // Structure is { investors: { cid: { username, ... } } } or direct map { cid: { ... } }
-        // Based on user input, it looks like a direct map of CIDs or a field holding the map.
-        // We return the raw object which acts as the map.
         return data.investors || data; 
     } catch (error) {
         logger.log('ERROR', `Failed to load PI Master List: ${error.message}`);
@@ -855,6 +817,6 @@ module.exports = {
     loadPIPageViews,
     loadWatchlistMembership,
     loadPIAlertHistory,
-    loadPopularInvestorMasterList, // [NEW]
+    loadPopularInvestorMasterList,
     loadPIWatchlistData,
 };

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -7,7 +7,7 @@
 const crypto = require('crypto');
 const { shouldTryProxy, recordProxyOutcome, getFailureCount, getMaxFailures } = require('../utils/proxy_circuit_breaker');
 const { FieldValue } = require('@google-cloud/firestore');
-const { notifyTaskEngineComplete, notifyTaskEngineProgress, notifyPIDataRefreshed } = require('../../old-generic-api/user-api/helpers/notifications/notification_helpers');
+const { notifyTaskEngineComplete, notifyTaskEngineProgress, notifyPIDataRefreshed } = require('../../api-v2/helpers/notification_helpers');
 const { conditionallyRunRootDataIndexer } = require('./root_data_indexer_helpers');
 
 const { 
@@ -405,7 +405,7 @@ async function finalizeOnDemandRequest(deps, config, taskData, isPI, success, to
     // 2. Trigger Computations (only if root data indexer completed successfully)
     if (indexerCompleted && pubsub && config.computationSystem) {
         const { triggerComputationWithDependencies } = require('../../computation-system/helpers/on_demand_helpers');
-        const { checkIfUserIsPI } = require('../../old-generic-api/user-api/helpers/core/user_status_helpers');
+        const { fetchPopularInvestorMasterList } = require('../../api-v2/helpers/data-fetchers/firestore.js');
         
         // Use userType from metadata if available, otherwise fall back to isPI
         const userType = metadata?.userType || (isPI ? 'POPULAR_INVESTOR' : 'SIGNED_IN_USER');
@@ -429,11 +429,15 @@ async function finalizeOnDemandRequest(deps, config, taskData, isPI, success, to
             
             // IMPORTANT: Check if this signed-in user is also a Popular Investor
             // If they are, we need to run PI computations as well
-            const rankEntry = await checkIfUserIsPI(db, cid, config, logger);
-            if (rankEntry) {
-                logger.log('INFO', `[On-Demand] Signed-in user ${cid} is also a Popular Investor. Adding PI computations.`);
-                compsSet.add('PopularInvestorProfileMetrics');
-                compsSet.add('SignedInUserPIPersonalizedMetrics');
+            try {
+                const piData = await fetchPopularInvestorMasterList(db, String(cid));
+                if (piData) {
+                    logger.log('INFO', `[On-Demand] Signed-in user ${cid} is also a Popular Investor. Adding PI computations.`);
+                    compsSet.add('PopularInvestorProfileMetrics');
+                    compsSet.add('SignedInUserPIPersonalizedMetrics');
+                }
+            } catch (e) {
+                // User is not a PI, continue with signed-in user computations only
             }
         } else {
             // Fallback to isPI-based logic for backward compatibility

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.630",
+  "version": "1.0.632",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -9,7 +9,6 @@
     "functions/task-engine/",
     "functions/core/",
     "functions/computation-system/",
-    "functions/old-generic-api/",
     "functions/api-v2/",
     "functions/dispatcher/",
     "functions/invalid-speculator-handler/",