bulltrackers-module 1.0.677 → 1.0.679

package/functions/alert-system/helpers/alert_helpers.js

@@ -16,12 +16,53 @@ const storage = new Storage(); // Singleton GCS Client
  */
 async function processAlertForPI(db, logger, piCid, alertType, computationMetadata, computationDate, dependencies = {}) {
   try {
+    // [FIX] Check if computation date is earlier than today (backfill protection)
+    const today = new Date().toISOString().split('T')[0]; // YYYY-MM-DD format
+    const isHistoricalData = computationDate < today;
+    
+    // If it's historical data, check if this alert already exists
+    if (isHistoricalData) {
+      // Check if we've already created alerts for this PI/date/alertType combination
+      const existingAlertsSnapshot = await db.collection('SignedInUsers')
+        .doc('_metadata') // Use metadata doc to track processed alerts
+        .collection('processed_alerts')
+        .where('piCid', '==', Number(piCid))
+        .where('computationDate', '==', computationDate)
+        .where('alertType', '==', alertType.id)
+        .limit(1)
+        .get();
+      
+      if (!existingAlertsSnapshot.empty) {
+        logger.log('INFO', `[processAlertForPI] Skipping duplicate alert for historical data: PI ${piCid}, date ${computationDate}, alert type ${alertType.id}`);
+        return;
+      }
+      
+      logger.log('WARN', `[processAlertForPI] Processing alert for historical data (backfill): PI ${piCid}, date ${computationDate}, alert type ${alertType.id}. This alert will only be sent to developers.`);
+    }
+    
     // 1. Get PI username from rankings or subscriptions
     const piUsername = await getPIUsername(db, piCid);
     
     // 2. Find all users subscribed to this PI and alert type
     // Use computationName (e.g., 'RiskScoreIncrease') to map to alertConfig keys
-    const subscriptions = await findSubscriptionsForPI(db, logger, piCid, alertType.computationName, computationDate, dependencies);
+    let subscriptions = await findSubscriptionsForPI(db, logger, piCid, alertType.computationName, computationDate, dependencies);
+    
+    // [FIX] If it's historical data, only send to developer accounts
+    if (isHistoricalData) {
+      const { isDeveloper } = require('../../api-v2/helpers/data-fetchers/firestore.js');
+      
+      // Filter subscriptions to only include developers
+      const devSubscriptions = [];
+      for (const subscription of subscriptions) {
+        const isDev = await isDeveloper(db, String(subscription.userCid));
+        if (isDev) {
+          devSubscriptions.push(subscription);
+        }
+      }
+      
+      subscriptions = devSubscriptions;
+      logger.log('INFO', `[processAlertForPI] Historical data: Filtered to ${subscriptions.length} developer subscriptions for PI ${piCid}, alert type ${alertType.id}`);
+    }
     
     if (subscriptions.length === 0) {
       logger.log('INFO', `[processAlertForPI] No subscriptions found for PI ${piCid}, alert type ${alertType.id}`);
@@ -38,8 +79,24 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
     const config = dependencies.config || {};
     const notificationPromises = [];
     
+    // Check watchlistAlerts preference for each user
+    const { manageNotificationPreferences } = require('../../api-v2/helpers/data-fetchers/firestore.js');
+    
     for (const subscription of subscriptions) {
       const userCid = subscription.userCid;
+      
+      // [FIX] Check user's watchlistAlerts preference before creating alert
+      try {
+        const prefs = await manageNotificationPreferences(db, userCid, 'get');
+        if (!prefs.watchlistAlerts) {
+          logger.log('DEBUG', `[processAlertForPI] User ${userCid} has watchlistAlerts disabled, skipping alert`);
+          continue; // Skip this user
+        }
+      } catch (prefError) {
+        logger.log('WARN', `[processAlertForPI] Error checking watchlistAlerts preference for user ${userCid}: ${prefError.message}`);
+        // Continue anyway - fail open for important alerts
+      }
+      
       const notificationId = `alert_${Date.now()}_${userCid}_${piCid}_${Math.random().toString(36).substring(2, 9)}`;
       
       const notificationData = {
@@ -129,6 +186,28 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
     
     logger.log('SUCCESS', `[processAlertForPI] Created ${notificationPromises.length} notifications for PI ${piCid}, alert type ${alertType.id}`);
     
+    // [FIX] Mark alert as processed to prevent duplicates on backfill
+    if (isHistoricalData) {
+      try {
+        await db.collection('SignedInUsers')
+          .doc('_metadata')
+          .collection('processed_alerts')
+          .doc(`${piCid}_${computationDate}_${alertType.id}`)
+          .set({
+            piCid: Number(piCid),
+            computationDate: computationDate,
+            alertType: alertType.id,
+            alertTypeName: alertType.name,
+            processedAt: FieldValue.serverTimestamp(),
+            notificationsSent: notificationPromises.length
+          });
+        logger.log('INFO', `[processAlertForPI] Marked historical alert as processed: PI ${piCid}, date ${computationDate}, alert type ${alertType.id}`);
+      } catch (markError) {
+        logger.log('WARN', `[processAlertForPI] Failed to mark alert as processed: ${markError.message}`);
+        // Don't throw - this is non-critical
+      }
+    }
+    
   } catch (error) {
     logger.log('ERROR', `[processAlertForPI] Error processing alert for PI ${piCid}`, error);
     throw error;
@@ -285,8 +364,29 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
               const isTestProbe = alertTypeId === 'TestSystemProbe';
               const isEnabled = item.alertConfig && item.alertConfig[configKey] === true;
               
-              // If it's the probe OR the user explicitly enabled it
-              if (isTestProbe || isEnabled) {
+              // [FIX] For TestSystemProbe, check user's testAlerts preference
+              let shouldSendAlert = false;
+              if (isTestProbe) {
+                // Check if user has testAlerts enabled in their notification preferences
+                try {
+                  const { manageNotificationPreferences } = require('../../api-v2/helpers/data-fetchers/firestore.js');
+                  const prefs = await manageNotificationPreferences(db, userCid, 'get');
+                  shouldSendAlert = prefs.testAlerts === true;
+                  
+                  if (!shouldSendAlert) {
+                    logger.log('DEBUG', `[findSubscriptionsForPI] User ${userCid} has testAlerts disabled, skipping TestSystemProbe alert`);
+                  }
+                } catch (prefError) {
+                  logger.log('WARN', `[findSubscriptionsForPI] Error checking testAlerts preference for user ${userCid}: ${prefError.message}`);
+                  // Default to not sending test alerts if we can't check preferences
+                  shouldSendAlert = false;
+                }
+              } else {
+                // For non-test alerts, use normal alert config check
+                shouldSendAlert = isEnabled;
+              }
+              
+              if (shouldSendAlert) {
                 subscriptions.push({
                   userCid: userCid,
                   piCid: piCid,

package/functions/api-v2/helpers/notification_helpers.js

@@ -18,6 +18,15 @@ const { FieldValue } = require('@google-cloud/firestore');
  */
 async function notifyTaskEngineComplete(db, logger, userCid, requestId, username, success, errorMessage, options = {}) {
     try {
+        // [FIX] Check user's notification preferences before sending
+        const { manageNotificationPreferences } = require('./data-fetchers/firestore.js');
+        const prefs = await manageNotificationPreferences(db, userCid, 'get');
+        
+        if (!prefs.syncProcesses) {
+            logger?.log('DEBUG', `[notifyTaskEngineComplete] User ${userCid} has syncProcesses disabled, skipping notification`);
+            return;
+        }
+        
         const notificationData = {
             type: 'sync',
             subType: 'complete',
@@ -42,6 +51,29 @@ async function notifyTaskEngineComplete(db, logger, userCid, requestId, username
             .doc(requestId)
             .set(notificationData);
 
+        // [FIX] Clean up progress notifications for this sync to stop showing loading icons
+        try {
+            const progressNotificationsQuery = await db.collection('SignedInUsers')
+                .doc(String(userCid))
+                .collection('notifications')
+                .where('metadata.requestId', '==', requestId)
+                .where('subType', '==', 'progress')
+                .get();
+            
+            const batch = db.batch();
+            progressNotificationsQuery.docs.forEach(doc => {
+                batch.delete(doc.ref);
+            });
+            
+            if (progressNotificationsQuery.size > 0) {
+                await batch.commit();
+                logger?.log('INFO', `[notifyTaskEngineComplete] Cleaned up ${progressNotificationsQuery.size} progress notifications for request ${requestId}`);
+            }
+        } catch (cleanupError) {
+            logger?.log('WARN', `[notifyTaskEngineComplete] Failed to clean up progress notifications: ${cleanupError.message}`);
+            // Don't throw - cleanup is non-critical
+        }
+
         logger?.log('INFO', `[notifyTaskEngineComplete] Notification sent for user ${userCid}, request ${requestId}`);
     } catch (error) {
         logger?.log('WARN', `[notifyTaskEngineComplete] Failed to send notification: ${error.message}`);
@@ -61,6 +93,15 @@ async function notifyTaskEngineComplete(db, logger, userCid, requestId, username
  */
 async function notifyTaskEngineProgress(db, logger, userCid, requestId, username, stage, dataType, options = {}) {
     try {
+        // [FIX] Check user's notification preferences before sending
+        const { manageNotificationPreferences } = require('./data-fetchers/firestore.js');
+        const prefs = await manageNotificationPreferences(db, userCid, 'get');
+        
+        if (!prefs.syncProcesses) {
+            logger?.log('DEBUG', `[notifyTaskEngineProgress] User ${userCid} has syncProcesses disabled, skipping notification`);
+            return;
+        }
+        
         const stageMessages = {
             'started': 'Data sync started',
             'portfolio_complete': 'Portfolio data fetched',
@@ -130,6 +171,15 @@ async function notifyPIDataRefreshed(db, logger, collectionRegistry, piCid, user
  */
 async function notifyComputationComplete(db, logger, userCid, requestId, computation, displayName, success, errorMessage, options = {}) {
     try {
+        // [FIX] Check user's notification preferences before sending
+        const { manageNotificationPreferences } = require('./data-fetchers/firestore.js');
+        const prefs = await manageNotificationPreferences(db, userCid, 'get');
+        
+        if (!prefs.userActionCompletions) {
+            logger?.log('DEBUG', `[notifyComputationComplete] User ${userCid} has userActionCompletions disabled, skipping notification`);
+            return;
+        }
+        
         const notificationData = {
             type: 'computation',
             subType: 'complete',

package/functions/fetch-popular-investors/helpers/fetch_helpers.js

@@ -8,6 +8,64 @@ const { IntelligentProxyManager } = require('../../core/utils/intelligent_proxy_
 const { IntelligentHeaderManager } = require('../../core/utils/intelligent_header_manager');
 const zlib = require('zlib');
 
+/**
+ * Fetches individual user rankings data by CID
+ * @param {string} cid - Customer ID
+ * @param {object} headers - Request headers to use
+ * @param {object} proxyManager - ProxyManager instance
+ * @param {object} logger - Logger instance
+ * @returns {object|null} - User rankings data or null if failed
+ */
+async function fetchIndividualUserRankings(cid, headers, proxyManager, logger) {
+    const individualUrl = `https://www.etoro.com/sapi/rankings/cid/${cid}/rankings/?Period=OneYearAgo`;
+    
+    try {
+        logger.log('INFO', `[PopularInvestorFetch] Fetching individual rankings for CID: ${cid}`);
+        
+        // Try with proxy first
+        try {
+            const response = await proxyManager.fetch(individualUrl, {
+                method: 'GET',
+                headers
+            });
+
+            if (response.ok) {
+                const data = await response.json();
+                if (data && data.Data) {
+                    logger.log('SUCCESS', `[PopularInvestorFetch] Successfully fetched individual rankings for CID: ${cid} via proxy`);
+                    return data.Data; // Return the Data object which matches the Items schema
+                }
+            }
+        } catch (proxyError) {
+            logger.log('WARN', `[PopularInvestorFetch] Proxy fetch failed for CID ${cid}: ${proxyError.message}`);
+        }
+        
+        // Fallback to direct fetch
+        try {
+            const directResponse = await fetch(individualUrl, {
+                method: 'GET',
+                headers
+            });
+
+            if (directResponse.ok) {
+                const data = await directResponse.json();
+                if (data && data.Data) {
+                    logger.log('SUCCESS', `[PopularInvestorFetch] Successfully fetched individual rankings for CID: ${cid} via direct fetch`);
+                    return data.Data;
+                }
+            }
+        } catch (directError) {
+            logger.log('WARN', `[PopularInvestorFetch] Direct fetch failed for CID ${cid}: ${directError.message}`);
+        }
+        
+        logger.log('ERROR', `[PopularInvestorFetch] Failed to fetch individual rankings for CID: ${cid} from all sources`);
+        return null;
+    } catch (error) {
+        logger.log('ERROR', `[PopularInvestorFetch] Error fetching individual rankings for CID ${cid}`, { errorMessage: error.message });
+        return null;
+    }
+}
+
 /**
  * Fetches the top Popular Investors and stores the raw result in Firestore.
  * @param {object} dependencies - Contains db, logger.
@@ -117,6 +175,90 @@ async function fetchAndStorePopularInvestors(config, dependencies) {
         await headerManager.flushPerformanceUpdates();
     }
 
+    // 5.5. Check for missing users from master list and fetch them individually
+    if (data && data.Items && Array.isArray(data.Items)) {
+        try {
+            logger.log('INFO', '[PopularInvestorFetch] Checking for missing users from master list...');
+            
+            // Get master list path
+            let masterListPath = 'system_state/popular_investor_master_list';
+            if (collectionRegistry && collectionRegistry.getCollectionPath) {
+                try {
+                    const registryPath = collectionRegistry.getCollectionPath('system', 'popularInvestorMasterList', {});
+                    masterListPath = registryPath;
+                } catch (e) {
+                    logger.log('WARN', `[PopularInvestorFetch] Failed to get master list path from registry, using default: ${e.message}`);
+                }
+            }
+            
+            const masterListRef = db.doc(masterListPath);
+            const masterListDoc = await masterListRef.get();
+            
+            if (masterListDoc.exists) {
+                const masterListData = masterListDoc.data();
+                const masterInvestors = masterListData.investors || {};
+                
+                // Build a Set of CIDs from the fetched data for fast lookup
+                const fetchedCids = new Set(data.Items.map(item => String(item.CustomerId)));
+                
+                // Identify missing CIDs
+                const masterCids = Object.keys(masterInvestors);
+                const missingCids = masterCids.filter(cid => !fetchedCids.has(cid));
+                
+                if (missingCids.length > 0) {
+                    logger.log('INFO', `[PopularInvestorFetch] Found ${missingCids.length} missing users from master list. Fetching individually...`);
+                    
+                    // Prepare headers for individual fetches
+                    const requestHeaders = {
+                        'Accept': 'application/json',
+                        'Referer': 'https://www.etoro.com/',
+                        ...(await headerManager.selectHeader()).header
+                    };
+                    
+                    // Fetch missing users with rate limiting
+                    const missingUserData = [];
+                    let successCount = 0;
+                    let failureCount = 0;
+                    
+                    for (const cid of missingCids) {
+                        const userData = await fetchIndividualUserRankings(cid, requestHeaders, proxyManager, logger);
+                        
+                        if (userData) {
+                            missingUserData.push(userData);
+                            successCount++;
+                        } else {
+                            failureCount++;
+                            logger.log('WARN', `[PopularInvestorFetch] Failed to fetch data for missing user CID: ${cid} (${masterInvestors[cid].username})`);
+                        }
+                        
+                        // Add small delay between requests to avoid rate limiting
+                        if (missingCids.length > 10 && missingCids.indexOf(cid) < missingCids.length - 1) {
+                            await new Promise(resolve => setTimeout(resolve, 200)); // 200ms delay
+                        }
+                    }
+                    
+                    // Append successfully fetched missing users to the main data
+                    if (missingUserData.length > 0) {
+                        data.Items.push(...missingUserData);
+                        data.TotalRows += missingUserData.length;
+                        logger.log('SUCCESS', `[PopularInvestorFetch] Successfully fetched ${successCount}/${missingCids.length} missing users. Updated Items array from ${data.Items.length - missingUserData.length} to ${data.Items.length} users.`);
+                    }
+                    
+                    if (failureCount > 0) {
+                        logger.log('WARN', `[PopularInvestorFetch] Failed to fetch ${failureCount}/${missingCids.length} missing users.`);
+                    }
+                } else {
+                    logger.log('INFO', '[PopularInvestorFetch] All users from master list are present in the main fetch. No missing users to fetch individually.');
+                }
+            } else {
+                logger.log('INFO', '[PopularInvestorFetch] Master list document does not exist yet. Skipping missing user check.');
+            }
+        } catch (missingUserError) {
+            logger.log('WARN', `[PopularInvestorFetch] Error while checking/fetching missing users: ${missingUserError.message}. Continuing with main fetch data.`);
+            // Non-critical error, continue with whatever data we have
+        }
+    }
+
     // 6. Final Validation & Storage
     if (data && data.Items && Array.isArray(data.Items)) {
         try {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.677",
+  "version": "1.0.679",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [