bulltrackers-module 1.0.777 → 1.0.779

package/functions/alert-system/helpers/alert_helpers.js

@@ -23,7 +23,7 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
     // [FIX] Check if computation date is earlier than today (backfill protection)
     const today = new Date().toISOString().split('T')[0]; // YYYY-MM-DD format
     const isHistoricalData = computationDate < today;
-    
+
     // If it's historical data, check if this alert already exists
     if (isHistoricalData) {
       // Check if we've already created alerts for this PI/date/alertType combination
@@ -35,26 +35,26 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
         .where('alertType', '==', alertType.id)
         .limit(1)
         .get();
-      
+
       if (!existingAlertsSnapshot.empty) {
         logger.log('INFO', `[processAlertForPI] Skipping duplicate alert for historical data: PI ${piCid}, date ${computationDate}, alert type ${alertType.id}`);
         return;
       }
-      
+
       logger.log('WARN', `[processAlertForPI] Processing alert for historical data (backfill): PI ${piCid}, date ${computationDate}, alert type ${alertType.id}. This alert will only be sent to developers.`);
     }
-    
+
     // 1. Get PI username from rankings or subscriptions
     const piUsername = await getPIUsername(db, piCid);
-    
+
     // 2. Find all users subscribed to this PI and alert type
     // Use computationName (e.g., 'RiskScoreIncrease') to map to alertConfig keys
     let subscriptions = await findSubscriptionsForPI(db, logger, piCid, alertType.computationName, computationDate, dependencies);
-    
+
     // [FIX] If it's historical data, only send to developer accounts
     if (isHistoricalData) {
       const { isDeveloper } = require('../../api-v2/helpers/data-fetchers/firestore.js');
-      
+
       // Filter subscriptions to only include developers
       const devSubscriptions = [];
       for (const subscription of subscriptions) {
@@ -63,32 +63,32 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
           devSubscriptions.push(subscription);
         }
       }
-      
+
       subscriptions = devSubscriptions;
       logger.log('INFO', `[processAlertForPI] Historical data: Filtered to ${subscriptions.length} developer subscriptions for PI ${piCid}, alert type ${alertType.id}`);
     }
-    
+
     if (subscriptions.length === 0) {
       logger.log('INFO', `[processAlertForPI] No subscriptions found for PI ${piCid}, alert type ${alertType.id}`);
       return;
     }
-    
+
     logger.log('INFO', `[processAlertForPI] Processing ${subscriptions.length} subscriptions for PI ${piCid}, alert type ${alertType.id}`);
-    
+
     // 3. Generate alert message
     const alertMessage = generateAlertMessage(alertType, piUsername, computationMetadata);
-    
+
     // 4. Create notifications for each subscribed user (using CID and collection registry)
     const { collectionRegistry } = dependencies;
     const config = dependencies.config || {};
     const notificationPromises = [];
-    
+
     // Check watchlistAlerts preference for each user
     const { manageNotificationPreferences } = require('../../api-v2/helpers/data-fetchers/firestore.js');
-    
+
     for (const subscription of subscriptions) {
       const userCid = subscription.userCid;
-      
+
       // [FIX] Check user's watchlistAlerts preference before creating alert
       try {
         const prefs = await manageNotificationPreferences(db, userCid, 'get');
@@ -100,16 +100,16 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
         logger.log('WARN', `[processAlertForPI] Error checking watchlistAlerts preference for user ${userCid}: ${prefError.message}`);
         // Continue anyway - fail open for important alerts
       }
-      
+
       // [NEW] Evaluate dynamic conditions based on user's subscription mode
       // Check if user is a developer for bypass
       const { isDeveloper } = require('../../api-v2/helpers/data-fetchers/firestore.js');
       const isDev = await isDeveloper(db, String(userCid));
-      
+
       // Get user's configuration from subscription
       const userDynamicConfig = subscription.dynamicConfig?.[alertType.configKey] || {};
       const userUseDynamic = subscription.useDynamic?.[alertType.configKey] === true;
-      
+
       // Evaluate conditions (will pass if static mode or no conditions)
       const evaluation = evaluateDynamicConditions(
         alertType,
@@ -119,20 +119,20 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
         isDev,
         logger
       );
-      
+
       if (!evaluation.passes) {
         logger.log('DEBUG', `[processAlertForPI] User ${userCid} dynamic conditions not met for ${alertType.id}: ${evaluation.reason}`);
         continue; // Skip this user
       }
-      
+
       if (evaluation.developerBypass) {
         logger.log('INFO', `[processAlertForPI] Developer ${userCid} bypass: ${evaluation.reason}`);
       } else {
         logger.log('DEBUG', `[processAlertForPI] User ${userCid} alert ${alertType.id} in ${evaluation.mode} mode`);
       }
-      
+
       const notificationId = `alert_${Date.now()}_${userCid}_${piCid}_${Math.random().toString(36).substring(2, 9)}`;
-      
+
       const notificationData = {
         id: notificationId,
         type: 'watchlistAlerts', // Use watchlistAlerts type for watchlist-based alerts
@@ -157,7 +157,7 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
           ...(computationMetadata || {})
         }
       };
-      
+
       // Write to alerts collection (not notifications) - alerts are separate from system notifications
       // Write directly to new path: SignedInUsers/{userCid}/alerts/{alertId}
       const alertData = {
@@ -176,7 +176,7 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
         computationName: alertType.computationName,
         ...(computationMetadata || {})
       };
-      
+
       // Combined promise: write to Firestore AND send FCM push notification
       const writeAndPushPromise = (async () => {
         // 1. Write to Firestore first (this is the source of truth)
@@ -185,7 +185,7 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
           .collection('alerts')
           .doc(notificationId)
           .set(alertData);
-        
+
         // 2. Send FCM push notification (non-blocking, don't fail if push fails)
         // This enables notifications even when user is offline
         try {
@@ -194,46 +194,70 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
           // Log but don't throw - FCM failure shouldn't fail the alert
           logger.log('WARN', `[processAlertForPI] FCM push failed for CID ${userCid}: ${fcmError.message}`);
         }
-        
+
         return { userCid, success: true };
       })().catch(err => {
         logger.log('ERROR', `[processAlertForPI] Failed to write alert for CID ${userCid}: ${err.message}`, err);
         throw err; // Re-throw so we know if writes are failing
       });
-      
+
       notificationPromises.push(writeAndPushPromise);
     }
-    
+
     // Wait for all notifications to be written and push notifications sent
     await Promise.all(notificationPromises);
-    
+
+    // 4b. Write to BigQuery pi_alert_history (for computation system and analytics)
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+      try {
+        const { ensurePIAlertHistoryTable, insertRowsWithMerge } = require('../../core/utils/bigquery_utils');
+        await ensurePIAlertHistoryTable(logger);
+        const datasetId = process.env.BIGQUERY_DATASET_ID || 'bulltrackers_data';
+        const now = new Date().toISOString();
+        const row = {
+          date: computationDate,
+          pi_id: Number(piCid),
+          alert_type: alertType.computationName || alertType.id,
+          triggered: true,
+          trigger_count: subscriptions.length,
+          triggered_for: subscriptions.map(s => String(s.userCid)),
+          metadata: computationMetadata && typeof computationMetadata === 'object' ? computationMetadata : {},
+          last_triggered: now,
+          last_updated: now
+        };
+        await insertRowsWithMerge(datasetId, 'pi_alert_history', [row], ['date', 'pi_id', 'alert_type'], logger);
+      } catch (bqErr) {
+        logger.log('WARN', `[processAlertForPI] pi_alert_history write failed: ${bqErr.message}`);
+      }
+    }
+
     // 5. Notify the PI themselves if they are a signed-in user (Optional feature)
     await notifyPIIfSignedIn(db, logger, piCid, alertType, subscriptions.length);
 
     // 6. Update global rootdata collection for computation system
     // (Wrap in try-catch to prevent crashing the alert if metrics fail)
     try {
-        const { runRootDataIndexer } = require('../../root-data-indexer/index');
-        const triggeredUserCids = subscriptions.map(s => s.userCid);
-        
-        // Update alert history root data by running root data indexer for the specific date
-        // The indexer will detect and update PIAlertHistoryData availability
-        const indexerConfig = {
-            availabilityCollection: 'root_data_availability',
-            targetDate: computationDate,
-            collections: {
-                piAlertHistory: 'PIAlertHistoryData'
-            }
-        };
-        
-        await runRootDataIndexer(indexerConfig, { db, logger });
-        logger.log('INFO', `[processAlertForPI] Updated root data indexer for date ${computationDate}`);
+      const { runRootDataIndexer } = require('../../root-data-indexer/index');
+      const triggeredUserCids = subscriptions.map(s => s.userCid);
+
+      // Update alert history root data by running root data indexer for the specific date
+      // The indexer will detect and update PIAlertHistoryData availability
+      const indexerConfig = {
+        availabilityCollection: 'root_data_availability',
+        targetDate: computationDate,
+        collections: {
+          piAlertHistory: 'PIAlertHistoryData'
+        }
+      };
+
+      await runRootDataIndexer(indexerConfig, { db, logger });
+      logger.log('INFO', `[processAlertForPI] Updated root data indexer for date ${computationDate}`);
     } catch (e) {
-        logger.log('WARN', `[processAlertForPI] Failed to update history rootdata: ${e.message}`);
+      logger.log('WARN', `[processAlertForPI] Failed to update history rootdata: ${e.message}`);
     }
-    
+
     logger.log('SUCCESS', `[processAlertForPI] Created ${notificationPromises.length} notifications for PI ${piCid}, alert type ${alertType.id}`);
-    
+
     // [FIX] Mark alert as processed to prevent duplicates on backfill
     if (isHistoricalData) {
       try {
@@ -255,7 +279,7 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
         // Don't throw - this is non-critical
       }
     }
-    
+
   } catch (error) {
     logger.log('ERROR', `[processAlertForPI] Error processing alert for PI ${piCid}`, error);
     throw error;
@@ -269,24 +293,24 @@ async function processAlertForPI(db, logger, piCid, alertType, computationMetada
  */
 async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computationDate, dependencies = {}) {
   const subscriptions = [];
-  
+
   // [DYNAMIC] Get configKey from alertType metadata instead of hardcoded mapping
   // The alertType is passed in dependencies and contains the configKey from computation metadata
   const alertType = dependencies.alertType;
   const configKey = alertType?.configKey;
-  
+
   if (!configKey) {
     logger.log('WARN', `[findSubscriptionsForPI] No configKey found for alert type: ${alertTypeId}`);
     return subscriptions;
   }
-  
+
   // Check for developer accounts with pretendSubscribedToAllAlerts flag enabled
   // This allows developers to test the alert system without manually configuring subscriptions
   try {
     const { isDeveloper } = require('../../api-v2/helpers/data-fetchers/firestore.js');
     const { fetchPopularInvestorMasterList } = require('../../api-v2/helpers/data-fetchers/firestore.js');
     const config = dependencies.config || {};
-    
+
     // Get PI username from master list
     let piUsername = `PI-${piCid}`;
     try {
@@ -295,7 +319,7 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
     } catch (e) {
       // PI not in master list, use fallback
     }
-    
+
     // Default alert config with all alert types enabled (for dev override)
     const allAlertsEnabledConfig = {
       increasedRisk: true,
@@ -307,22 +331,22 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
       behavioralAnomaly: true,
       testSystemProbe: true // Test alerts for developers
     };
-    
+
     // Check all developer accounts
     const devOverridesCollection = config.devOverridesCollection || 'dev_overrides';
     const devOverridesSnapshot = await db.collection(devOverridesCollection).get();
-    
+
     for (const devOverrideDoc of devOverridesSnapshot.docs) {
       const devUserCid = Number(devOverrideDoc.id);
-      
+
       // Verify this is actually a developer account (security check) - using api-v2 helper
       const isDev = await isDeveloper(db, String(devUserCid));
       if (!isDev) {
         continue;
       }
-      
+
       const devOverrideData = devOverrideDoc.data();
-      
+
       // Check if this developer has the pretendSubscribedToAllAlerts flag enabled
       if (devOverrideData.enabled === true && devOverrideData.pretendSubscribedToAllAlerts === true) {
         // Add this developer as a subscription for this PI and alert type
@@ -334,7 +358,7 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
           watchlistName: 'Dev Override - All Alerts',
           alertConfig: allAlertsEnabledConfig
         });
-        
+
         logger.log('INFO', `[findSubscriptionsForPI] DEV OVERRIDE: Added developer ${devUserCid} to subscriptions for PI ${piCid}, alert type ${alertTypeId}`);
       }
     }
@@ -342,19 +366,19 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
     // Don't fail the entire function if dev override check fails
     logger.log('WARN', `[findSubscriptionsForPI] Error checking dev overrides: ${error.message}`);
   }
-  
+
   // Step 1: Load WatchlistMembershipData/{date} to find which users have this PI in their watchlist
   const piCidStr = String(piCid);
   let userCids = [];
-  
+
   try {
     const membershipRef = db.collection('WatchlistMembershipData').doc(computationDate);
     const membershipDoc = await membershipRef.get();
-    
+
     if (membershipDoc.exists) {
       const membershipData = membershipDoc.data();
       const piMembership = membershipData[piCidStr];
-      
+
       if (piMembership && piMembership.users && Array.isArray(piMembership.users)) {
         userCids = piMembership.users.map(cid => String(cid));
         logger.log('INFO', `[findSubscriptionsForPI] Found ${userCids.length} users with PI ${piCid} in watchlist from WatchlistMembershipData/${computationDate}`);
@@ -372,29 +396,29 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
     // Return subscriptions array which may contain dev overrides
     return subscriptions;
   }
-  
+
   // Step 2: For each user, read their watchlists from SignedInUsers/{cid}/watchlists
   // Read directly from new path (no migration needed)
   for (const userCidStr of userCids) {
     try {
       const userCid = Number(userCidStr);
-      
+
       // Read all watchlists for this user from new path
       const watchlistsSnapshot = await db.collection('SignedInUsers')
         .doc(String(userCid))
         .collection('watchlists')
         .get();
-      
+
       if (watchlistsSnapshot.empty) {
         continue;
       }
-      
+
       // Get watchlists from snapshot
       const watchlists = watchlistsSnapshot.docs.map(doc => ({
         id: doc.id,
         ...doc.data()
       }));
-      
+
       // Step 3: Check each watchlist for the PI and alert config
       for (const watchlistData of watchlists) {
         if (watchlistData.type === 'static' && watchlistData.items && Array.isArray(watchlistData.items)) {
@@ -403,12 +427,12 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
               // Check if this alert type is enabled
               const isTestProbe = alertTypeId === 'TestSystemProbe';
               const isEnabled = item.alertConfig && item.alertConfig[configKey] === true;
-              
+
               // [FIX] Check if this is a test alert (respect testAlerts preference)
               // Load alert type from dependencies to check isTest flag
               const alertType = dependencies.alertType;
               const isTestAlert = alertType && alertType.isTest === true;
-              
+
               let shouldSendAlert = false;
               if (isTestAlert) {
                 // Check if user has testAlerts enabled in their notification preferences
@@ -416,7 +440,7 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
                   const { manageNotificationPreferences } = require('../../api-v2/helpers/data-fetchers/firestore.js');
                   const prefs = await manageNotificationPreferences(db, userCid, 'get');
                   shouldSendAlert = prefs.testAlerts === true;
-                  
+
                   if (!shouldSendAlert) {
                     logger.log('DEBUG', `[findSubscriptionsForPI] User ${userCid} has testAlerts disabled, skipping test alert ${alertTypeId}`);
                   }
@@ -429,7 +453,7 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
                 // For non-test alerts, use normal alert config check
                 shouldSendAlert = isEnabled;
               }
-              
+
               if (shouldSendAlert) {
                 subscriptions.push({
                   userCid: userCid,
@@ -451,7 +475,7 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
       continue;
     }
   }
-  
+
   logger.log('INFO', `[findSubscriptionsForPI] Found ${subscriptions.length} subscriptions for PI ${piCid}, alert type ${alertTypeId}`);
   return subscriptions;
 }
@@ -477,22 +501,22 @@ async function getPIUsername(db, piCid) {
     // Try to get from master list first (single source of truth) - using api-v2 helper
     const { fetchPopularInvestorMasterList } = require('../../api-v2/helpers/data-fetchers/firestore.js');
     const piData = await fetchPopularInvestorMasterList(db, String(piCid));
-    
+
     if (piData && piData.username) {
       return piData.username;
     }
-    
+
     // Fallback: try to get from any subscription
     const subscriptionsSnapshot = await db.collection('watchlist_subscriptions')
       .where('piCid', '==', Number(piCid))
       .limit(1)
       .get();
-    
+
     if (!subscriptionsSnapshot.empty) {
       const subData = subscriptionsSnapshot.docs[0].data();
       return subData.piUsername || `PI-${piCid}`;
     }
-    
+
     return `PI-${piCid}`;
   } catch (error) {
     return `PI-${piCid}`;
@@ -506,14 +530,14 @@ async function notifyPIIfSignedIn(db, logger, piCid, alertType, alertCount) {
   try {
     const userRef = db.collection('signedInUsers').doc(String(piCid));
     const userDoc = await userRef.get();
-    
-    if (!userDoc.exists) return; 
+
+    if (!userDoc.exists) return;
 
     const notificationRef = db.collection('signedInUsers')
       .doc(String(piCid))
       .collection('user_alerts_metrics') // Changed from user_alerts/triggered to avoid collision
       .doc(`trigger_${Date.now()}_${alertType.id}`);
-    
+
     await notificationRef.set({
       alertType: alertType.id,
       alertTypeName: alertType.name,
@@ -522,7 +546,7 @@ async function notifyPIIfSignedIn(db, logger, piCid, alertType, alertCount) {
       computationDate: new Date().toISOString().split('T')[0],
       createdAt: FieldValue.serverTimestamp()
     });
-    
+
     logger.log('INFO', `[notifyPIIfSignedIn] Notified PI ${piCid} that ${alertCount} users received ${alertType.id} alert`);
   } catch (error) {
     // Silent fail - non-critical
@@ -573,12 +597,12 @@ function tryDecompress(data) {
 function readComputationResults(docData) {
   try {
     const decompressed = tryDecompress(docData);
-    
+
     if (decompressed && typeof decompressed === 'object') {
       if (decompressed.cids && Array.isArray(decompressed.cids)) {
         const userDataKeys = Object.keys(decompressed)
           .filter(key => key !== 'cids' && key !== 'metadata' && /^\d+$/.test(key));
-        
+
         if (userDataKeys.length > 0) {
           return {
             cids: decompressed.cids,
@@ -595,11 +619,11 @@ function readComputationResults(docData) {
           };
         }
       }
-      
+
       const cids = Object.keys(decompressed)
         .filter(key => /^\d+$/.test(key))
         .map(key => Number(key));
-      
+
       if (cids.length > 0) {
         return {
           cids: cids,
@@ -633,14 +657,14 @@ async function readComputationResultsWithShards(db, docData, docRef, logger = nu
       try {
         const bucketName = docData.gcsBucket || docData.gcsUri.split('/')[2];
         const fileName = docData.gcsPath || docData.gcsUri.split('/').slice(3).join('/');
-        
+
         if (logger) {
           logger.log('INFO', `[AlertSystem] Reading computation results from GCS: ${fileName}`);
         }
-        
+
         // Stream download is memory efficient for large files
         const [fileContent] = await storage.bucket(bucketName).file(fileName).download();
-        
+
         // Assume Gzip (as writer does it), if fails try plain
         let decompressedData;
         try {
@@ -649,7 +673,7 @@ async function readComputationResultsWithShards(db, docData, docRef, logger = nu
           // Fallback for uncompressed GCS files
           decompressedData = JSON.parse(fileContent.toString('utf8'));
         }
-        
+
         // Process the decompressed data through readComputationResults
         return readComputationResults(decompressedData);
       } catch (gcsErr) {
@@ -666,7 +690,7 @@ async function readComputationResultsWithShards(db, docData, docRef, logger = nu
     if (docData._sharded === true && docData._shardCount) {
       const shardsCol = docRef.collection('_shards');
       const shardsSnapshot = await shardsCol.get();
-      
+
       if (!shardsSnapshot.empty) {
         let mergedData = {};
         for (const shardDoc of shardsSnapshot.docs) {