bulltrackers-module 1.0.467 → 1.0.469

package/functions/alert-system/index.js

@@ -112,6 +112,13 @@ async function handleAlertTrigger(message, context, config, dependencies) {
     
     logger.log('SUCCESS', `[AlertTrigger] Completed processing ${computationName}: ${processedCount} successful, ${errorCount} errors`);
     
+    // After processing alerts, check for "all clear" notifications for static watchlists
+    // This runs after all alert computations are processed
+    if (processedCount > 0 || results.cids.length > 0) {
+      // Check if any PIs were processed but didn't trigger alerts
+      await checkAndSendAllClearNotifications(db, logger, results.cids, date, config);
+    }
+    
   } catch (error) {
     logger.log('ERROR', '[AlertTrigger] Fatal error processing alert trigger', error);
     throw error; // Re-throw for Pub/Sub retry
@@ -134,12 +141,23 @@ async function handleComputationResultWrite(change, context, config, dependencie
   }
   
   try {
-    // 1. Check if this is an alert computation
-    if (!isAlertComputation(computationName)) {
-      logger.log('DEBUG', `[AlertTrigger] Not an alert computation: ${computationName}`);
+    // 1. Check if this is an alert computation OR PopularInvestorProfileMetrics
+    const isProfileMetrics = computationName === 'PopularInvestorProfileMetrics';
+    if (!isAlertComputation(computationName) && !isProfileMetrics) {
+      logger.log('DEBUG', `[AlertTrigger] Not an alert computation or profile metrics: ${computationName}`);
       return;
     }
     
+    // If it's PopularInvestorProfileMetrics, check for all-clear notifications only
+    if (isProfileMetrics) {
+      const docData = change.after.data();
+      const results = readComputationResults(docData);
+      if (results.cids && results.cids.length > 0) {
+        await checkAndSendAllClearNotifications(db, logger, results.cids, date, config);
+      }
+      return; // Don't process as alert computation
+    }
+    
     const alertType = getAlertTypeByComputation(computationName);
     if (!alertType) {
       logger.log('WARN', `[AlertTrigger] Alert type not found for computation: ${computationName}`);
@@ -188,17 +206,189 @@ async function handleComputationResultWrite(change, context, config, dependencie
     
     logger.log('SUCCESS', `[AlertTrigger] Completed processing ${computationName}: ${processedCount} successful, ${errorCount} errors`);
     
+    // After processing alerts, check for "all clear" notifications for static watchlists
+    if (processedCount > 0 || results.cids.length > 0) {
+      // Check if any PIs were processed but didn't trigger alerts
+      await checkAndSendAllClearNotifications(db, logger, results.cids, date, config);
+    }
+    
   } catch (error) {
     logger.log('ERROR', `[AlertTrigger] Fatal error processing ${computationName}`, error);
     // Don't throw - we don't want to retry the entire computation write
   }
 }
 
+/**
+ * Check for PIs in static watchlists that were processed but didn't trigger alerts
+ * Send "all clear" notifications to users who have these PIs in their static watchlists
+ * This should be called after PopularInvestorProfileMetrics is computed (not just alert computations)
+ */
+async function checkAndSendAllClearNotifications(db, logger, processedPICids, computationDate, config) {
+  try {
+    const today = computationDate || new Date().toISOString().split('T')[0];
+    
+    if (!processedPICids || processedPICids.length === 0) {
+      return; // No PIs to check
+    }
+    
+    // Get all static watchlists that contain any of the processed PIs
+    const watchlistsCollection = config.watchlistsCollection || 'watchlists';
+    const watchlistsSnapshot = await db.collection(watchlistsCollection).get();
+    
+    const usersToNotify = new Map(); // userCid -> Map of piCid -> {username, ...}
+    
+    for (const userDoc of watchlistsSnapshot.docs) {
+      const userCid = userDoc.id;
+      const userListsSnapshot = await userDoc.ref.collection('lists').get();
+      
+      for (const listDoc of userListsSnapshot.docs) {
+        const listData = listDoc.data();
+        
+        // Only check static watchlists
+        if (listData.type !== 'static' || !listData.items || !Array.isArray(listData.items)) {
+          continue;
+        }
+        
+        // Check if any processed PI is in this watchlist
+        for (const item of listData.items) {
+          const piCid = Number(item.cid);
+          if (processedPICids.includes(piCid)) {
+            // Check if this PI triggered any alerts today
+            const hasAlerts = await checkIfPIHasAlertsToday(db, userCid, piCid, today);
+            
+            if (!hasAlerts) {
+              // No alerts triggered - send "all clear" notification
+              if (!usersToNotify.has(userCid)) {
+                usersToNotify.set(userCid, new Map());
+              }
+              usersToNotify.get(userCid).set(piCid, {
+                cid: piCid,
+                username: item.username || `PI-${piCid}`
+              });
+            }
+          }
+        }
+      }
+    }
+    
+    // Send notifications to all users
+    let totalNotifications = 0;
+    for (const [userCid, pisMap] of usersToNotify.entries()) {
+      for (const [piCid, pi] of pisMap.entries()) {
+        await sendAllClearNotification(db, logger, userCid, pi.cid, pi.username, today);
+        totalNotifications++;
+      }
+    }
+    
+    if (totalNotifications > 0) {
+      logger.log('INFO', `[checkAndSendAllClearNotifications] Sent ${totalNotifications} all-clear notifications to ${usersToNotify.size} users`);
+    }
+    
+  } catch (error) {
+    logger.log('ERROR', `[checkAndSendAllClearNotifications] Error checking all-clear notifications`, error);
+    // Don't throw - this is a non-critical feature
+  }
+}
+
+/**
+ * Check if a PI has any alerts for a user today
+ */
+async function checkIfPIHasAlertsToday(db, userCid, piCid, date) {
+  try {
+    const alertsRef = db.collection('user_alerts')
+      .doc(String(userCid))
+      .collection('alerts');
+    
+    // Check if there are any alerts for this PI today
+    const todayStart = new Date(date);
+    todayStart.setHours(0, 0, 0, 0);
+    const todayEnd = new Date(date);
+    todayEnd.setHours(23, 59, 59, 999);
+    
+    const snapshot = await alertsRef
+      .where('piCid', '==', Number(piCid))
+      .where('computationDate', '==', date)
+      .limit(1)
+      .get();
+    
+    return !snapshot.empty;
+  } catch (error) {
+    // If we can't check, assume there are alerts (safer)
+    return true;
+  }
+}
+
+/**
+ * Send an "all clear" notification to a user
+ */
+async function sendAllClearNotification(db, logger, userCid, piCid, piUsername, date) {
+  try {
+    const { FieldValue } = require('@google-cloud/firestore');
+    
+    // Check if we already sent an all-clear notification for this PI today
+    const existingRef = db.collection('user_alerts')
+      .doc(String(userCid))
+      .collection('alerts')
+      .where('piCid', '==', Number(piCid))
+      .where('alertType', '==', 'all_clear')
+      .where('computationDate', '==', date)
+      .limit(1);
+    
+    const existingSnapshot = await existingRef.get();
+    if (!existingSnapshot.empty) {
+      // Already sent notification for this PI today
+      return;
+    }
+    
+    // Create the notification using the notification system
+    const notificationRef = db.collection('user_notifications')
+      .doc(String(userCid))
+      .collection('notifications')
+      .doc();
+    
+    await notificationRef.set({
+      id: notificationRef.id,
+      type: 'alert',
+      title: 'All Clear',
+      message: `${piUsername} was processed, all clear today!`,
+      piCid: Number(piCid),
+      piUsername: piUsername,
+      alertType: 'all_clear',
+      computationDate: date,
+      read: false,
+      createdAt: FieldValue.serverTimestamp(),
+      metadata: {
+        message: 'User was processed, all clear today!'
+      }
+    });
+    
+    // Update notification counter
+    const counterRef = db.collection('user_notifications')
+      .doc(String(userCid))
+      .collection('counters')
+      .doc(date);
+    
+    await counterRef.set({
+      date: date,
+      unreadCount: FieldValue.increment(1),
+      totalCount: FieldValue.increment(1),
+      lastUpdated: FieldValue.serverTimestamp()
+    }, { merge: true });
+    
+    logger.log('INFO', `[sendAllClearNotification] Sent all-clear notification to user ${userCid} for PI ${piCid}`);
+    
+  } catch (error) {
+    logger.log('ERROR', `[sendAllClearNotification] Error sending all-clear notification`, error);
+    // Don't throw - this is non-critical
+  }
+}
+
 /**
  * Export the trigger handlers
  */
 module.exports = {
   handleAlertTrigger,
-  handleComputationResultWrite
+  handleComputationResultWrite,
+  checkAndSendAllClearNotifications
 };
 

package/functions/core/utils/firestore_utils.js

@@ -397,10 +397,42 @@ async function getSignedInUsersToUpdate(dependencies, config) {
             targets.push({ cid: String(cid), username });
         });
         
-        // Now filter out users who were updated in the last 18 hours
-        // Check user_sync_requests for last completed sync
+        // Now filter out users who were updated today
+        // Check actual portfolio/history snapshot dates, not just sync request timestamps
         const filteredTargets = [];
+        const today = new Date().toISOString().split('T')[0];
         const checkPromises = targets.map(async (target) => {
+            // Check if user has portfolio or history data for today
+            const blockId = `${Math.floor(parseInt(target.cid) / 1000000)}M`;
+            
+            // Check portfolio snapshot for today
+            const portfolioSnapshotRef = db.collection(collectionName)
+                .doc(blockId)
+                .collection('snapshots')
+                .doc(today)
+                .collection('parts')
+                .limit(1);
+            
+            const portfolioSnapshot = await portfolioSnapshotRef.get();
+            
+            // Check history snapshot for today
+            const historyCollection = config.signedInUserHistoryCollection || process.env.FIRESTORE_COLLECTION_SIGNED_IN_USER_HISTORY || 'signed_in_user_history';
+            const historySnapshotRef = db.collection(historyCollection)
+                .doc(blockId)
+                .collection('snapshots')
+                .doc(today)
+                .collection('parts')
+                .limit(1);
+            
+            const historySnapshot = await historySnapshotRef.get();
+            
+            // If user has data for today, skip them (they're already up-to-date)
+            if (!portfolioSnapshot.empty || !historySnapshot.empty) {
+                skippedCount++;
+                return null; // Skip this user - already updated today
+            }
+            
+            // Also check sync requests as a fallback (in case data exists but snapshot check failed)
             const syncRef = db.collection('user_sync_requests')
                 .doc(target.cid)
                 .collection('global')
@@ -418,7 +450,7 @@ async function getSignedInUsersToUpdate(dependencies, config) {
                 }
             }
             
-            return target; // Include this user
+            return target; // Include this user - needs update
         });
         
         const results = await Promise.all(checkPromises);

package/functions/etoro-price-fetcher/helpers/handler_helpers.js

@@ -49,6 +49,51 @@ exports.fetchAndStorePrices = async (config, dependencies) => {
         const batchPromises = [];
         for (const shardId in shardUpdates) { const docRef = db.collection(priceCollectionName).doc(shardId); const payload = shardUpdates[shardId]; batchPromises.push(docRef.update(payload)); }
         await Promise.all(batchPromises);
+        
+        // Extract all dates from the price data and create a date tracking document
+        const priceDatesSet = new Set();
+        const AUGUST_2025 = '2025-08-01';
+        
+        for (const instrumentData of results) {
+            const dailyData = instrumentData?.ClosingPrices?.Daily;
+            const weeklyData = instrumentData?.ClosingPrices?.Weekly;
+            const monthlyData = instrumentData?.ClosingPrices?.Monthly;
+            
+            if (dailyData?.Date) {
+                const dateKey = dailyData.Date.substring(0, 10);
+                if (dateKey >= AUGUST_2025) {
+                    priceDatesSet.add(dateKey);
+                }
+            }
+            if (weeklyData?.Date) {
+                const dateKey = weeklyData.Date.substring(0, 10);
+                if (dateKey >= AUGUST_2025) {
+                    priceDatesSet.add(dateKey);
+                }
+            }
+            if (monthlyData?.Date) {
+                const dateKey = monthlyData.Date.substring(0, 10);
+                if (dateKey >= AUGUST_2025) {
+                    priceDatesSet.add(dateKey);
+                }
+            }
+        }
+        
+        // Write date tracking document
+        const today = new Date().toISOString().split('T')[0];
+        const dateTrackingRef = db.collection('pricedatastoreddates').doc(today);
+        const priceDatesArray = Array.from(priceDatesSet).sort();
+        
+        await dateTrackingRef.set({
+            fetchDate: today,
+            datesAvailable: priceDatesArray,
+            dateCount: priceDatesArray.length,
+            instrumentsProcessed: results.length,
+            storedAt: FieldValue.serverTimestamp()
+        });
+        
+        logger.log('INFO', `[PriceFetcherHelpers] Wrote price date tracking document for ${today} with ${priceDatesArray.length} dates (from August 2025 onwards)`);
+        
         const successMessage = `Successfully processed and saved daily prices for ${results.length} instruments to ${batchPromises.length} shards.`;
         logger.log('SUCCESS', `[PriceFetcherHelpers] ${successMessage}`);
         return { success: true, message: successMessage, instrumentsProcessed: results.length };

package/functions/root-data-indexer/index.js

@@ -75,35 +75,61 @@ exports.runRootDataIndexer = async (config, dependencies) => {
     const scanMode = targetDate ? 'SINGLE_DATE' : 'FULL_SCAN';
     logger.log('INFO', `[RootDataIndexer] Starting Root Data Availability Scan... Mode: ${scanMode}`, { targetDate });
     
-    // 1. Price Availability (Global Scan Mode Only)
-    // If running for a single targetDate, we skip the global shard sampling to save time.
+    // 1. Price Availability - Read from date tracking documents
+    // Find the latest pricedatastoreddates document and extract available dates
     const priceAvailabilitySet = new Set();
     
-    if (!targetDate) {
-        try {
-            // Path: asset_prices/shard_*
-            const priceCollectionRef = db.collection(PRICE_COLLECTION_NAME);
-            const priceShardsSnapshot = await priceCollectionRef.limit(10).get();
+    try {
+        // Get the latest price date tracking document
+        const dateTrackingRef = db.collection('pricedatastoreddates')
+            .orderBy('fetchDate', 'desc')
+            .limit(1);
+        
+        const dateTrackingSnapshot = await dateTrackingRef.get();
+        
+        if (!dateTrackingSnapshot.empty) {
+            const latestTrackingDoc = dateTrackingSnapshot.docs[0].data();
+            const datesAvailable = latestTrackingDoc.datesAvailable || [];
             
-            if (!priceShardsSnapshot.empty) {
-                // Sample up to 10 shards and extract date keys from them
-                for (const shardDoc of priceShardsSnapshot.docs) {
-                    if (shardDoc.id.startsWith('shard_')) {
-                        const data = shardDoc.data();
-                        Object.values(data).forEach(instrument => { 
-                            if (instrument && instrument.prices) { 
-                                Object.keys(instrument.prices).forEach(dateKey => { 
-                                    if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) { 
-                                        priceAvailabilitySet.add(dateKey); 
-                                    } 
-                                }); 
-                            } 
-                        });
+            // Add all dates from the tracking document
+            datesAvailable.forEach(dateKey => {
+                if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) {
+                    priceAvailabilitySet.add(dateKey);
+                }
+            });
+            
+            logger.log('INFO', `[RootDataIndexer] Loaded ${priceAvailabilitySet.size} price dates from tracking document (fetchDate: ${latestTrackingDoc.fetchDate})`);
+        } else {
+            logger.log('WARN', '[RootDataIndexer] No price date tracking documents found. Falling back to empty set.');
+        }
+    } catch (e) {
+        logger.log('ERROR', '[RootDataIndexer] Failed to load price date tracking document.', { error: e.message });
+        // Fallback: try to sample shards if tracking document fails
+        if (!targetDate) {
+            try {
+                const priceCollectionRef = db.collection(PRICE_COLLECTION_NAME);
+                const priceShardsSnapshot = await priceCollectionRef.limit(10).get();
+                
+                if (!priceShardsSnapshot.empty) {
+                    for (const shardDoc of priceShardsSnapshot.docs) {
+                        if (shardDoc.id.startsWith('shard_')) {
+                            const data = shardDoc.data();
+                            Object.values(data).forEach(instrument => { 
+                                if (instrument && instrument.prices) { 
+                                    Object.keys(instrument.prices).forEach(dateKey => { 
+                                        if (/^\d{4}-\d{2}-\d{2}$/.test(dateKey)) { 
+                                            priceAvailabilitySet.add(dateKey); 
+                                        } 
+                                    }); 
+                                } 
+                            });
+                        }
                     }
+                    logger.log('INFO', `[RootDataIndexer] Fallback: Loaded ${priceAvailabilitySet.size} price dates from shard sampling.`);
                 }
+            } catch (fallbackError) {
+                logger.log('ERROR', '[RootDataIndexer] Fallback shard sampling also failed.', { error: fallbackError.message });
             }
-        } catch (e) {
-            logger.log('ERROR', '[RootDataIndexer] Failed to sample price shards.', { error: e.message });
         }
     }
 
@@ -130,9 +156,9 @@ exports.runRootDataIndexer = async (config, dependencies) => {
     const promises = datesToScan.map(dateStr => limit(async () => {
         try {
             // Define Time Range for Social Query (Full Day UTC)
-            const dayStart = new Date(dateStr);
-            const dayEnd = new Date(dateStr);
-            dayEnd.setHours(23, 59, 59, 999);
+            // Use UTC methods to ensure correct timezone handling
+            const dayStart = new Date(dateStr + 'T00:00:00.000Z');
+            const dayEnd = new Date(dateStr + 'T23:59:59.999Z');
 
             const availability = {
                 date: dateStr,
@@ -250,17 +276,33 @@ exports.runRootDataIndexer = async (config, dependencies) => {
             let foundSignedInSocial = false;
 
             if (!universalSocialSnap.empty) {
+                logger.log('DEBUG', `[RootDataIndexer/${dateStr}] Found ${universalSocialSnap.docs.length} social posts in query`);
                 universalSocialSnap.docs.forEach(doc => {
                     const path = doc.ref.path;
+                    const data = doc.data();
+                    const fetchedAt = data.fetchedAt;
+                    
                     // Use includes() to match collection name anywhere in path (more robust)
                     // Path format: {collectionName}/{userId}/posts/{postId}
-                    if (path.includes(`/${PI_SOCIAL_COLL_NAME}/`) || path.startsWith(`${PI_SOCIAL_COLL_NAME}/`)) {
+                    // Firestore paths don't have leading slash, so check both with and without
+                    const piMatchPattern = `${PI_SOCIAL_COLL_NAME}/`;
+                    const signedInMatchPattern = `${SIGNED_IN_SOCIAL_COLL_NAME}/`;
+                    
+                    if (path.includes(piMatchPattern) || path.startsWith(piMatchPattern)) {
                         foundPISocial = true;
+                        logger.log('DEBUG', `[RootDataIndexer/${dateStr}] ✓ Found PI social: ${path}`);
                     }
-                    if (path.includes(`/${SIGNED_IN_SOCIAL_COLL_NAME}/`) || path.startsWith(`${SIGNED_IN_SOCIAL_COLL_NAME}/`)) {
+                    if (path.includes(signedInMatchPattern) || path.startsWith(signedInMatchPattern)) {
                         foundSignedInSocial = true;
+                        const fetchedAtStr = fetchedAt ? (fetchedAt.toDate ? fetchedAt.toDate().toISOString() : String(fetchedAt)) : 'missing';
+                        logger.log('DEBUG', `[RootDataIndexer/${dateStr}] ✓ Found signed-in social: ${path}, fetchedAt: ${fetchedAtStr}`);
+                    } else if (!path.includes(piMatchPattern)) {
+                        // Log paths that don't match either pattern to help debug
+                        logger.log('DEBUG', `[RootDataIndexer/${dateStr}] ✗ Path doesn't match: ${path} (looking for: "${signedInMatchPattern}" or "${piMatchPattern}")`);
                     }
                 });
+            } else {
+                logger.log('DEBUG', `[RootDataIndexer/${dateStr}] No social posts found in query (dayStart: ${dayStart.toISOString()}, dayEnd: ${dayEnd.toISOString()})`);
             }
 
             // --- Assign to Availability ---

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.467",
+  "version": "1.0.469",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [