bulltrackers-module 1.0.692 → 1.0.694

package/functions/api-v2/helpers/data-fetchers/firestore.js

@@ -3028,12 +3028,18 @@ const getWatchlistTriggerCounts = async (db, userId, watchlistId) => {
 
 /**
  * Query PIs matching dynamic watchlist criteria
- * Reads computation results and filters ALL per-user data (not just triggered CIDs)
+ * 
+ * IMPORTANT: This function only evaluates PIs based on the MOST RECENT available data.
+ * The timeRange parameter controls how far back we look to FIND data (in case today's
+ * computation hasn't run yet), but we DO NOT aggregate matches across multiple dates.
+ * 
+ * If a PI matched criteria 3 days ago but doesn't match on the most recent data,
+ * they will NOT be included - they "dropped off" the watchlist.
  * 
  * @param {Object} db - Firestore instance
  * @param {string} computationName - Name of the computation to query
  * @param {Object} parameters - Threshold parameters from watchlist config (e.g., {minChange: 1, minRiskLevel: 5})
- * @param {string} timeRange - Time range (today, last_7_days, last_30_days)
+ * @param {string} timeRange - Time range to look back for data availability (today, last_7_days, last_30_days)
  * @param {number} limit - Maximum number of results
  * @returns {Promise<Object>} Matching PIs with their values
  */
@@ -3041,7 +3047,7 @@ const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}
     try {
         console.log(`[queryDynamicWatchlistMatches] Querying ${computationName} with params:`, parameters, `timeRange: ${timeRange}`);
         
-        // Determine date range based on timeRange
+        // Determine how far back to look for data availability
         const endDate = new Date();
         const startDate = new Date();
         
@@ -3059,24 +3065,22 @@ const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}
                 startDate.setDate(startDate.getDate() - 7);
         }
         
-        // Build list of dates to query (most recent first)
+        // Build list of dates to check (most recent first)
         const dates = [];
         for (let d = new Date(endDate); d >= startDate; d.setDate(d.getDate() - 1)) {
             dates.push(d.toISOString().split('T')[0]);
         }
         
-        console.log(`[queryDynamicWatchlistMatches] Checking dates:`, dates);
+        console.log(`[queryDynamicWatchlistMatches] Looking for most recent data in date range:`, dates);
         
-        const matchingPIs = new Map(); // cid -> match data
+        // Find the MOST RECENT date that has computation data
+        let mostRecentDate = null;
+        let docRef = null;
+        let docSnapshot = null;
         
-        // Query each date
         for (const dateStr of dates) {
             try {
-                // Try both 'alerts' and 'popular-investor' result paths
-                let docSnapshot = null;
-                let docRef = null;
-                
-                // First try alerts path
+                // Try alerts path first
                 docRef = db.collection('unified_insights')
                     .doc(dateStr)
                     .collection('results')
@@ -3098,118 +3102,134 @@ const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}
                     docSnapshot = await docRef.get();
                 }
                 
-                if (!docSnapshot.exists) {
-                    console.log(`[queryDynamicWatchlistMatches] No data for ${dateStr}`);
-                    continue;
+                if (docSnapshot.exists) {
+                    mostRecentDate = dateStr;
+                    console.log(`[queryDynamicWatchlistMatches] Found most recent data on ${dateStr}`);
+                    break; // Stop searching - we found the most recent data
                 }
-                
-                console.log(`[queryDynamicWatchlistMatches] Found data for ${dateStr}`);
-                
-                const docData = docSnapshot.data();
-                
-                // Check for sharded data
-                let allUserData = {};
-                
-                if (docData._sharded === true) {
-                    // Read from shards
-                    const shardsSnapshot = await docRef.collection('_shards').get();
-                    console.log(`[queryDynamicWatchlistMatches] Found ${shardsSnapshot.size} shards`);
-                    
-                    for (const shardDoc of shardsSnapshot.docs) {
-                        const shardData = shardDoc.data();
-                        // Each shard contains CID keys with their data
-                        Object.entries(shardData).forEach(([key, value]) => {
-                            // Skip metadata keys
-                            if (key.startsWith('_') || key === 'cids' || key === 'metadata') return;
-                            // Check if key is a CID (numeric string)
-                            if (/^\d+$/.test(key)) {
-                                allUserData[key] = value;
-                            }
-                        });
+            } catch (dateErr) {
+                console.warn(`[queryDynamicWatchlistMatches] Error checking date ${dateStr}: ${dateErr.message}`);
+                continue;
+            }
+        }
+        
+        // If no data found in the entire range, return empty
+        if (!mostRecentDate || !docSnapshot || !docSnapshot.exists) {
+            console.log(`[queryDynamicWatchlistMatches] No computation data found for ${computationName} in date range`);
+            return {
+                success: true,
+                matches: [],
+                count: 0,
+                totalScanned: 0,
+                dateRange: {
+                    start: startDate.toISOString().split('T')[0],
+                    end: endDate.toISOString().split('T')[0]
+                },
+                dataDate: null,
+                computationName,
+                parameters,
+                message: `No computation data found for ${computationName} in the selected time range`
+            };
+        }
+        
+        const docData = docSnapshot.data();
+        
+        // Read all per-user data from the most recent date
+        let allUserData = {};
+        
+        if (docData._sharded === true) {
+            // Read from shards
+            const shardsSnapshot = await docRef.collection('_shards').get();
+            console.log(`[queryDynamicWatchlistMatches] Found ${shardsSnapshot.size} shards for ${mostRecentDate}`);
+            
+            for (const shardDoc of shardsSnapshot.docs) {
+                const shardData = shardDoc.data();
+                Object.entries(shardData).forEach(([key, value]) => {
+                    // Skip metadata keys, only include CID keys
+                    if (key.startsWith('_') || key === 'cids' || key === 'metadata') return;
+                    if (/^\d+$/.test(key)) {
+                        allUserData[key] = value;
                     }
-                } else {
-                    // Data is in the document itself
-                    Object.entries(docData).forEach(([key, value]) => {
-                        // Skip metadata keys
-                        if (key.startsWith('_') || key === 'cids' || key === 'metadata' || key === 'globalMetadata') return;
-                        // Check if key is a CID (numeric string)
-                        if (/^\d+$/.test(key)) {
-                            allUserData[key] = value;
-                        }
-                    });
+                });
+            }
+        } else {
+            // Data is in the document itself
+            Object.entries(docData).forEach(([key, value]) => {
+                if (key.startsWith('_') || key === 'cids' || key === 'metadata' || key === 'globalMetadata') return;
+                if (/^\d+$/.test(key)) {
+                    allUserData[key] = value;
                 }
-                
-                console.log(`[queryDynamicWatchlistMatches] Found ${Object.keys(allUserData).length} PIs in computation data`);
-                
-                // Process ALL PIs (not just those in cids array)
-                for (const [piCidStr, piData] of Object.entries(allUserData)) {
-                    const piCid = Number(piCidStr);
-                    
-                    // Skip if already matched from a more recent date
-                    if (matchingPIs.has(piCid)) continue;
-                    
-                    // Skip if error data
-                    if (piData.error) continue;
-                    
-                    // Apply computation-specific filtering based on parameters
-                    const passesFilter = checkPIMatchesCriteria(computationName, piData, parameters);
-                    
-                    if (passesFilter.passes) {
-                        // Get PI username from master list
-                        let username = `PI-${piCid}`;
-                        try {
-                            const piProfile = await fetchPopularInvestorMasterList(db, String(piCid));
-                            if (piProfile && piProfile.username) {
-                                username = piProfile.username;
-                            }
-                        } catch (e) {
-                            // Use default username
-                        }
-                        
-                        matchingPIs.set(piCid, {
-                            cid: piCid,
-                            username,
-                            matchedAt: dateStr,
-                            matchValue: passesFilter.matchValue,
-                            currentValue: passesFilter.currentValue,
-                            previousValue: passesFilter.previousValue,
-                            change: passesFilter.change,
-                            metadata: {
-                                ...piData,
-                                computationDate: dateStr
-                            }
-                        });
-                        
-                        // Stop if we've hit the limit
-                        if (matchingPIs.size >= limit) break;
-                    }
+            });
+        }
+        
+        const totalPIs = Object.keys(allUserData).length;
+        console.log(`[queryDynamicWatchlistMatches] Evaluating ${totalPIs} PIs from ${mostRecentDate} against criteria`);
+        
+        // Filter PIs that match the criteria on this most recent date
+        const matchingPIs = [];
+        const cidsToLookup = [];
+        
+        for (const [piCidStr, piData] of Object.entries(allUserData)) {
+            const piCid = Number(piCidStr);
+            
+            // Skip error data
+            if (piData.error) continue;
+            
+            // Check if PI matches the criteria
+            const filterResult = checkPIMatchesCriteria(computationName, piData, parameters);
+            
+            if (filterResult.passes) {
+                cidsToLookup.push({ piCid, piData, filterResult });
+            }
+        }
+        
+        console.log(`[queryDynamicWatchlistMatches] ${cidsToLookup.length} PIs match criteria, fetching usernames...`);
+        
+        // Fetch usernames for matching PIs (batch for efficiency)
+        for (const { piCid, piData, filterResult } of cidsToLookup.slice(0, limit)) {
+            let username = `PI-${piCid}`;
+            try {
+                const piProfile = await fetchPopularInvestorMasterList(db, String(piCid));
+                if (piProfile && piProfile.username) {
+                    username = piProfile.username;
                 }
-                
-                // If we found enough matches, stop looking at older dates
-                if (matchingPIs.size >= limit) break;
-                
-            } catch (dateErr) {
-                console.warn(`[queryDynamicWatchlistMatches] Error processing date ${dateStr}: ${dateErr.message}`);
-                continue;
+            } catch (e) {
+                // Use default username
             }
+            
+            matchingPIs.push({
+                cid: piCid,
+                username,
+                matchedAt: mostRecentDate,
+                matchValue: filterResult.matchValue,
+                currentValue: filterResult.currentValue,
+                previousValue: filterResult.previousValue,
+                change: filterResult.change,
+                metadata: {
+                    ...piData,
+                    computationDate: mostRecentDate
+                }
+            });
         }
         
-        // Sort by match value (descending) and return
-        const sortedMatches = Array.from(matchingPIs.values())
+        // Sort by match value (descending)
+        const sortedMatches = matchingPIs
             .sort((a, b) => Math.abs(b.matchValue) - Math.abs(a.matchValue))
             .slice(0, limit);
         
-        console.log(`[queryDynamicWatchlistMatches] Returning ${sortedMatches.length} matches`);
+        console.log(`[queryDynamicWatchlistMatches] Returning ${sortedMatches.length} matches from ${mostRecentDate}`);
         
         return {
             success: true,
             matches: sortedMatches,
             count: sortedMatches.length,
-            totalScanned: Object.keys(matchingPIs).length,
+            totalScanned: totalPIs,
+            totalMatching: cidsToLookup.length,
             dateRange: {
                 start: startDate.toISOString().split('T')[0],
                 end: endDate.toISOString().split('T')[0]
             },
+            dataDate: mostRecentDate, // The actual date the data is from
             computationName,
             parameters
         };

package/functions/computation-system/persistence/ResultCommitter.js

@@ -420,7 +420,8 @@ async function writeSingleResult(result, docRef, name, dateContext, category, lo
             }
             
             const rootUpdate = updates.find(u => u.ref.path === docRef.path && u.type !== 'DELETE');
-            if (rootUpdate) { rootUpdate.options = { merge: rootMergeOption }; }
+            // FIX: Always use merge: false to ensure old fields (like _compressed/payload) are wiped
+            if (rootUpdate) { rootUpdate.options = { merge: false }; }
 
             const writes = updates.filter(u => u.type !== 'DELETE').length;
             const deletes = updates.filter(u => u.type === 'DELETE').length;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.692",
+  "version": "1.0.694",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [

package/functions/alert-system/helpers/alert_type_registry.js

@@ -1,172 +0,0 @@
-/**
- * file: alert-system/helpers/alert_type_registry.js
- */
-const ALERT_TYPES = {
-  increasedRisk: {
-    id: 'increasedRisk',
-    name: 'Increased Risk',
-    description: 'Alert when a Popular Investor\'s risk score increases',
-    computationName: 'RiskScoreIncrease',
-    category: 'alerts',
-    messageTemplate: '{piUsername}\'s risk score increased by {change} points (from {previous} to {current})',
-    severity: 'high',
-    enabled: true
-  },
-  volatilityChanges: {
-    id: 'volatilityChanges',
-    name: 'Significant Volatility',
-    description: 'Alert when a Popular Investor\'s portfolio volatility exceeds 50%',
-    computationName: 'SignificantVolatility',
-    category: 'alerts',
-    messageTemplate: '{piUsername}\'s portfolio volatility is {volatility}% (threshold: {threshold}%)',
-    severity: 'medium',
-    enabled: true
-  },
-  newSector: {
-    id: 'newSector',
-    name: 'New Sector Entry',
-    description: 'Alert when a Popular Investor enters a new sector',
-    computationName: 'NewSectorExposure',
-    category: 'alerts',
-    messageTemplate: '{piUsername} entered new sector(s): {sectorName}',
-    severity: 'low',
-    enabled: true
-  },
-  increasedPositionSize: {
-    id: 'increasedPositionSize',
-    name: 'Increased Position Size',
-    description: 'Alert when a Popular Investor significantly increases a position size (>5%)',
-    computationName: 'PositionInvestedIncrease',
-    category: 'alerts',
-    messageTemplate: '{piUsername} increased position size for {symbol} by {diff}% (from {prev}% to {curr}%)',
-    severity: 'medium',
-    enabled: true
-  },
-  newSocialPost: {
-    id: 'newSocialPost',
-    name: 'New Social Post',
-    description: 'Alert when a Popular Investor makes a new social post',
-    computationName: 'NewSocialPost',
-    category: 'alerts',
-    messageTemplate: '{piUsername} posted a new update: {title}',
-    severity: 'low',
-    enabled: true
-  },
-  // [NEW] Behavioral Anomaly Registration
-  behavioralAnomaly: {
-    id: 'behavioralAnomaly',
-    name: 'Behavioral Anomaly',
-    description: 'Alert when a Popular Investor deviates significantly from their baseline behavior',
-    computationName: 'BehavioralAnomaly',
-    category: 'alerts',
-    // Uses metadata keys from Behaviour.js: primaryDriver, driverSignificance, anomalyScore
-    messageTemplate: 'Behavioral Alert for {piUsername}: {primaryDriver} Deviation ({driverSignificance}) detected.',
-    severity: 'high',
-    enabled: true
-  },
-  testSystemProbe: {
-    id: 'testSystemProbe',
-    name: 'Test System Probe',
-    description: 'Always-on debug alert',
-    computationName: 'TestSystemProbe',
-    category: 'alerts',
-    messageTemplate: 'Probe Triggered for {status} at {timestamp}', 
-    severity: 'info',
-    enabled: true
-  },
-};
-
-/**
- * Get alert type by ID
- */
-function getAlertType(alertTypeId) {
-  return ALERT_TYPES[alertTypeId] || null;
-}
-
-/**
- * Get alert type by computation name
- */
-function getAlertTypeByComputation(computationName) {
-  for (const [id, alertType] of Object.entries(ALERT_TYPES)) {
-    if (alertType.computationName === computationName) {
-      return alertType;
-    }
-  }
-  return null;
-}
-
-/**
- * Get all enabled alert types
- */
-function getAllAlertTypes() {
-  return Object.values(ALERT_TYPES).filter(type => type.enabled);
-}
-
-/**
- * Check if a computation is an alert computation
- */
-function isAlertComputation(computationName) {
-  return getAlertTypeByComputation(computationName) !== null;
-}
-
-/**
- * Generate alert message from template and metadata
- */
-function generateAlertMessage(alertType, piUsername, metadata = {}) {
-  let message = alertType.messageTemplate;
-  
-  // Replace placeholders
-  message = message.replace('{piUsername}', piUsername || 'Unknown');
-  message = message.replace('{count}', metadata.count || metadata.positions?.length || metadata.moveCount || 0);
-  message = message.replace('{change}', metadata.change || metadata.changePercent || metadata.diff || 'N/A');
-  message = message.replace('{previous}', metadata.previous || metadata.previousValue || metadata.prev || metadata.previousRisk || 'N/A');
-  message = message.replace('{current}', metadata.current || metadata.currentValue || metadata.curr || metadata.currentRisk || 'N/A');
-  message = message.replace('{sectorName}', metadata.sectorName || (metadata.newExposures && metadata.newExposures.length > 0 ? metadata.newExposures.join(', ') : 'Unknown'));
-  message = message.replace('{ticker}', metadata.ticker || metadata.symbol || 'Unknown');
-  
-  // Format numeric values
-  message = message.replace('{volatility}', metadata.volatility ? `${(metadata.volatility * 100).toFixed(1)}` : 'N/A');
-  message = message.replace('{threshold}', metadata.threshold ? `${(metadata.threshold * 100).toFixed(0)}` : 'N/A');
-  message = message.replace('{diff}', metadata.diff ? `${metadata.diff.toFixed(1)}` : 'N/A');
-  message = message.replace('{prev}', metadata.prev ? `${metadata.prev.toFixed(1)}` : 'N/A');
-  message = message.replace('{curr}', metadata.curr ? `${metadata.curr.toFixed(1)}` : 'N/A');
-  message = message.replace('{title}', metadata.title || 'New Update');
-  
-  // [FIX] Probe Placeholders (Missing in original)
-  message = message.replace('{status}', metadata.status || 'Unknown Status');
-  message = message.replace('{timestamp}', metadata.timestamp || new Date().toISOString());
-
-  // [NEW] Behavioral Anomaly Placeholders
-  message = message.replace('{primaryDriver}', metadata.primaryDriver || 'Unknown Factor');
-  message = message.replace('{driverSignificance}', metadata.driverSignificance || 'N/A');
-  message = message.replace('{anomalyScore}', metadata.anomalyScore || 'N/A');
-  
-  // Handle positions list if available
-  if (metadata.positions && Array.isArray(metadata.positions) && metadata.positions.length > 0) {
-    const positionsList = metadata.positions
-      .slice(0, 3)
-      .map(p => p.ticker || p.instrumentId)
-      .join(', ');
-    message = message.replace('{positions}', positionsList);
-  }
-  
-  // Handle moves array for PositionInvestedIncrease
-  if (metadata.moves && Array.isArray(metadata.moves) && metadata.moves.length > 0) {
-    const firstMove = metadata.moves[0];
-    message = message.replace('{symbol}', firstMove.symbol || 'Unknown');
-    message = message.replace('{diff}', firstMove.diff ? `${firstMove.diff.toFixed(1)}` : 'N/A');
-    message = message.replace('{prev}', firstMove.prev ? `${firstMove.prev.toFixed(1)}` : 'N/A');
-    message = message.replace('{curr}', firstMove.curr ? `${firstMove.curr.toFixed(1)}` : 'N/A');
-  }
-  
-  return message;
-}
-
-module.exports = {
-  ALERT_TYPES,
-  getAlertType,
-  getAlertTypeByComputation,
-  getAllAlertTypes,
-  isAlertComputation,
-  generateAlertMessage
-};