bulltrackers-module 1.0.707 → 1.0.710

package/functions/api-v2/helpers/data-fetchers/firestore.js

@@ -3025,29 +3025,27 @@ const getWatchlistTriggerCounts = async (db, userId, watchlistId) => {
         throw error;
     }
 };
-
 /**
- * Query PIs matching dynamic watchlist criteria
- * 
- * IMPORTANT: This function only evaluates PIs based on the MOST RECENT available data.
- * The timeRange parameter controls how far back we look to FIND data (in case today's
- * computation hasn't run yet), but we DO NOT aggregate matches across multiple dates.
- * 
- * If a PI matched criteria 3 days ago but doesn't match on the most recent data,
- * they will NOT be included - they "dropped off" the watchlist.
- * 
- * @param {Object} db - Firestore instance
+ * Query PIs matching dynamic watchlist criteria over a time range.
+ * * UPDATED LOGIC: 
+ * Instead of only looking at the latest data snapshot, this function scans 
+ * the entire requested time range (e.g. 7 days).
+ * * It returns any PI who met the criteria AT LEAST ONCE during the period.
+ * It provides a history of their matches, allowing the UI to show:
+ * - When they first appeared (firstMatchedAt)
+ * - If they are no longer matching (droppedOffAt)
+ * - Their value history over the period
+ * * @param {Object} db - Firestore instance
  * @param {string} computationName - Name of the computation to query
- * @param {Object} parameters - Threshold parameters from watchlist config (e.g., {minChange: 1, minRiskLevel: 5})
- * @param {string} timeRange - Time range to look back for data availability (today, last_7_days, last_30_days)
+ * @param {Object} parameters - Threshold parameters (e.g., {minChange: 1})
+ * @param {string} timeRange - Time range (today, last_7_days, last_30_days)
  * @param {number} limit - Maximum number of results
- * @returns {Promise<Object>} Matching PIs with their values
  */
 const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}, timeRange = 'last_7_days', limit = 100) => {
     try {
-        console.log(`[queryDynamicWatchlistMatches] Querying ${computationName} with params:`, parameters, `timeRange: ${timeRange}`);
+        console.log(`[queryDynamicWatchlistMatches] Aggregating ${computationName} over ${timeRange}`);
         
-        // Determine how far back to look for data availability
+        // 1. Determine Date Range
         const endDate = new Date();
         const startDate = new Date();
         
@@ -3055,42 +3053,35 @@ const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}
             case 'today':
                 // Just today
                 break;
-            case 'last_7_days':
-                startDate.setDate(startDate.getDate() - 7);
-                break;
             case 'last_30_days':
                 startDate.setDate(startDate.getDate() - 30);
                 break;
+            case 'last_7_days':
             default:
                 startDate.setDate(startDate.getDate() - 7);
         }
         
-        // Build list of dates to check (most recent first)
+        // Build list of dates to check (Newest -> Oldest)
         const dates = [];
         for (let d = new Date(endDate); d >= startDate; d.setDate(d.getDate() - 1)) {
             dates.push(d.toISOString().split('T')[0]);
         }
         
-        console.log(`[queryDynamicWatchlistMatches] Looking for most recent data in date range:`, dates);
-        
-        // Find the MOST RECENT date that has computation data
-        let mostRecentDate = null;
-        let docRef = null;
-        let docSnapshot = null;
-        
-        for (const dateStr of dates) {
+        // 2. Fetch Data for ALL Dates in Parallel
+        // We do not stop at the first match anymore.
+        const datePromises = dates.map(async (dateStr) => {
             try {
-                // Try alerts path first
-                docRef = db.collection('unified_insights')
+                // Try alerts path first (primary location for alert computations)
+                let docRef = db.collection('unified_insights')
                     .doc(dateStr)
                     .collection('results')
                     .doc('alerts')
                     .collection('computations')
                     .doc(computationName);
                 
-                docSnapshot = await docRef.get();
+                let docSnapshot = await docRef.get();
                 
-                // If not found, try popular-investor path
+                // Fallback to popular-investor path
                 if (!docSnapshot.exists) {
                     docRef = db.collection('unified_insights')
                         .doc(dateStr)
@@ -3098,141 +3089,176 @@ const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}
                         .doc('popular-investor')
                         .collection('computations')
                         .doc(computationName);
-                    
                     docSnapshot = await docRef.get();
                 }
-                
-                if (docSnapshot.exists) {
-                    mostRecentDate = dateStr;
-                    console.log(`[queryDynamicWatchlistMatches] Found most recent data on ${dateStr}`);
-                    break; // Stop searching - we found the most recent data
+
+                if (!docSnapshot.exists) return null;
+
+                const docData = docSnapshot.data();
+                let dayData = {};
+
+                // Handle Sharding
+                if (docData._sharded === true) {
+                    const shardsSnapshot = await docRef.collection('_shards').get();
+                    for (const shardDoc of shardsSnapshot.docs) {
+                        const shardContent = shardDoc.data();
+                        Object.entries(shardContent).forEach(([key, value]) => {
+                            if (!key.startsWith('_') && key !== 'cids' && /^\d+$/.test(key)) {
+                                dayData[key] = value;
+                            }
+                        });
+                    }
+                } else {
+                    // Standard Document
+                    Object.entries(docData).forEach(([key, value]) => {
+                        if (!key.startsWith('_') && key !== 'cids' && /^\d+$/.test(key)) {
+                            dayData[key] = value;
+                        }
+                    });
                 }
-            } catch (dateErr) {
-                console.warn(`[queryDynamicWatchlistMatches] Error checking date ${dateStr}: ${dateErr.message}`);
-                continue;
+                
+                return { date: dateStr, data: dayData };
+
+            } catch (err) {
+                console.warn(`[queryDynamicWatchlistMatches] Error fetching date ${dateStr}: ${err.message}`);
+                return null;
             }
-        }
-        
-        // If no data found in the entire range, return empty
-        if (!mostRecentDate || !docSnapshot || !docSnapshot.exists) {
-            console.log(`[queryDynamicWatchlistMatches] No computation data found for ${computationName} in date range`);
-            return {
-                success: true,
-                matches: [],
-                count: 0,
-                totalScanned: 0,
-                dateRange: {
-                    start: startDate.toISOString().split('T')[0],
-                    end: endDate.toISOString().split('T')[0]
-                },
-                dataDate: null,
-                computationName,
-                parameters,
-                message: `No computation data found for ${computationName} in the selected time range`
-            };
-        }
-        
-        const docData = docSnapshot.data();
+        });
+
+        // Wait for all days to load
+        const rawResults = (await Promise.all(datePromises)).filter(r => r !== null);
         
-        // Read all per-user data from the most recent date
-        let allUserData = {};
+        // 3. Aggregate Matches Per User
+        // Map: piCid -> { firstMatchedAt, lastMatchedAt, history: [], ... }
+        const piAggregates = new Map();
         
-        if (docData._sharded === true) {
-            // Read from shards
-            const shardsSnapshot = await docRef.collection('_shards').get();
-            console.log(`[queryDynamicWatchlistMatches] Found ${shardsSnapshot.size} shards for ${mostRecentDate}`);
-            
-            for (const shardDoc of shardsSnapshot.docs) {
-                const shardData = shardDoc.data();
-                Object.entries(shardData).forEach(([key, value]) => {
-                    // Skip metadata keys, only include CID keys
-                    if (key.startsWith('_') || key === 'cids' || key === 'metadata') return;
-                    if (/^\d+$/.test(key)) {
-                        allUserData[key] = value;
+        // Process dates from Oldest -> Newest to build timeline correctly
+        // (rawResults is Newest->Oldest, so we reverse it)
+        const timeline = rawResults.reverse(); 
+
+        for (const dayEntry of timeline) {
+            const { date, data } = dayEntry;
+
+            for (const [piCidStr, piData] of Object.entries(data)) {
+                if (piData.error) continue;
+
+                const piCid = Number(piCidStr);
+                
+                // Check if this PI matches the criteria ON THIS SPECIFIC DAY
+                const filterResult = checkPIMatchesCriteria(computationName, piData, parameters);
+
+                if (filterResult.passes) {
+                    if (!piAggregates.has(piCid)) {
+                        piAggregates.set(piCid, {
+                            cid: piCid,
+                            firstMatchedAt: date, // Earliest date in range they matched
+                            lastMatchedAt: date,  // Will update as we go forward
+                            droppedOffAt: null,   // Will calculate later
+                            matchCount: 0,
+                            history: [],          // Array of { date, value, change }
+                            latestData: null      // Store latest raw data for metadata
+                        });
+                    }
+
+                    const agg = piAggregates.get(piCid);
+                    agg.lastMatchedAt = date;
+                    agg.matchCount++;
+                    agg.latestData = piData; // Update to keep the most recent data packet
+                    
+                    agg.history.push({
+                        date: date,
+                        matched: true,
+                        value: filterResult.matchValue,
+                        change: filterResult.change
+                    });
+                } else {
+                    // PI exists in data but DOES NOT match criteria (e.g. Risk score dropped back down)
+                    if (piAggregates.has(piCid)) {
+                        const agg = piAggregates.get(piCid);
+                        // Record the non-match event in history if they had previously matched
+                        agg.history.push({
+                            date: date,
+                            matched: false,
+                            value: filterResult.matchValue, // Value that failed criteria
+                            change: filterResult.change
+                        });
                     }
-                });
-            }
-        } else {
-            // Data is in the document itself
-            Object.entries(docData).forEach(([key, value]) => {
-                if (key.startsWith('_') || key === 'cids' || key === 'metadata' || key === 'globalMetadata') return;
-                if (/^\d+$/.test(key)) {
-                    allUserData[key] = value;
                 }
-            });
-        }
-        
-        const totalPIs = Object.keys(allUserData).length;
-        console.log(`[queryDynamicWatchlistMatches] Evaluating ${totalPIs} PIs from ${mostRecentDate} against criteria`);
-        
-        // Filter PIs that match the criteria on this most recent date
-        const matchingPIs = [];
-        const cidsToLookup = [];
-        
-        for (const [piCidStr, piData] of Object.entries(allUserData)) {
-            const piCid = Number(piCidStr);
-            
-            // Skip error data
-            if (piData.error) continue;
-            
-            // Check if PI matches the criteria
-            const filterResult = checkPIMatchesCriteria(computationName, piData, parameters);
-            
-            if (filterResult.passes) {
-                cidsToLookup.push({ piCid, piData, filterResult });
             }
         }
-        
-        console.log(`[queryDynamicWatchlistMatches] ${cidsToLookup.length} PIs match criteria, fetching usernames...`);
-        
-        // Fetch usernames for matching PIs (batch for efficiency)
-        for (const { piCid, piData, filterResult } of cidsToLookup.slice(0, limit)) {
-            let username = `PI-${piCid}`;
+
+        // 4. Calculate Status (Dropped Off, Current) & Fetch Usernames
+        const results = [];
+        const todayStr = new Date().toISOString().split('T')[0];
+        const lastDataDate = rawResults.length > 0 ? rawResults[rawResults.length - 1].date : todayStr; // The "Current" date of the system
+
+        for (const [cid, agg] of piAggregates) {
+            // Determine if "Dropped Off"
+            // They dropped off if their last match was BEFORE the most recent data date we have
+            let isCurrent = agg.lastMatchedAt === lastDataDate;
+            let droppedOffAt = isCurrent ? null : agg.history[agg.history.length - 1]?.date;
+
+            // Fetch Username (Optimistic/Lazy)
+            let username = `PI-${cid}`;
             try {
-                const piProfile = await fetchPopularInvestorMasterList(db, String(piCid));
-                if (piProfile && piProfile.username) {
-                    username = piProfile.username;
-                }
-            } catch (e) {
-                // Use default username
-            }
-            
-            matchingPIs.push({
-                cid: piCid,
-                username,
-                matchedAt: mostRecentDate,
-                matchValue: filterResult.matchValue,
-                currentValue: filterResult.currentValue,
-                previousValue: filterResult.previousValue,
-                change: filterResult.change,
-                metadata: {
-                    ...piData,
-                    computationDate: mostRecentDate
-                }
+                // Assuming this is cached or fast enough. For 1500 users this might be slow loop.
+                // In production, better to fetch all usernames in one batch or store in the computation data.
+                // Falling back to "Unknown" if costly, or rely on frontend to fetch details.
+                const piProfile = await fetchPopularInvestorMasterList(db, String(cid)).catch(() => null);
+                if (piProfile) username = piProfile.username;
+            } catch (e) {}
+
+            results.push({
+                cid: cid,
+                username: username,
+                
+                // Aggregated Stats
+                firstMatchedAt: agg.firstMatchedAt,
+                lastMatchedAt: agg.lastMatchedAt,
+                droppedOffAt: droppedOffAt, // If set, they matched previously but don't match now
+                isCurrentlyMatching: isCurrent,
+                matchCount: agg.matchCount,
+                
+                // Visualization Data
+                history: agg.history, 
+                
+                // Latest Snapshot Values (for sorting/display)
+                latestValue: agg.history[agg.history.length - 1]?.value,
+                
+                // Metadata for debug/details
+                metadata: agg.latestData
             });
         }
-        
-        // Sort by match value (descending)
-        const sortedMatches = matchingPIs
-            .sort((a, b) => Math.abs(b.matchValue) - Math.abs(a.matchValue))
-            .slice(0, limit);
-        
-        console.log(`[queryDynamicWatchlistMatches] Returning ${sortedMatches.length} matches from ${mostRecentDate}`);
-        
+
+        // 5. Sort Results
+        // Priority: Currently Matching > Recently Dropped Off
+        // Secondary: Match Value (High Risk/High Change first)
+        results.sort((a, b) => {
+            if (a.isCurrentlyMatching !== b.isCurrentlyMatching) {
+                return a.isCurrentlyMatching ? -1 : 1;
+            }
+            // If both current or both dropped, sort by latest value magnitude
+            return Math.abs(b.latestValue) - Math.abs(a.latestValue);
+        });
+
+        const limitedResults = results.slice(0, limit);
+
+        console.log(`[queryDynamicWatchlistMatches] Found ${results.length} unique PIs matching at least once.`);
+
         return {
             success: true,
-            matches: sortedMatches,
-            count: sortedMatches.length,
-            totalScanned: totalPIs,
-            totalMatching: cidsToLookup.length,
+            matches: limitedResults,
+            count: limitedResults.length,
+            totalUniqueMatches: results.length,
             dateRange: {
                 start: startDate.toISOString().split('T')[0],
                 end: endDate.toISOString().split('T')[0]
             },
-            dataDate: mostRecentDate, // The actual date the data is from
+            dataDate: lastDataDate, // Most recent date found in system
             computationName,
             parameters
         };
+
     } catch (error) {
         console.error(`[queryDynamicWatchlistMatches] Error: ${error.message}`, error);
         throw error;

package/functions/computation-system/executors/StandardExecutor.js

@@ -2,7 +2,7 @@
  * @fileoverview Executor for "Standard" (User-Level) calculations.
  * REFACTORED: Hoisted data loading, centralized Series/Root logic.
  * UPDATED: Calls getResult() on computations to trigger summary logging.
- * FIXED: Force-increments shard indexes after every flush to prevent batch overwrites.
+ * FIXED: Removed redundant shard index increment that caused shard gaps (0, 2, 4...).
  */
 const { normalizeName, getEarliestDataDates } = require('../utils/utils');
 const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
@@ -279,7 +279,7 @@ class StandardExecutor {
                 data = inst.results || {};
             }
 
-            // Track active calculations to increment their shards later
+            // Track active calculations (for debugging/logging only now)
             if (Object.keys(data).length > 0) {
                 activeCalcs.push(name);
             }
@@ -302,16 +302,10 @@ class StandardExecutor {
 
         const res = await commitResults(transformedState, dateStr, passName, config, deps, skipStatus, { flushMode: mode, shardIndexes: shardMap, isInitialWrite: isInitial });
         
-        // Update shardMap from result
+        // Update shardMap from result. 
+        // ResultCommitter now returns the CORRECT nextShardIndex (e.g. if it wrote shard_0, it returns 1).
         if (res.shardIndexes) Object.assign(shardMap, res.shardIndexes);
         
-        // [FIX 2] Force increment shard indexes for active calculations.
-        // This ensures the NEXT batch writes to a NEW shard (e.g. results_1) 
-        // instead of overwriting the current one (results_0).
-        activeCalcs.forEach(name => {
-            shardMap[name] = (shardMap[name] || 0) + 1;
-        });
-
         return res;
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.707",
+  "version": "1.0.710",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [