bulltrackers-module 1.0.709 → 1.0.712

package/functions/api-v2/helpers/data-fetchers/firestore.js

@@ -5,6 +5,7 @@ const { dispatchSyncRequest } = require('../task_engine_helper.js');
 const { sanitizeCid, sanitizeDocId } = require('../security_utils.js');
 const crypto = require('crypto');
 const zlib = require('zlib');
+const { query: bigqueryQuery } = require('../../../core/utils/bigquery_utils');
 
 const storage = new Storage(); // Singleton GCS Client
 
@@ -3027,27 +3028,23 @@ const getWatchlistTriggerCounts = async (db, userId, watchlistId) => {
 };
 
 /**
- * Query PIs matching dynamic watchlist criteria
- * 
- * IMPORTANT: This function only evaluates PIs based on the MOST RECENT available data.
- * The timeRange parameter controls how far back we look to FIND data (in case today's
- * computation hasn't run yet), but we DO NOT aggregate matches across multiple dates.
- * 
- * If a PI matched criteria 3 days ago but doesn't match on the most recent data,
- * they will NOT be included - they "dropped off" the watchlist.
- * 
- * @param {Object} db - Firestore instance
+ * Query PIs matching dynamic watchlist criteria over a time range.
+ * UPDATED LOGIC: 
+ * - Scans the entire requested time range (e.g. 7 days).
+ * - Tracks history of matches vs non-matches.
+ * - droppedOffAt is now an ARRAY of dates where the user stopped matching.
+ * - Handles users disappearing from the dataset as a "drop-off" event.
+ * * @param {Object} db - Firestore instance
  * @param {string} computationName - Name of the computation to query
- * @param {Object} parameters - Threshold parameters from watchlist config (e.g., {minChange: 1, minRiskLevel: 5})
- * @param {string} timeRange - Time range to look back for data availability (today, last_7_days, last_30_days)
+ * @param {Object} parameters - Threshold parameters (e.g., {minChange: 1})
+ * @param {string} timeRange - Time range (today, last_7_days, last_30_days)
  * @param {number} limit - Maximum number of results
- * @returns {Promise<Object>} Matching PIs with their values
  */
 const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}, timeRange = 'last_7_days', limit = 100) => {
     try {
-        console.log(`[queryDynamicWatchlistMatches] Querying ${computationName} with params:`, parameters, `timeRange: ${timeRange}`);
+        console.log(`[queryDynamicWatchlistMatches] Aggregating ${computationName} over ${timeRange}`);
         
-        // Determine how far back to look for data availability
+        // 1. Determine Date Range
         const endDate = new Date();
         const startDate = new Date();
         
@@ -3055,42 +3052,59 @@ const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}
             case 'today':
                 // Just today
                 break;
-            case 'last_7_days':
-                startDate.setDate(startDate.getDate() - 7);
-                break;
             case 'last_30_days':
                 startDate.setDate(startDate.getDate() - 30);
                 break;
+            case 'last_7_days':
             default:
                 startDate.setDate(startDate.getDate() - 7);
         }
         
-        // Build list of dates to check (most recent first)
+        const startDateStr = startDate.toISOString().split('T')[0];
+        const endDateStr = endDate.toISOString().split('T')[0];
+        
+        // 2. Try BigQuery first (if enabled)
+        if (process.env.BIGQUERY_ENABLED !== 'false') {
+            try {
+                const bigqueryResult = await queryDynamicWatchlistMatchesBigQuery(
+                    computationName,
+                    parameters,
+                    startDateStr,
+                    endDateStr,
+                    limit,
+                    db
+                );
+                if (bigqueryResult) {
+                    console.log(`[queryDynamicWatchlistMatches] Successfully queried from BigQuery`);
+                    return bigqueryResult;
+                }
+            } catch (bqError) {
+                console.warn(`[queryDynamicWatchlistMatches] BigQuery query failed, falling back to Firestore: ${bqError.message}`);
+                // Fall through to Firestore logic
+            }
+        }
+        
+        // 3. Fallback to Firestore (original logic)
+        // Build list of dates to check (Newest -> Oldest)
         const dates = [];
         for (let d = new Date(endDate); d >= startDate; d.setDate(d.getDate() - 1)) {
             dates.push(d.toISOString().split('T')[0]);
         }
         
-        console.log(`[queryDynamicWatchlistMatches] Looking for most recent data in date range:`, dates);
-        
-        // Find the MOST RECENT date that has computation data
-        let mostRecentDate = null;
-        let docRef = null;
-        let docSnapshot = null;
-        
-        for (const dateStr of dates) {
+        // 4. Fetch Data for ALL Dates in Parallel
+        const datePromises = dates.map(async (dateStr) => {
             try {
-                // Try alerts path first
-                docRef = db.collection('unified_insights')
+                // Try alerts path first (primary location for alert computations)
+                let docRef = db.collection('unified_insights')
                     .doc(dateStr)
                     .collection('results')
                     .doc('alerts')
                     .collection('computations')
                     .doc(computationName);
                 
-                docSnapshot = await docRef.get();
+                let docSnapshot = await docRef.get();
                 
-                // If not found, try popular-investor path
+                // Fallback to popular-investor path
                 if (!docSnapshot.exists) {
                     docRef = db.collection('unified_insights')
                         .doc(dateStr)
@@ -3098,138 +3112,206 @@ const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}
                         .doc('popular-investor')
                         .collection('computations')
                         .doc(computationName);
-                    
                     docSnapshot = await docRef.get();
                 }
-                
-                if (docSnapshot.exists) {
-                    mostRecentDate = dateStr;
-                    console.log(`[queryDynamicWatchlistMatches] Found most recent data on ${dateStr}`);
-                    break; // Stop searching - we found the most recent data
+
+                if (!docSnapshot.exists) return null;
+
+                const docData = docSnapshot.data();
+                let dayData = {};
+
+                // Handle Sharding
+                if (docData._sharded === true) {
+                    const shardsSnapshot = await docRef.collection('_shards').get();
+                    for (const shardDoc of shardsSnapshot.docs) {
+                        const shardContent = shardDoc.data();
+                        Object.entries(shardContent).forEach(([key, value]) => {
+                            if (!key.startsWith('_') && key !== 'cids' && /^\d+$/.test(key)) {
+                                dayData[key] = value;
+                            }
+                        });
+                    }
+                } else {
+                    // Standard Document
+                    Object.entries(docData).forEach(([key, value]) => {
+                        if (!key.startsWith('_') && key !== 'cids' && /^\d+$/.test(key)) {
+                            dayData[key] = value;
+                        }
+                    });
                 }
-            } catch (dateErr) {
-                console.warn(`[queryDynamicWatchlistMatches] Error checking date ${dateStr}: ${dateErr.message}`);
-                continue;
+                
+                return { date: dateStr, data: dayData };
+
+            } catch (err) {
+                console.warn(`[queryDynamicWatchlistMatches] Error fetching date ${dateStr}: ${err.message}`);
+                return null;
             }
-        }
-        
-        // If no data found in the entire range, return empty
-        if (!mostRecentDate || !docSnapshot || !docSnapshot.exists) {
-            console.log(`[queryDynamicWatchlistMatches] No computation data found for ${computationName} in date range`);
-            return {
-                success: true,
-                matches: [],
-                count: 0,
-                totalScanned: 0,
-                dateRange: {
-                    start: startDate.toISOString().split('T')[0],
-                    end: endDate.toISOString().split('T')[0]
-                },
-                dataDate: null,
-                computationName,
-                parameters,
-                message: `No computation data found for ${computationName} in the selected time range`
-            };
-        }
+        });
+
+        // Wait for all days to load
+        // rawResults is sorted Newest -> Oldest (matches 'dates' order)
+        const rawResults = (await Promise.all(datePromises)).filter(r => r !== null);
         
-        const docData = docSnapshot.data();
+        // 5. Aggregate Matches Per User
+        // Map: piCid -> { firstMatchedAt, lastMatchedAt, history: [], ... }
+        const piAggregates = new Map();
         
-        // Read all per-user data from the most recent date
-        let allUserData = {};
+        // Process dates from Oldest -> Newest to build timeline correctly
+        // Note: .reverse() mutates the array in place, so rawResults becomes Oldest->Newest
+        const timeline = rawResults.reverse();
         
-        if (docData._sharded === true) {
-            // Read from shards
-            const shardsSnapshot = await docRef.collection('_shards').get();
-            console.log(`[queryDynamicWatchlistMatches] Found ${shardsSnapshot.size} shards for ${mostRecentDate}`);
+        for (const dayEntry of timeline) {
+            const { date, data } = dayEntry;
+            const seenCidsThisDay = new Set();
             
-            for (const shardDoc of shardsSnapshot.docs) {
-                const shardData = shardDoc.data();
-                Object.entries(shardData).forEach(([key, value]) => {
-                    // Skip metadata keys, only include CID keys
-                    if (key.startsWith('_') || key === 'cids' || key === 'metadata') return;
-                    if (/^\d+$/.test(key)) {
-                        allUserData[key] = value;
+            // A. Process Users Present in the Daily File
+            for (const [piCidStr, piData] of Object.entries(data)) {
+                if (piData.error) continue;
+                
+                const piCid = Number(piCidStr);
+                seenCidsThisDay.add(piCid);
+                
+                const filterResult = checkPIMatchesCriteria(computationName, piData, parameters);
+                
+                if (filterResult.passes) {
+                    // Initialize if new
+                    if (!piAggregates.has(piCid)) {
+                        piAggregates.set(piCid, {
+                            cid: piCid,
+                            firstMatchedAt: date,
+                            lastMatchedAt: date,
+                            matchCount: 0,
+                            history: [],
+                            latestData: null
+                        });
                     }
-                });
+                    
+                    const agg = piAggregates.get(piCid);
+                    agg.lastMatchedAt = date;
+                    agg.matchCount++;
+                    agg.latestData = piData;
+                    
+                    agg.history.push({
+                        date: date,
+                        matched: true,
+                        value: filterResult.matchValue,
+                        change: filterResult.change
+                    });
+                } else {
+                    // User exists in data but DOES NOT match criteria
+                    if (piAggregates.has(piCid)) {
+                        const agg = piAggregates.get(piCid);
+                        // Update metadata to show why they failed (current value)
+                        agg.latestData = piData;
+                        
+                        agg.history.push({
+                            date: date,
+                            matched: false,
+                            value: filterResult.matchValue,
+                            change: filterResult.change
+                        });
+                    }
+                }
             }
-        } else {
-            // Data is in the document itself
-            Object.entries(docData).forEach(([key, value]) => {
-                if (key.startsWith('_') || key === 'cids' || key === 'metadata' || key === 'globalMetadata') return;
-                if (/^\d+$/.test(key)) {
-                    allUserData[key] = value;
+            
+            // B. Process Missing Users (Implicit Drop-off)
+            // If a user was tracked previously but is missing today, record as non-match
+            for (const [cid, agg] of piAggregates) {
+                if (!seenCidsThisDay.has(cid)) {
+                    agg.history.push({
+                        date: date,
+                        matched: false,
+                        value: null, // Value unknown/missing
+                        change: null
+                    });
                 }
-            });
+            }
         }
         
-        const totalPIs = Object.keys(allUserData).length;
-        console.log(`[queryDynamicWatchlistMatches] Evaluating ${totalPIs} PIs from ${mostRecentDate} against criteria`);
-        
-        // Filter PIs that match the criteria on this most recent date
-        const matchingPIs = [];
-        const cidsToLookup = [];
+        // 6. Calculate Status (Dropped Off, Current) & Fetch Usernames
+        const results = [];
+        const todayStr = new Date().toISOString().split('T')[0];
+        // Since rawResults was reversed, the last element is the Newest date
+        const lastDataDate = timeline.length > 0 ? timeline[timeline.length - 1].date : todayStr;
         
-        for (const [piCidStr, piData] of Object.entries(allUserData)) {
-            const piCid = Number(piCidStr);
-            
-            // Skip error data
-            if (piData.error) continue;
+        for (const [cid, agg] of piAggregates) {
+            const history = agg.history;
+            const lastEntry = history[history.length - 1];
             
-            // Check if PI matches the criteria
-            const filterResult = checkPIMatchesCriteria(computationName, piData, parameters);
+            // Is Currently Matching?
+            // Must be matched=true AND on the most recent data date available
+            const isCurrent = lastEntry.matched && lastEntry.date === lastDataDate;
             
-            if (filterResult.passes) {
-                cidsToLookup.push({ piCid, piData, filterResult });
-            }
-        }
-        
-        console.log(`[queryDynamicWatchlistMatches] ${cidsToLookup.length} PIs match criteria, fetching usernames...`);
-        
-        // Fetch usernames for matching PIs (batch for efficiency)
-        for (const { piCid, piData, filterResult } of cidsToLookup.slice(0, limit)) {
-            let username = `PI-${piCid}`;
-            try {
-                const piProfile = await fetchPopularInvestorMasterList(db, String(piCid));
-                if (piProfile && piProfile.username) {
-                    username = piProfile.username;
+            // Calculate Drop Off Dates
+            // Find all transitions from True -> False
+            const droppedOffAt = [];
+            for (let i = 1; i < history.length; i++) {
+                const prev = history[i - 1];
+                const curr = history[i];
+                if (prev.matched && !curr.matched) {
+                    droppedOffAt.push(curr.date);
                 }
-            } catch (e) {
-                // Use default username
             }
             
-            matchingPIs.push({
-                cid: piCid,
-                username,
-                matchedAt: mostRecentDate,
-                matchValue: filterResult.matchValue,
-                currentValue: filterResult.currentValue,
-                previousValue: filterResult.previousValue,
-                change: filterResult.change,
-                metadata: {
-                    ...piData,
-                    computationDate: mostRecentDate
-                }
+            // Fetch Username (Optimistic)
+            let username = `PI-${cid}`;
+            if (db) {
+                try {
+                    const piProfile = await fetchPopularInvestorMasterList(db, String(cid)).catch(() => null);
+                    if (piProfile) username = piProfile.username;
+                } catch (e) {}
+            }
+            
+            results.push({
+                cid: cid,
+                username: username,
+                
+                // Aggregated Stats
+                firstMatchedAt: agg.firstMatchedAt,
+                lastMatchedAt: agg.lastMatchedAt,
+                
+                // [UPDATED] Array of dates where they stopped matching
+                droppedOffAt: droppedOffAt,
+                
+                isCurrentlyMatching: isCurrent,
+                matchCount: agg.matchCount,
+                
+                // Visualization Data
+                history: agg.history,
+                
+                // Latest Snapshot Values
+                latestValue: history[history.length - 1]?.value,
+                
+                // Metadata
+                metadata: agg.latestData
             });
         }
         
-        // Sort by match value (descending)
-        const sortedMatches = matchingPIs
-            .sort((a, b) => Math.abs(b.matchValue) - Math.abs(a.matchValue))
-            .slice(0, limit);
+        // 7. Sort Results
+        // Priority: Currently Matching > Recently Dropped Off
+        // Secondary: Match Value magnitude
+        results.sort((a, b) => {
+            if (a.isCurrentlyMatching !== b.isCurrentlyMatching) {
+                return a.isCurrentlyMatching ? -1 : 1;
+            }
+            // If both same status, sort by magnitude of value (risk, change, etc)
+            return Math.abs(b.latestValue || 0) - Math.abs(a.latestValue || 0);
+        });
+        
+        const limitedResults = results.slice(0, limit);
         
-        console.log(`[queryDynamicWatchlistMatches] Returning ${sortedMatches.length} matches from ${mostRecentDate}`);
+        console.log(`[queryDynamicWatchlistMatches] Found ${results.length} unique PIs matching at least once.`);
         
         return {
             success: true,
-            matches: sortedMatches,
-            count: sortedMatches.length,
-            totalScanned: totalPIs,
-            totalMatching: cidsToLookup.length,
+            matches: limitedResults,
+            count: limitedResults.length,
+            totalUniqueMatches: results.length,
             dateRange: {
-                start: startDate.toISOString().split('T')[0],
-                end: endDate.toISOString().split('T')[0]
+                start: startDateStr,
+                end: endDateStr
             },
-            dataDate: mostRecentDate, // The actual date the data is from
+            dataDate: lastDataDate,
             computationName,
             parameters
         };

package/functions/computation-system/data/CachedDataLoader.js

@@ -189,7 +189,28 @@ class CachedDataLoader {
         return getRelevantShardRefs(this.config, this.deps, targetInstrumentIds); 
     }
 
-    async loadPriceShard(docRef) { 
+    async loadPriceShard(docRef) {
+        // Check if this is a BigQuery marker
+        if (docRef && docRef._bigquery === true) {
+            // Load all price data from BigQuery
+            try {
+                const { queryAssetPrices } = require('../../core/utils/bigquery_utils');
+                const priceData = await queryAssetPrices(null, null, null, this.deps.logger);
+                
+                if (priceData && Object.keys(priceData).length > 0) {
+                    this.deps.logger.log('INFO', `[CachedDataLoader] ✅ Loaded ${Object.keys(priceData).length} instruments from BigQuery`);
+                    return priceData;
+                }
+                
+                // If BigQuery returns empty, fallback to Firestore
+                this.deps.logger.log('WARN', `[CachedDataLoader] BigQuery returned no price data, falling back to Firestore`);
+            } catch (bqError) {
+                this.deps.logger.log('WARN', `[CachedDataLoader] BigQuery price load failed, falling back to Firestore: ${bqError.message}`);
+                // Fall through to Firestore
+            }
+        }
+        
+        // Firestore fallback (original logic)
         try { 
             const snap = await docRef.get(); 
             return snap.exists ? this._tryDecompress(snap.data()) : {}; 

package/functions/computation-system/data/DependencyFetcher.js

@@ -44,11 +44,36 @@ function tryDecompress(payload) {
 
 /**
  * Fetches, decompresses, and reassembles (if sharded or on GCS) a single result document.
+ * NEW: For non-alert, non-page computations, tries BigQuery first (cheaper, no sharding/compression).
  */
 async function fetchSingleResult(db, config, dateStr, name, category) {
     const { resultsCollection = 'computation_results', resultsSubcollection = 'results', computationsSubcollection = 'computations' } = config;
     const log = config.logger || console;
 
+    // NEW STRATEGY: Check if this is an alert or page computation
+    // We need to check the manifest to determine this, but we can infer from category
+    // For now, we'll try BigQuery first for all non-alert computations (alerts are in 'alerts' category)
+    const isAlertComputation = category === 'alerts';
+    // Page computations are typically in 'popular-investor' category but have isPage flag
+    // For now, we'll try BigQuery for all non-alert computations
+    
+    // Try BigQuery first for non-alert computations (reduces Firestore reads)
+    if (!isAlertComputation && process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryComputationResult } = require('../../core/utils/bigquery_utils');
+            const bigqueryResult = await queryComputationResult(name, category, dateStr, log);
+            
+            if (bigqueryResult && !isDataEmpty(bigqueryResult)) {
+                log.log('INFO', `[DependencyFetcher] ✅ Using BigQuery for ${name} (${dateStr}, ${category})`);
+                return bigqueryResult;
+            }
+        } catch (bqError) {
+            log.log('WARN', `[DependencyFetcher] BigQuery fetch failed for ${name}, falling back to Firestore: ${bqError.message}`);
+            // Fall through to Firestore
+        }
+    }
+
+    // Fallback to Firestore (for alerts, pages, or if BigQuery fails)
     const docRef = db.collection(resultsCollection).doc(dateStr)
                      .collection(resultsSubcollection).doc(category)
                      .collection(computationsSubcollection).doc(name);
@@ -238,6 +263,9 @@ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config,
         d.setUTCDate(d.getUTCDate() - 1);
         dates.push(d.toISOString().slice(0, 10));
     }
+    
+    const startDateStr = dates[dates.length - 1]; // Oldest date
+    const queryEndDateStr = dates[0]; // Newest date (for BigQuery query)
 
     // [DEBUG] Log the manifest lookup and resolved categories
     logger.log('INFO', `[DependencyFetcher] 🔍 ManifestLookup has ${Object.keys(manifestLookup).length} entries`);
@@ -248,6 +276,96 @@ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config,
         logger.log('INFO', `[DependencyFetcher] 📍 '${rawName}' -> category='${category}' -> Path: ${samplePath}`);
     }
 
+    // =========================================================================
+    // BIGQUERY FIRST: Try batch query for all dates at once
+    // =========================================================================
+    if (process.env.BIGQUERY_ENABLED !== 'false') {
+        try {
+            const { queryComputationResultsRange } = require('../../core/utils/bigquery_utils');
+            
+            // Query each computation in parallel
+            const bigqueryPromises = calcNames.map(async (rawName) => {
+                const norm = normalizeName(rawName);
+                const category = manifestLookup[norm] || 'analytics';
+                
+                const bigqueryRows = await queryComputationResultsRange(
+                    rawName,
+                    category,
+                    startDateStr,
+                    queryEndDateStr,
+                    logger
+                );
+                
+                if (bigqueryRows && bigqueryRows.length > 0) {
+                    logger.log('INFO', `[DependencyFetcher] ✅ Using BigQuery for ${rawName} series: ${bigqueryRows.length} dates`);
+                    
+                    // Map BigQuery results to results structure
+                    for (const row of bigqueryRows) {
+                        if (row.data && !isDataEmpty(row.data)) {
+                            results[norm][row.date] = row.data;
+                        }
+                    }
+                    
+                    return { name: rawName, found: bigqueryRows.length };
+                }
+                
+                return { name: rawName, found: 0 };
+            });
+            
+            const bigqueryResults = await Promise.all(bigqueryPromises);
+            const totalFound = bigqueryResults.reduce((sum, r) => sum + r.found, 0);
+            
+            if (totalFound > 0) {
+                logger.log('INFO', `[DependencyFetcher] ✅ BigQuery retrieved ${totalFound} computation result records across ${calcNames.length} computations`);
+                
+                // Fill in any missing dates from Firestore (fallback)
+                const missingOps = [];
+                for (const dateStr of dates) {
+                    for (const rawName of calcNames) {
+                        const norm = normalizeName(rawName);
+                        // Only fetch if we don't have this date already
+                        if (!results[norm] || !results[norm][dateStr]) {
+                            const category = manifestLookup[norm] || 'analytics';
+                            missingOps.push(async () => {
+                                const val = await fetchSingleResult(db, { ...config, logger }, dateStr, rawName, category);
+                                if (val && !isDataEmpty(val)) {
+                                    results[norm][dateStr] = val;
+                                }
+                            });
+                        }
+                    }
+                }
+                
+                // Fetch missing dates from Firestore
+                if (missingOps.length > 0) {
+                    logger.log('INFO', `[DependencyFetcher] 📂 Fetching ${missingOps.length} missing dates from Firestore (fallback)`);
+                    const BATCH_SIZE = 20;
+                    for (let i = 0; i < missingOps.length; i += BATCH_SIZE) {
+                        await Promise.all(missingOps.slice(i, i + BATCH_SIZE).map(fn => fn()));
+                    }
+                }
+                
+                // Log final summary
+                for (const rawName of calcNames) {
+                    const norm = normalizeName(rawName);
+                    const foundDates = Object.keys(results[norm] || {});
+                    logger.log('INFO', `[DependencyFetcher] ✅ '${rawName}' found data for ${foundDates.length}/${lookbackDays} days (BigQuery + Firestore)`);
+                }
+                
+                return results;
+            } else {
+                logger.log('INFO', `[DependencyFetcher] ⚠️ BigQuery returned no results, falling back to Firestore`);
+            }
+        } catch (bqError) {
+            logger.log('WARN', `[DependencyFetcher] BigQuery series query failed, falling back to Firestore: ${bqError.message}`);
+        }
+    }
+    
+    // =========================================================================
+    // FIRESTORE FALLBACK: Original logic (backwards compatibility)
+    // =========================================================================
+    logger.log('INFO', `[DependencyFetcher] 📂 Using Firestore for computation result series: ${calcNames.length} calcs x ${lookbackDays} days`);
+
     // Build Fetch Operations
     const ops = [];
     for (const dateStr of dates) {