bulltrackers-module 1.0.689 → 1.0.690

package/functions/api-v2/helpers/data-fetchers/firestore.js

@@ -3028,16 +3028,18 @@ const getWatchlistTriggerCounts = async (db, userId, watchlistId) => {
 
 /**
  * Query PIs matching dynamic watchlist criteria
+ * Reads computation results and filters ALL per-user data (not just triggered CIDs)
+ * 
  * @param {Object} db - Firestore instance
  * @param {string} computationName - Name of the computation to query
- * @param {Object} conditions - Threshold conditions to filter by
- * @param {string} timeRange - Time range (today, last_7_days, last_30_days, all_time)
+ * @param {Object} parameters - Threshold parameters from watchlist config (e.g., {minChange: 1, minRiskLevel: 5})
+ * @param {string} timeRange - Time range (today, last_7_days, last_30_days)
  * @param {number} limit - Maximum number of results
  * @returns {Promise<Object>} Matching PIs with their values
  */
-const queryDynamicWatchlistMatches = async (db, computationName, conditions = {}, timeRange = 'last_7_days', limit = 100) => {
+const queryDynamicWatchlistMatches = async (db, computationName, parameters = {}, timeRange = 'last_7_days', limit = 100) => {
     try {
-        const { readComputationResultsWithShards } = require('../../alert-system/helpers/alert_helpers');
+        console.log(`[queryDynamicWatchlistMatches] Querying ${computationName} with params:`, parameters, `timeRange: ${timeRange}`);
         
         // Determine date range based on timeRange
         const endDate = new Date();
@@ -3053,75 +3055,108 @@ const queryDynamicWatchlistMatches = async (db, computationName, conditions = {}
             case 'last_30_days':
                 startDate.setDate(startDate.getDate() - 30);
                 break;
-            case 'all_time':
-                startDate.setDate(startDate.getDate() - 365); // Max 1 year lookback
-                break;
             default:
                 startDate.setDate(startDate.getDate() - 7);
         }
         
-        // Build list of dates to query
+        // Build list of dates to query (most recent first)
         const dates = [];
         for (let d = new Date(endDate); d >= startDate; d.setDate(d.getDate() - 1)) {
             dates.push(d.toISOString().split('T')[0]);
         }
         
-        const matchingPIs = new Map(); // cid -> { matchValue, matchedAt, metadata }
+        console.log(`[queryDynamicWatchlistMatches] Checking dates:`, dates);
+        
+        const matchingPIs = new Map(); // cid -> match data
         
-        // Query each date until we find results
+        // Query each date
         for (const dateStr of dates) {
             try {
-                const docRef = db.collection('unified_insights')
+                // Try both 'alerts' and 'popular-investor' result paths
+                let docSnapshot = null;
+                let docRef = null;
+                
+                // First try alerts path
+                docRef = db.collection('unified_insights')
                     .doc(dateStr)
                     .collection('results')
                     .doc('alerts')
                     .collection('computations')
                     .doc(computationName);
                 
-                const docSnapshot = await docRef.get();
+                docSnapshot = await docRef.get();
+                
+                // If not found, try popular-investor path
+                if (!docSnapshot.exists) {
+                    docRef = db.collection('unified_insights')
+                        .doc(dateStr)
+                        .collection('results')
+                        .doc('popular-investor')
+                        .collection('computations')
+                        .doc(computationName);
+                    
+                    docSnapshot = await docRef.get();
+                }
+                
+                if (!docSnapshot.exists) {
+                    console.log(`[queryDynamicWatchlistMatches] No data for ${dateStr}`);
+                    continue;
+                }
                 
-                if (!docSnapshot.exists) continue;
+                console.log(`[queryDynamicWatchlistMatches] Found data for ${dateStr}`);
                 
                 const docData = docSnapshot.data();
-                const results = await readComputationResultsWithShards(db, docData, docRef, null);
                 
-                if (!results.cids || results.cids.length === 0) continue;
+                // Check for sharded data
+                let allUserData = {};
                 
-                // Process each PI
-                for (const piCid of results.cids) {
-                    // Skip if already processed
-                    if (matchingPIs.has(piCid)) continue;
+                if (docData._sharded === true) {
+                    // Read from shards
+                    const shardsSnapshot = await docRef.collection('_shards').get();
+                    console.log(`[queryDynamicWatchlistMatches] Found ${shardsSnapshot.size} shards`);
                     
-                    const piData = results.perUserData?.[piCid] || results.metadata || {};
+                    for (const shardDoc of shardsSnapshot.docs) {
+                        const shardData = shardDoc.data();
+                        // Each shard contains CID keys with their data
+                        Object.entries(shardData).forEach(([key, value]) => {
+                            // Skip metadata keys
+                            if (key.startsWith('_') || key === 'cids' || key === 'metadata') return;
+                            // Check if key is a CID (numeric string)
+                            if (/^\d+$/.test(key)) {
+                                allUserData[key] = value;
+                            }
+                        });
+                    }
+                } else {
+                    // Data is in the document itself
+                    Object.entries(docData).forEach(([key, value]) => {
+                        // Skip metadata keys
+                        if (key.startsWith('_') || key === 'cids' || key === 'metadata' || key === 'globalMetadata') return;
+                        // Check if key is a CID (numeric string)
+                        if (/^\d+$/.test(key)) {
+                            allUserData[key] = value;
+                        }
+                    });
+                }
+                
+                console.log(`[queryDynamicWatchlistMatches] Found ${Object.keys(allUserData).length} PIs in computation data`);
+                
+                // Process ALL PIs (not just those in cids array)
+                for (const [piCidStr, piData] of Object.entries(allUserData)) {
+                    const piCid = Number(piCidStr);
                     
-                    // Apply condition filtering
-                    let passesConditions = true;
-                    let matchValue = 0;
+                    // Skip if already matched from a more recent date
+                    if (matchingPIs.has(piCid)) continue;
                     
-                    // Extract match value based on computation type
-                    if (piData.change !== undefined) {
-                        matchValue = piData.change;
-                    } else if (piData.current !== undefined) {
-                        matchValue = piData.current;
-                    } else if (piData.riskScore !== undefined) {
-                        matchValue = piData.riskScore;
-                    } else if (piData.volatility !== undefined) {
-                        matchValue = piData.volatility * 100; // Convert to percentage
-                    } else if (piData.diff !== undefined) {
-                        matchValue = piData.diff;
-                    }
+                    // Skip if error data
+                    if (piData.error) continue;
                     
-                    // Apply threshold conditions
-                    if (conditions.minValue !== undefined && matchValue < conditions.minValue) {
-                        passesConditions = false;
-                    }
-                    if (conditions.maxValue !== undefined && matchValue > conditions.maxValue) {
-                        passesConditions = false;
-                    }
+                    // Apply computation-specific filtering based on parameters
+                    const passesFilter = checkPIMatchesCriteria(computationName, piData, parameters);
                     
-                    if (passesConditions) {
+                    if (passesFilter.passes) {
                         // Get PI username from master list
-                        let username = piData.piUsername || piData.username || `PI-${piCid}`;
+                        let username = `PI-${piCid}`;
                         try {
                             const piProfile = await fetchPopularInvestorMasterList(db, String(piCid));
                             if (piProfile && piProfile.username) {
@@ -3132,14 +3167,13 @@ const queryDynamicWatchlistMatches = async (db, computationName, conditions = {}
                         }
                         
                         matchingPIs.set(piCid, {
-                            cid: Number(piCid),
+                            cid: piCid,
                             username,
                             matchedAt: dateStr,
-                            matchValue,
-                            previousValue: piData.previous || piData.prev,
-                            trend: piData.previous !== undefined 
-                                ? (matchValue > piData.previous ? 'up' : matchValue < piData.previous ? 'down' : 'stable')
-                                : undefined,
+                            matchValue: passesFilter.matchValue,
+                            currentValue: passesFilter.currentValue,
+                            previousValue: passesFilter.previousValue,
+                            change: passesFilter.change,
                             metadata: {
                                 ...piData,
                                 computationDate: dateStr
@@ -3151,11 +3185,11 @@ const queryDynamicWatchlistMatches = async (db, computationName, conditions = {}
                     }
                 }
                 
-                // Break out of date loop if we have enough results
+                // If we found enough matches, stop looking at older dates
                 if (matchingPIs.size >= limit) break;
                 
             } catch (dateErr) {
-                console.warn(`Error processing date ${dateStr} for dynamic watchlist: ${dateErr.message}`);
+                console.warn(`[queryDynamicWatchlistMatches] Error processing date ${dateStr}: ${dateErr.message}`);
                 continue;
             }
         }
@@ -3165,22 +3199,135 @@ const queryDynamicWatchlistMatches = async (db, computationName, conditions = {}
             .sort((a, b) => Math.abs(b.matchValue) - Math.abs(a.matchValue))
             .slice(0, limit);
         
+        console.log(`[queryDynamicWatchlistMatches] Returning ${sortedMatches.length} matches`);
+        
         return {
             success: true,
             matches: sortedMatches,
             count: sortedMatches.length,
+            totalScanned: Object.keys(matchingPIs).length,
             dateRange: {
                 start: startDate.toISOString().split('T')[0],
                 end: endDate.toISOString().split('T')[0]
             },
-            computationName
+            computationName,
+            parameters
         };
     } catch (error) {
-        console.error(`Error querying dynamic watchlist matches: ${error}`);
+        console.error(`[queryDynamicWatchlistMatches] Error: ${error.message}`, error);
         throw error;
     }
 };
 
+/**
+ * Check if a PI's data matches the watchlist criteria
+ * Handles different computation types with their specific data formats
+ */
+function checkPIMatchesCriteria(computationName, piData, parameters) {
+    const result = { passes: false, matchValue: 0, currentValue: null, previousValue: null, change: null };
+    
+    switch (computationName) {
+        case 'RiskScoreIncrease': {
+            // Data format: { change, currentRisk, previousRisk, isBaselineReset }
+            const change = piData.change || 0;
+            const currentRisk = piData.currentRisk;
+            const previousRisk = piData.previousRisk;
+            const minChange = parameters.minChange ?? 0;
+            const minRiskLevel = parameters.minRiskLevel ?? 0;
+            
+            result.currentValue = currentRisk;
+            result.previousValue = previousRisk;
+            result.change = change;
+            result.matchValue = currentRisk; // Sort by current risk level
+            
+            // Pass if change >= minChange AND currentRisk >= minRiskLevel
+            if (change >= minChange && currentRisk >= minRiskLevel) {
+                result.passes = true;
+            }
+            break;
+        }
+        
+        case 'SignificantVolatility': {
+            // Data format: { volatility, threshold, samples }
+            const volatility = piData.volatility || 0;
+            const minVolatility = parameters.volatilityThreshold ?? 50;
+            
+            result.currentValue = volatility;
+            result.matchValue = volatility;
+            
+            if (volatility >= minVolatility) {
+                result.passes = true;
+            }
+            break;
+        }
+        
+        case 'PositionInvestedIncrease': {
+            // Data format: { moveCount, moves: [{symbol, prev, curr, diff}], isBaselineReset }
+            const moveCount = piData.moveCount || 0;
+            const moves = piData.moves || [];
+            const minIncrease = parameters.minIncrease ?? 5;
+            
+            // Find the largest position increase
+            const maxMove = moves.reduce((max, m) => m.diff > max ? m.diff : max, 0);
+            
+            result.currentValue = moveCount;
+            result.matchValue = maxMove;
+            
+            if (maxMove >= minIncrease) {
+                result.passes = true;
+            }
+            break;
+        }
+        
+        case 'NewSectorExposure': {
+            // Data format: { currentSectors, previousSectors, newExposures, isBaselineReset }
+            const newExposures = piData.newExposures || [];
+            
+            result.currentValue = newExposures.length;
+            result.matchValue = newExposures.length;
+            
+            if (newExposures.length > 0) {
+                result.passes = true;
+            }
+            break;
+        }
+        
+        case 'NewSocialPost': {
+            // Data format: { hasNewPost, latestPostDate, postCount, title }
+            const hasNewPost = piData.hasNewPost || false;
+            
+            result.currentValue = hasNewPost ? 1 : 0;
+            result.matchValue = hasNewPost ? 1 : 0;
+            
+            if (hasNewPost) {
+                result.passes = true;
+            }
+            break;
+        }
+        
+        case 'BehavioralAnomaly': {
+            // Data format: { anomalyScore, primaryDriver, driverSignificance }
+            const anomalyScore = piData.anomalyScore || 0;
+            const minScore = parameters.minAnomalyScore ?? 0;
+            
+            result.currentValue = anomalyScore;
+            result.matchValue = anomalyScore;
+            
+            if (anomalyScore >= minScore) {
+                result.passes = true;
+            }
+            break;
+        }
+        
+        default:
+            // Generic fallback - pass if there's any data
+            result.passes = true;
+            result.matchValue = 0;
+    }
+    
+    return result;
+}
+
 /**
  * Subscribe to all PIs in a watchlist
  */

package/functions/api-v2/routes/watchlists.js

@@ -40,11 +40,8 @@ const watchlistSubscribeSchema = z.object({
 
 const dynamicMatchesSchema = z.object({
     computationName: z.string().min(1).max(200),
-    conditions: z.object({
-        minValue: z.number().optional(),
-        maxValue: z.number().optional()
-    }).optional(),
-    timeRange: z.enum(['today', 'last_7_days', 'last_30_days', 'all_time']).optional(),
+    parameters: z.record(z.any()).optional(), // Flexible parameters object for thresholds
+    timeRange: z.enum(['today', 'last_7_days', 'last_30_days']).optional(),
     limit: z.number().int().min(1).max(500).optional()
 });
 
@@ -234,15 +231,20 @@ router.post('/dynamic/query', async (req, res, next) => {
         // Validate input
         const validatedData = dynamicMatchesSchema.parse(req.body);
         
-        const { db } = req.dependencies;
+        const { db, logger } = req.dependencies;
+        
+        logger?.log('INFO', `[POST /watchlists/dynamic/query] Querying ${validatedData.computationName} with params:`, validatedData.parameters);
+        
         const result = await queryDynamicWatchlistMatches(
             db,
             validatedData.computationName,
-            validatedData.conditions || {},
+            validatedData.parameters || {},
             validatedData.timeRange || 'last_7_days',
             validatedData.limit || 100
         );
         
+        logger?.log('INFO', `[POST /watchlists/dynamic/query] Found ${result.count} matches`);
+        
         res.json(result);
     } catch (error) {
         if (error instanceof z.ZodError) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.689",
+  "version": "1.0.690",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [