bulltrackers-module 1.0.573 → 1.0.575

package/functions/alert-system/helpers/alert_helpers.js

@@ -212,14 +212,14 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
         return subscriptions;
       }
     } else {
-      logger.log('WARN', `[findSubscriptionsForPI] WatchlistMembershipData/${computationDate} not found, falling back to scanning all watchlists`);
-      // Fallback to legacy method if membership data doesn't exist
-      return await findSubscriptionsForPILegacy(db, logger, piCid, alertTypeId, configKey);
+      logger.log('WARN', `[findSubscriptionsForPI] WatchlistMembershipData/${computationDate} not found, returning existing subscriptions (dev overrides only)`);
+      // Return subscriptions array which may contain dev overrides
+      return subscriptions;
     }
   } catch (error) {
     logger.log('ERROR', `[findSubscriptionsForPI] Error loading WatchlistMembershipData: ${error.message}`);
-    // Fallback to legacy method on error
-    return await findSubscriptionsForPILegacy(db, logger, piCid, alertTypeId, configKey);
+    // Return subscriptions array which may contain dev overrides
+    return subscriptions;
   }
   
   // Step 2: For each user, read their watchlists from SignedInUsers/{cid}/watchlists
@@ -298,51 +298,6 @@ async function findSubscriptionsForPI(db, logger, piCid, alertTypeId, computatio
   return subscriptions;
 }
 
-/**
- * Legacy fallback method - scans all watchlists (used if WatchlistMembershipData is not available)
- */
-async function findSubscriptionsForPILegacy(db, logger, piCid, alertTypeId, configKey) {
-  const subscriptions = [];
-  
-  logger.log('INFO', `[findSubscriptionsForPILegacy] Using legacy method to find subscriptions for PI ${piCid}`);
-  
-  // Get all watchlists from legacy path
-  const watchlistsCollection = db.collection('watchlists');
-  const watchlistsSnapshot = await watchlistsCollection.get();
-  
-  for (const userDoc of watchlistsSnapshot.docs) {
-    const userCid = Number(userDoc.id);
-    const userListsSnapshot = await userDoc.ref.collection('lists').get();
-    
-    for (const listDoc of userListsSnapshot.docs) {
-      const listData = listDoc.data();
-      
-      // Check static watchlists
-      if (listData.type === 'static' && listData.items && Array.isArray(listData.items)) {
-        for (const item of listData.items) {
-          if (Number(item.cid) === Number(piCid)) {
-            const isTestProbe = alertTypeId === 'TestSystemProbe';
-            const isEnabled = item.alertConfig && item.alertConfig[configKey] === true;
-            
-            if (isTestProbe || isEnabled) {
-              subscriptions.push({
-                userCid: userCid,
-                piCid: piCid,
-                piUsername: item.username || `PI-${piCid}`,
-                watchlistId: listDoc.id,
-                watchlistName: listData.name || 'Unnamed Watchlist',
-                alertConfig: item.alertConfig
-              });
-              break;
-            }
-          }
-        }
-      }
-    }
-  }
-  
-  return subscriptions;
-}
 
 /**
  * Check if alert should trigger based on thresholds

package/functions/generic-api/user-api/helpers/data/computation_helpers.js

@@ -9,6 +9,7 @@ const { getEffectiveCid, getDevOverride } = require('../dev/dev_helpers');
 
 /**
  * Check if a PI exists in a computation date
+ * Reads from the new path format: /unified_insights/YYYY-MM-DD/results/popular-investor/computations/PopularInvestorProfileMetrics/pages/{cid}
  * Returns { found: boolean, profileData: object | null, computationData: object | null }
  * @param {object} db - Firestore instance
  * @param {string} insightsCollection - Insights collection name
@@ -23,72 +24,45 @@ const { getEffectiveCid, getDevOverride } = require('../dev/dev_helpers');
  */
 async function checkPiInComputationDate(db, insightsCollection, resultsSub, compsSub, category, computationName, dateStr, cidStr, logger) {
     try {
-        const computationRef = db.collection(insightsCollection)
+        // New path format: /unified_insights/YYYY-MM-DD/results/popular-investor/computations/PopularInvestorProfileMetrics/pages/{cid}
+        const pageRef = db.collection(insightsCollection)
             .doc(dateStr)
             .collection(resultsSub)
             .doc(category)
             .collection(compsSub)
-            .doc(computationName);
+            .doc(computationName)
+            .collection('pages')
+            .doc(cidStr);
         
-        const computationDoc = await computationRef.get();
+        const pageDoc = await pageRef.get();
         
-        if (!computationDoc.exists) {
+        if (!pageDoc.exists) {
+            logger.log('INFO', `[checkPiInComputationDate] Page document not found for CID ${cidStr} in date ${dateStr}`);
             return { found: false, profileData: null, computationData: null };
         }
         
-        const rawData = computationDoc.data();
-        let computationData = null;
+        const rawData = pageDoc.data();
+        let profileData = null;
         
-        // Check if data is sharded
-        if (rawData._sharded === true && rawData._shardCount) {
-            const shardsCol = computationRef.collection('_shards');
-            const shardCount = rawData._shardCount;
-            
-            logger.log('INFO', `[checkPiInComputationDate] Reading ${shardCount} shards for date ${dateStr}`);
-            
-            computationData = {};
-            
-            // Read all shards (shard_0, shard_1, ..., shard_N-1)
-            for (let i = 0; i < shardCount; i++) {
-                const shardDoc = await shardsCol.doc(`shard_${i}`).get();
-                if (shardDoc.exists) {
-                    const shardData = shardDoc.data();
-                    Object.assign(computationData, shardData);
-                } else {
-                    logger.log('WARN', `[checkPiInComputationDate] Shard shard_${i} missing for date ${dateStr}`);
-                }
-            }
-        } else {
-            // Data is in the main document (compressed or raw)
-            computationData = tryDecompress(rawData);
-            
-            // Handle string decompression result
-            if (typeof computationData === 'string') {
-                try {
-                    computationData = JSON.parse(computationData);
-                } catch (e) {
-                    logger.log('WARN', `[checkPiInComputationDate] Failed to parse decompressed string for date ${dateStr}:`, e.message);
-                    return { found: false, profileData: null, computationData: null };
-                }
+        // Decompress if needed
+        profileData = tryDecompress(rawData);
+        
+        // Handle string decompression result
+        if (typeof profileData === 'string') {
+            try {
+                profileData = JSON.parse(profileData);
+            } catch (e) {
+                logger.log('WARN', `[checkPiInComputationDate] Failed to parse decompressed string for CID ${cidStr} on date ${dateStr}:`, e.message);
+                return { found: false, profileData: null, computationData: null };
             }
         }
         
-        // Check if CID exists in the computation data
-        if (computationData && typeof computationData === 'object' && !Array.isArray(computationData)) {
-            // Filter out metadata keys that start with underscore
-            const cids = Object.keys(computationData).filter(key => !key.startsWith('_'));
-            
-            // Check if the requested CID exists
-            const profileData = computationData[cidStr];
-            if (profileData) {
-                logger.log('INFO', `[checkPiInComputationDate] Found CID ${cidStr} in date ${dateStr} (total CIDs: ${cids.length})`);
-                return { found: true, profileData, computationData };
-            } else {
-                logger.log('INFO', `[checkPiInComputationDate] CID ${cidStr} not found in date ${dateStr} (total CIDs: ${cids.length})`);
-                return { found: false, profileData: null, computationData };
-            }
+        if (profileData && typeof profileData === 'object') {
+            logger.log('INFO', `[checkPiInComputationDate] Found profile data for CID ${cidStr} in date ${dateStr}`);
+            // Return the profile data - computationData is set to null since we're reading individual pages
+            return { found: true, profileData, computationData: null };
         } else {
-            logger.log('WARN', `[checkPiInComputationDate] Computation data is not an object for date ${dateStr}`);
+            logger.log('WARN', `[checkPiInComputationDate] Profile data is not an object for CID ${cidStr} on date ${dateStr}`);
             return { found: false, profileData: null, computationData: null };
         }
     } catch (error) {
@@ -174,18 +148,38 @@ async function getUserComputations(req, res, dependencies, config) {
                 const dateStr = checkDate.toISOString().split('T')[0];
                 
                 try {
-                    // Check if computation document exists for this date
-                    const computationRef = db.collection(insightsCollection)
-                        .doc(dateStr)
-                        .collection(resultsSub)
-                        .doc(category)
-                        .collection(compsSub)
-                        .doc(firstCompName);
+                    // For PopularInvestorProfileMetrics, check pages subcollection directly
+                    // For other computations, check main computation document
+                    let shouldCheck = false;
                     
-                    const computationDoc = await computationRef.get();
+                    if (firstCompName === 'PopularInvestorProfileMetrics') {
+                        // Check if page document exists for this user
+                        const pageRef = db.collection(insightsCollection)
+                            .doc(dateStr)
+                            .collection(resultsSub)
+                            .doc(category)
+                            .collection(compsSub)
+                            .doc(firstCompName)
+                            .collection('pages')
+                            .doc(String(effectiveCid));
+                        
+                        const pageDoc = await pageRef.get();
+                        shouldCheck = pageDoc.exists;
+                    } else {
+                        // Check if computation document exists for this date
+                        const computationRef = db.collection(insightsCollection)
+                            .doc(dateStr)
+                            .collection(resultsSub)
+                            .doc(category)
+                            .collection(compsSub)
+                            .doc(firstCompName);
+                        
+                        const computationDoc = await computationRef.get();
+                        shouldCheck = computationDoc.exists;
+                    }
                     
-                    if (computationDoc.exists) {
-                        // Computation exists, verify user is in it (same as data-status)
+                    if (shouldCheck) {
+                        // Computation/page exists, verify user is in it (same as data-status)
                         const { found } = await checkPiInComputationDate(
                             db,
                             insightsCollection,
@@ -249,94 +243,128 @@ async function getUserComputations(req, res, dependencies, config) {
             
             for (const compName of computationNames) {
                 try {
-                    const docRef = db.collection(insightsCollection)
-                        .doc(date)
-                        .collection(resultsSub)
-                        .doc(category)
-                        .collection(compsSub)
-                        .doc(compName);
-                    
-                    const doc = await docRef.get();
+                    let userResult = null;
                     
-                    if (doc.exists) {
-                        const rawData = doc.data();
-                        let data = tryDecompress(rawData);
+                    // Special handling for PopularInvestorProfileMetrics - read from pages subcollection
+                    if (compName === 'PopularInvestorProfileMetrics') {
+                        const pageRef = db.collection(insightsCollection)
+                            .doc(date)
+                            .collection(resultsSub)
+                            .doc(category)
+                            .collection(compsSub)
+                            .doc(compName)
+                            .collection('pages')
+                            .doc(String(effectiveCid));
                         
-                        if (typeof data === 'string') {
-                            try {
-                                data = JSON.parse(data);
-                            } catch (e) {
-                                logger.log('WARN', `[getUserComputations] Failed to parse decompressed string for ${compName} on ${date}:`, e.message);
-                                data = null;
-                            }
-                        }
+                        const pageDoc = await pageRef.get();
                         
-                        if (data && data._sharded === true && data._shardCount) {
-                            const shardsCol = docRef.collection('_shards');
-                            const shardsSnapshot = await shardsCol.get();
+                        if (pageDoc.exists) {
+                            const rawData = pageDoc.data();
+                            let data = tryDecompress(rawData);
                             
-                            if (!shardsSnapshot.empty) {
-                                data = {};
-                                for (const shardDoc of shardsSnapshot.docs) {
-                                    const shardData = shardDoc.data();
-                                    Object.assign(data, shardData);
+                            if (typeof data === 'string') {
+                                try {
+                                    data = JSON.parse(data);
+                                } catch (e) {
+                                    logger.log('WARN', `[getUserComputations] Failed to parse decompressed string for ${compName} page on ${date}:`, e.message);
+                                    data = null;
                                 }
-                            } else {
-                                data = null;
                             }
-                        }
-                        
-                        // [FIX] Handle meta computations (global results) vs standard computations (user-specific)
-                        // Use metadata from schema collection to determine computation type
-                        let userResult = null;
-                        if (data && typeof data === 'object') {
-                            const metadata = computationMetadata[compName];
-                            const isMetaComputation = metadata && metadata.type === 'meta';
                             
-                            if (isMetaComputation) {
-                                // Meta computation: return entire data object (global results)
+                            if (data && typeof data === 'object') {
                                 userResult = data;
-                            } else {
-                                // Standard computation: extract user-specific result
-                                const userCidKey = String(effectiveCid);
-                                userResult = data.hasOwnProperty(userCidKey) ? data[userCidKey] : null;
                             }
                         }
+                    } else {
+                        // Standard path for other computations
+                        const docRef = db.collection(insightsCollection)
+                            .doc(date)
+                            .collection(resultsSub)
+                            .doc(category)
+                            .collection(compsSub)
+                            .doc(compName);
                         
-                        if (isDevOverrideActive && (compName === 'SignedInUserProfileMetrics' || compName === 'SignedInUserCopiedPIs')) {
-                            if (compName === 'SignedInUserCopiedPIs') {
-                                userResult = {
-                                    current: devOverride.fakeCopiedPIs,
-                                    past: [],
-                                    all: devOverride.fakeCopiedPIs
-                                };
-                                logger.log('INFO', `[getUserComputations] Applied DEV OVERRIDE to SignedInUserCopiedPIs for user ${userCid}`);
-                            } else if (compName === 'SignedInUserProfileMetrics' && userResult && userResult.copiedPIs) {
-                                const fakeMirrors = devOverride.fakeCopiedPIs.map(cid => ({
-                                    cid: Number(cid),
-                                    username: `PI-${cid}`,
-                                    invested: 0,
-                                    netProfit: 0,
-                                    value: 0,
-                                    pendingClosure: false,
-                                    isRanked: false
-                                }));
+                        const doc = await docRef.get();
+                        
+                        if (doc.exists) {
+                            const rawData = doc.data();
+                            let data = tryDecompress(rawData);
+                            
+                            if (typeof data === 'string') {
+                                try {
+                                    data = JSON.parse(data);
+                                } catch (e) {
+                                    logger.log('WARN', `[getUserComputations] Failed to parse decompressed string for ${compName} on ${date}:`, e.message);
+                                    data = null;
+                                }
+                            }
+                            
+                            if (data && data._sharded === true && data._shardCount) {
+                                const shardsCol = docRef.collection('_shards');
+                                const shardsSnapshot = await shardsCol.get();
                                 
-                                userResult = {
-                                    ...userResult,
-                                    copiedPIs: {
-                                        chartType: 'cards',
-                                        data: fakeMirrors
+                                if (!shardsSnapshot.empty) {
+                                    data = {};
+                                    for (const shardDoc of shardsSnapshot.docs) {
+                                        const shardData = shardDoc.data();
+                                        Object.assign(data, shardData);
                                     }
-                                };
-                                logger.log('INFO', `[getUserComputations] Applied DEV OVERRIDE to SignedInUserProfileMetrics.copiedPIs for user ${userCid}`);
+                                } else {
+                                    data = null;
+                                }
+                            }
+                            
+                            // [FIX] Handle meta computations (global results) vs standard computations (user-specific)
+                            // Use metadata from schema collection to determine computation type
+                            if (data && typeof data === 'object') {
+                                const metadata = computationMetadata[compName];
+                                const isMetaComputation = metadata && metadata.type === 'meta';
+                                
+                                if (isMetaComputation) {
+                                    // Meta computation: return entire data object (global results)
+                                    userResult = data;
+                                } else {
+                                    // Standard computation: extract user-specific result
+                                    const userCidKey = String(effectiveCid);
+                                    userResult = data.hasOwnProperty(userCidKey) ? data[userCidKey] : null;
+                                }
                             }
                         }
-                        
-                        if (userResult) {
-                            results[date][compName] = userResult;
+                    }
+                    
+                    if (isDevOverrideActive && (compName === 'SignedInUserProfileMetrics' || compName === 'SignedInUserCopiedPIs')) {
+                        if (compName === 'SignedInUserCopiedPIs') {
+                            userResult = {
+                                current: devOverride.fakeCopiedPIs,
+                                past: [],
+                                all: devOverride.fakeCopiedPIs
+                            };
+                            logger.log('INFO', `[getUserComputations] Applied DEV OVERRIDE to SignedInUserCopiedPIs for user ${userCid}`);
+                        } else if (compName === 'SignedInUserProfileMetrics' && userResult && userResult.copiedPIs) {
+                            const fakeMirrors = devOverride.fakeCopiedPIs.map(cid => ({
+                                cid: Number(cid),
+                                username: `PI-${cid}`,
+                                invested: 0,
+                                netProfit: 0,
+                                value: 0,
+                                pendingClosure: false,
+                                isRanked: false
+                            }));
+                            
+                            userResult = {
+                                ...userResult,
+                                copiedPIs: {
+                                    chartType: 'cards',
+                                    data: fakeMirrors
+                                }
+                            };
+                            logger.log('INFO', `[getUserComputations] Applied DEV OVERRIDE to SignedInUserProfileMetrics.copiedPIs for user ${userCid}`);
                         }
                     }
+                    
+                    if (userResult) {
+                        results[date][compName] = userResult;
+                    }
                 } catch (err) {
                     logger.log('WARN', `[getUserComputations] Error fetching ${compName} for ${date}`, err);
                 }

package/functions/generic-api/user-api/helpers/profile/pi_profile_helpers.js

@@ -138,33 +138,14 @@ async function getPiProfile(req, res, dependencies, config) {
         if (!foundDate || !profileData) {
             logger.log('WARN', `[getPiProfile] CID ${cid} not found in any checked dates: ${checkedDates.join(', ')}`);
             
-            // Try to get sample data from the latest date to show what CIDs are available
-            const latestResult = await checkPiInComputationDate(
-                db,
-                insightsCollection,
-                resultsSub,
-                compsSub,
-                category,
-                computationName,
-                latestDate,
-                cidStr,
-                logger
-            );
-            
-            const allAvailableCids = latestResult.computationData && typeof latestResult.computationData === 'object' && !Array.isArray(latestResult.computationData)
-                ? Object.keys(latestResult.computationData)
-                    .filter(key => !key.startsWith('_'))
-                    .sort()
-                : [];
-            
+            // Note: With the new pages subcollection structure, we can't enumerate all available CIDs
+            // from a single document, so we skip that debug information
             return res.status(404).json({ 
                 error: "Profile data not found",
                 message: `Popular Investor ${cid} does not exist in computation results for the last ${maxDaysBackForPi + 1} days. This PI may not have been processed recently.`,
                 debug: {
                     searchedCid: cidStr,
                     checkedDates: checkedDates,
-                    totalCidsInLatestDocument: allAvailableCids.length,
-                    sampleAvailableCids: allAvailableCids.slice(0, 20),
                     latestDate: latestDate
                 }
             });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.573",
+  "version": "1.0.575",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [