bulltrackers-module 1.0.548 → 1.0.549

package/functions/computation-system/data/CachedDataLoader.js

@@ -15,6 +15,7 @@ const {
     loadPIPageViews,              // [NEW]
     loadWatchlistMembership: loadWatchlistMembershipData, // [NEW] Renamed to avoid conflict
     loadPIAlertHistory,            // [NEW]
+    loadPIWatchlistData,           // [NEW] PI-centric watchlist data
     loadPopularInvestorMasterList // [NEW]
 } = require('../utils/data_loader');
 const zlib = require('zlib'); 
@@ -33,6 +34,7 @@ class CachedDataLoader {
             pageViews: new Map(), // [NEW]
             watchlistMembership: new Map(), // [NEW]
             alertHistory: new Map(),// [NEW]
+            piWatchlistData: new Map(), // [NEW] PI-centric watchlist data cache (keyed by piCid)
             piMasterList: null // [NEW] Singleton cache (not date dependent)
         }; 
     }
@@ -125,6 +127,17 @@ class CachedDataLoader {
         return promise;
     }
 
+    // [NEW] Load PI-Centric Watchlist Data
+    // Loads watchlist data from PopularInvestors/{piCid}/watchlistData/current
+    // [FIX] Cache promises to prevent race conditions when multiple users request same data
+    async loadPIWatchlistData(piCid) {
+        const piCidStr = String(piCid);
+        if (this.cache.piWatchlistData.has(piCidStr)) return this.cache.piWatchlistData.get(piCidStr);
+        const promise = loadPIWatchlistData(this.config, this.deps, piCidStr);
+        this.cache.piWatchlistData.set(piCidStr, promise);
+        return promise;
+    }
+
     async getPriceShardReferences() { 
         return getPriceShardRefs(this.config, this.deps); 
     }

package/functions/computation-system/layers/extractors.js

@@ -808,6 +808,102 @@ class WatchlistMembershipExtractor {
     }
 }
 
+/**
+ * [NEW] Extractor for PI-Centric Watchlist Data
+ * Access via: loader.loadPIWatchlistData(piCid) or context-specific loading
+ * Schema: { totalUsers, userCids: [], dailyAdditions: { date: { count, userCids: [], timestamp } }, lastUpdated }
+ */
+class PIWatchlistDataExtractor {
+    /**
+     * Get total users who have this PI in their watchlist
+     * @param {Object} piWatchlistData - The PI watchlist data from PopularInvestors/{piCid}/watchlistData/current
+     * @returns {number} Total users count or 0 if not available
+     */
+    static getTotalUsers(piWatchlistData) {
+        if (!piWatchlistData) return 0;
+        return piWatchlistData.totalUsers || 0;
+    }
+
+    /**
+     * Get array of user CIDs who have this PI in their watchlist
+     * @param {Object} piWatchlistData - The PI watchlist data from PopularInvestors/{piCid}/watchlistData/current
+     * @returns {string[]} Array of user CIDs, or empty array if not available
+     */
+    static getUserCids(piWatchlistData) {
+        if (!piWatchlistData || !Array.isArray(piWatchlistData.userCids)) return [];
+        return piWatchlistData.userCids.map(cid => String(cid));
+    }
+
+    /**
+     * Check if a specific user has this PI in their watchlist
+     * @param {Object} piWatchlistData - The PI watchlist data from PopularInvestors/{piCid}/watchlistData/current
+     * @param {string|number} userId - User CID
+     * @returns {boolean} True if user has PI in watchlist
+     */
+    static hasUser(piWatchlistData, userId) {
+        const userCids = this.getUserCids(piWatchlistData);
+        return userCids.includes(String(userId));
+    }
+
+    /**
+     * Get daily additions data for a specific date
+     * @param {Object} piWatchlistData - The PI watchlist data from PopularInvestors/{piCid}/watchlistData/current
+     * @param {string} date - Date string in YYYY-MM-DD format
+     * @returns {Object|null} Daily addition data { count, userCids: [], timestamp } or null if not found
+     */
+    static getDailyAdditions(piWatchlistData, date) {
+        if (!piWatchlistData || !piWatchlistData.dailyAdditions || typeof piWatchlistData.dailyAdditions !== 'object') {
+            return null;
+        }
+        return piWatchlistData.dailyAdditions[date] || null;
+    }
+
+    /**
+     * Get count of users who added this PI to their watchlist on a specific date
+     * @param {Object} piWatchlistData - The PI watchlist data from PopularInvestors/{piCid}/watchlistData/current
+     * @param {string} date - Date string in YYYY-MM-DD format
+     * @returns {number} Count of users who added on this date, or 0 if not available
+     */
+    static getDailyAdditionCount(piWatchlistData, date) {
+        const dailyData = this.getDailyAdditions(piWatchlistData, date);
+        return dailyData?.count || 0;
+    }
+
+    /**
+     * Get user CIDs who added this PI to their watchlist on a specific date
+     * @param {Object} piWatchlistData - The PI watchlist data from PopularInvestors/{piCid}/watchlistData/current
+     * @param {string} date - Date string in YYYY-MM-DD format
+     * @returns {string[]} Array of user CIDs who added on this date, or empty array if not available
+     */
+    static getDailyAdditionUserCids(piWatchlistData, date) {
+        const dailyData = this.getDailyAdditions(piWatchlistData, date);
+        if (!dailyData || !Array.isArray(dailyData.userCids)) return [];
+        return dailyData.userCids.map(cid => String(cid));
+    }
+
+    /**
+     * Get all dates with daily addition data
+     * @param {Object} piWatchlistData - The PI watchlist data from PopularInvestors/{piCid}/watchlistData/current
+     * @returns {string[]} Array of date strings (YYYY-MM-DD format), or empty array if not available
+     */
+    static getAllDates(piWatchlistData) {
+        if (!piWatchlistData || !piWatchlistData.dailyAdditions || typeof piWatchlistData.dailyAdditions !== 'object') {
+            return [];
+        }
+        return Object.keys(piWatchlistData.dailyAdditions);
+    }
+
+    /**
+     * Get last updated timestamp
+     * @param {Object} piWatchlistData - The PI watchlist data from PopularInvestors/{piCid}/watchlistData/current
+     * @returns {Object|null} Timestamp object or null if not available
+     */
+    static getLastUpdated(piWatchlistData) {
+        if (!piWatchlistData) return null;
+        return piWatchlistData.lastUpdated || null;
+    }
+}
+
 /**
  * [NEW] Extractor for PI Alert History Data
  * Access via: context.globalData.alertHistory
@@ -995,6 +1091,7 @@ module.exports = {
     RatingsExtractor,
     PageViewsExtractor,
     WatchlistMembershipExtractor,
+    PIWatchlistDataExtractor,
     AlertHistoryExtractor,
     PIMasterListExtractor
 };

package/functions/computation-system/utils/data_loader.js

@@ -774,6 +774,38 @@ async function loadPIAlertHistory(config, deps, dateString) {
     }
 }
 
+/** Stage 17: Load PI-Centric Watchlist Data
+ * Loads watchlist data from PopularInvestors/{piCid}/watchlistData/current
+ * This provides time-series data of watchlist additions per PI
+ */
+async function loadPIWatchlistData(config, deps, piCid) {
+    const { db, logger, calculationUtils } = deps;
+    const { withRetry } = calculationUtils;
+    const piCidStr = String(piCid);
+    
+    logger.log('INFO', `Loading PI Watchlist Data for PI ${piCid}`);
+    
+    try {
+        const docRef = db.collection('PopularInvestors')
+            .doc(piCidStr)
+            .collection('watchlistData')
+            .doc('current');
+        
+        const docSnap = await withRetry(() => docRef.get(), `getPIWatchlistData(${piCidStr})`);
+        
+        if (!docSnap.exists) { 
+            logger.log('WARN', `PI Watchlist Data not found for PI ${piCidStr}`); 
+            return null; 
+        }
+        
+        const data = tryDecompress(docSnap.data());
+        return data; // Returns { totalUsers, userCids: [], dailyAdditions: { date: { count, userCids, timestamp } }, lastUpdated }
+    } catch (error) {
+        logger.log('ERROR', `Failed to load PI Watchlist Data for PI ${piCidStr}: ${error.message}`);
+        return null;
+    }
+}
+
 // [NEW] Load Popular Investor Master List
 async function loadPopularInvestorMasterList(config, deps) {
     const { db, logger, calculationUtils } = deps;
@@ -823,5 +855,6 @@ module.exports = {
     loadPIPageViews,
     loadWatchlistMembership,
     loadPIAlertHistory,
-    loadPopularInvestorMasterList // [NEW]
+    loadPopularInvestorMasterList, // [NEW]
+    loadPIWatchlistData,
 };

package/functions/generic-api/user-api/helpers/rootdata/rootdata_aggregation_helpers.js

@@ -116,6 +116,7 @@ async function updatePageViewsRootData(db, logger, piCid, userCid, date) {
  * Update Watchlist Membership rootdata collection
  * Aggregates watchlist membership from SignedInUsers/{cid}/watchlists into WatchlistMembershipData/{date}
  * This should be called when a PI is added/removed from a watchlist
+ * This provides a single point of reference for all watchlists made in that day
  */
 async function updateWatchlistMembershipRootData(db, logger, piCid, userCid, isPublic, date, action = 'add') {
     try {
@@ -161,8 +162,9 @@ async function updateWatchlistMembershipRootData(db, logger, piCid, userCid, isP
             }
         }
         
-        // Update the data
+        // Update the data - ensure date field is set for rootdata indexer
         const updateData = {
+            date: date, // Ensure date field exists for rootdata indexer
             [piCidStr]: {
                 totalUsers: piMembership.totalUsers,
                 users: piMembership.users,
@@ -182,6 +184,89 @@ async function updateWatchlistMembershipRootData(db, logger, piCid, userCid, isP
     }
 }
 
+/**
+ * Update Popular Investor watchlist data
+ * Tracks watchlist additions per PI in PopularInvestors/{piCid}/watchlistData
+ * This provides a time series of watchlist additions over time per popular investor
+ */
+async function updatePIWatchlistData(db, logger, piCid, userCid, date, action = 'add') {
+    try {
+        const piCidStr = String(piCid);
+        const userCidStr = String(userCid);
+        const piWatchlistRef = db.collection('PopularInvestors').doc(piCidStr).collection('watchlistData').doc('current');
+        
+        const piWatchlistDoc = await piWatchlistRef.get();
+        const existingData = piWatchlistDoc.exists ? piWatchlistDoc.data() : {};
+        
+        // Initialize structure if needed
+        const watchlistData = existingData || {
+            totalUsers: 0,
+            userCids: [],
+            dailyAdditions: {},
+            lastUpdated: FieldValue.serverTimestamp()
+        };
+        
+        if (action === 'add') {
+            // Add user CID if not already present
+            if (!watchlistData.userCids || !Array.isArray(watchlistData.userCids)) {
+                watchlistData.userCids = [];
+            }
+            if (!watchlistData.userCids.includes(userCidStr)) {
+                watchlistData.userCids.push(userCidStr);
+                watchlistData.totalUsers = watchlistData.userCids.length;
+            }
+            
+            // Track daily additions with timestamp
+            if (!watchlistData.dailyAdditions || typeof watchlistData.dailyAdditions !== 'object') {
+                watchlistData.dailyAdditions = {};
+            }
+            
+            // Initialize date entry if needed
+            if (!watchlistData.dailyAdditions[date]) {
+                watchlistData.dailyAdditions[date] = {
+                    count: 0,
+                    userCids: [],
+                    timestamp: FieldValue.serverTimestamp()
+                };
+            }
+            
+            // Add user to this date's additions if not already present
+            const dateEntry = watchlistData.dailyAdditions[date];
+            if (!dateEntry.userCids.includes(userCidStr)) {
+                dateEntry.userCids.push(userCidStr);
+                dateEntry.count = dateEntry.userCids.length;
+                dateEntry.timestamp = FieldValue.serverTimestamp();
+            }
+            
+        } else if (action === 'remove') {
+            // Remove user CID
+            if (watchlistData.userCids && Array.isArray(watchlistData.userCids)) {
+                watchlistData.userCids = watchlistData.userCids.filter(cid => cid !== userCidStr);
+                watchlistData.totalUsers = watchlistData.userCids.length;
+            }
+            
+            // Update daily additions for this date
+            if (watchlistData.dailyAdditions && watchlistData.dailyAdditions[date]) {
+                const dateEntry = watchlistData.dailyAdditions[date];
+                if (dateEntry.userCids && Array.isArray(dateEntry.userCids)) {
+                    dateEntry.userCids = dateEntry.userCids.filter(cid => cid !== userCidStr);
+                    dateEntry.count = dateEntry.userCids.length;
+                    dateEntry.timestamp = FieldValue.serverTimestamp();
+                }
+            }
+        }
+        
+        // Update the document
+        watchlistData.lastUpdated = FieldValue.serverTimestamp();
+        await piWatchlistRef.set(watchlistData, { merge: true });
+        
+        logger.log('INFO', `[updatePIWatchlistData] Updated watchlist data for PI ${piCid} (action: ${action}, user: ${userCid})`);
+    } catch (error) {
+        logger.log('ERROR', `[updatePIWatchlistData] Error updating PI watchlist data for PI ${piCid}:`, error);
+        // Don't throw - this is a non-critical aggregation
+    }
+}
+
 /**
  * Update PI Alert History rootdata collection
  * Records alert triggers in PIAlertHistoryData/{date}
@@ -286,6 +371,7 @@ module.exports = {
     updateRatingsRootData,
     updatePageViewsRootData,
     updateWatchlistMembershipRootData,
+    updatePIWatchlistData,
     updateAlertHistoryRootData,
     updateAllClearAlertHistory
 };

package/functions/generic-api/user-api/helpers/watchlist/watchlist_generation_helpers.js

@@ -272,6 +272,21 @@ async function autoGenerateWatchlist(req, res, dependencies, config) {
             }
         );
         
+        // Update global rootdata collections for computation system
+        if (watchlistItems && watchlistItems.length > 0) {
+            const { updateWatchlistMembershipRootData, updatePIWatchlistData } = require('../rootdata/rootdata_aggregation_helpers');
+            const today = new Date().toISOString().split('T')[0];
+            const isPublic = false; // Auto-generated watchlists are always private
+            
+            for (const item of watchlistItems) {
+                const piCid = String(item.cid);
+                // Update global WatchlistMembershipData/{date} document
+                await updateWatchlistMembershipRootData(db, logger, piCid, userCid, isPublic, today, 'add');
+                // Update PI-centric PopularInvestors/{piCid}/watchlistData
+                await updatePIWatchlistData(db, logger, piCid, userCid, today, 'add');
+            }
+        }
+        
         logger.log('SUCCESS', `[autoGenerateWatchlist] Created auto-generated watchlist ${watchlistId} with ${watchlistItems.length} items for user ${userCid}`);
         
         return res.status(200).json({

package/functions/generic-api/user-api/helpers/watchlist/watchlist_management_helpers.js

@@ -141,22 +141,37 @@ async function createWatchlist(req, res, dependencies, config) {
             watchlistData.dynamicConfig = dynamicConfig;
         }
         
-        const watchlistRef = db.collection(watchlistsCollection)
-            .doc(String(userCid))
-            .collection('lists')
-            .doc(watchlistId);
-        
-        await watchlistRef.set(watchlistData);
+        // Write to new SignedInUsers path with dual-write to legacy path
+        const { writeWithMigration } = require('../core/path_resolution_helpers');
+        await writeWithMigration(
+            db,
+            'signedInUsers',
+            'watchlists',
+            { cid: userCid },
+            watchlistData,
+            {
+                isCollection: true,
+                merge: false,
+                dataType: 'watchlists',
+                config,
+                collectionRegistry: dependencies.collectionRegistry,
+                documentId: watchlistId,
+                dualWrite: true
+            }
+        );
         
         // Update global rootdata collection for computation system if static watchlist has items
         if (type === 'static' && items && items.length > 0) {
-            const { updateWatchlistMembershipRootData } = require('../rootdata/rootdata_aggregation_helpers');
+            const { updateWatchlistMembershipRootData, updatePIWatchlistData } = require('../rootdata/rootdata_aggregation_helpers');
             const today = new Date().toISOString().split('T')[0];
             const isPublic = visibility === 'public';
             
             for (const item of items) {
                 const piCid = String(item.cid);
+                // Update global WatchlistMembershipData/{date} document
                 await updateWatchlistMembershipRootData(db, logger, piCid, userCid, isPublic, today, 'add');
+                // Update PI-centric PopularInvestors/{piCid}/watchlistData
+                await updatePIWatchlistData(db, logger, piCid, userCid, today, 'add');
             }
         }
         
@@ -193,19 +208,28 @@ async function updateWatchlist(req, res, dependencies, config) {
     }
     
     try {
-        const watchlistsCollection = config.watchlistsCollection || 'watchlists';
-        const watchlistRef = db.collection(watchlistsCollection)
-            .doc(String(userCid))
-            .collection('lists')
-            .doc(id);
-        
-        const watchlistDoc = await watchlistRef.get();
+        // Read from new path with migration fallback
+        const { readWithMigration } = require('../core/path_resolution_helpers');
+        const readResult = await readWithMigration(
+            db,
+            'signedInUsers',
+            'watchlists',
+            { cid: userCid },
+            {
+                isCollection: true,
+                dataType: 'watchlists',
+                config,
+                logger,
+                collectionRegistry: dependencies.collectionRegistry,
+                documentId: id
+            }
+        );
         
-        if (!watchlistDoc.exists) {
+        if (!readResult || !readResult.exists) {
             return res.status(404).json({ error: "Watchlist not found" });
         }
         
-        const existingData = watchlistDoc.data();
+        const existingData = readResult.data();
         
         // Verify ownership
         if (existingData.createdBy !== Number(userCid)) {
@@ -262,7 +286,7 @@ async function updateWatchlist(req, res, dependencies, config) {
             
             // Update global rootdata collection for computation system
             // Compare old and new items to find added/removed PIs
-            const { updateWatchlistMembershipRootData } = require('../rootdata/rootdata_aggregation_helpers');
+            const { updateWatchlistMembershipRootData, updatePIWatchlistData } = require('../rootdata/rootdata_aggregation_helpers');
             const oldItems = existingData.items || [];
             const newItems = items || [];
             const oldPiCids = new Set(oldItems.map(item => String(item.cid)));
@@ -274,7 +298,10 @@ async function updateWatchlist(req, res, dependencies, config) {
             for (const newItem of newItems) {
                 const piCid = String(newItem.cid);
                 if (!oldPiCids.has(piCid)) {
+                    // Update global WatchlistMembershipData/{date} document
                     await updateWatchlistMembershipRootData(db, logger, piCid, userCid, isPublic, today, 'add');
+                    // Update PI-centric PopularInvestors/{piCid}/watchlistData
+                    await updatePIWatchlistData(db, logger, piCid, userCid, today, 'add');
                 }
             }
             
@@ -282,7 +309,10 @@ async function updateWatchlist(req, res, dependencies, config) {
             for (const oldItem of oldItems) {
                 const piCid = String(oldItem.cid);
                 if (!newPiCids.has(piCid)) {
+                    // Update global WatchlistMembershipData/{date} document
                     await updateWatchlistMembershipRootData(db, logger, piCid, userCid, isPublic, today, 'remove');
+                    // Update PI-centric PopularInvestors/{piCid}/watchlistData
+                    await updatePIWatchlistData(db, logger, piCid, userCid, today, 'remove');
                 }
             }
         }
@@ -291,16 +321,35 @@ async function updateWatchlist(req, res, dependencies, config) {
             updates.dynamicConfig = dynamicConfig;
         }
         
-        await watchlistRef.update(updates);
+        // Merge updates with existing data
+        const updatedData = { ...existingData, ...updates };
+        
+        // Write to new SignedInUsers path with dual-write to legacy path
+        const { writeWithMigration } = require('../core/path_resolution_helpers');
+        await writeWithMigration(
+            db,
+            'signedInUsers',
+            'watchlists',
+            { cid: userCid },
+            updatedData,
+            {
+                isCollection: true,
+                merge: true,
+                dataType: 'watchlists',
+                config,
+                collectionRegistry: dependencies.collectionRegistry,
+                documentId: id,
+                dualWrite: true
+            }
+        );
         
         logger.log('SUCCESS', `[updateWatchlist] Updated watchlist ${id} for user ${userCid}`);
         
-        const updatedDoc = await watchlistRef.get();
         return res.status(200).json({
             success: true,
             watchlist: {
-                id: updatedDoc.id,
-                ...updatedDoc.data()
+                id: id,
+                ...updatedData
             }
         });
         
@@ -343,8 +392,41 @@ async function deleteWatchlist(req, res, dependencies, config) {
             return res.status(403).json({ error: "You can only delete your own watchlists" });
         }
         
-        // Delete watchlist
-        await watchlistRef.delete();
+        // Update global rootdata collections before deleting
+        // Remove all PIs from global tracking if this is a static watchlist with items
+        if (watchlistData.type === 'static' && watchlistData.items && watchlistData.items.length > 0) {
+            const { updateWatchlistMembershipRootData, updatePIWatchlistData } = require('../rootdata/rootdata_aggregation_helpers');
+            const today = new Date().toISOString().split('T')[0];
+            const isPublic = watchlistData.visibility === 'public';
+            
+            for (const item of watchlistData.items) {
+                const piCid = String(item.cid);
+                // Remove from global WatchlistMembershipData/{date} document
+                await updateWatchlistMembershipRootData(db, logger, piCid, userCid, isPublic, today, 'remove');
+                // Remove from PI-centric PopularInvestors/{piCid}/watchlistData
+                await updatePIWatchlistData(db, logger, piCid, userCid, today, 'remove');
+            }
+        }
+        
+        // Delete watchlist from both new and legacy paths
+        const { writeWithMigration } = require('../core/path_resolution_helpers');
+        // Delete from new path
+        const newPathRef = db.collection('SignedInUsers')
+            .doc(String(userCid))
+            .collection('watchlists')
+            .doc(id);
+        await newPathRef.delete().catch(err => {
+            logger.log('WARN', `[deleteWatchlist] Failed to delete from new path: ${err.message}`);
+        });
+        
+        // Delete from legacy path if it exists
+        const legacyPathRef = db.collection(config.watchlistsCollection || 'watchlists')
+            .doc(String(userCid))
+            .collection('lists')
+            .doc(id);
+        await legacyPathRef.delete().catch(err => {
+            logger.log('WARN', `[deleteWatchlist] Failed to delete from legacy path: ${err.message}`);
+        });
         
         // Remove from public watchlists if it was public
         if (watchlistData.visibility === 'public') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.548",
+  "version": "1.0.549",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [