bulltrackers-module 1.0.501 → 1.0.502

package/functions/generic-api/user-api/helpers/on_demand_fetch_helpers.js

@@ -89,13 +89,10 @@ async function requestPiFetch(req, res, dependencies, config) {
         const requestId = `req_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
         const now = new Date();
         
-        // Store request
-        const requestRef = db.collection('pi_fetch_requests')
-            .doc(String(piCidNum))
-            .collection('requests')
-            .doc(requestId);
+        const { getCollectionPath, writeDual, extractCollectionName } = require('./collection_helpers');
+        const { collectionRegistry } = dependencies;
         
-        await requestRef.set({
+        const requestData = {
             requestId,
             userCid: requestUserCid, // Use effective CID
             actualUserCid: Number(userCid), // Track actual developer CID for audit
@@ -105,35 +102,92 @@ async function requestPiFetch(req, res, dependencies, config) {
             isImpersonating: isImpersonating || false,
             createdAt: FieldValue.serverTimestamp(),
             updatedAt: FieldValue.serverTimestamp()
-        });
+        };
         
-        // Update user rate limit (use effective CID)
-        const userRequestRef = db.collection('pi_fetch_requests')
-            .doc(String(piCidNum))
-            .collection('user_requests')
-            .doc(String(requestUserCid));
+        // Store request in new structure: popularInvestors/{piCid}/fetchRequests/{requestId}
+        if (collectionRegistry) {
+            const newPath = getCollectionPath(collectionRegistry, 'popularInvestors', 'fetchRequests', {
+                piCid: String(piCidNum)
+            }) + `/${requestId}`;
+            
+            const legacyPath = `pi_fetch_requests/${piCidNum}/requests/${requestId}`;
+            
+            await writeDual(db, newPath, legacyPath, requestData);
+        } else {
+            // Fallback to legacy only
+            const requestRef = db.collection('pi_fetch_requests')
+                .doc(String(piCidNum))
+                .collection('requests')
+                .doc(requestId);
+            
+            await requestRef.set(requestData);
+        }
         
-        await userRequestRef.set({
+        // Update user rate limit (use effective CID)
+        const userRequestData = {
             userCid: requestUserCid,
             piCid: piCidNum,
             lastRequestedAt: FieldValue.serverTimestamp(),
             requestCount: FieldValue.increment(1),
             updatedAt: FieldValue.serverTimestamp()
-        }, { merge: true });
+        };
         
-        // Update global rate limit
-        const globalRef = db.collection('pi_fetch_requests')
-            .doc(String(piCidNum))
-            .collection('global')
-            .doc('latest');
+        if (collectionRegistry) {
+            const newUserPath = getCollectionPath(collectionRegistry, 'popularInvestors', 'userFetchRequests', {
+                piCid: String(piCidNum)
+            }) + `/${requestUserCid}`;
+            
+            const legacyUserPath = `pi_fetch_requests/${piCidNum}/user_requests/${requestUserCid}`;
+            
+            await writeDual(db, newUserPath, legacyUserPath, userRequestData, { merge: true });
+        } else {
+            const userRequestRef = db.collection('pi_fetch_requests')
+                .doc(String(piCidNum))
+                .collection('user_requests')
+                .doc(String(requestUserCid));
+            
+            await userRequestRef.set(userRequestData, { merge: true });
+        }
         
-        await globalRef.set({
+        // Update global rate limit
+        const globalData = {
             piCid: piCidNum,
             lastRequestedAt: FieldValue.serverTimestamp(),
             lastRequestedBy: requestUserCid, // Use effective CID
             totalRequests: FieldValue.increment(1),
             updatedAt: FieldValue.serverTimestamp()
-        }, { merge: true });
+        };
+        
+        if (collectionRegistry) {
+            const newGlobalPath = getCollectionPath(collectionRegistry, 'popularInvestors', 'fetchStatus', {
+                piCid: String(piCidNum)
+            });
+            
+            const legacyGlobalPath = `pi_fetch_requests/${piCidNum}/global/latest`;
+            
+            await writeDual(db, newGlobalPath, legacyGlobalPath, globalData, { merge: true });
+        } else {
+            const globalRef = db.collection('pi_fetch_requests')
+                .doc(String(piCidNum))
+                .collection('global')
+                .doc('latest');
+            
+            await globalRef.set(globalData, { merge: true });
+        }
+        
+        // Get request ref for later updates
+        let requestRef;
+        if (collectionRegistry) {
+            const newPath = getCollectionPath(collectionRegistry, 'popularInvestors', 'fetchRequests', {
+                piCid: String(piCidNum)
+            }) + `/${requestId}`;
+            requestRef = db.doc(newPath);
+        } else {
+            requestRef = db.collection('pi_fetch_requests')
+                .doc(String(piCidNum))
+                .collection('requests')
+                .doc(requestId);
+        }
         
         // Publish to task engine - use on-demand topic for API requests
         const topicName = config.taskEngine?.PUBSUB_TOPIC_USER_FETCH_ONDEMAND || 
@@ -268,13 +322,41 @@ async function getPiFetchStatus(req, res, dependencies, config) {
         }
         
         // Data doesn't exist, check for pending requests
-        const requestsRef = db.collection('pi_fetch_requests')
-            .doc(String(piCidNum))
-            .collection('requests')
-            .orderBy('createdAt', 'desc')
-            .limit(1);
+        const { getCollectionPath, extractCollectionName, readWithFallback } = require('./collection_helpers');
+        const { collectionRegistry } = dependencies;
         
-        const requestsSnapshot = await requestsRef.get();
+        let requestsSnapshot = null;
+        
+        // Try new structure first
+        if (collectionRegistry) {
+            try {
+                const newPath = getCollectionPath(collectionRegistry, 'popularInvestors', 'fetchRequests', {
+                    piCid: String(piCidNum)
+                });
+                const collectionName = extractCollectionName(newPath);
+                
+                const requestsRef = db.collection(collectionName)
+                    .doc(String(piCidNum))
+                    .collection('fetchRequests')
+                    .orderBy('createdAt', 'desc')
+                    .limit(1);
+                
+                requestsSnapshot = await requestsRef.get();
+            } catch (newError) {
+                logger.log('WARN', `[getPiFetchStatus] New structure failed, trying legacy: ${newError.message}`);
+            }
+        }
+        
+        // Fallback to legacy structure
+        if (!requestsSnapshot || requestsSnapshot.empty) {
+            const requestsRef = db.collection('pi_fetch_requests')
+                .doc(String(piCidNum))
+                .collection('requests')
+                .orderBy('createdAt', 'desc')
+                .limit(1);
+            
+            requestsSnapshot = await requestsRef.get();
+        }
         
         if (!requestsSnapshot.empty) {
             const latestRequest = requestsSnapshot.docs[0].data();

package/functions/generic-api/user-api/helpers/review_helpers.js

@@ -321,8 +321,33 @@ async function submitReview(req, res, dependencies, config) {
 
         // 3. Check if review already exists (for update) - use effective CID for review ID
         const reviewId = `${effectiveCid}_${piCid}`;
-        const existingReview = await db.collection(reviewsCollection).doc(reviewId).get();
-        const isUpdate = existingReview.exists;
+        
+        const { getCollectionPath, writeDual, readWithFallback } = require('./collection_helpers');
+        const { collectionRegistry } = dependencies;
+        
+        // Check existing review in new or legacy structure
+        let existingReview = null;
+        let isUpdate = false;
+        
+        if (collectionRegistry) {
+            const newPath = getCollectionPath(collectionRegistry, 'popularInvestors', 'reviews', {
+                piCid: String(piCid)
+            }) + `/${reviewId}`;
+            
+            const legacyPath = `${reviewsCollection}/${reviewId}`;
+            
+            const result = await readWithFallback(db, newPath, legacyPath);
+            if (result && result.data) {
+                existingReview = { exists: true, data: () => result.data };
+                isUpdate = true;
+            }
+        }
+        
+        // Fallback to legacy check
+        if (!existingReview) {
+            existingReview = await db.collection(reviewsCollection).doc(reviewId).get();
+            isUpdate = existingReview.exists;
+        }
 
         // 4. Store/Update Review (store effective CID, but track actual CID for audit)
         const reviewData = {
@@ -341,7 +366,18 @@ async function submitReview(req, res, dependencies, config) {
             reviewData.createdAt = FieldValue.serverTimestamp();
         }
 
-        await db.collection(reviewsCollection).doc(reviewId).set(reviewData, { merge: true });
+        // Write to both new and legacy structures
+        if (collectionRegistry) {
+            const newPath = getCollectionPath(collectionRegistry, 'popularInvestors', 'reviews', {
+                piCid: String(piCid)
+            }) + `/${reviewId}`;
+            
+            const legacyPath = `${reviewsCollection}/${reviewId}`;
+            
+            await writeDual(db, newPath, legacyPath, reviewData, { merge: true });
+        } else {
+            await db.collection(reviewsCollection).doc(reviewId).set(reviewData, { merge: true });
+        }
 
         logger.log('INFO', `[Review] User ${userCid} ${isUpdate ? 'updated' : 'submitted'} review for PI ${piCid}`);
         return res.status(200).json({ 
@@ -368,13 +404,39 @@ async function getReviews(req, res, dependencies, config) {
 
     try {
         const piCidNum = Number(piCid);
+        const { getCollectionPath, extractCollectionName } = require('./collection_helpers');
+        const { collectionRegistry } = dependencies;
         
-        // Query reviews for this PI
-        const snapshot = await db.collection(reviewsCollection)
-            .where('piCid', '==', piCidNum)
-            .orderBy('createdAt', 'desc')
-            .limit(100) // Increased limit
-            .get();
+        let snapshot = null;
+        
+        // Try new structure first
+        if (collectionRegistry) {
+            try {
+                const newPath = getCollectionPath(collectionRegistry, 'popularInvestors', 'reviews', {
+                    piCid: String(piCidNum)
+                });
+                const collectionName = extractCollectionName(newPath);
+                
+                const reviewsRef = db.collection(collectionName)
+                    .doc(String(piCidNum))
+                    .collection('reviews')
+                    .orderBy('createdAt', 'desc')
+                    .limit(100);
+                
+                snapshot = await reviewsRef.get();
+            } catch (newError) {
+                logger.log('WARN', `[getReviews] New structure failed, trying legacy: ${newError.message}`);
+            }
+        }
+        
+        // Fallback to legacy structure
+        if (!snapshot || snapshot.empty) {
+            snapshot = await db.collection(reviewsCollection)
+                .where('piCid', '==', piCidNum)
+                .orderBy('createdAt', 'desc')
+                .limit(100)
+                .get();
+        }
 
         const reviews = [];
         let totalStars = 0;
@@ -448,14 +510,36 @@ async function getUserReview(req, res, dependencies, config) {
     }
 
     try {
+        const { getCollectionPath, readWithFallback } = require('./collection_helpers');
+        const { collectionRegistry } = dependencies;
         const reviewId = `${userCid}_${piCid}`;
-        const reviewDoc = await db.collection(reviewsCollection).doc(reviewId).get();
-
-        if (!reviewDoc.exists) {
-            return res.status(200).json({ exists: false, review: null });
+        
+        let reviewData = null;
+        
+        // Try new structure first
+        if (collectionRegistry) {
+            const newPath = getCollectionPath(collectionRegistry, 'popularInvestors', 'reviews', {
+                piCid: String(piCid)
+            }) + `/${reviewId}`;
+            
+            const legacyPath = `${reviewsCollection}/${reviewId}`;
+            
+            const result = await readWithFallback(db, newPath, legacyPath);
+            if (result && result.data) {
+                reviewData = result.data;
+            }
         }
-
-        const data = reviewDoc.data();
+        
+        // Fallback to legacy
+        if (!reviewData) {
+            const reviewDoc = await db.collection(reviewsCollection).doc(reviewId).get();
+            if (!reviewDoc.exists) {
+                return res.status(200).json({ exists: false, review: null });
+            }
+            reviewData = reviewDoc.data();
+        }
+        
+        const data = reviewData;
         const review = {
             id: reviewDoc.id,
             rating: data.rating,

package/functions/generic-api/user-api/helpers/subscription_helpers.js

@@ -67,14 +67,41 @@ async function subscribeToAlerts(req, res, dependencies, config) {
             lastAlertAt: null
         };
         
-        // Store subscription
-        const subscriptionsCollection = config.watchlistSubscriptionsCollection || 'watchlist_subscriptions';
-        const subscriptionRef = db.collection(subscriptionsCollection)
-            .doc(String(userCid))
-            .collection('alerts')
-            .doc(String(piCid));
-        
-        await subscriptionRef.set(subscriptionData, { merge: true });
+        const { getCollectionPath, writeDual } = require('./collection_helpers');
+        const { collectionRegistry } = dependencies;
+        
+        // Determine storage location based on whether watchlistId is present
+        if (watchlistId && collectionRegistry) {
+            // Per-watchlist subscription: signedInUsers/{cid}/watchlists/{watchlistId}/subscriptions/{piCid}
+            const newPath = getCollectionPath(collectionRegistry, 'signedInUsers', 'watchlistSubscriptions', {
+                cid: String(userCid),
+                watchlistId: watchlistId
+            }) + `/${piCid}`;
+            
+            const legacyPath = `watchlist_subscriptions/${userCid}/alerts/${piCid}`;
+            
+            await writeDual(db, newPath, legacyPath, subscriptionData, { merge: true });
+        } else {
+            // Global subscription: signedInUsers/{cid}/subscriptions/{piCid}
+            if (collectionRegistry) {
+                const newPath = getCollectionPath(collectionRegistry, 'signedInUsers', 'subscriptions', {
+                    cid: String(userCid)
+                }) + `/${piCid}`;
+                
+                const legacyPath = `watchlist_subscriptions/${userCid}/alerts/${piCid}`;
+                
+                await writeDual(db, newPath, legacyPath, subscriptionData, { merge: true });
+            } else {
+                // Fallback to legacy only
+                const subscriptionsCollection = config.watchlistSubscriptionsCollection || 'watchlist_subscriptions';
+                const subscriptionRef = db.collection(subscriptionsCollection)
+                    .doc(String(userCid))
+                    .collection('alerts')
+                    .doc(String(piCid));
+                
+                await subscriptionRef.set(subscriptionData, { merge: true });
+            }
+        }
         
         logger.log('SUCCESS', `[subscribeToAlerts] User ${userCid} subscribed to alerts for PI ${piCid} in watchlist ${watchlistId}`);
         
@@ -104,44 +131,72 @@ async function updateSubscription(req, res, dependencies, config) {
     }
     
     try {
-        const subscriptionsCollection = config.watchlistSubscriptionsCollection || 'watchlist_subscriptions';
-        const subscriptionRef = db.collection(subscriptionsCollection)
-            .doc(String(userCid))
-            .collection('alerts')
-            .doc(String(piCid));
-        
-        const subscriptionDoc = await subscriptionRef.get();
+        const { getCollectionPath, readWithFallback, writeDual } = require('./collection_helpers');
+        const { collectionRegistry } = dependencies;
+        
+        // Try to find subscription in new structure first, then legacy
+        let subscriptionData = null;
+        let subscriptionRef = null;
+        let newPath = null;
+        let legacyPath = `watchlist_subscriptions/${userCid}/alerts/${piCid}`;
+        
+        // Check global subscriptions first
+        if (collectionRegistry) {
+            newPath = getCollectionPath(collectionRegistry, 'signedInUsers', 'subscriptions', {
+                cid: String(userCid)
+            }) + `/${piCid}`;
+            
+            const result = await readWithFallback(db, newPath, legacyPath);
+            if (result && result.data) {
+                subscriptionData = result.data;
+            }
+        }
         
-        if (!subscriptionDoc.exists) {
-            return res.status(404).json({ error: "Subscription not found" });
+        // Fallback to legacy if not found
+        if (!subscriptionData) {
+            const subscriptionsCollection = config.watchlistSubscriptionsCollection || 'watchlist_subscriptions';
+            subscriptionRef = db.collection(subscriptionsCollection)
+                .doc(String(userCid))
+                .collection('alerts')
+                .doc(String(piCid));
+            
+            const subscriptionDoc = await subscriptionRef.get();
+            if (!subscriptionDoc.exists) {
+                return res.status(404).json({ error: "Subscription not found" });
+            }
+            subscriptionData = subscriptionDoc.data();
         }
         
         const updates = {};
-        
         if (alertTypes !== undefined) {
             updates.alertTypes = alertTypes;
         }
-        
         if (thresholds !== undefined) {
             updates.thresholds = thresholds;
         }
-        
         if (Object.keys(updates).length === 0) {
             return res.status(400).json({ error: "No updates provided" });
         }
         
         updates.updatedAt = FieldValue.serverTimestamp();
         
-        await subscriptionRef.update(updates);
+        // Update both structures if using new path, otherwise just legacy
+        if (newPath && collectionRegistry) {
+            const updatedData = { ...subscriptionData, ...updates };
+            await writeDual(db, newPath, legacyPath, updatedData, { merge: true });
+        } else if (subscriptionRef) {
+            await subscriptionRef.update(updates);
+        }
         
         logger.log('SUCCESS', `[updateSubscription] Updated subscription for user ${userCid}, PI ${piCid}`);
         
-        const updatedDoc = await subscriptionRef.get();
+        // Return updated data
+        const finalData = { ...subscriptionData, ...updates };
         return res.status(200).json({
             success: true,
             subscription: {
-                id: updatedDoc.id,
-                ...updatedDoc.data()
+                piCid: Number(piCid),
+                ...finalData
             }
         });
         
@@ -165,6 +220,54 @@ async function unsubscribeFromAlerts(req, res, dependencies, config) {
     }
     
     try {
+        const { getCollectionPath, readWithFallback } = require('./collection_helpers');
+        const { collectionRegistry } = dependencies;
+        
+        const legacyPath = `watchlist_subscriptions/${userCid}/alerts/${piCid}`;
+        let newPath = null;
+        let found = false;
+        
+        // Check global subscriptions first
+        if (collectionRegistry) {
+            newPath = getCollectionPath(collectionRegistry, 'signedInUsers', 'subscriptions', {
+                cid: String(userCid)
+            }) + `/${piCid}`;
+            
+            const newDoc = await db.doc(newPath).get();
+            if (newDoc.exists) {
+                await db.doc(newPath).delete();
+                found = true;
+            }
+        }
+        
+        // Also check per-watchlist subscriptions (need to search all watchlists)
+        if (collectionRegistry && !found) {
+            const { getCollectionPath } = require('./collection_helpers');
+            const watchlistsPath = getCollectionPath(collectionRegistry, 'signedInUsers', 'watchlists', {
+                cid: String(userCid)
+            });
+            const watchlistsRef = db.collection(watchlistsPath.split('/')[0])
+                .doc(String(userCid))
+                .collection('watchlists');
+            
+            const watchlistsSnapshot = await watchlistsRef.get();
+            for (const watchlistDoc of watchlistsSnapshot.docs) {
+                const watchlistId = watchlistDoc.id;
+                const watchlistSubPath = getCollectionPath(collectionRegistry, 'signedInUsers', 'watchlistSubscriptions', {
+                    cid: String(userCid),
+                    watchlistId: watchlistId
+                }) + `/${piCid}`;
+                
+                const subDoc = await db.doc(watchlistSubPath).get();
+                if (subDoc.exists) {
+                    await db.doc(watchlistSubPath).delete();
+                    found = true;
+                    break;
+                }
+            }
+        }
+        
+        // Delete from legacy structure
         const subscriptionsCollection = config.watchlistSubscriptionsCollection || 'watchlist_subscriptions';
         const subscriptionRef = db.collection(subscriptionsCollection)
             .doc(String(userCid))
@@ -172,13 +275,15 @@ async function unsubscribeFromAlerts(req, res, dependencies, config) {
             .doc(String(piCid));
         
         const subscriptionDoc = await subscriptionRef.get();
+        if (subscriptionDoc.exists) {
+            await subscriptionRef.delete();
+            found = true;
+        }
         
-        if (!subscriptionDoc.exists) {
+        if (!found) {
             return res.status(404).json({ error: "Subscription not found" });
         }
         
-        await subscriptionRef.delete();
-        
         logger.log('SUCCESS', `[unsubscribeFromAlerts] User ${userCid} unsubscribed from alerts for PI ${piCid}`);
         
         return res.status(200).json({
@@ -205,19 +310,99 @@ async function getUserSubscriptions(req, res, dependencies, config) {
     }
     
     try {
+        const { getCollectionPath, extractCollectionName } = require('./collection_helpers');
+        const { collectionRegistry } = dependencies;
+        
+        const subscriptions = [];
+        const seenPiCids = new Set();
+        
+        // 1. Read from global subscriptions (new structure)
+        if (collectionRegistry) {
+            try {
+                const globalSubsPath = getCollectionPath(collectionRegistry, 'signedInUsers', 'subscriptions', {
+                    cid: String(userCid)
+                });
+                const collectionName = extractCollectionName(globalSubsPath);
+                
+                const globalSubsRef = db.collection(collectionName)
+                    .doc(String(userCid))
+                    .collection('subscriptions');
+                
+                const globalSnapshot = await globalSubsRef.get();
+                globalSnapshot.forEach(doc => {
+                    const data = doc.data();
+                    subscriptions.push({
+                        piCid: Number(doc.id),
+                        ...data
+                    });
+                    seenPiCids.add(Number(doc.id));
+                });
+            } catch (newError) {
+                logger.log('WARN', `[getUserSubscriptions] Failed to read global subscriptions from new structure: ${newError.message}`);
+            }
+            
+            // 2. Read from per-watchlist subscriptions
+            try {
+                const watchlistsPath = getCollectionPath(collectionRegistry, 'signedInUsers', 'watchlists', {
+                    cid: String(userCid)
+                });
+                const watchlistsCollectionName = extractCollectionName(watchlistsPath);
+                
+                const watchlistsRef = db.collection(watchlistsCollectionName)
+                    .doc(String(userCid))
+                    .collection('watchlists');
+                
+                const watchlistsSnapshot = await watchlistsRef.get();
+                
+                for (const watchlistDoc of watchlistsSnapshot.docs) {
+                    const watchlistId = watchlistDoc.id;
+                    const watchlistSubsPath = getCollectionPath(collectionRegistry, 'signedInUsers', 'watchlistSubscriptions', {
+                        cid: String(userCid),
+                        watchlistId: watchlistId
+                    });
+                    const watchlistSubsCollectionName = extractCollectionName(watchlistSubsPath);
+                    
+                    const watchlistSubsRef = db.collection(watchlistSubsCollectionName)
+                        .doc(String(userCid))
+                        .collection('watchlists')
+                        .doc(watchlistId)
+                        .collection('subscriptions');
+                    
+                    const watchlistSubsSnapshot = await watchlistSubsRef.get();
+                    watchlistSubsSnapshot.forEach(doc => {
+                        const piCid = Number(doc.id);
+                        if (!seenPiCids.has(piCid)) {
+                            const data = doc.data();
+                            subscriptions.push({
+                                piCid,
+                                watchlistId,
+                                ...data
+                            });
+                            seenPiCids.add(piCid);
+                        }
+                    });
+                }
+            } catch (watchlistError) {
+                logger.log('WARN', `[getUserSubscriptions] Failed to read watchlist subscriptions: ${watchlistError.message}`);
+            }
+        }
+        
+        // 3. Fallback to legacy structure for any missing subscriptions
         const subscriptionsCollection = config.watchlistSubscriptionsCollection || 'watchlist_subscriptions';
         const subscriptionsRef = db.collection(subscriptionsCollection)
             .doc(String(userCid))
             .collection('alerts');
         
-        const snapshot = await subscriptionsRef.get();
-        
-        const subscriptions = [];
-        snapshot.forEach(doc => {
-            subscriptions.push({
-                piCid: Number(doc.id),
-                ...doc.data()
-            });
+        const legacySnapshot = await subscriptionsRef.get();
+        legacySnapshot.forEach(doc => {
+            const piCid = Number(doc.id);
+            if (!seenPiCids.has(piCid)) {
+                subscriptions.push({
+                    piCid,
+                    ...doc.data()
+                });
+                seenPiCids.add(piCid);
+            }
         });
         
         return res.status(200).json({