bulltrackers-module 1.0.621 → 1.0.623

package/functions/api-v2/helpers/data-fetchers/firestore.js

@@ -1,6 +1,7 @@
 // Firestore helper functions for fetching data from collections
-const { FieldValue } = require('@google-cloud/firestore');
+const { FieldValue, Timestamp } = require('@google-cloud/firestore');
 const { dispatchSyncRequest } = require('../task_engine_helper.js');
+const { sanitizeCid, sanitizeDocId } = require('../security_utils.js');
 const crypto = require('crypto');
 
 // 1. Fetch latest stored snapshots of user data from a user-centric collection
@@ -34,11 +35,14 @@ const crypto = require('crypto');
  */
 const latestUserCentricSnapshot = async (firestore, userId, collectionName, dataType, userType, documentName, fields = []) => {
     try {
-        const baseRef = firestore.collection(userType).doc(userId).collection(collectionName);
+        // Sanitize user inputs to prevent injection attacks
+        const sanitizedUserId = sanitizeCid(userId);
+        const sanitizedDocName = documentName ? sanitizeDocId(documentName) : null;
+        const baseRef = firestore.collection(userType).doc(sanitizedUserId).collection(collectionName);
 
-        if (documentName) {
+        if (sanitizedDocName) {
             // SCENARIO 1: Single Document
-            let docRef = baseRef.doc(documentName);
+            let docRef = baseRef.doc(sanitizedDocName);
             // Apply field selection if fields are specified (Firestore projection)
             if (fields.length > 0) {
                 docRef = docRef.select(...fields);
@@ -85,14 +89,14 @@ const pageCollection = async (firestore, dateStr, computationName, userId, lookb
             const docRef = firestore.collection('unified_insights').doc(dateKey)    
                 .collection('results').doc('popular-investor')
                 .collection('computations').doc(computationName)
-                .collection('pages').doc(userId);
+                .collection('pages').doc(sanitizedUserId);
             const docSnapshot = await docRef.get();
             if (docSnapshot.exists) {
                 results.push({ date: dateKey, data: docSnapshot.data() });
             }   
         }
         if (results.length === 0) {
-            throw new Error(`No page data found for User ID ${userId} in computation ${computationName} within the last ${lookbackDays} days`);
+            throw new Error(`No page data found for User ID ${sanitizedUserId} in computation ${sanitizedComputationName} within the last ${lookbackDays} days`);
         }
         // Sort results by date descending (newest first) so the latest data is always first
         results.sort((a, b) => {
@@ -370,9 +374,12 @@ const manageUserWatchlist = async (db, userId, instruction, payload = {}) => {
     const todayStr = new Date().toISOString().split('T')[0]; // "2026-01-09"
     const batch = db.batch();
     
+    // Sanitize user inputs
+    const sanitizedUserId = sanitizeCid(userId);
+    
     // 1. Determine Watchlist ID and Reference
-    const watchlistId = payload.id || `watchlist_${Date.now()}_${Math.random().toString(16).substr(2, 8)}`;
-    const userDocRef = db.collection('SignedInUsers').doc(userId).collection('watchlists').doc(watchlistId);
+    const watchlistId = payload.id ? sanitizeDocId(payload.id) : `watchlist_${Date.now()}_${Math.random().toString(16).substr(2, 8)}`;
+    const userDocRef = db.collection('SignedInUsers').doc(sanitizedUserId).collection('watchlists').doc(watchlistId);
 
     try {
         let addedItems = [];   // Items newly added (needs Master Log + Current Counter Increment)
@@ -397,7 +404,7 @@ const manageUserWatchlist = async (db, userId, instruction, payload = {}) => {
             const newDocData = {
                 ...payload,
                 id: watchlistId,
-                createdBy: userId,
+                createdBy: sanitizedUserId,
                 createdAt: FieldValue.serverTimestamp(),
                 updatedAt: FieldValue.serverTimestamp(),
                 copyCount: 0,
@@ -447,7 +454,7 @@ const manageUserWatchlist = async (db, userId, instruction, payload = {}) => {
                 totalUsers: FieldValue.increment(1),
                 privateWatchlistCount: FieldValue.increment(isPrivate ? 1 : 0),
                 publicWatchlistCount: FieldValue.increment(isPrivate ? 0 : 1),
-                users: FieldValue.arrayUnion(userId)
+                users: FieldValue.arrayUnion(sanitizedUserId)
             }, { merge: true });
 
             // B. Update PI Real-time Counter (Increment)
@@ -459,12 +466,12 @@ const manageUserWatchlist = async (db, userId, instruction, payload = {}) => {
             batch.set(piCounterRef, {
                 lastUpdated: FieldValue.serverTimestamp(),
                 totalUsers: FieldValue.increment(1),
-                userCids: FieldValue.arrayUnion(userId),
+                userCids: FieldValue.arrayUnion(sanitizedUserId),
                 // Nested Map Update for Daily stats
                 [dailyField]: {
                     count: FieldValue.increment(1),
                     timestamp: FieldValue.serverTimestamp(),
-                    userCids: FieldValue.arrayUnion(userId)
+                    userCids: FieldValue.arrayUnion(sanitizedUserId)
                 }
             }, { merge: true });
         });
@@ -618,10 +625,20 @@ const lookupCidByEmail = async (firestore, userEmail, firebaseUid = null) => {
                 const normalizedEmails = emails.map(e => String(e).toLowerCase().trim());
                 
                 if (normalizedEmails.includes(normalizedEmail)) {
-                    // Optional: Validate Firebase UID if provided
-                    if (firebaseUid && verificationData.firebaseUids && Array.isArray(verificationData.firebaseUids)) {
-                        if (!verificationData.firebaseUids.includes(firebaseUid)) {
-                            console.warn(`[lookupCidByEmail] Email ${userEmail} matches CID ${cid} but Firebase UID ${firebaseUid} not in firebaseUids array`);
+                    // Validate and enforce Firebase UID if provided
+                    if (firebaseUid) {
+                        if (verificationData.firebaseUids && Array.isArray(verificationData.firebaseUids)) {
+                            // New users: strict enforcement
+                            if (!verificationData.firebaseUids.includes(firebaseUid)) {
+                                throw new Error(`Firebase UID not authorized for this CID`);
+                            }
+                        } else {
+                            // Legacy users: populate firebaseUids array (migration)
+                            // Note: A single CID can have multiple emails/UIDs for different accounts
+                            await verificationRef.update({
+                                firebaseUids: FieldValue.arrayUnion(firebaseUid)
+                            });
+                            console.log(`[Migration] Added Firebase UID ${firebaseUid} to CID ${cid}`);
                         }
                     }
                     
@@ -666,10 +683,20 @@ const lookupCidByEmail = async (firestore, userEmail, firebaseUid = null) => {
                 if (normalizedEmails.includes(normalizedEmail)) {
                     const cid = verificationData.etoroCID || Number(userDoc.id);
                     
-                    // Optional: Validate Firebase UID if provided
-                    if (firebaseUid && verificationData.firebaseUids && Array.isArray(verificationData.firebaseUids)) {
-                        if (!verificationData.firebaseUids.includes(firebaseUid)) {
-                            console.warn(`[lookupCidByEmail] Email ${userEmail} matches CID ${cid} but Firebase UID ${firebaseUid} not in firebaseUids array`);
+                    // Validate and enforce Firebase UID if provided
+                    if (firebaseUid) {
+                        if (verificationData.firebaseUids && Array.isArray(verificationData.firebaseUids)) {
+                            // New users: strict enforcement
+                            if (!verificationData.firebaseUids.includes(firebaseUid)) {
+                                throw new Error(`Firebase UID not authorized for this CID`);
+                            }
+                        } else {
+                            // Legacy users: populate firebaseUids array (migration)
+                            // Note: A single CID can have multiple emails/UIDs for different accounts
+                            await verificationRef.update({
+                                firebaseUids: FieldValue.arrayUnion(firebaseUid)
+                            });
+                            console.log(`[Migration] Added Firebase UID ${firebaseUid} to CID ${cid}`);
                         }
                     }
                     
@@ -734,8 +761,7 @@ const requestPopularInvestorAddition = async (firestore, userId, piId, piUsernam
             piUsername,
             requestedBy: userId,
             requestedAt: FieldValue.serverTimestamp(),
-            ttl: admin.firestore.Timestamp.fromDate(ttlDate)
-
+            ttl: Timestamp.fromDate(ttlDate)
         });
         const globalRequestRef = firestore.collection('PiAdditionRequestLogs').doc(todayStr).collection('requests').doc(piId);
         await globalRequestRef.set({
@@ -933,14 +959,18 @@ const hasUserCopiedPopularInvestor = async (firestore, userId, popularInvestorId
 // 3. A GLOBAL REVIEWS COLLECTION LIKE /PiReviews/YYYY-MM-DD/document_name containing a map of the reviews made each day, who made them, for which PI and text content
 // 4. We need a migration strategy to pull across the existing stored data from the old collection path into the new collection path upon read.
 const manageReviews = async (db, userId, action, params = {}) => {
+    // Sanitize user inputs
+    const sanitizedUserId = sanitizeCid(userId);
     const { piId } = params;
+    const sanitizedPiId = sanitizeCid(piId);
+    
     const batch = db.batch();
     const todayStr = new Date().toISOString().split('T')[0];
     
-    // Paths
-    const newReviewPath = db.collection('SignedInUsers').doc(userId).collection('reviews').doc(piId.toString());
-    const piReviewPath = db.collection('PopularInvestors').doc(piId.toString()).collection('reviews').doc(`${userId}_${piId}`);
-    const legacyReviewPath = db.collection('pi_reviews').doc(`${userId}_${piId}`);
+    // Paths - all user inputs sanitized
+    const newReviewPath = db.collection('SignedInUsers').doc(sanitizedUserId).collection('reviews').doc(sanitizedPiId);
+    const piReviewPath = db.collection('PopularInvestors').doc(sanitizedPiId).collection('reviews').doc(`${sanitizedUserId}_${sanitizedPiId}`);
+    const legacyReviewPath = db.collection('pi_reviews').doc(`${sanitizedUserId}_${sanitizedPiId}`);
     const globalLogPath = db.collection('PiReviews').doc(todayStr).collection('shards').doc('daily_log');
 
     try {
@@ -2048,9 +2078,87 @@ const searchPopularInvestors = async (db, queryStr) => {
 // NEW: SYNC & RATE LIMITS (Rec 7, 9, 15)
 // ==========================================
 
-const checkSyncRateLimits = async (db, targetId, requesterId, isDev) => {
-    if (isDev) return { allowed: true };
+const checkSyncRateLimits = async (db, targetId, requesterId, isDev, maxRequests = 1, windowMs = 6 * 60 * 60 * 1000) => {
+    // Developers get higher limits (100 requests per hour globally, 10 per target) but still limited
+    if (isDev) {
+        const DEV_GLOBAL_LIMIT_MS = 60 * 60 * 1000; // 1 hour
+        const DEV_GLOBAL_MAX_REQUESTS = 100; // Total requests per hour
+        const DEV_PER_TARGET_LIMIT_MS = 60 * 60 * 1000; // 1 hour
+        const DEV_PER_TARGET_MAX_REQUESTS = 10; // Requests per target per hour
+        const now = Date.now();
+        
+        // Check global developer limit (total requests across all targets)
+        const globalDevRef = db.collection('developer_rate_limits').doc(String(requesterId)).collection('global').doc('latest');
+        const globalDevSnap = await globalDevRef.get();
+        
+        let globalRequestCount = 0;
+        let globalWindowStart = now;
+        
+        if (globalDevSnap.exists) {
+            const globalData = globalDevSnap.data();
+            globalRequestCount = globalData.requestCount || 0;
+            globalWindowStart = globalData.windowStart?.toMillis() || now;
+            
+            // Reset window if expired
+            if (now - globalWindowStart >= DEV_GLOBAL_LIMIT_MS) {
+                globalRequestCount = 0;
+                globalWindowStart = now;
+            }
+        }
+        
+        // Check per-target limit (prevent spamming specific users)
+        const sanitizedTargetId = sanitizeCid(targetId);
+        const perTargetRef = db.collection('user_sync_requests').doc(sanitizedTargetId).collection('developer_limits').doc(String(requesterId));
+        const perTargetSnap = await perTargetRef.get();
+        
+        let perTargetCount = 0;
+        let perTargetWindowStart = now;
+        
+        if (perTargetSnap.exists) {
+            const perTargetData = perTargetSnap.data();
+            perTargetCount = perTargetData.requestCount || 0;
+            perTargetWindowStart = perTargetData.windowStart?.toMillis() || now;
+            
+            // Reset window if expired
+            if (now - perTargetWindowStart >= DEV_PER_TARGET_LIMIT_MS) {
+                perTargetCount = 0;
+                perTargetWindowStart = now;
+            }
+            
+            // Check per-target limit
+            if (perTargetCount >= DEV_PER_TARGET_MAX_REQUESTS) {
+                const waitMinutes = Math.ceil((perTargetWindowStart + DEV_PER_TARGET_LIMIT_MS - now) / 60000);
+                return {
+                    allowed: false,
+                    message: `Developer rate limit exceeded for this target. Try again in ${waitMinutes} minutes.`
+                };
+            }
+        }
+        
+        // Check global limit
+        if (globalRequestCount >= DEV_GLOBAL_MAX_REQUESTS) {
+            const waitMinutes = Math.ceil((globalWindowStart + DEV_GLOBAL_LIMIT_MS - now) / 60000);
+            return {
+                allowed: false,
+                message: `Developer global rate limit exceeded. Try again in ${waitMinutes} minutes.`
+            };
+        }
+        
+        // Update both counters
+        await globalDevRef.set({
+            requestCount: globalRequestCount + 1,
+            windowStart: Timestamp.fromMillis(globalWindowStart)
+        }, { merge: true });
+        
+        await perTargetRef.set({
+            requestCount: perTargetCount + 1,
+            windowStart: Timestamp.fromMillis(perTargetWindowStart)
+        }, { merge: true });
+        
+        return { allowed: true };
+    }
 
+    // Regular users: 1 request per 6 hours
     const RATE_LIMIT_MS = 6 * 60 * 60 * 1000; // 6 hours
     const now = Date.now();
     
@@ -2681,33 +2789,42 @@ const getWatchlistTriggerCounts = async (db, userId, watchlistId) => {
             const piCid = String(item.cid);
             let totalCount = 0;
             
-            // Check each of the last 7 days
-            for (const dateStr of dates) {
-                try {
-                    const dayDoc = await db.collection('PIAlertHistoryData')
+            // Check each of the last 7 days - PARALLELIZED to fix N+1 query problem
+            const dayDocs = await Promise.all(
+                dates.map(dateStr => 
+                    db.collection('PIAlertHistoryData')
                         .doc(dateStr)
-                        .get();
+                        .get()
+                        .catch(e => {
+                            console.warn(`Error fetching date ${dateStr} for PI ${piCid}: ${e.message}`);
+                            return null; // Return null on error instead of throwing
+                        })
+                )
+            );
+            
+            // Process all day documents
+            for (const dayDoc of dayDocs) {
+                if (!dayDoc || !dayDoc.exists) continue;
+                
+                try {
+                    const dayData = dayDoc.data();
+                    const piData = dayData[piCid];
                     
-                    if (dayDoc.exists) {
-                        const dayData = dayDoc.data();
-                        const piData = dayData[piCid];
-                        
-                        if (piData) {
-                            // Check all alert types for this PI
-                            for (const [alertType, alertData] of Object.entries(piData)) {
-                                if (alertType === 'lastUpdated' || typeof alertData !== 'object') continue;
-                                
-                                // Check if triggeredFor array contains this user
-                                const triggeredFor = alertData.triggeredFor || [];
-                                if (Array.isArray(triggeredFor) && triggeredFor.includes(userCidStr)) {
-                                    totalCount += alertData.count || 1; // Count each trigger
-                                }
+                    if (piData) {
+                        // Check all alert types for this PI
+                        for (const [alertType, alertData] of Object.entries(piData)) {
+                            if (alertType === 'lastUpdated' || typeof alertData !== 'object') continue;
+                            
+                            // Check if triggeredFor array contains this user
+                            const triggeredFor = alertData.triggeredFor || [];
+                            if (Array.isArray(triggeredFor) && triggeredFor.includes(userCidStr)) {
+                                totalCount += alertData.count || 1; // Count each trigger
                             }
                         }
                     }
                 } catch (e) {
-                    // Skip this day if error (document might not exist)
-                    console.warn(`Error checking date ${dateStr} for PI ${piCid}: ${e.message}`);
+                    // Skip this day if error processing
+                    console.warn(`Error processing day data for PI ${piCid}: ${e.message}`);
                 }
             }
             

package/functions/api-v2/helpers/security_utils.js

@@ -0,0 +1,110 @@
+/**
+ * Security utility functions for input validation and sanitization
+ */
+
+/**
+ * Sanitizes a CID (Customer ID) to prevent injection attacks
+ * @param {string|number} cid - The CID to sanitize
+ * @returns {string} - Sanitized CID
+ * @throws {Error} - If CID is invalid
+ */
+function sanitizeCid(cid) {
+    if (cid === null || cid === undefined) {
+        throw new Error('CID is required');
+    }
+    
+    // Convert to string and remove all non-numeric characters
+    const sanitized = String(cid).replace(/[^0-9]/g, '');
+    
+    // Validate length and format
+    if (!sanitized || sanitized.length === 0) {
+        throw new Error('Invalid CID format: must contain at least one digit');
+    }
+    
+    if (sanitized.length > 20) {
+        throw new Error('Invalid CID format: too long (max 20 digits)');
+    }
+    
+    return sanitized;
+}
+
+/**
+ * Sanitizes a document ID to prevent path traversal attacks
+ * @param {string} docId - The document ID to sanitize
+ * @returns {string} - Sanitized document ID
+ * @throws {Error} - If document ID is invalid
+ */
+function sanitizeDocId(docId) {
+    if (!docId || typeof docId !== 'string') {
+        throw new Error('Document ID is required and must be a string');
+    }
+    
+    // Remove path traversal attempts and other dangerous characters
+    const sanitized = docId
+        .replace(/\.\./g, '') // Remove .. (path traversal)
+        .replace(/\//g, '') // Remove / (path separator)
+        .replace(/\\/g, '') // Remove \ (path separator)
+        .trim();
+    
+    if (!sanitized || sanitized.length === 0) {
+        throw new Error('Invalid document ID: empty after sanitization');
+    }
+    
+    if (sanitized.length > 200) {
+        throw new Error('Invalid document ID: too long (max 200 characters)');
+    }
+    
+    return sanitized;
+}
+
+/**
+ * Validates and sanitizes a username
+ * @param {string} username - The username to validate
+ * @returns {string} - Sanitized username
+ * @throws {Error} - If username is invalid
+ */
+function sanitizeUsername(username) {
+    if (!username || typeof username !== 'string') {
+        throw new Error('Username is required and must be a string');
+    }
+    
+    const sanitized = username.trim();
+    
+    if (sanitized.length === 0) {
+        throw new Error('Username cannot be empty');
+    }
+    
+    if (sanitized.length > 100) {
+        throw new Error('Username too long (max 100 characters)');
+    }
+    
+    // Allow alphanumeric, underscore, hyphen, and dot
+    if (!/^[a-zA-Z0-9._-]+$/.test(sanitized)) {
+        throw new Error('Username contains invalid characters');
+    }
+    
+    return sanitized;
+}
+
+/**
+ * Validates batch operation size
+ * @param {number} size - The size of the batch operation
+ * @param {number} maxSize - Maximum allowed size (default: 1000)
+ * @throws {Error} - If batch size exceeds limit
+ */
+function validateBatchSize(size, maxSize = 1000) {
+    if (typeof size !== 'number' || size < 0) {
+        throw new Error('Batch size must be a non-negative number');
+    }
+    
+    if (size > maxSize) {
+        throw new Error(`Batch operation too large. Maximum ${maxSize} operations allowed, got ${size}`);
+    }
+}
+
+module.exports = {
+    sanitizeCid,
+    sanitizeDocId,
+    sanitizeUsername,
+    validateBatchSize
+};

package/functions/api-v2/helpers/timeout_utils.js

@@ -0,0 +1,42 @@
+/**
+ * Utility functions for request timeout protection
+ */
+
+/**
+ * Wraps a promise with a timeout
+ * @param {Promise} promise - The promise to wrap
+ * @param {number} ms - Timeout in milliseconds (default: 10000)
+ * @param {string} errorMessage - Custom error message (optional)
+ * @returns {Promise} - Promise that rejects on timeout
+ */
+function withTimeout(promise, ms = 10000, errorMessage = 'Request timeout') {
+    return Promise.race([
+        promise,
+        new Promise((_, reject) => 
+            setTimeout(() => reject(new Error(`${errorMessage} (${ms}ms)`)), ms)
+        )
+    ]);
+}
+
+/**
+ * Creates a timeout wrapper with a specific duration
+ * @param {number} ms - Timeout in milliseconds
+ * @returns {Function} - Function that wraps promises with the specified timeout
+ */
+function createTimeoutWrapper(ms) {
+    return (promise, errorMessage) => withTimeout(promise, ms, errorMessage);
+}
+
+// Pre-configured timeout wrappers for common use cases
+const timeouts = {
+    short: createTimeoutWrapper(5000),   // 5 seconds
+    medium: createTimeoutWrapper(15000),  // 15 seconds
+    long: createTimeoutWrapper(30000),    // 30 seconds
+    veryLong: createTimeoutWrapper(60000) // 60 seconds
+};
+
+module.exports = {
+    withTimeout,
+    createTimeoutWrapper,
+    timeouts
+};

package/functions/api-v2/index.js

@@ -6,6 +6,7 @@
 const express = require('express');
 const cors = require('cors');
 const rateLimit = require('express-rate-limit');
+const { v4: uuidv4 } = require('uuid');
 const createRouter = require('./routes/index.js');
 const errorHandler = require('./middleware/error_handler.js');
 
@@ -27,6 +28,10 @@ function createApiV2App(config, dependencies) {
     const app = express();
     const { logger } = dependencies;
     
+    // Trust proxy - Required when behind a load balancer/proxy (e.g., Google Cloud Functions)
+    // This allows express-rate-limit to correctly identify client IPs from X-Forwarded-For headers
+    app.set('trust proxy', true);
+    
     // CORS Configuration - Restrict to specific origins
     app.use(cors({
         origin: function (origin, callback) {
@@ -66,6 +71,13 @@ function createApiV2App(config, dependencies) {
     app.use('/verification', authLimiter);
     app.use('/', apiLimiter);
     
+    // Request ID tracking middleware (for debugging and error tracking)
+    app.use((req, res, next) => {
+        req.requestId = uuidv4();
+        res.setHeader('X-Request-ID', req.requestId);
+        next();
+    });
+    
     app.use(express.json());
     
     // Health Check

package/functions/api-v2/middleware/error_handler.js

@@ -6,6 +6,7 @@
 module.exports = (err, req, res, next) => {
     // Log full error internally for debugging
     console.error('[Error Handler]', {
+        requestId: req.requestId,
         error: err.message,
         stack: err.stack,
         path: req.path,

package/functions/api-v2/middleware/identity_middleware.js

@@ -113,7 +113,8 @@ const resolveUserIdentity = async (req, res, next) => {
         // SECURITY: For private routes, require Firebase Auth to prevent IDOR attacks
         if (!isPublic && !authenticatedUserCid && !hasFirebaseAuth) {
             // Private route without authentication - reject immediately
-            console.warn(`[Identity] Security violation: Private route ${req.path} accessed without Firebase Auth`);
+            // This is expected behavior - the security system is working correctly
+            console.log(`[Identity] Rejected unauthorized access to private route ${req.path} (no Firebase Auth token provided)`);
             return res.status(401).json({ 
                 error: "Authentication required. Please provide a valid Firebase ID token in the Authorization header." 
             });