bulltrackers-module 1.0.592 → 1.0.593

package/functions/old-generic-api/user-api/helpers/fetch/on_demand_fetch_helpers.js

@@ -0,0 +1,615 @@
+/**
+ * @fileoverview On-Demand Popular Investor Data Fetching
+ * Allows users to request missing PI data with rate limiting
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+const crypto = require('crypto');
+const { tryDecompress } = require('../data_helpers');
+
+const RATE_LIMIT_HOURS = 1;
+const RATE_LIMIT_MS = RATE_LIMIT_HOURS * 60 * 60 * 1000;
+
+/**
+ * Request on-demand fetch for a Popular Investor
+ * POST /user/pi/:piCid/request-fetch
+ */
+async function requestPiFetch(req, res, dependencies, config) {
+    const { db, logger, pubsub } = dependencies;
+    const { piCid } = req.params;
+    
+    // Get userCid from query (same pattern as other endpoints)
+    const userCid = req.query?.userCid;
+    
+    if (!userCid) {
+        return res.status(400).json({ 
+            success: false,
+            error: "Missing userCid",
+            message: "Please provide userCid as a query parameter"
+        });
+    }
+    
+    // Check for dev override impersonation (same pattern as other endpoints)
+    const { getEffectiveCid, getDevOverride } = require('../dev/dev_helpers');
+    const effectiveCid = await getEffectiveCid(db, userCid, config, logger);
+    const devOverride = await getDevOverride(db, userCid, config, logger);
+    const isImpersonating = devOverride && devOverride.enabled && devOverride.impersonateCid && effectiveCid !== Number(userCid);
+    
+    // Use effective CID for the request (impersonated or actual)
+    const requestUserCid = Number(effectiveCid);
+    
+    const piCidNum = Number(piCid);
+    if (isNaN(piCidNum) || piCidNum <= 0) {
+        return res.status(400).json({
+            success: false,
+            error: "Invalid PI CID",
+            message: "Please provide a valid Popular Investor CID"
+        });
+    }
+    
+    try {
+        // Check if this is a developer account (bypass rate limits for developers, even when impersonating)
+        const { isDeveloperAccount } = require('../dev/dev_helpers');
+        const isDeveloper = isDeveloperAccount(userCid);
+        
+        let rateLimitCheck = { allowed: true }; // Default to allowed for developers
+        
+        if (!isDeveloper) {
+            // Check rate limits (use effective CID for non-developers)
+            rateLimitCheck = await checkRateLimits(db, requestUserCid, piCidNum, logger);
+            
+            if (!rateLimitCheck.allowed) {
+                return res.status(429).json({
+                    success: false,
+                    error: "Rate limit exceeded",
+                    message: rateLimitCheck.message,
+                    rateLimit: {
+                        userCanRequestAgainAt: rateLimitCheck.userCanRequestAgainAt,
+                        globalCanRequestAgainAt: rateLimitCheck.globalCanRequestAgainAt,
+                        lastRequestedBy: rateLimitCheck.lastRequestedBy,
+                        lastRequestedAt: rateLimitCheck.lastRequestedAt
+                    }
+                });
+            }
+        } else {
+            logger.log('INFO', `[requestPiFetch] Developer account ${userCid} bypassing rate limits${isImpersonating ? ` (impersonating ${requestUserCid})` : ''}`);
+        }
+        
+        // Get PI username from rankings (needed for task engine)
+        const piUsername = await getPiUsername(db, piCidNum, config, logger);
+        if (!piUsername) {
+            return res.status(404).json({
+                success: false,
+                error: "PI not found",
+                message: `Popular Investor ${piCidNum} not found in rankings. Cannot fetch data.`
+            });
+        }
+        
+        // Create request document
+        const requestId = `req_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
+        const now = new Date();
+        
+        // Store request
+        const requestRef = db.collection('pi_fetch_requests')
+            .doc(String(piCidNum))
+            .collection('requests')
+            .doc(requestId);
+        
+        await requestRef.set({
+            requestId,
+            userCid: requestUserCid, // Use effective CID
+            actualUserCid: Number(userCid), // Track actual developer CID for audit
+            piCid: piCidNum,
+            piUsername,
+            status: 'queued',
+            isImpersonating: isImpersonating || false,
+            createdAt: FieldValue.serverTimestamp(),
+            updatedAt: FieldValue.serverTimestamp()
+        });
+        
+        // Update user rate limit (use effective CID)
+        // Use new path: PopularInvestors/{piCid}/userFetchRequests/{userCid}
+        const userRequestRef = db.collection('PopularInvestors')
+            .doc(String(piCidNum))
+            .collection('userFetchRequests')
+            .doc(String(requestUserCid));
+        
+        await userRequestRef.set({
+            userCid: requestUserCid,
+            piCid: piCidNum,
+            lastRequestedAt: FieldValue.serverTimestamp(),
+            requestCount: FieldValue.increment(1),
+            updatedAt: FieldValue.serverTimestamp()
+        }, { merge: true });
+        
+        // Update global rate limit
+        const globalRef = db.collection('pi_fetch_requests')
+            .doc(String(piCidNum))
+            .collection('global')
+            .doc('latest');
+        
+        await globalRef.set({
+            piCid: piCidNum,
+            lastRequestedAt: FieldValue.serverTimestamp(),
+            lastRequestedBy: requestUserCid, // Use effective CID
+            totalRequests: FieldValue.increment(1),
+            updatedAt: FieldValue.serverTimestamp()
+        }, { merge: true });
+        
+        // Publish to task engine - use on-demand topic for API requests
+        const topicName = config.taskEngine?.PUBSUB_TOPIC_USER_FETCH_ONDEMAND || 
+                         config.pubsubTopicUserFetchOnDemand || 
+                         'etoro-user-fetch-topic-ondemand';
+        const topic = pubsub.topic(topicName);
+        
+        const message = {
+            type: 'POPULAR_INVESTOR_UPDATE',
+            cid: piCidNum,
+            username: piUsername,
+            priority: 'high',
+            source: 'on_demand',
+            requestId,
+            requestedBy: requestUserCid, // Use effective CID
+            actualRequestedBy: Number(userCid), // Track actual developer CID
+            userType: 'POPULAR_INVESTOR', // Explicitly set userType for task engine
+            metadata: {
+                onDemand: true,
+                targetCid: piCidNum, // Target specific user for optimization
+                requestedAt: now.toISOString(),
+                isImpersonating: isImpersonating || false,
+                userType: 'POPULAR_INVESTOR' // Also in metadata for consistency
+            }
+        };
+        
+        const messageId = await topic.publishMessage({
+            data: Buffer.from(JSON.stringify(message))
+        });
+        
+        // Update request with message ID
+        await requestRef.update({
+            taskMessageId: messageId,
+            updatedAt: FieldValue.serverTimestamp()
+        });
+        
+        logger.log('INFO', `[requestPiFetch] User ${userCid}${isImpersonating ? ` (impersonating ${requestUserCid})` : ''} requested fetch for PI ${piCidNum} (${piUsername}). Request ID: ${requestId}`);
+        
+        return res.status(200).json({
+            success: true,
+            requestId,
+            status: 'queued',
+            message: 'Request queued successfully. Data will be available shortly.',
+            estimatedTime: '2-5 minutes',
+            rateLimit: {
+                userCanRequestAgainAt: new Date(Date.now() + RATE_LIMIT_MS).toISOString(),
+                globalCanRequestAgainAt: new Date(Date.now() + RATE_LIMIT_MS).toISOString()
+            }
+        });
+        
+    } catch (error) {
+        logger.log('ERROR', `[requestPiFetch] Error requesting fetch for PI ${piCidNum}`, error);
+        return res.status(500).json({
+            success: false,
+            error: "Internal server error",
+            message: error.message
+        });
+    }
+}
+
+/**
+ * Check fetch status for a Popular Investor
+ * GET /user/pi/:piCid/fetch-status
+ */
+async function getPiFetchStatus(req, res, dependencies, config) {
+    const { db, logger } = dependencies;
+    const { piCid } = req.params;
+    const userCid = req.query?.userCid; // Optional - for rate limit info
+    
+    const piCidNum = Number(piCid);
+    if (isNaN(piCidNum) || piCidNum <= 0) {
+        return res.status(400).json({
+            success: false,
+            error: "Invalid PI CID"
+        });
+    }
+    
+    try {
+        // First check if data exists by checking computation results directly
+        const insightsCollection = config.unifiedInsightsCollection || 'unified_insights';
+        const resultsSub = config.resultsSubcollection || 'results';
+        const compsSub = config.computationsSubcollection || 'computations';
+        
+        // Check last 7 days for PI data
+        const today = new Date();
+        let latestDate = null;
+        
+        for (let i = 0; i < 7; i++) {
+            const checkDate = new Date(today);
+            checkDate.setDate(checkDate.getDate() - i);
+            const dateStr = checkDate.toISOString().split('T')[0];
+            
+            const docRef = db.collection(insightsCollection)
+                .doc(dateStr)
+                .collection(resultsSub)
+                .doc('popular-investor')
+                .collection(compsSub)
+                .doc('PopularInvestorProfileMetrics');
+            
+            const doc = await docRef.get();
+            if (doc.exists) {
+                const { tryDecompress } = require('../data_helpers');
+                const data = tryDecompress(doc.data());
+                
+                if (data && data[String(piCidNum)]) {
+                    latestDate = dateStr;
+                    break;
+                }
+            }
+        }
+        
+        if (latestDate) {
+            // Data exists, check if it contains this PI
+            const docRef = db.collection(insightsCollection)
+                .doc(latestDate)
+                .collection(resultsSub)
+                .doc('popular-investor')
+                .collection(compsSub)
+                .doc('PopularInvestorProfileMetrics');
+            
+            const doc = await docRef.get();
+            if (doc.exists) {
+                const data = tryDecompress(doc.data());
+                
+                if (data && data[String(piCidNum)]) {
+                    return res.status(200).json({
+                        success: true,
+                        dataAvailable: true,
+                        latestDate,
+                        message: "Data is available"
+                    });
+                }
+            }
+        }
+        
+        // Data doesn't exist, check for pending requests
+        const requestsRef = db.collection('pi_fetch_requests')
+            .doc(String(piCidNum))
+            .collection('requests')
+            .orderBy('createdAt', 'desc')
+            .limit(1);
+        
+        const requestsSnapshot = await requestsRef.get();
+        
+        if (!requestsSnapshot.empty) {
+            const latestRequest = requestsSnapshot.docs[0].data();
+            let status = latestRequest.status || 'queued';
+            
+            // If status is 'indexing' or 'computing', check if computation results are now available
+            if (status === 'indexing' || status === 'computing') {
+                // Re-check if computation results exist
+                const checkDate = new Date();
+                for (let i = 0; i < 2; i++) { // Check today and yesterday
+                    const dateStr = new Date(checkDate);
+                    dateStr.setDate(checkDate.getDate() - i);
+                    const dateStrFormatted = dateStr.toISOString().split('T')[0];
+                    
+                    const docRef = db.collection(insightsCollection)
+                        .doc(dateStrFormatted)
+                        .collection(resultsSub)
+                        .doc('popular-investor')
+                        .collection(compsSub)
+                        .doc('PopularInvestorProfileMetrics');
+                    
+                    const doc = await docRef.get();
+                    if (doc.exists) {
+                        const docData = doc.data();
+                        let mergedData = null;
+                        
+                        // Check if data is sharded
+                        if (docData._sharded === true && docData._shardCount) {
+                            // Data is stored in shards - read all shards and merge
+                            const shardsCol = docRef.collection('_shards');
+                            const shardsSnapshot = await shardsCol.get();
+                            
+                            if (!shardsSnapshot.empty) {
+                                mergedData = {};
+                                for (const shardDoc of shardsSnapshot.docs) {
+                                    const shardData = shardDoc.data();
+                                    Object.assign(mergedData, shardData);
+                                }
+                            }
+                        } else {
+                            // Data is in the main document (compressed or not)
+                            mergedData = tryDecompress(docData);
+                            
+                            // Handle string decompression result
+                            if (typeof mergedData === 'string') {
+                                try {
+                                    mergedData = JSON.parse(mergedData);
+                                } catch (e) {
+                                    logger.log('WARN', `[getPiFetchStatus] Failed to parse decompressed string for date ${dateStrFormatted}:`, e.message);
+                                    mergedData = null;
+                                }
+                            }
+                        }
+                        
+                        if (mergedData && typeof mergedData === 'object' && mergedData[String(piCidNum)]) {
+                            // Computation completed! Update status
+                            status = 'completed';
+                            await requestsSnapshot.docs[0].ref.update({
+                                status: 'completed',
+                                completedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                                updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                            });
+                            break;
+                        }
+                    }
+                }
+            }
+            
+            // Check if request is stale (stuck in processing state for too long)
+            // Set to 2 minutes to prevent indefinite polling when computation system crashes
+            const { FieldValue } = require('@google-cloud/firestore');
+            const STALE_THRESHOLD_MS = 2 * 60 * 1000; // 2 minutes
+            const processingStates = ['processing', 'dispatched', 'indexing', 'computing', 'queued'];
+            const isProcessingState = processingStates.includes(status);
+            
+            let isStale = false;
+            if (isProcessingState) {
+                const now = Date.now();
+                const createdAt = latestRequest.createdAt?.toDate?.()?.getTime() || 
+                                 latestRequest.createdAt?.toMillis?.() || null;
+                const dispatchedAt = latestRequest.dispatchedAt?.toDate?.()?.getTime() || 
+                                    latestRequest.dispatchedAt?.toMillis?.() || null;
+                const startedAt = latestRequest.startedAt?.toDate?.()?.getTime() || 
+                                 latestRequest.startedAt?.toMillis?.() || null;
+                const updatedAt = latestRequest.updatedAt?.toDate?.()?.getTime() || 
+                                 latestRequest.updatedAt?.toMillis?.() || null;
+                
+                // Use the most recent timestamp to determine age
+                const referenceTime = startedAt || dispatchedAt || createdAt || updatedAt;
+                
+                if (referenceTime && (now - referenceTime) > STALE_THRESHOLD_MS) {
+                    // Before marking as stale, do one final check for computation results
+                    const finalCheckDate = new Date();
+                    let foundResults = false;
+                    for (let i = 0; i < 2; i++) {
+                        const dateStr = new Date(finalCheckDate);
+                        dateStr.setDate(finalCheckDate.getDate() - i);
+                        const dateStrFormatted = dateStr.toISOString().split('T')[0];
+                        
+                        const docRef = db.collection(insightsCollection)
+                            .doc(dateStrFormatted)
+                            .collection(resultsSub)
+                            .doc('popular-investor')
+                            .collection(compsSub)
+                            .doc('PopularInvestorProfileMetrics');
+                        
+                        const doc = await docRef.get();
+                        if (doc.exists) {
+                            const { tryDecompress } = require('../data_helpers');
+                            const data = tryDecompress(doc.data());
+                            if (data && typeof data === 'object' && data[String(piCidNum)]) {
+                                foundResults = true;
+                                status = 'completed';
+                                await requestsSnapshot.docs[0].ref.update({
+                                    status: 'completed',
+                                    completedAt: FieldValue.serverTimestamp(),
+                                    updatedAt: FieldValue.serverTimestamp()
+                                });
+                                logger.log('INFO', `[getPiFetchStatus] Found computation results on stale check, marked as completed`);
+                                break;
+                            }
+                        }
+                    }
+                    
+                    if (!foundResults) {
+                        isStale = true;
+                        logger.log('WARN', `[getPiFetchStatus] Detected stale request ${latestRequest.requestId} for PI ${piCidNum}. Status: ${status}, Age: ${Math.round((now - referenceTime) / 60000)} minutes`);
+                    }
+                }
+            }
+            
+            // If stale, mark as failed to stop polling
+            if (isStale) {
+                status = 'failed';
+                const requestDocRef = requestsSnapshot.docs[0].ref;
+                try {
+                    await requestDocRef.update({
+                        status: 'failed',
+                        error: 'Request timed out - task may have failed to process. Please try again.',
+                        failedAt: FieldValue.serverTimestamp(),
+                        updatedAt: FieldValue.serverTimestamp()
+                    });
+                    logger.log('INFO', `[getPiFetchStatus] Marked stale request ${latestRequest.requestId} as failed`);
+                } catch (updateErr) {
+                    logger.log('WARN', `[getPiFetchStatus] Failed to update stale request status`, updateErr);
+                }
+            }
+            
+            const response = {
+                success: true,
+                dataAvailable: status === 'completed',
+                status,
+                requestId: latestRequest.requestId,
+                startedAt: latestRequest.startedAt?.toDate?.()?.toISOString() || null,
+                createdAt: latestRequest.createdAt?.toDate?.()?.toISOString() || null,
+                estimatedCompletion: latestRequest.startedAt 
+                    ? new Date(new Date(latestRequest.startedAt.toDate()).getTime() + 10 * 60 * 1000).toISOString() // 10 min for computation
+                    : null
+            };
+            
+            // Include error details if status is failed
+            if (status === 'failed') {
+                response.error = isStale 
+                    ? 'Request timed out - task may have failed to process. Please try again.'
+                    : (latestRequest.error || 'Unknown error occurred');
+                response.failedAt = latestRequest.failedAt?.toDate?.()?.toISOString() || 
+                                   (isStale ? new Date().toISOString() : null);
+            }
+            
+            // Include raw data status if computing
+            if (status === 'computing') {
+                response.rawDataStoredAt = latestRequest.rawDataStoredAt?.toDate?.()?.toISOString() || null;
+                response.message = 'Raw data stored, computation in progress...';
+            }
+            
+            return res.status(200).json(response);
+        }
+        
+        // No request found, check if user can request
+        let canRequest = true;
+        let rateLimitInfo = null;
+        
+        if (userCid) {
+            // Check if this is a developer account (bypass rate limits for developers)
+            const { isDeveloperAccount } = require('../dev/dev_helpers');
+            const isDeveloper = isDeveloperAccount(userCid);
+            
+            if (isDeveloper) {
+                // Developer accounts bypass rate limits
+                canRequest = true;
+                rateLimitInfo = {
+                    bypassed: true,
+                    reason: 'developer_account'
+                };
+            } else {
+                const rateLimitCheck = await checkRateLimits(db, userCid, piCidNum, logger);
+                canRequest = rateLimitCheck.allowed;
+                rateLimitInfo = {
+                    userCanRequestAgainAt: rateLimitCheck.userCanRequestAgainAt,
+                    globalCanRequestAgainAt: rateLimitCheck.globalCanRequestAgainAt
+                };
+            }
+        }
+        
+        return res.status(200).json({
+            success: true,
+            dataAvailable: false,
+            status: 'not_requested',
+            canRequest,
+            rateLimit: rateLimitInfo
+        });
+        
+    } catch (error) {
+        logger.log('ERROR', `[getPiFetchStatus] Error checking fetch status for PI ${piCidNum}`, error);
+        return res.status(500).json({
+            success: false,
+            error: "Internal server error",
+            message: error.message
+        });
+    }
+}
+
+/**
+ * Check rate limits for user and global
+ * NOTE: Developer accounts should bypass this check (check isDeveloperAccount before calling)
+ */
+async function checkRateLimits(db, userCid, piCid, logger) {
+    const now = Date.now();
+    
+    // Check user rate limit - use new path: PopularInvestors/{piCid}/userFetchRequests/{userCid}
+    const userRequestRef = db.collection('PopularInvestors')
+        .doc(String(piCid))
+        .collection('userFetchRequests')
+        .doc(String(userCid));
+    
+    const userRequestDoc = await userRequestRef.get();
+    let userCanRequestAgainAt = null;
+    let userBlocked = false;
+    
+    if (userRequestDoc.exists) {
+        const userData = userRequestDoc.data();
+        const lastRequestedAt = userData.lastRequestedAt?.toDate?.()?.getTime() || userData.lastRequestedAt?.toMillis?.() || null;
+        
+        if (lastRequestedAt && (now - lastRequestedAt) < RATE_LIMIT_MS) {
+            userBlocked = true;
+            userCanRequestAgainAt = new Date(lastRequestedAt + RATE_LIMIT_MS).toISOString();
+        }
+    }
+    
+    // Check global rate limit
+    const globalRef = db.collection('pi_fetch_requests')
+        .doc(String(piCid))
+        .collection('global')
+        .doc('latest');
+    
+    const globalDoc = await globalRef.get();
+    let globalCanRequestAgainAt = null;
+    let globalBlocked = false;
+    let lastRequestedBy = null;
+    let lastRequestedAt = null;
+    
+    if (globalDoc.exists) {
+        const globalData = globalDoc.data();
+        const globalLastRequestedAt = globalData.lastRequestedAt?.toDate?.()?.getTime() || globalData.lastRequestedAt?.toMillis?.() || null;
+        
+        if (globalLastRequestedAt && (now - globalLastRequestedAt) < RATE_LIMIT_MS) {
+            globalBlocked = true;
+            globalCanRequestAgainAt = new Date(globalLastRequestedAt + RATE_LIMIT_MS).toISOString();
+            lastRequestedBy = globalData.lastRequestedBy;
+            lastRequestedAt = new Date(globalLastRequestedAt).toISOString();
+        }
+    }
+    
+    // Global limit is more restrictive
+    if (globalBlocked) {
+        const minutesRemaining = Math.ceil((new Date(globalCanRequestAgainAt).getTime() - now) / (60 * 1000));
+        return {
+            allowed: false,
+            message: `This Popular Investor was recently requested. Please try again in ${minutesRemaining} minute${minutesRemaining !== 1 ? 's' : ''}.`,
+            userCanRequestAgainAt,
+            globalCanRequestAgainAt,
+            lastRequestedBy,
+            lastRequestedAt
+        };
+    }
+    
+    if (userBlocked) {
+        const minutesRemaining = Math.ceil((new Date(userCanRequestAgainAt).getTime() - now) / (60 * 1000));
+        return {
+            allowed: false,
+            message: `You have recently requested this Popular Investor. Please try again in ${minutesRemaining} minute${minutesRemaining !== 1 ? 's' : ''}.`,
+            userCanRequestAgainAt,
+            globalCanRequestAgainAt,
+            lastRequestedBy,
+            lastRequestedAt
+        };
+    }
+    
+    return {
+        allowed: true,
+        userCanRequestAgainAt: new Date(now + RATE_LIMIT_MS).toISOString(),
+        globalCanRequestAgainAt: new Date(now + RATE_LIMIT_MS).toISOString()
+    };
+}
+
+/**
+ * Get PI username from master list (single source of truth)
+ * Uses the popular investor master list instead of rankings
+ */
+async function getPiUsername(db, piCid, config, logger) {
+    try {
+        // Use master list helper instead of rankings
+        const { getPIUsernameFromMasterList } = require('../core/user_status_helpers');
+        const collectionRegistry = config.collectionRegistry || null;
+        
+        const username = await getPIUsernameFromMasterList(db, piCid, collectionRegistry, logger);
+        
+        if (username) {
+            logger.log('INFO', `[getPiUsername] Found username "${username}" for PI ${piCid} from master list`);
+            return username;
+        }
+        
+        logger.log('WARN', `[getPiUsername] PI ${piCid} not found in master list`);
+        return null;
+    } catch (error) {
+        logger.log('ERROR', `[getPiUsername] Error fetching username for PI ${piCid}`, error);
+        return null;
+    }
+}
+
+module.exports = {
+    requestPiFetch,
+    getPiFetchStatus,
+    getPiUsername
+};