bulltrackers-module 1.0.451 → 1.0.453

package/functions/core/utils/firestore_utils.js

@@ -297,17 +297,47 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
         const items = data.Items || [];
 
         // Map to { cid, username }
-        const targets = items.map(item => ({
+        const allTargets = items.map(item => ({
             cid: String(item.CustomerId),
             username: item.UserName
         }));
 
-        if (targets.length > 0) {
-            logger.log('INFO', `[Core Utils Debug] First PI Target Mapped: ${JSON.stringify(targets[0])}`);
+        // Filter out PIs updated in the last 18 hours
+        const HOURS_THRESHOLD = 18;
+        const thresholdMs = HOURS_THRESHOLD * 60 * 60 * 1000;
+        const now = Date.now();
+        
+        const filteredTargets = [];
+        let skippedCount = 0;
+        
+        // Check each PI's last sync time
+        for (const target of allTargets) {
+            const syncRef = db.collection('user_sync_requests')
+                .doc(target.cid)
+                .collection('global')
+                .doc('latest');
+            
+            const syncDoc = await syncRef.get();
+            if (syncDoc.exists) {
+                const syncData = syncDoc.data();
+                const lastRequestedAt = syncData.lastRequestedAt?.toDate?.()?.getTime() || 
+                                      syncData.lastRequestedAt?.toMillis?.() || null;
+                
+                if (lastRequestedAt && (now - lastRequestedAt) < thresholdMs) {
+                    skippedCount++;
+                    continue; // Skip this PI
+                }
+            }
+            
+            filteredTargets.push(target); // Include this PI
         }
 
-        logger.log('INFO', `[Core Utils] Found ${targets.length} Popular Investors from ${rankingsDate} ranking${rankingsDate !== today ? ` (fallback from ${today})` : ''}.`);
-        return targets;
+        if (filteredTargets.length > 0) {
+            logger.log('INFO', `[Core Utils Debug] First PI Target Mapped: ${JSON.stringify(filteredTargets[0])}`);
+        }
+
+        logger.log('INFO', `[Core Utils] Found ${allTargets.length} Popular Investors from ${rankingsDate} ranking${rankingsDate !== today ? ` (fallback from ${today})` : ''}. Skipped ${skippedCount} recently updated. ${filteredTargets.length} will be updated.`);
+        return filteredTargets;
 
     } catch (error) {
         logger.log('ERROR', '[Core Utils] Error getting Popular Investors', { errorMessage: error.message });
@@ -316,13 +346,19 @@ async function getPopularInvestorsToUpdate(dependencies, config) {
 }
 
 /**
- * [NEW] Fetches all Signed-In Users for daily update.
+ * [NEW] Fetches Signed-In Users for daily update.
+ * Skips users updated in the last 18 hours to avoid duplicate refreshes.
  */
 async function getSignedInUsersToUpdate(dependencies, config) {
     const { db, logger } = dependencies;
     const collectionName = config.signedInUsersCollection || process.env.FIRESTORE_COLLECTION_SIGNED_IN_USER_PORTFOLIOS || 'signed_in_users';
+    
+    // 18 hours threshold - skip users updated more recently than this
+    const HOURS_THRESHOLD = 18;
+    const thresholdMs = HOURS_THRESHOLD * 60 * 60 * 1000;
+    const now = Date.now();
 
-    logger.log('INFO', `[Core Utils] Getting Signed-In Users to update from ${collectionName}...`);
+    logger.log('INFO', `[Core Utils] Getting Signed-In Users to update from ${collectionName} (skipping users updated in last ${HOURS_THRESHOLD} hours)...`);
 
     try {
         const snapshot = await db.collection(collectionName).get();
@@ -333,17 +369,63 @@ async function getSignedInUsersToUpdate(dependencies, config) {
         }
 
         const targets = [];
+        let skippedCount = 0;
+        
         snapshot.forEach(doc => {
             const data = doc.data();
-            // Assuming the doc ID is the CID, or it's a field 'cid'
             const cid = data.cid || doc.id; 
             const username = data.username || 'Unknown';
-            // Only push if we have a valid CID
-            if(cid) targets.push({ cid: String(cid), username });
+            
+            if (!cid) return;
+            
+            // Check if user was recently updated (check sync requests and portfolio timestamps)
+            // First check sync requests for last update time
+            let shouldSkip = false;
+            
+            // Check user_sync_requests for last completed sync
+            // We'll check this asynchronously, but for now we'll check portfolio data timestamps
+            // The portfolio data is stored in snapshots with dates, so we can check the latest snapshot date
+            const portfolioCollection = collectionName; // Same collection
+            const blockId = '19M';
+            const today = new Date().toISOString().split('T')[0];
+            
+            // For efficiency, we'll check the sync requests collection for the last update time
+            // This is a synchronous check we can do here
+            // Actually, we need to make this async, so we'll do a batch check after
+            
+            // For now, include all users - we'll filter in the dispatcher
+            targets.push({ cid: String(cid), username });
+        });
+        
+        // Now filter out users who were updated in the last 18 hours
+        // Check user_sync_requests for last completed sync
+        const filteredTargets = [];
+        const checkPromises = targets.map(async (target) => {
+            const syncRef = db.collection('user_sync_requests')
+                .doc(target.cid)
+                .collection('global')
+                .doc('latest');
+            
+            const syncDoc = await syncRef.get();
+            if (syncDoc.exists) {
+                const syncData = syncDoc.data();
+                const lastRequestedAt = syncData.lastRequestedAt?.toDate?.()?.getTime() || 
+                                      syncData.lastRequestedAt?.toMillis?.() || null;
+                
+                if (lastRequestedAt && (now - lastRequestedAt) < thresholdMs) {
+                    skippedCount++;
+                    return null; // Skip this user
+                }
+            }
+            
+            return target; // Include this user
         });
+        
+        const results = await Promise.all(checkPromises);
+        const finalTargets = results.filter(t => t !== null);
 
-        logger.log('INFO', `[Core Utils] Found ${targets.length} Signed-In Users.`);
-        return targets;
+        logger.log('INFO', `[Core Utils] Found ${targets.length} Signed-In Users. Skipped ${skippedCount} recently updated. ${finalTargets.length} will be updated.`);
+        return finalTargets;
 
     } catch (error) {
         logger.log('ERROR', '[Core Utils] Error getting Signed-In Users', { errorMessage: error.message });

package/functions/generic-api/user-api/helpers/user_sync_helpers.js

@@ -0,0 +1,358 @@
+/**
+ * @fileoverview User Sync Request Helpers
+ * Allows users to request on-demand sync for their own profile or Popular Investor profiles
+ * Rate limiting: 1 update per user per 6 hours (global, not per-requester)
+ * Developer accounts bypass rate limits
+ */
+
+const { FieldValue } = require('@google-cloud/firestore');
+const crypto = require('crypto');
+
+const RATE_LIMIT_HOURS = 6;
+const RATE_LIMIT_MS = RATE_LIMIT_HOURS * 60 * 60 * 1000;
+
+/**
+ * Request on-demand sync for a user (signed-in user or Popular Investor)
+ * POST /user/:userCid/sync
+ */
+async function requestUserSync(req, res, dependencies, config) {
+    const { db, logger, pubsub } = dependencies;
+    const { userCid: targetUserCid } = req.params;
+    
+    // Get requesting userCid from query
+    const requestingUserCid = req.query?.userCid;
+    
+    if (!requestingUserCid) {
+        return res.status(400).json({ 
+            success: false,
+            error: "Missing userCid",
+            message: "Please provide userCid as a query parameter"
+        });
+    }
+    
+    // Check for dev override impersonation
+    const { getEffectiveCid, getDevOverride } = require('./dev_helpers');
+    const effectiveCid = await getEffectiveCid(db, requestingUserCid, config, logger);
+    const devOverride = await getDevOverride(db, requestingUserCid, config, logger);
+    const isImpersonating = devOverride && devOverride.enabled && devOverride.impersonateCid && effectiveCid !== Number(requestingUserCid);
+    
+    const targetCidNum = Number(targetUserCid);
+    if (isNaN(targetCidNum) || targetCidNum <= 0) {
+        return res.status(400).json({
+            success: false,
+            error: "Invalid user CID",
+            message: "Please provide a valid user CID"
+        });
+    }
+    
+    try {
+        // Check if this is a developer account (bypass rate limits for developers)
+        const { isDeveloperAccount } = require('./dev_helpers');
+        const isDeveloper = isDeveloperAccount(requestingUserCid);
+        
+        let rateLimitCheck = { allowed: true }; // Default to allowed for developers
+        
+        if (!isDeveloper) {
+            // Check global rate limit (applies to target user, not requester)
+            rateLimitCheck = await checkRateLimits(db, targetCidNum, logger);
+            
+            if (!rateLimitCheck.allowed) {
+                return res.status(429).json({
+                    success: false,
+                    error: "Rate limit exceeded",
+                    message: rateLimitCheck.message,
+                    rateLimit: {
+                        canRequestAgainAt: rateLimitCheck.canRequestAgainAt,
+                        lastRequestedAt: rateLimitCheck.lastRequestedAt
+                    }
+                });
+            }
+        } else {
+            logger.log('INFO', `[requestUserSync] Developer account ${requestingUserCid} bypassing rate limits${isImpersonating ? ` (impersonating ${effectiveCid})` : ''}`);
+        }
+        
+        // Determine user type (PI or signed-in user)
+        // Check if user is in rankings (PI) or has signed-in user data
+        const { checkIfUserIsPI } = require('./data_helpers');
+        const isPI = await checkIfUserIsPI(db, targetCidNum, config, logger);
+        
+        // Get username - for PIs, get from rankings; for signed-in users, we'll need to fetch from their profile
+        let username = null;
+        if (isPI) {
+            // Get PI username from rankings
+            const { findLatestRankingsDate } = require('./data_helpers');
+            const rankingsCollection = config.popularInvestorRankingsCollection || 'popular_investor_rankings';
+            const rankingsDate = await findLatestRankingsDate(db, rankingsCollection, 30);
+            
+            if (rankingsDate) {
+                const rankingsRef = db.collection(rankingsCollection).doc(rankingsDate);
+                const rankingsDoc = await rankingsRef.get();
+                if (rankingsDoc.exists) {
+                    const rankingsData = rankingsDoc.data();
+                    const rankingsItems = rankingsData.Items || [];
+                    const rankingEntry = rankingsItems.find(item => Number(item.CustomerId) === targetCidNum);
+                    if (rankingEntry) {
+                        username = rankingEntry.UserName || rankingEntry.username || null;
+                    }
+                }
+            }
+        } else {
+            // For signed-in users, try to get username from verification data
+            const signedInUsersCollection = config.signedInUsersCollection || 'signed_in_users';
+            const userDoc = await db.collection(signedInUsersCollection).doc(String(targetCidNum)).get();
+            if (userDoc.exists) {
+                const userData = userDoc.data();
+                username = userData.username || userData.verification?.username || null;
+            }
+        }
+        
+        if (!username) {
+            logger.log('WARN', `[requestUserSync] Could not find username for user ${targetCidNum}, will use CID as fallback`);
+            username = String(targetCidNum);
+        }
+        
+        // Create request document
+        const requestId = `sync_${Date.now()}_${crypto.randomBytes(4).toString('hex')}`;
+        const now = new Date();
+        
+        // Store request
+        const requestRef = db.collection('user_sync_requests')
+            .doc(String(targetCidNum))
+            .collection('requests')
+            .doc(requestId);
+        
+        await requestRef.set({
+            requestId,
+            targetUserCid: targetCidNum,
+            requestedBy: Number(requestingUserCid),
+            effectiveRequestedBy: effectiveCid,
+            username,
+            userType: isPI ? 'POPULAR_INVESTOR' : 'SIGNED_IN_USER',
+            status: 'queued',
+            isImpersonating: isImpersonating || false,
+            createdAt: FieldValue.serverTimestamp(),
+            updatedAt: FieldValue.serverTimestamp()
+        });
+        
+        // Update global rate limit (for target user)
+        const globalRef = db.collection('user_sync_requests')
+            .doc(String(targetCidNum))
+            .collection('global')
+            .doc('latest');
+        
+        await globalRef.set({
+            targetUserCid: targetCidNum,
+            lastRequestedAt: FieldValue.serverTimestamp(),
+            lastRequestedBy: Number(requestingUserCid),
+            totalRequests: FieldValue.increment(1),
+            updatedAt: FieldValue.serverTimestamp()
+        }, { merge: true });
+        
+        // Publish to task engine
+        const topicName = config.taskEngine?.PUBSUB_TOPIC_USER_FETCH || 'etoro-user-fetch-topic';
+        const topic = pubsub.topic(topicName);
+        
+        const message = {
+            type: isPI ? 'POPULAR_INVESTOR_UPDATE' : 'ON_DEMAND_USER_UPDATE',
+            cid: targetCidNum,
+            username: username,
+            priority: 'high',
+            source: 'on_demand_sync',
+            requestId,
+            requestedBy: Number(requestingUserCid),
+            effectiveRequestedBy: effectiveCid,
+            metadata: {
+                onDemand: true,
+                requestedAt: now.toISOString(),
+                isImpersonating: isImpersonating || false
+            }
+        };
+        
+        const messageId = await topic.publishMessage({
+            data: Buffer.from(JSON.stringify(message))
+        });
+        
+        // Update request with message ID
+        await requestRef.update({
+            taskMessageId: messageId,
+            status: 'dispatched',
+            dispatchedAt: FieldValue.serverTimestamp(),
+            updatedAt: FieldValue.serverTimestamp()
+        });
+        
+        logger.log('INFO', `[requestUserSync] User ${requestingUserCid}${isImpersonating ? ` (impersonating ${effectiveCid})` : ''} requested sync for ${isPI ? 'PI' : 'signed-in user'} ${targetCidNum} (${username}). Request ID: ${requestId}`);
+        
+        return res.status(200).json({
+            success: true,
+            requestId,
+            status: 'dispatched',
+            message: 'Sync request queued successfully. Data will be updated shortly.',
+            estimatedTime: '2-5 minutes',
+            rateLimit: {
+                canRequestAgainAt: new Date(Date.now() + RATE_LIMIT_MS).toISOString()
+            }
+        });
+        
+    } catch (error) {
+        logger.log('ERROR', `[requestUserSync] Error requesting sync for user ${targetCidNum}`, error);
+        return res.status(500).json({
+            success: false,
+            error: "Internal server error",
+            message: error.message
+        });
+    }
+}
+
+/**
+ * Get sync status for a user
+ * GET /user/:userCid/sync-status
+ */
+async function getUserSyncStatus(req, res, dependencies, config) {
+    const { db, logger } = dependencies;
+    const { userCid: targetUserCid } = req.params;
+    const requestingUserCid = req.query?.userCid; // Optional - for rate limit info
+    
+    const targetCidNum = Number(targetUserCid);
+    if (isNaN(targetCidNum) || targetCidNum <= 0) {
+        return res.status(400).json({
+            success: false,
+            error: "Invalid user CID"
+        });
+    }
+    
+    try {
+        // Check for latest request
+        const requestsRef = db.collection('user_sync_requests')
+            .doc(String(targetCidNum))
+            .collection('requests')
+            .orderBy('createdAt', 'desc')
+            .limit(1);
+        
+        const requestsSnapshot = await requestsRef.get();
+        
+        if (!requestsSnapshot.empty) {
+            const latestRequest = requestsSnapshot.docs[0].data();
+            let status = latestRequest.status || 'queued';
+            
+            const response = {
+                success: true,
+                status,
+                requestId: latestRequest.requestId,
+                startedAt: latestRequest.startedAt?.toDate?.()?.toISOString() || null,
+                createdAt: latestRequest.createdAt?.toDate?.()?.toISOString() || null,
+                dispatchedAt: latestRequest.dispatchedAt?.toDate?.()?.toISOString() || null,
+                completedAt: latestRequest.completedAt?.toDate?.()?.toISOString() || null,
+                estimatedCompletion: latestRequest.dispatchedAt 
+                    ? new Date(new Date(latestRequest.dispatchedAt.toDate()).getTime() + 5 * 60 * 1000).toISOString() // 5 min estimate
+                    : null
+            };
+            
+            // Include error details if status is failed
+            if (status === 'failed') {
+                response.error = latestRequest.error || 'Unknown error occurred';
+                response.failedAt = latestRequest.failedAt?.toDate?.()?.toISOString() || null;
+            }
+            
+            // Include raw data status if processing
+            if (status === 'processing' || status === 'indexing' || status === 'computing') {
+                response.rawDataStoredAt = latestRequest.rawDataStoredAt?.toDate?.()?.toISOString() || null;
+                response.message = status === 'indexing' ? 'Raw data stored, indexing data...' : 
+                                 status === 'computing' ? 'Raw data stored, computation in progress...' :
+                                 'Processing data...';
+            }
+            
+            return res.status(200).json(response);
+        }
+        
+        // No request found, check if user can request
+        let canRequest = true;
+        let rateLimitInfo = null;
+        
+        if (requestingUserCid) {
+            // Check if this is a developer account (bypass rate limits for developers)
+            const { isDeveloperAccount } = require('./dev_helpers');
+            const isDeveloper = isDeveloperAccount(requestingUserCid);
+            
+            if (isDeveloper) {
+                // Developer accounts bypass rate limits
+                canRequest = true;
+                rateLimitInfo = {
+                    bypassed: true,
+                    reason: 'developer_account'
+                };
+            } else {
+                const rateLimitCheck = await checkRateLimits(db, targetCidNum, logger);
+                canRequest = rateLimitCheck.allowed;
+                rateLimitInfo = {
+                    canRequestAgainAt: rateLimitCheck.canRequestAgainAt
+                };
+            }
+        }
+        
+        return res.status(200).json({
+            success: true,
+            status: 'not_requested',
+            canRequest,
+            rateLimit: rateLimitInfo
+        });
+        
+    } catch (error) {
+        logger.log('ERROR', `[getUserSyncStatus] Error checking sync status for user ${targetCidNum}`, error);
+        return res.status(500).json({
+            success: false,
+            error: "Internal server error",
+            message: error.message
+        });
+    }
+}
+
+/**
+ * Check global rate limits for a user
+ * NOTE: Developer accounts should bypass this check (check isDeveloperAccount before calling)
+ */
+async function checkRateLimits(db, targetUserCid, logger) {
+    const now = Date.now();
+    
+    // Check global rate limit (applies to target user, not requester)
+    const globalRef = db.collection('user_sync_requests')
+        .doc(String(targetUserCid))
+        .collection('global')
+        .doc('latest');
+    
+    const globalDoc = await globalRef.get();
+    let canRequestAgainAt = null;
+    let blocked = false;
+    let lastRequestedAt = null;
+    
+    if (globalDoc.exists) {
+        const globalData = globalDoc.data();
+        const globalLastRequestedAt = globalData.lastRequestedAt?.toDate?.()?.getTime() || globalData.lastRequestedAt?.toMillis?.() || null;
+        
+        if (globalLastRequestedAt && (now - globalLastRequestedAt) < RATE_LIMIT_MS) {
+            blocked = true;
+            canRequestAgainAt = new Date(globalLastRequestedAt + RATE_LIMIT_MS).toISOString();
+            lastRequestedAt = new Date(globalLastRequestedAt).toISOString();
+        }
+    }
+    
+    if (blocked) {
+        const hoursRemaining = Math.ceil((new Date(canRequestAgainAt).getTime() - now) / (60 * 60 * 1000));
+        return {
+            allowed: false,
+            message: `This user was recently synced. Please try again in ${hoursRemaining} hour${hoursRemaining !== 1 ? 's' : ''}.`,
+            canRequestAgainAt,
+            lastRequestedAt
+        };
+    }
+    
+    return {
+        allowed: true,
+        canRequestAgainAt: new Date(now + RATE_LIMIT_MS).toISOString()
+    };
+}
+
+module.exports = {
+    requestUserSync,
+    getUserSyncStatus
+};
+

package/functions/generic-api/user-api/index.js

@@ -11,6 +11,7 @@ const { subscribeToAlerts, updateSubscription, unsubscribeFromAlerts, getUserSub
 const { setDevOverride, getDevOverrideStatus } = require('./helpers/dev_helpers');
 const { getAlertTypes, getUserAlerts, getAlertCount, markAlertRead, markAllAlertsRead, deleteAlert } = require('./helpers/alert_helpers');
 const { requestPiFetch, getPiFetchStatus } = require('./helpers/on_demand_fetch_helpers');
+const { requestUserSync, getUserSyncStatus } = require('./helpers/user_sync_helpers');
 
 module.exports = (dependencies, config) => {
     const router = express.Router();
@@ -34,6 +35,10 @@ module.exports = (dependencies, config) => {
     router.post('/pi/:piCid/request-fetch', (req, res) => requestPiFetch(req, res, dependencies, config));
     router.get('/pi/:piCid/fetch-status', (req, res) => getPiFetchStatus(req, res, dependencies, config));
     
+    // --- User Sync (for profile pages) ---
+    router.post('/:userCid/sync', (req, res) => requestUserSync(req, res, dependencies, config));
+    router.get('/:userCid/sync-status', (req, res) => getUserSyncStatus(req, res, dependencies, config));
+    
     // Signed-In User Personalized Routes
     // Note: Router is mounted at /user, so these routes should be /me/* not /user/me/*
     router.get('/me/hedges', (req, res) => getUserRecommendations(req, res, dependencies, config, 'hedges'));

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -46,19 +46,37 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
 
     logger.log('INFO', `[PI Update] Starting update for User: ${username || 'unknown'} (${cid || 'unknown'})${requestId ? ` [Request: ${requestId}]` : ''}`);
     
-    // Update request status if this is an on-demand request
-    if (requestId && source === 'on_demand' && db) {
+    // Update request status if this is an on-demand request (PI fetch or sync)
+    if (requestId && (source === 'on_demand' || source === 'on_demand_sync') && db) {
         try {
-            const requestRef = db.collection('pi_fetch_requests')
+            // Check if it's a sync request first
+            const syncRequestRef = db.collection('user_sync_requests')
                 .doc(String(cid))
                 .collection('requests')
                 .doc(requestId);
             
-            await requestRef.update({
-                status: 'processing',
-                startedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
-                updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
-            });
+            const syncRequestDoc = await syncRequestRef.get();
+            
+            if (syncRequestDoc.exists) {
+                // This is a sync request
+                await syncRequestRef.update({
+                    status: 'processing',
+                    startedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                    updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                });
+            } else {
+                // This is a PI fetch request
+                const requestRef = db.collection('pi_fetch_requests')
+                    .doc(String(cid))
+                    .collection('requests')
+                    .doc(requestId);
+                
+                await requestRef.update({
+                    status: 'processing',
+                    startedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                    updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                });
+            }
         } catch (err) {
             logger.log('WARN', `[PI Update] Failed to update request status for ${requestId}`, err);
         }
@@ -299,14 +317,31 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
 
         logger.log('SUCCESS', `[PI Update] Completed full update for ${username}`);
         
-        // Update request status and trigger computation if this is an on-demand request
-        if (requestId && source === 'on_demand' && db) {
+        // Update request status and trigger computation if this is an on-demand request (PI fetch or sync)
+        if (requestId && (source === 'on_demand' || source === 'on_demand_sync') && db) {
             try {
-                const requestRef = db.collection('pi_fetch_requests')
+                // Check if it's a sync request first
+                const syncRequestRef = db.collection('user_sync_requests')
                     .doc(String(cid))
                     .collection('requests')
                     .doc(requestId);
                 
+                const syncRequestDoc = await syncRequestRef.get();
+                let requestRef = null;
+                let isSyncRequest = false;
+                
+                if (syncRequestDoc.exists) {
+                    // This is a sync request
+                    requestRef = syncRequestRef;
+                    isSyncRequest = true;
+                } else {
+                    // This is a PI fetch request
+                    requestRef = db.collection('pi_fetch_requests')
+                        .doc(String(cid))
+                        .collection('requests')
+                        .doc(requestId);
+                }
+                
                 // Update status to indicate raw data is stored, indexing and computation will be triggered
                 await requestRef.update({
                     status: 'indexing',
@@ -392,14 +427,29 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
     } catch (error) {
         logger.log('ERROR', `[PI Update] Failed for ${username}`, error);
         
-        // Update request status to failed if this is an on-demand request
-        if (requestId && source === 'on_demand' && db) {
+        // Update request status to failed if this is an on-demand request (PI fetch or sync)
+        if (requestId && (source === 'on_demand' || source === 'on_demand_sync') && db) {
             try {
-                const requestRef = db.collection('pi_fetch_requests')
+                // Check if it's a sync request first
+                const syncRequestRef = db.collection('user_sync_requests')
                     .doc(String(cid))
                     .collection('requests')
                     .doc(requestId);
                 
+                const syncRequestDoc = await syncRequestRef.get();
+                let requestRef = null;
+                
+                if (syncRequestDoc.exists) {
+                    // This is a sync request
+                    requestRef = syncRequestRef;
+                } else {
+                    // This is a PI fetch request
+                    requestRef = db.collection('pi_fetch_requests')
+                        .doc(String(cid))
+                        .collection('requests')
+                        .doc(requestId);
+                }
+                
                 await requestRef.update({
                     status: 'failed',
                     error: error.message || 'Unknown error',
@@ -423,13 +473,40 @@ async function handlePopularInvestorUpdate(taskData, config, dependencies) {
  * Handles the On-Demand update for a Signed-In User.
  */
 async function handleOnDemandUserUpdate(taskData, config, dependencies) {
-    const { cid, username } = taskData;
-    const { ETORO_API_PORTFOLIO_URL, ETORO_API_HISTORY_URL } = config;
+    const { cid, username, requestId, source } = taskData;
+    
+    // [FIX] Destructure dependencies first
+    const { logger, proxyManager, batchManager, headerManager, db } = dependencies;
+    
+    // Validate and set API URLs with defaults and fallbacks
+    const ETORO_API_PORTFOLIO_URL = config.ETORO_API_PORTFOLIO_URL || process.env.ETORO_API_PORTFOLIO_URL || 'https://www.etoro.com/sapi/portfolios/portfolio';
+    const ETORO_API_HISTORY_URL = config.ETORO_API_HISTORY_URL || process.env.ETORO_API_HISTORY_URL || 'https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat';
     
-    // [FIX] Destructure headerManager
-    const { logger, proxyManager, batchManager, headerManager } = dependencies;
+    if (!ETORO_API_PORTFOLIO_URL || ETORO_API_PORTFOLIO_URL === 'undefined') {
+        const errorMsg = 'ETORO_API_PORTFOLIO_URL is not configured. Check taskEngine config.';
+        logger.log('ERROR', `[On-Demand Update] ${errorMsg}`);
+        throw new Error(errorMsg);
+    }
 
-    logger.log('INFO', `[On-Demand Update] Fetching Portfolio & History for Signed-In User: ${username} (${cid})`);
+    logger.log('INFO', `[On-Demand Update] Fetching Portfolio & History for Signed-In User: ${username} (${cid})${requestId ? ` [Request: ${requestId}]` : ''}`);
+    
+    // Update request status if this is a sync request
+    if (requestId && source === 'on_demand_sync' && db) {
+        try {
+            const requestRef = db.collection('user_sync_requests')
+                .doc(String(cid))
+                .collection('requests')
+                .doc(requestId);
+            
+            await requestRef.update({
+                status: 'processing',
+                startedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+            });
+        } catch (err) {
+            logger.log('WARN', `[On-Demand Update] Failed to update request status for ${requestId}`, err);
+        }
+    }
     
     const today = new Date().toISOString().split('T')[0];
     const blockId = '19M'; 
@@ -568,8 +645,49 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
         }
 
         logger.log('SUCCESS', `[On-Demand Update] Complete for ${username}`);
+        
+        // Update request status to completed if this is a sync request
+        if (requestId && source === 'on_demand_sync' && db) {
+            try {
+                const requestRef = db.collection('user_sync_requests')
+                    .doc(String(cid))
+                    .collection('requests')
+                    .doc(requestId);
+                
+                await requestRef.update({
+                    status: 'completed',
+                    completedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                    updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                });
+                logger.log('INFO', `[On-Demand Update] Updated sync request ${requestId} to completed`);
+            } catch (err) {
+                logger.log('WARN', `[On-Demand Update] Failed to update request status to completed for ${requestId}`, err);
+            }
+        }
     } catch (error) {
         logger.log('ERROR', `[On-Demand Update] Failed for ${username}`, error);
+        
+        // Update request status to failed if this is a sync request
+        if (requestId && source === 'on_demand_sync' && db) {
+            try {
+                const requestRef = db.collection('user_sync_requests')
+                    .doc(String(cid))
+                    .collection('requests')
+                    .doc(requestId);
+                
+                const errorMessage = error.message || 'Unknown error occurred';
+                await requestRef.update({
+                    status: 'failed',
+                    error: errorMessage,
+                    failedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                    updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
+                });
+                logger.log('INFO', `[On-Demand Update] Updated sync request ${requestId} to failed: ${errorMessage}`);
+            } catch (err) {
+                logger.log('WARN', `[On-Demand Update] Failed to update request status to failed for ${requestId}`, err);
+            }
+        }
+        
         throw error;
     } finally {
         if (headerManager && headerId !== 'fallback') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.451",
+  "version": "1.0.453",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [