bulltrackers-module 1.0.491 → 1.0.493

package/functions/generic-api/user-api/helpers/data_helpers.js

@@ -891,6 +891,7 @@ async function getUserComputations(req, res, dependencies, config) {
         
         // If mode is 'latest', try to find the latest available date for the first computation
         // Use effectiveCid for computation lookup
+        // CRITICAL: Only look back 7 days as per requirements
         if (mode === 'latest') {
             const firstCompName = computationNames[0];
             const latestDate = await findLatestComputationDate(
@@ -901,7 +902,7 @@ async function getUserComputations(req, res, dependencies, config) {
                 category, 
                 firstCompName, 
                 effectiveCid, 
-                30
+                7 // Only look back 7 days as per requirements
             );
             
             if (latestDate) {
@@ -910,14 +911,15 @@ async function getUserComputations(req, res, dependencies, config) {
                     logger.log('INFO', `[getUserComputations] Using fallback date ${latestDate} for effective CID ${effectiveCid} (today: ${today})`);
                 }
             } else {
-                // No data found, return empty
+                // No data found after 7 days - return empty (frontend will use fallback)
+                logger.log('WARN', `[getUserComputations] No computation data found for CID ${effectiveCid} in last 7 days. Frontend will use fallback.`);
                 return res.status(200).json({
                     status: 'success',
                     userCid: String(effectiveCid),
                     mode,
                     computations: computationNames,
                     data: {},
-                    isFallback: false,
+                    isFallback: true, // Mark as fallback since no data found
                     requestedDate: today,
                     isImpersonating: isImpersonating || false,
                     actualCid: Number(userCid)
@@ -955,10 +957,38 @@ async function getUserComputations(req, res, dependencies, config) {
                     if (doc.exists) {
                         // Decompress if needed (handles byte string storage)
                         const rawData = doc.data();
-                        const data = tryDecompress(rawData);
+                        let data = tryDecompress(rawData);
+                        
+                        // Handle string decompression result
+                        if (typeof data === 'string') {
+                            try {
+                                data = JSON.parse(data);
+                            } catch (e) {
+                                logger.log('WARN', `[getUserComputations] Failed to parse decompressed string for ${compName} on ${date}:`, e.message);
+                                data = null;
+                            }
+                        }
+                        
+                        // Check if data is sharded
+                        if (data && data._sharded === true && data._shardCount) {
+                            // Data is stored in shards - read all shards and merge
+                            const shardsCol = docRef.collection('_shards');
+                            const shardsSnapshot = await shardsCol.get();
+                            
+                            if (!shardsSnapshot.empty) {
+                                data = {};
+                                for (const shardDoc of shardsSnapshot.docs) {
+                                    const shardData = shardDoc.data();
+                                    Object.assign(data, shardData);
+                                }
+                            } else {
+                                data = null; // Sharded but no shards found
+                            }
+                        }
+                        
                         // Filter by user CID - computation results are stored as { cid: result }
                         // Use effectiveCid for lookup
-                        let userResult = data[String(effectiveCid)];
+                        let userResult = data && typeof data === 'object' ? data[String(effectiveCid)] : null;
                         
                         // Apply dev override for computations that include copied PIs (use actual userCid for override check)
                         if (isDevOverrideActive && (compName === 'SignedInUserProfileMetrics' || compName === 'SignedInUserCopiedPIs')) {

package/functions/generic-api/user-api/helpers/on_demand_fetch_helpers.js

@@ -5,6 +5,7 @@
 
 const { FieldValue } = require('@google-cloud/firestore');
 const crypto = require('crypto');
+const { tryDecompress } = require('./data_helpers');
 
 const RATE_LIMIT_HOURS = 1;
 const RATE_LIMIT_MS = RATE_LIMIT_HOURS * 60 * 60 * 1000;
@@ -253,7 +254,6 @@ async function getPiFetchStatus(req, res, dependencies, config) {
             
             const doc = await docRef.get();
             if (doc.exists) {
-                const { tryDecompress } = require('./data_helpers');
                 const data = tryDecompress(doc.data());
                 
                 if (data && data[String(piCidNum)]) {
@@ -316,7 +316,6 @@ async function getPiFetchStatus(req, res, dependencies, config) {
                             }
                         } else {
                             // Data is in the main document (compressed or not)
-                            const { tryDecompress } = require('./data_helpers');
                             mergedData = tryDecompress(docData);
                             
                             // Handle string decompression result

package/functions/task-engine/handler_creator.js

@@ -235,12 +235,27 @@ async function handleRequest(message, context, configObj, dependencies) {
                 await handlePopularInvestorUpdate(taskData, configObj, dependencies);
                 break;
             case 'ON_DEMAND_USER_UPDATE':
-                const onDemandData = data || payload;
-                if (!onDemandData.cid || !onDemandData.username) {
-                    logger.log('ERROR', `[TaskEngine] ON_DEMAND_USER_UPDATE missing required fields`, { data: onDemandData });
+                // For ON_DEMAND_USER_UPDATE, the entire payload IS the task data
+                // (not wrapped in a 'data' field like other task types)
+                // Extract task data from payload, excluding 'type'
+                const onDemandTaskData = data || {
+                    cid: payload.cid,
+                    username: payload.username,
+                    requestId: payload.requestId,
+                    source: payload.source,
+                    requestedBy: payload.requestedBy,
+                    effectiveRequestedBy: payload.effectiveRequestedBy,
+                    metadata: payload.metadata,
+                    priority: payload.priority,
+                    data: payload.data // Include nested data object (includeSocial, since, etc.)
+                };
+                
+                if (!onDemandTaskData.cid || !onDemandTaskData.username) {
+                    logger.log('ERROR', `[TaskEngine] ON_DEMAND_USER_UPDATE missing required fields (cid or username)`, { payload, onDemandTaskData });
                     return;
                 }
-                await handleOnDemandUserUpdate(onDemandData, configObj, dependencies);
+                
+                await handleOnDemandUserUpdate(onDemandTaskData, configObj, dependencies);
                 break;
             case 'SOCIAL_INSTRUMENT_FETCH':
             case 'SOCIAL_PI_FETCH':

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -500,6 +500,9 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
     // [FIX] Destructure dependencies first
     const { logger, proxyManager, batchManager, headerManager, db } = dependencies;
     
+    // Import notification helper once at the top
+    const { notifyTaskEngineComplete } = require('../../generic-api/user-api/helpers/notification_helpers');
+    
     // Validate and set API URLs with defaults and fallbacks
     const ETORO_API_PORTFOLIO_URL = config.ETORO_API_PORTFOLIO_URL || process.env.ETORO_API_PORTFOLIO_URL || 'https://www.etoro.com/sapi/portfolios/portfolio';
     const ETORO_API_HISTORY_URL = config.ETORO_API_HISTORY_URL || process.env.ETORO_API_HISTORY_URL || 'https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat';
@@ -558,6 +561,48 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
     let fetchSuccess = false;
     let portfolioFetched = false;
     let historyFetched = false;
+    let criticalError = null; // Track critical errors that should fail the sync
+
+    // Helper function to mark sync as failed and send notification
+    const markSyncFailed = async (errorMessage, stage) => {
+        if (requestId && source === 'on_demand_sync' && db) {
+            try {
+                const requestRef = db.collection('user_sync_requests')
+                    .doc(String(cid))
+                    .collection('requests')
+                    .doc(requestId);
+                
+                await requestRef.update({
+                    status: 'failed',
+                    error: errorMessage,
+                    failedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                    updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                    failedStage: stage
+                });
+                
+                // Send error notification
+                if (metadata?.requestingUserCid) {
+                    try {
+                        await notifyTaskEngineComplete(
+                            db,
+                            logger,
+                            metadata.requestingUserCid,
+                            requestId,
+                            username,
+                            false,
+                            errorMessage
+                        );
+                    } catch (notifErr) {
+                        logger.log('WARN', `[On-Demand Update] Failed to send error notification`, notifErr);
+                    }
+                }
+                
+                logger.log('ERROR', `[On-Demand Update] Marked sync ${requestId} as failed at stage: ${stage}. Error: ${errorMessage}`);
+            } catch (updateErr) {
+                logger.log('ERROR', `[On-Demand Update] Failed to mark sync as failed for ${requestId}`, updateErr);
+            }
+        }
+    };
 
     try {
         // Portfolio Fetch (only if portfolioOnly is true)
@@ -597,7 +642,10 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                     portfolioSuccess = true;
                 } catch (fetchErr) {
                     logger.log('ERROR', `[On-Demand] Direct fetch also failed for ${username}`, fetchErr);
-                    throw new Error(`Failed to fetch portfolio for ${cid}. Direct: ${fetchErr.message}`);
+                    const errorMsg = `Failed to fetch portfolio for ${cid}. Direct: ${fetchErr.message}`;
+                    criticalError = new Error(errorMsg);
+                    await markSyncFailed(errorMsg, 'portfolio_fetch');
+                    throw criticalError;
                 }
             }
 
@@ -608,7 +656,15 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
             fetchSuccess = true;
             portfolioFetched = true;
 
-            await batchManager.addToPortfolioBatch(String(cid), blockId, today, portfolioData, 'signed_in_user');
+            try {
+                await batchManager.addToPortfolioBatch(String(cid), blockId, today, portfolioData, 'signed_in_user');
+            } catch (batchErr) {
+                const errorMsg = `Failed to store portfolio data: ${batchErr.message}`;
+                logger.log('ERROR', `[On-Demand] ${errorMsg}`, batchErr);
+                criticalError = new Error(errorMsg);
+                await markSyncFailed(errorMsg, 'portfolio_storage');
+                throw criticalError;
+            }
         } else {
             logger.log('INFO', `[On-Demand Update] Skipping portfolio fetch (portfolioOnly=false) for ${username}`);
         }
@@ -660,23 +716,49 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
         }
         
         if (historySuccess && historyRes.ok) {
-            const historyData = await historyRes.json();
-            historyData.fetchedAt = new Date();
-            
-            const VALID_REASONS = [0, 1, 5]; 
-            if (historyData.PublicHistoryPositions) {
-                historyData.PublicHistoryPositions = historyData.PublicHistoryPositions.filter(
-                    p => VALID_REASONS.includes(p.CloseReason)
-                );
+            try {
+                const historyData = await historyRes.json();
+                historyData.fetchedAt = new Date();
+                
+                const VALID_REASONS = [0, 1, 5]; 
+                if (historyData.PublicHistoryPositions) {
+                    historyData.PublicHistoryPositions = historyData.PublicHistoryPositions.filter(
+                        p => VALID_REASONS.includes(p.CloseReason)
+                    );
+                }
+                
+                try {
+                    await batchManager.addToTradingHistoryBatch(String(cid), blockId, today, historyData, 'signed_in_user');
+                    historyFetched = true;
+                } catch (batchErr) {
+                    const errorMsg = `Failed to store history data: ${batchErr.message}`;
+                    logger.log('ERROR', `[On-Demand] ${errorMsg}`, batchErr);
+                    criticalError = new Error(errorMsg);
+                    await markSyncFailed(errorMsg, 'history_storage');
+                    throw criticalError;
+                }
+            } catch (parseErr) {
+                const errorMsg = `Failed to parse history data: ${parseErr.message}`;
+                logger.log('ERROR', `[On-Demand] ${errorMsg}`, parseErr);
+                criticalError = new Error(errorMsg);
+                await markSyncFailed(errorMsg, 'history_parse');
+                throw criticalError;
             }
-                await batchManager.addToTradingHistoryBatch(String(cid), blockId, today, historyData, 'signed_in_user');
-                historyFetched = true;
+        } else {
+            // History fetch failed - this is critical if portfolioOnly is true
+            if (portfolioOnly) {
+                const errorMsg = `History fetch failed for ${username} (status: ${historyRes?.status || 'unknown'})`;
+                logger.log('ERROR', `[On-Demand Update] ${errorMsg}`);
+                criticalError = new Error(errorMsg);
+                await markSyncFailed(errorMsg, 'history_fetch');
+                throw criticalError;
             } else {
-                logger.log('WARN', `[On-Demand Update] History fetch failed for ${username} (${historyRes.status})`);
+                logger.log('WARN', `[On-Demand Update] History fetch failed for ${username} (${historyRes?.status || 'unknown'})`);
             }
-        } else {
-            logger.log('INFO', `[On-Demand Update] Skipping history fetch (portfolioOnly=false) for ${username}`);
         }
+    } else {
+        logger.log('INFO', `[On-Demand Update] Skipping history fetch (portfolioOnly=false) for ${username}`);
+    }
 
         // Fetch social data if requested (for user signup or explicit request)
         let socialFetched = false;
@@ -764,7 +846,6 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
         // Send notification to requesting user if this is an on-demand sync
         if (requestId && source === 'on_demand_sync' && metadata?.requestingUserCid) {
             try {
-                const { notifyTaskEngineComplete } = require('../../generic-api/user-api/helpers/notification_helpers');
                 const success = portfolioFetched || historyFetched; // At least one should succeed
                 await notifyTaskEngineComplete(
                     db,
@@ -816,14 +897,12 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                         updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
                     });
                 } catch (indexerError) {
-                    logger.log('ERROR', `[On-Demand Update] Failed to run root data indexer for ${today}`, indexerError);
-                    // Continue anyway - computation might still work if index already exists
-                    // But update status to indicate we tried
-                    await requestRef.update({
-                        status: 'computing',
-                        indexerError: indexerError.message,
-                        updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
-                    });
+                    const errorMsg = `Failed to run root data indexer: ${indexerError.message}`;
+                    logger.log('ERROR', `[On-Demand Update] ${errorMsg}`, indexerError);
+                    // Root data indexing is critical - if it fails, computations won't know data exists
+                    criticalError = new Error(errorMsg);
+                    await markSyncFailed(errorMsg, 'root_data_indexing');
+                    throw criticalError;
                 }
                 
                 // Trigger computations with dependency chain resolution
@@ -870,22 +949,30 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                     // Don't mark as completed yet - wait for computation to finish
                     // The status will remain 'computing' until computation completes
                 } else {
-                    logger.log('WARN', `[On-Demand Update] PubSub not available, cannot trigger computation`);
-                    // Mark as completed if we can't trigger computation
-                    await requestRef.update({
-                        status: 'completed',
-                        completedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
-                        updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
-                    });
+                    const errorMsg = 'PubSub not available, cannot trigger computation';
+                    logger.log('ERROR', `[On-Demand Update] ${errorMsg}`);
+                    criticalError = new Error(errorMsg);
+                    await markSyncFailed(errorMsg, 'computation_trigger');
+                    throw criticalError;
                 }
             } catch (err) {
-                logger.log('WARN', `[On-Demand Update] Failed to update request status or trigger computation for ${requestId}`, err);
+                // If this is a critical error we already handled, re-throw it
+                if (criticalError) {
+                    throw criticalError;
+                }
+                // Otherwise, this is an unexpected error in the computation triggering block
+                const errorMsg = `Failed to trigger computations: ${err.message}`;
+                logger.log('ERROR', `[On-Demand Update] ${errorMsg}`, err);
+                criticalError = new Error(errorMsg);
+                await markSyncFailed(errorMsg, 'computation_trigger');
+                throw criticalError;
             }
         }
     } catch (error) {
         logger.log('ERROR', `[On-Demand Update] Failed for ${username}`, error);
         
         // Update request status to failed if this is a sync request
+        // Only update if we haven't already marked it as failed (to avoid duplicate updates)
         if (requestId && source === 'on_demand_sync' && db) {
             try {
                 const requestRef = db.collection('user_sync_requests')
@@ -893,14 +980,39 @@ async function handleOnDemandUserUpdate(taskData, config, dependencies) {
                     .collection('requests')
                     .doc(requestId);
                 
-                const errorMessage = error.message || 'Unknown error occurred';
-                await requestRef.update({
-                    status: 'failed',
-                    error: errorMessage,
-                    failedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
-                    updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp()
-                });
-                logger.log('INFO', `[On-Demand Update] Updated sync request ${requestId} to failed: ${errorMessage}`);
+                // Check current status to avoid overwriting if already marked as failed
+                const currentRequest = await requestRef.get();
+                if (currentRequest.exists && currentRequest.data().status !== 'failed') {
+                    const errorMessage = error.message || 'Unknown error occurred';
+                    const failedStage = error.failedStage || 'unknown';
+                    
+                    await requestRef.update({
+                        status: 'failed',
+                        error: errorMessage,
+                        failedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                        updatedAt: require('@google-cloud/firestore').FieldValue.serverTimestamp(),
+                        failedStage: failedStage
+                    });
+                    
+                    // Send error notification if not already sent
+                    if (metadata?.requestingUserCid) {
+                        try {
+                            await notifyTaskEngineComplete(
+                                db,
+                                logger,
+                                metadata.requestingUserCid,
+                                requestId,
+                                username,
+                                false,
+                                errorMessage
+                            );
+                        } catch (notifErr) {
+                            logger.log('WARN', `[On-Demand Update] Failed to send error notification`, notifErr);
+                        }
+                    }
+                    
+                    logger.log('INFO', `[On-Demand Update] Updated sync request ${requestId} to failed: ${errorMessage} (stage: ${failedStage})`);
+                }
             } catch (err) {
                 logger.log('WARN', `[On-Demand Update] Failed to update request status to failed for ${requestId}`, err);
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.491",
+  "version": "1.0.493",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [