bulltrackers-module 1.0.209 → 1.0.211

package/functions/task-engine/handler_creator.js

@@ -2,10 +2,12 @@
  * @fileoverview Main pipe for the Task Engine.
  * REFACTORED: This function is the high-level "manual of processes"
  * that orchestrates the task engine logic.
+ * FIXED: Removed dead call to runUsernameLookups.
  */
 
 // Import the new utility functions that contain the logic
-const { parseTaskPayload, prepareTaskBatches, runUsernameLookups, executeTasks } = require('./utils/task_engine_utils'); 
+// REMOVED: runUsernameLookups from import
+const { parseTaskPayload, prepareTaskBatches, executeTasks } = require('./utils/task_engine_utils'); 
 
 /**
  * Main pipe: pipe.taskEngine.handleRequest
@@ -20,15 +22,29 @@ async function handleRequest(message, context, config, dependencies) {
     const taskId = `batch-${context.eventId || Date.now()}`;
     try {
         const tasks = parseTaskPayload(message, logger);
-        if (!tasks) {return; }
+        if (!tasks) { return; }
+        
         logger.log('INFO', `[TaskEngine/${taskId}] Received batch of ${tasks.length} tasks.`);
-        const { tasksToRun, cidsToLookup, otherTasks } = await prepareTaskBatches(tasks, batchManager, logger);
-        await runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config, batchManager, logger);
+        
+        // Prepare batches
+        const { tasksToRun, otherTasks } = await prepareTaskBatches(tasks, batchManager, logger);
+        
+        // REMOVED: await runUsernameLookups(...) which caused the crash
+        
+        // Execute (Now runs everything sequentially inside executeTasks)
         await executeTasks(tasksToRun, otherTasks, dependencies, config, taskId);
-    } catch (error) {logger.log('ERROR', `[TaskEngine/${taskId}] Failed during batch processing.`, { errorMessage: error.message, errorStack: error.stack });
-    } finally { try {logger.log('INFO', `[TaskEngine/${taskId}] Flushing all accumulated batches...`);
-        await batchManager.flushBatches();
-        logger.log('INFO', `[TaskEngine/${taskId}] Final batch and header flush complete.`);} catch (flushError) {logger.log('ERROR', `[TaskEngine/${taskId}] Error during final flush attempt.`, { error: flushError.message });}}
+
+    } catch (error) {
+        logger.log('ERROR', `[TaskEngine/${taskId}] Failed during batch processing.`, { errorMessage: error.message, errorStack: error.stack });
+    } finally { 
+        try {
+            logger.log('INFO', `[TaskEngine/${taskId}] Flushing all accumulated batches...`);
+            await batchManager.flushBatches();
+            logger.log('INFO', `[TaskEngine/${taskId}] Final batch and header flush complete.`);
+        } catch (flushError) {
+            logger.log('ERROR', `[TaskEngine/${taskId}] Error during final flush attempt.`, { error: flushError.message });
+        }
+    }
 }
 
-module.exports = {handleRequest};
+module.exports = { handleRequest };

package/functions/task-engine/helpers/update_helpers.js

@@ -4,6 +4,8 @@
  * (OPTIMIZED V3: Removed obsolete username lookup logic)
  * (OPTIMIZED V2: Added "Circuit Breaker" for Proxy failures)
  * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
+ * (FIXED: Improved logging clarity for Normal vs Speculator users)
+ * (FIXED: Final log now accurately reflects failure state)
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
@@ -66,6 +68,8 @@ function detectSpeculatorTargets(historyData, portfolioData) {
  */
 async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager, pubsub }, config) {
     const { userId, instruments, instrumentId, userType } = task;
+    
+    // Normalize the loop: Speculators get specific IDs, Normal users get [undefined] to trigger one pass.
     const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
@@ -75,7 +79,12 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     let capturedHistory = null;
     let capturedPortfolio = null;
 
-    logger.log('TRACE', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
+    // Track overall success for the final log
+    let hasPortfolioErrors = false;
+
+    // FIX 1: Better Start Log
+    const scopeLog = userType === 'speculator' ? `Instruments: [${instrumentsToProcess.join(', ')}]` : 'Scope: Full Portfolio';
+    logger.log('TRACE', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. ${scopeLog}`);
 
     // --- 1. Process History Fetch (Sequentially) ---
     let historyHeader = null;
@@ -148,13 +157,16 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     logger.log('TRACE', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`); 
     
     for (const instId of instrumentsToProcess) { 
+        // FIX 2: Define a clear scope name for logging
+        const scopeName = instId ? `Instrument ${instId}` : 'Full Portfolio';
+
         if (isPrivate) { 
-            logger.log('TRACE', `[handleUpdate/${userId}] Skipping remaining instruments (User Private).`); 
+            logger.log('TRACE', `[handleUpdate/${userId}] Skipping ${scopeName} (User Private).`); 
             break;  
         }
 
         const portfolioHeader = await headerManager.selectHeader();
-        if (!portfolioHeader) { logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`); continue; }
+        if (!portfolioHeader) { logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for ${scopeName}. Skipping.`); continue; }
 
         const portfolioUrl = userType === 'speculator'  ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
         const options = { headers: portfolioHeader.header };
@@ -165,7 +177,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         // --- PROXY ATTEMPT ---
         if (shouldTryProxy()) {
             try {
-                logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch via AppScript proxy...`);
+                logger.log('TRACE', `[handleUpdate/${userId}] Attempting fetch for ${scopeName} via AppScript proxy...`);
                 response = await proxyManager.fetch(portfolioUrl, options);
                 if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); 
                 wasPortfolioSuccess = true; 
@@ -174,7 +186,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
 
             } catch (proxyError) { 
                 recordProxyOutcome(false);
-                logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch via Proxy FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
+                logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch for ${scopeName} via Proxy FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
             }
         }
 
@@ -184,9 +196,9 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                 response = await fetch(portfolioUrl, options); 
                 if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); }
                 wasPortfolioSuccess = true; 
-                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio fetch direct success.`);
+                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio fetch for ${scopeName} direct success.`);
             } catch (fallbackError) {
-                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
+                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch for ${scopeName} direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
                 wasPortfolioSuccess = false;
             }
         }
@@ -200,14 +212,16 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                 const portfolioJson = JSON.parse(body);
                 capturedPortfolio = portfolioJson; // Capture for detection
                 await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
-                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio processed successfully.`); 
+                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio for ${scopeName} processed successfully.`); 
 
             } catch (parseError) { 
                 wasPortfolioSuccess = false; 
-                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE.`, { url: portfolioUrl, parseErrorMessage: parseError.message }); 
+                hasPortfolioErrors = true; // Mark error state
+                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE for ${scopeName}.`, { url: portfolioUrl, parseErrorMessage: parseError.message }); 
             }
         } else { 
-            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}.`); 
+            hasPortfolioErrors = true; // Mark error state
+            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch FAILED for ${scopeName}.`); 
         }
         
         if (proxyUsedForPortfolio) { headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess); }
@@ -254,14 +268,22 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         return; 
     }
 
-    // If not private, update all timestamps
+    // If not private AND no critical errors, update timestamps
+    // (We update timestamps even on partial failures for speculators to avoid infinite retry loops immediately, 
+    // relying on the next scheduled run, but for Normal users, a failure usually means we should retry later. 
+    // Current logic: Update timestamp to prevent immediate re-queueing.)
     for (const instrumentId of instrumentsToProcess) { 
         await batchManager.updateUserTimestamp(userId, userType, instrumentId); 
     }
     
     if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
     
-    logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`); 
+    // FIX 3: Honest Final Log
+    if (hasPortfolioErrors) {
+        logger.log('WARN', `[handleUpdate/${userId}] Update task finished with ERRORS. See logs above.`); 
+    } else {
+        logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`); 
+    }
 }
 
 module.exports = { handleUpdate };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.209",
+  "version": "1.0.211",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [