bulltrackers-module 1.0.168 → 1.0.169

package/functions/task-engine/helpers/discover_helpers.js

@@ -1,7 +1,14 @@
+/*
+ * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/discover_helpers.js
+ * (REFACTORED: Now includes node-fetch fallback)
+ * (REFACTORED: Added verbose, traceable logging)
+ */
+
 /**
  * @fileoverview Sub-pipe: pipe.taskEngine.handleDiscover
  * REFACTORED: Now stateless and receives dependencies.
  * --- MODIFIED: Passes username to the 'verify' task. ---
+ * --- MODIFIED: Added node-fetch fallback and verbose logging. ---
  */
 
 /**
@@ -15,56 +22,164 @@ async function handleDiscover(task, taskId, dependencies, config) {
     const { logger, headerManager, proxyManager, pubsub, batchManager } = dependencies;
     const { cids, blockId, instrument, userType } = task;
     const url = `${config.ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
+    
     const selectedHeader = await headerManager.selectHeader();
-    if (!selectedHeader) throw new Error("Could not select a header.");
+    if (!selectedHeader) {
+        logger.log('ERROR', `[DISCOVER/${taskId}] Could not select a header. Aborting task.`);
+        throw new Error("Could not select a header.");
+    }
+
     let wasSuccess = false;
+    let proxyUsed = true;
+    const logPrefix = `[DISCOVER/${taskId}]`; // --- REFACTOR 2: VERBOSE LOGGING ---
+
+    try { // Outer try for the whole operation
+        let response;
+        const options = {
+            method: 'POST',
+            headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },
+            body: JSON.stringify(cids),
+        };
+        
+        logger.log('INFO', `${logPrefix} Starting discovery for ${cids.length} CIDs. Block: ${blockId}, Type: ${userType}`);
+
+        try {
+            // --- REFACTOR 3: ADD FALLBACK ---
+            logger.log('TRACE', `${logPrefix} Attempting discovery fetch via AppScript proxy...`);
+            response = await proxyManager.fetch(url, options);
+            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
+            
+        } catch (proxyError) {
+            logger.log('WARN', `${logPrefix} AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
+                error: proxyError.message,
+                source: 'AppScript'
+            });
+            proxyUsed = false;
+            
+            try {
+                response = await fetch(url, options); // Direct node-fetch
+                if (!response.ok) {
+                    const errorText = await response.text();
+                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
+                }
+                
+            } catch (fallbackError) {
+                logger.log('ERROR', `${logPrefix} Direct node-fetch fallback FAILED.`, {
+                    error: fallbackError.message,
+                    source: 'eToro/Network'
+                });
+                throw fallbackError; // Throw to be caught by outer try
+            }
+            // --- END REFACTOR 3 ---
+        }
+
+        // --- If we are here, `response` is valid ---
+        
+        // Step 1. Discover Speculators
+        if (userType === 'speculator') {
+            batchManager.addProcessedSpeculatorCids(cids);
+            logger.log('INFO', `${logPrefix} Added ${cids.length} speculator CIDs to the in-memory set to be flushed.`);
+        }
+        
+        const body = await response.text();
+        let publicUsers;
+        try {
+            // --- REFACTOR 4: LOG RAW RESPONSE ON PARSE FAILURE ---
+            publicUsers = JSON.parse(body);
+        } catch (parseError) {
+            logger.log('ERROR', `${logPrefix} FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
+                parseErrorMessage: parseError.message,
+                rawResponseText: body 
+            });
+            throw new Error(`Failed to parse JSON response from discovery API. Body: ${body.substring(0, 200)}`);
+        }
+        // --- END REFACTOR 4 ---
+
+        if (!Array.isArray(publicUsers)) {
+             logger.log('WARN', `${logPrefix} API returned non-array response. Type: ${typeof publicUsers}`);
+             wasSuccess = true; // API call worked, data was just empty/weird
+             return;
+        }
+        
+        wasSuccess = true; // Mark as success *after* parsing
+        logger.log('INFO', `${logPrefix} API call successful, found ${publicUsers.length} public users.`);
 
-    // Step 1. Discover Speculators
-    try {if (userType === 'speculator') {batchManager.addProcessedSpeculatorCids(cids);logger.log('INFO', `[DISCOVER] Added ${cids.length} speculator CIDs to the in-memory set to be flushed.`);}
-        const response = await proxyManager.fetch(url, {method: 'POST',headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },body: JSON.stringify(cids),});
-        if (!response.ok) {throw new Error(`API status ${response.status}`);}
-        wasSuccess = true;
-        const publicUsers = await response.json();
-        if (!Array.isArray(publicUsers)) return;
         const oneMonthAgo = new Date();
         oneMonthAgo.setMonth(oneMonthAgo.getMonth() - 1);
-        // Step 2. We know a user is "active" if their risk score is NOT 0 and they logged in, in the last month. This covers the edge case that a user IS active but does NOT hold positions (cannot hold no positions and have 0 risk score)
-        const preliminaryActiveUsers = publicUsers.filter(user => new Date(user.Value.LastActivity) > oneMonthAgo &&user.Value.DailyGain !== 0 &&user.Value.Exposure !== 0 &&user.Value.RiskScore !== 0);
+        
+        // Step 2. Filter active users
+        const preliminaryActiveUsers = publicUsers.filter(user => 
+            new Date(user.Value.LastActivity) > oneMonthAgo &&
+            user.Value.DailyGain !== 0 &&
+            user.Value.Exposure !== 0 &&
+            user.Value.RiskScore !== 0
+        );
+        logger.log('INFO', `${logPrefix} Found ${preliminaryActiveUsers.length} preliminary active users.`);
+
         let finalActiveUsers = [];
         const invalidCidsToLog = [];
-        // Step 3. Users that were passed INTO the discovery API but did not come OUT OF the discovery API, are private accounts, the API filters this for us, so mark them as such to avoid processing them again.
-        if (userType === 'speculator') {const publicUserCids = new Set(publicUsers.map(u => u.CID));
-            if (publicUserCids.size > 0 && publicUserCids.size < cids.length) {const privateUserCids = cids.filter(cid => !publicUserCids.has(cid));invalidCidsToLog.push(...privateUserCids);}
+        
+        // Step 3. Log private/inactive users
+        if (userType === 'speculator') {
+            const publicUserCids = new Set(publicUsers.map(u => u.CID));
+            if (publicUserCids.size > 0 && publicUserCids.size < cids.length) {
+                const privateUserCids = cids.filter(cid => !publicUserCids.has(cid));
+                invalidCidsToLog.push(...privateUserCids);
+                logger.log('INFO', `${logPrefix} Found ${privateUserCids.length} private users (in request but not response).`);
+            }
             const activeUserCids = new Set(preliminaryActiveUsers.map(u => u.CID));
             const inactiveUserCids = publicUsers.filter(u => !activeUserCids.has(u.CID)).map(u => u.CID);
             invalidCidsToLog.push(...inactiveUserCids);
-            logger.log('INFO', `[DISCOVER] Applying new speculator pre-filter to ${preliminaryActiveUsers.length} active users.`);
+            logger.log('INFO', `${logPrefix} Found ${inactiveUserCids.length} inactive users.`);
+            
+            // Step 4. Apply speculator heuristic
             const nonSpeculatorCids = [];
-            // Step 4. For the remaining active users, apply heuristic filters to weed out non-speculators.
             for (const user of preliminaryActiveUsers) {
                 const v = user.Value;
                 const totalLeverage = (v.MediumLeveragePct || 0) + (v.HighLeveragePct || 0);
-                const isLikelySpeculator = ((v.Trades || 0) > 500 ||(v.TotalTradedInstruments || 0) > 50 ||totalLeverage > 50 ||(v.WeeklyDD || 0) < -25);
-                if (isLikelySpeculator) {finalActiveUsers.push(user);} else {nonSpeculatorCids.push(user.CID);}}
+                const isLikelySpeculator = ((v.Trades || 0) > 500 || (v.TotalTradedInstruments || 0) > 50 || totalLeverage > 50 || (v.WeeklyDD || 0) < -25);
+                
+                if (isLikelySpeculator) {
+                    finalActiveUsers.push(user);
+                } else {
+                    nonSpeculatorCids.push(user.CID);
+                }
+            }
             invalidCidsToLog.push(...nonSpeculatorCids);
-            logger.log('INFO', `[DISCOVER] Pre-filter complete. ${finalActiveUsers.length} users passed. ${nonSpeculatorCids.length} users failed heuristic.`);
+            logger.log('INFO', `${logPrefix} Speculator pre-filter complete. ${finalActiveUsers.length} users passed. ${nonSpeculatorCids.length} users failed heuristic.`);
+            
             if (invalidCidsToLog.length > 0) {
                 await pubsub.topic(config.PUBSUB_TOPIC_INVALID_SPECULATOR_LOG).publishMessage({ json: { invalidCids: invalidCidsToLog } });
-                logger.log('INFO', `[DISCOVER] Reported ${invalidCidsToLog.length} invalid (private, inactive, or failed heuristic) speculator IDs.`);
+                logger.log('INFO', `${logPrefix} Reported ${invalidCidsToLog.length} invalid (private, inactive, or failed heuristic) speculator IDs.`);
             }
-        // Step 5. Remaining users are public active non speculators
+        // Step 5. Non-speculators are just active users
         } else { 
             finalActiveUsers = preliminaryActiveUsers;
         }
-        // Step 6. Publish 'verify' task for all active users found
+        
+        // Step 6. Publish 'verify' task
         if (finalActiveUsers.length > 0) {
-            const verificationTask = {type: 'verify',users: finalActiveUsers.map(u => ({ cid: u.CID, isBronze: u.Value.IsBronze,username: u.Value.UserName})), blockId, instrument, userType};
+            const verificationTask = {
+                type: 'verify',
+                users: finalActiveUsers.map(u => ({ cid: u.CID, isBronze: u.Value.IsBronze, username: u.Value.UserName })),
+                blockId,
+                instrument,
+                userType
+            };
             await pubsub.topic(config.PUBSUB_TOPIC_USER_FETCH).publishMessage({ json: { tasks: [verificationTask] } });
-            logger.log('INFO', `[DISCOVER] Verification message published was : ${JSON.stringify({ tasks: [verificationTask] })} `);
-            logger.log('INFO', `[DISCOVER] Chaining to 'verify' task for ${finalActiveUsers.length} active users.`);
+            logger.log('SUCCESS', `${logPrefix} Chaining to 'verify' task for ${finalActiveUsers.length} active users.`);
+        } else {
+            logger.log('INFO', `${logPrefix} No active users found to verify.`);
         }
+
+    } catch (err) {
+        logger.log('ERROR', `${logPrefix} FATAL error processing discovery task.`, { errorMessage: err.message, errorStack: err.stack });
+        wasSuccess = false; // Ensure it's marked as failure
     } finally {
-        if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess);
+        if (selectedHeader && proxyUsed) {
+            // Only update performance if the proxy was used
+            headerManager.updatePerformance(selectedHeader.id, wasSuccess);
+        }
     }
 }
 

package/functions/task-engine/helpers/update_helpers.js

@@ -1,33 +1,24 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
- * (MODIFIED: To conditionally fetch history API once per user per batch)
- * (MODIFIED: `lookupUsernames` runs batches in parallel)
- * (MODIFIED: `handleUpdate` fetches history and all portfolios in parallel)
- * (MODIFIED: `handleUpdate` now uses batchManager for history cache)
- * (FIXED: Added try/catch around JSON.parse to log raw HTML error pages)
- * (FIXED: Improved logging for proxy failures)
+ * (REFACTORED: Removed all concurrency from `handleUpdate` and `lookupUsernames`)
+ * (REFACTORED: Added node-fetch fallback for all API calls)
+ * (REFACTORED: Added verbose, user-centric logging for all operations)
  */
 
-/**
- * @fileoverview Sub-pipe: pipe.taskEngine.handleUpdate
- * REFACTORED: Now stateless and receives dependencies.
- * OPTIMIZED: Removed immediate batch commit for speculator timestamp fix.
- * --- MODIFIED: Fetches portfolio AND trade history in parallel. ---
- * --- MODIFIED: Includes helper to look up usernames from CIDs. ---
- * --- MODIFIED: Conditionally fetches history only once per user per batch. ---
- */
 const { FieldValue } = require('@google-cloud/firestore');
 const pLimit = require('p-limit');
 
 /**
- * (MODIFIED: Runs lookup batches in parallel)
+ * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
  */
 async function lookupUsernames(cids, { logger, headerManager, proxyManager }, config) {
     if (!cids?.length) return [];
     logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
     
-    // Use a new config value, falling back to 5
-    const limit = pLimit(config.USERNAME_LOOKUP_CONCURRENCY || 5);
+    // --- REFACTOR 1: REMOVE CONCURRENCY ---
+    const limit = pLimit(1);
+    // --- END REFACTOR 1 ---
+    
     const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
     
     const batches = [];
@@ -35,26 +26,67 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
         batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number));
     }
 
-    const batchPromises = batches.map(batch => limit(async () => {
+    const batchPromises = batches.map((batch, index) => limit(async () => {
+        const batchId = `batch-${index + 1}`;
+        logger.log('INFO', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`);
+        
         const header = await headerManager.selectHeader();
         if (!header) {
-            logger.log('ERROR', '[lookupUsernames] Could not select a header.');
-            return null; // Return null to skip this batch
+            logger.log('ERROR', `[lookupUsernames/${batchId}] Could not select a header.`);
+            return null;
         }
         
-        let success = false;
+        let wasSuccess = false;
+        let proxyUsed = true;
+        let response;
+        const url = `${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
+        const options = { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) };
+
         try {
-            const res = await proxyManager.fetch(`${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`, { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) });
-            if (!res.ok) throw new Error(`API status ${res.status}`);
-            const data = await res.json();
-            success = true;
-            logger.log('DEBUG', 'Looked up usernames', { batch: batch.slice(0, 5) }); // Log only a few
-            return data; // Return data on success
-        } catch (err) {
-            logger.log('WARN', `[lookupUsernames] Failed batch`, { error: err.message });
-            return null; // Return null on failure
+            // --- REFACTOR 3: ADD FALLBACK ---
+            logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
+            response = await proxyManager.fetch(url, options);
+            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
+            
+            wasSuccess = true;
+            logger.log('INFO', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`);
+            
+        } catch (proxyError) {
+            logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
+                error: proxyError.message,
+                source: 'AppScript'
+            });
+            
+            proxyUsed = false; // Don't penalize header for proxy failure
+            
+            try {
+                response = await fetch(url, options); // Direct node-fetch
+                if (!response.ok) {
+                    const errorText = await response.text();
+                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
+                }
+                logger.log('INFO', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`);
+                
+            } catch (fallbackError) {
+                logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, {
+                    error: fallbackError.message,
+                    source: 'eToro/Network'
+                });
+                return null; // Give up on this batch
+            }
+            // --- END REFACTOR 3 ---
         } finally {
-            headerManager.updatePerformance(header.id, success);
+            if (proxyUsed) {
+                headerManager.updatePerformance(header.id, wasSuccess);
+            }
+        }
+
+        try {
+            const data = await response.json();
+            return data;
+        } catch (parseError) {
+            logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message });
+            return null;
         }
     }));
 
@@ -62,7 +94,7 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
     
     const allUsers = results
         .filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value))
-        .flatMap(r => r.value); // Flatten all successful batch results
+        .flatMap(r => r.value); 
 
     logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
     return allUsers;
@@ -70,173 +102,197 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
 
 
 /**
- * (MODIFIED: Fetches history and all portfolios in parallel)
- * (MODIFIED: Uses batchManager for history cache)
+ * (REFACTORED: Fully sequential, verbose logging, node-fetch fallback)
  */
-async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) { // <--- REMOVED historyFetchedForUser
+async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) {
     const { userId, instruments, instrumentId, userType } = task;
     const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
+    let isPrivate = false;
+
+    // --- REFACTOR 2: ADD VERBOSE LOGGING (with User ID) ---
+    logger.log('INFO', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
 
+    // --- 1. Process History Fetch (Sequentially) ---
     let historyHeader = null;
     let wasHistorySuccess = false;
-    let historyFetchPromise = null;
-    let isPrivate = false;
+    let proxyUsedForHistory = true;
 
     try {
-        // --- 1. Prepare History Fetch (if needed) ---
-        // (MODIFIED: Use batchManager's cross-invocation cache)
         if (!batchManager.checkAndSetHistoryFetched(userId)) {
-            // This user has NOT been fetched in the last 10 mins (by this instance)
+            logger.log('INFO', `[handleUpdate/${userId}] Attempting history fetch.`);
             historyHeader = await headerManager.selectHeader();
-            if (historyHeader) {
-                // No need to add to a local set, batchManager did it.
-                const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
-                historyFetchPromise = proxyManager.fetch(historyUrl, { headers: historyHeader.header });
+            if (!historyHeader) {
+                logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
             } else {
-                logger.log('WARN', `[handleUpdate] Could not select history header for ${userId}. History will be skipped for this task.`);
+                const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
+                const options = { headers: historyHeader.header };
+                let response;
+
+                try {
+                    // --- REFACTOR 3: ADD FALLBACK ---
+                    logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
+                    response = await proxyManager.fetch(historyUrl, options);
+                    if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
+                    wasHistorySuccess = true;
+                    
+                } catch (proxyError) {
+                    logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
+                        error: proxyError.message,
+                        source: 'AppScript'
+                    });
+                    proxyUsedForHistory = false;
+
+                    try {
+                        response = await fetch(historyUrl, options); // Direct node-fetch
+                        if (!response.ok) {
+                            const errorText = await response.text();
+                            throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
+                        }
+                        wasHistorySuccess = true; // Fallback succeeded
+                        
+                    } catch (fallbackError) {
+                        logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, {
+                            error: fallbackError.message,
+                            source: 'eToro/Network'
+                        });
+                        wasHistorySuccess = false;
+                    }
+                    // --- END REFACTOR 3 ---
+                }
+
+                if (wasHistorySuccess) {
+                    logger.log('INFO', `[handleUpdate/${userId}] History fetch successful.`);
+                    const data = await response.json();
+                    await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
+                }
             }
         } else {
-            logger.log('TRACE', `[handleUpdate] History fetch for ${userId} skipped (already fetched by this instance).`);
+            logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched by this instance).`);
         }
-
-        // --- 2. Prepare All Portfolio Fetches ---
-        const portfolioRequests = [];
-        for (const instId of instrumentsToProcess) {
-            const portfolioHeader = await headerManager.selectHeader();
-            if (!portfolioHeader) throw new Error(`Could not select portfolio header for ${userId}`);
-            
-            const portfolioUrl = userType === 'speculator' 
-                ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}`
-                : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
-            
-            portfolioRequests.push({
-                instrumentId: instId,
-                url: portfolioUrl,
-                header: portfolioHeader,
-                promise: proxyManager.fetch(portfolioUrl, { headers: portfolioHeader.header })
-            });
+    } catch (err) {
+        logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message });
+        wasHistorySuccess = false;
+    } finally {
+        if (historyHeader && proxyUsedForHistory) {
+            headerManager.updatePerformance(historyHeader.id, wasHistorySuccess);
         }
-        
-        // --- 3. Execute All API Calls in Parallel ---
-        const allPromises = [
-            ...(historyFetchPromise ? [historyFetchPromise] : []),
-            ...portfolioRequests.map(r => r.promise)
-        ];
-        const allResults = await Promise.allSettled(allPromises);
-
-        // --- 4. Process History Result ---
-        let resultIndex = 0;
-        if (historyFetchPromise) {
-            const historyRes = allResults[resultIndex++];
-            if (historyRes.status === 'fulfilled' && historyRes.value.ok) {
-                const data = await historyRes.value.json();
-                wasHistorySuccess = true;
-                await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
-            } else {
-                logger.log('WARN', `[handleUpdate] History fetch failed for ${userId}`, { error: historyRes.reason || `status ${historyRes.value?.status}` });
-            }
+    }
+    
+    // --- 2. Process Portfolio Fetches (Sequentially) ---
+    logger.log('INFO', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`);
+    
+    for (const instId of instrumentsToProcess) {
+        if (isPrivate) {
+             logger.log('INFO', `[handleUpdate/${userId}] Skipping remaining instruments because user was marked as private.`);
+             break; 
         }
 
-        // --- 5. Process Portfolio Results ---
-        for (let i = 0; i < portfolioRequests.length; i++) {
-            const requestInfo = portfolioRequests[i];
-            const portfolioRes = allResults[resultIndex++];
-            let wasPortfolioSuccess = false;
-
-            if (portfolioRes.status === 'fulfilled' && portfolioRes.value.ok) {
-                const body = await portfolioRes.value.text();
-                if (body.includes("user is PRIVATE")) {
-                    isPrivate = true;
-                    logger.log('WARN', `User ${userId} is private. Removing from updates.`);
-                    break; // Stop processing more portfolios for this private user
-                }
+        const portfolioHeader = await headerManager.selectHeader();
+        if (!portfolioHeader) {
+            logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`);
+            continue;
+        }
 
-                // --- START OF THE FIX ---
-                try {
-                    // Try to parse the body as JSON
-                    const portfolioJson = JSON.parse(body);
-                    
-                    // If successful, proceed as normal
-                    wasPortfolioSuccess = true;
-                    await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, requestInfo.instrumentId);
-                    logger.log('DEBUG', 'Processing portfolio for user (Success)', { userId, portfolioUrl: requestInfo.url });
-                    logger.log('DEBUG', 'Response returned (parsed OK)', { body } , 'for user' , { userId });
-
-                } catch (parseError) {
-                    // IT FAILED. This means 'body' is NOT JSON. It's the HTML block page.
-                    wasPortfolioSuccess = false; // Mark as failure
-                    logger.log('ERROR', `[handleUpdate] FAILED TO PARSE RESPONSE. RAW BODY:`, {
-                        userId: userId,
-                        url: requestInfo.url,
-                        parseErrorMessage: parseError.message,
-                        rawResponseText: body // <--- THIS WILL LOG THE FULL HTML RESPONSE
-                    });
-                }
-                // --- END OF THE FIX ---
+        const portfolioUrl = userType === 'speculator' 
+            ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}`
+            : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+        
+        const options = { headers: portfolioHeader.header };
+        let response;
+        let wasPortfolioSuccess = false;
+        let proxyUsedForPortfolio = true;
 
-            } else {
-                // --- IMPROVED ERROR LOGGING FOR PROXY FAILURES ---
-                let errorLog = {};
-                if (portfolioRes.status === 'rejected') {
-                    // Promise.allSettled rejected (e.g., unhandled exception)
-                    errorLog = { 
-                        error: "Promise rejected", 
-                        reason: portfolioRes.reason?.message || portfolioRes.reason, 
-                        stack: portfolioRes.reason?.stack 
-                    };
-                } else {
-                    // Proxy returned ok: false
-                    const responseValue = portfolioRes.value || {};
-                    // Get the detailed error object from the proxy manager (if it exists)
-                    const errorDetails = responseValue.error || { message: `status ${responseValue.status}`, details: "No error object provided." };
-                    errorLog = { 
-                        error: "Proxy fetch failed", 
-                        errorMessage: errorDetails.message, 
-                        errorDetails: errorDetails.details  // This will log the raw error from the proxy
-                    };
+        try {
+            // --- REFACTOR 3: ADD FALLBACK ---
+            logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch for instId ${instId} via AppScript proxy...`);
+            response = await proxyManager.fetch(portfolioUrl, options);
+            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
+            wasPortfolioSuccess = true;
+
+        } catch (proxyError) {
+            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
+                error: proxyError.message,
+                source: 'AppScript'
+            });
+            proxyUsedForPortfolio = false;
+            
+            try {
+                response = await fetch(portfolioUrl, options); // Direct node-fetch
+                if (!response.ok) {
+                    const errorText = await response.text();
+                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
                 }
+                wasPortfolioSuccess = true; // Fallback succeeded
                 
-                logger.log('WARN', `Failed to fetch portfolio (Proxy Error or Rejected)`, { 
-                    userId, 
-                    url: requestInfo.url, 
-                    ...errorLog 
+            } catch (fallbackError) {
+                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} direct fallback FAILED.`, {
+                    error: fallbackError.message,
+                    source: 'eToro/Network'
                 });
-                // --- END IMPROVED LOGGING ---
+                wasPortfolioSuccess = false;
             }
-            // Update performance for this specific header
-            headerManager.updatePerformance(requestInfo.header.id, wasPortfolioSuccess);
+            // --- END REFACTOR 3 ---
         }
 
-        // --- 6. Handle Private Users & Timestamps ---
-        if (isPrivate) {
-            logger.log('WARN', `User ${userId} is private. Removing from updates.`);
-            for (const instrumentId of instrumentsToProcess) {
-                await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
+        // --- 4. Process Portfolio Result (with verbose, raw logging) ---
+        if (wasPortfolioSuccess) {
+            const body = await response.text();
+            if (body.includes("user is PRIVATE")) {
+                isPrivate = true;
+                logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`);
+                break; // Stop processing more portfolios for this private user
             }
-            const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
-            for (const instrumentId of instrumentsToProcess) {
-                const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
-                // This is not batched, but it's a rare event.
-                await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
+
+            try {
+                const portfolioJson = JSON.parse(body);
+                await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
+                logger.log('INFO', `[handleUpdate/${userId}] Successfully processed portfolio for instId ${instId}.`);
+                
+            } catch (parseError) {
+                // --- REFACTOR 4: RETURN EXACT PAGE RESPONSE ---
+                wasPortfolioSuccess = false; // Mark as failure
+                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
+                    url: portfolioUrl,
+                    parseErrorMessage: parseError.message,
+                    rawResponseText: body // <--- THIS LOGS THE FULL HTML/ERROR RESPONSE
+                });
+                // --- END REFACTOR 4 ---
             }
-            return; // Don't update timestamps
+        } else {
+             logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}. No response to process.`);
         }
+        
+        if (proxyUsedForPortfolio) {
+            headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess);
+        }
+    } // --- End of sequential portfolio loop ---
 
-        // If not private, update all timestamps
+    // --- 5. Handle Private Users & Timestamps ---
+    if (isPrivate) {
+        logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
         for (const instrumentId of instrumentsToProcess) {
-            await batchManager.updateUserTimestamp(userId, userType, instrumentId);
+            await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId);
         }
-        if (userType === 'speculator') {
-            await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
+        const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
+        for (const instrumentId of instrumentsToProcess) {
+            const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
+            await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
         }
+        return; 
+    }
 
-    } finally {
-        if (historyHeader) { // historyHeader is only set if a fetch was attempted
-            headerManager.updatePerformance(historyHeader.id, wasHistorySuccess);
-        }
+    // If not private, update all timestamps
+    for (const instrumentId of instrumentsToProcess) {
+        await batchManager.updateUserTimestamp(userId, userType, instrumentId);
     }
+    if (userType === 'speculator') {
+        await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
+    }
+    
+    logger.log('INFO', `[handleUpdate/${userId}] Update task finished successfully.`);
+    // 'finally' block for header flushing is handled by the main handler_creator.js
 }
 
 module.exports = { handleUpdate, lookupUsernames };

package/functions/task-engine/helpers/verify_helpers.js

@@ -1,28 +1,142 @@
+/*
+ * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/verify_helpers.js
+ * (REFACTORED: `handleVerify` now runs sequentially to prevent throttling)
+ * (REFACTORED: `fetchAndVerifyUser` now includes node-fetch fallback)
+ * (REFACTORED: Added verbose, user-centric logging for all operations)
+ */
 const { FieldValue } = require('@google-cloud/firestore');
 
-async function fetchAndVerifyUser(user, { logger, headerManager, proxyManager }, { userType, SPECULATOR_INSTRUMENTS_ARRAY }) {
+/**
+ * (REFACTORED: Now includes taskId for logging, and full fallback logic)
+ */
+async function fetchAndVerifyUser(user, { logger, headerManager, proxyManager }, { userType, SPECULATOR_INSTRUMENTS_ARRAY }, taskId) {
+    const cid = user.cid;
+    const logPrefix = `[VERIFY/${taskId}/${cid}]`; // --- REFACTOR 2: VERBOSE LOGGING ---
+
     const selectedHeader = await headerManager.selectHeader();
-    if (!selectedHeader) return null;
+    if (!selectedHeader) {
+        logger.log('WARN', `${logPrefix} Could not select a header. Skipping user.`);
+        return null;
+    }
+
     let wasSuccess = false;
-    try { const res = await proxyManager.fetch(`${process.env.ETORO_API_PORTFOLIO_URL}?cid=${user.cid}`, { headers: selectedHeader.header }); if (!res.ok) return null;
+    let proxyUsed = true;
+            
+    try { // Outer try for the whole operation
+        let response;
+        const url = `${process.env.ETORO_API_PORTFOLIO_URL}?cid=${cid}`;
+        const options = { headers: selectedHeader.header };
+
+        try {
+            // --- REFACTOR 3: ADD FALLBACK ---
+            logger.log('TRACE', `${logPrefix} Attempting portfolio fetch via AppScript proxy...`);
+            response = await proxyManager.fetch(url, options);
+            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
+            
+        } catch (proxyError) {
+            logger.log('WARN', `${logPrefix} AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
+                error: proxyError.message,
+                source: 'AppScript'
+            });
+            proxyUsed = false;
+
+            try {
+                response = await fetch(url, options); // Direct node-fetch
+                if (!response.ok) {
+                    const errorText = await response.text();
+                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
+                }
+                // Fallback succeeded, but we don't set wasSuccess yet,
+                // as we still need to parse the body.
+                
+            } catch (fallbackError) {
+                logger.log('ERROR', `${logPrefix} Direct node-fetch fallback FAILED.`, {
+                    error: fallbackError.message,
+                    source: 'eToro/Network'
+                });
+                throw fallbackError; // Throw to be caught by outer try
+            }
+            // --- END REFACTOR 3 ---
+        }
+
+        // --- If we are here, `response` is valid ---
+        let portfolioData;
+        const body = await response.text();
+        
+        try {
+            // --- REFACTOR 4: LOG RAW RESPONSE ON PARSE FAILURE ---
+            portfolioData = JSON.parse(body);
+        } catch (parseError) {
+            logger.log('ERROR', `${logPrefix} FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
+                parseErrorMessage: parseError.message,
+                rawResponseText: body 
+            });
+            throw new Error(`Failed to parse JSON for user ${cid}.`);
+        }
+        // --- END REFACTOR 4 ---
+
+        // --- Original logic ---
+        if (userType === 'speculator') {
+            const instruments = portfolioData.AggregatedPositions.map(p => p.InstrumentID).filter(id => SPECULATOR_INSTRUMENTS_ARRAY.includes(id));
+            if (!instruments.length) {
+                logger.log('TRACE', `${logPrefix} Verified user, but not a speculator (no matching assets).`);
+                wasSuccess = true; // API call *worked*
+                return null;
+            }
+            logger.log('INFO', `${logPrefix} Verified as SPECULATOR.`);
+            wasSuccess = true;
+            return { type: 'speculator', userId: cid, isBronze: user.isBronze, username: user.username, updateData: { instruments, lastVerified: new Date(), lastHeldSpeculatorAsset: new Date() } };
+        }
+
+        logger.log('INFO', `${logPrefix} Verified as NORMAL user.`);
         wasSuccess = true;
-        const portfolioData = await res.json();
-        if (userType === 'speculator') { const instruments = portfolioData.AggregatedPositions.map(p => p.InstrumentID) .filter(id => SPECULATOR_INSTRUMENTS_ARRAY.includes(id));
-            if (!instruments.length) return null; return { type: 'speculator', userId: user.cid, isBronze: user.isBronze, username: user.username, updateData: { instruments, lastVerified: new Date(), lastHeldSpeculatorAsset: new Date() } }; }
-        return { type: 'normal', userId: user.cid, isBronze: user.isBronze, username: user.username, updateData: { lastVerified: new Date() } };
-    } catch (err) { logger.log('WARN', `[VERIFY] Error processing user ${user.cid}`, { errorMessage: err.message }); return null;
-    } finally { if (selectedHeader) headerManager.updatePerformance(selectedHeader.id, wasSuccess); }
+        return { type: 'normal', userId: cid, isBronze: user.isBronze, username: user.username, updateData: { lastVerified: new Date() } };
+        
+    } catch (err) {
+        // This catches proxy, fallback, or parse errors
+        logger.log('WARN', `${logPrefix} Error processing user.`, { errorMessage: err.message });
+        wasSuccess = false; // Ensure it's marked as failure
+        return null;
+    } finally {
+        if (selectedHeader && proxyUsed) {
+            // Only update performance if the proxy was used
+            headerManager.updatePerformance(selectedHeader.id, wasSuccess);
+        }
+    }
 }
 
+/**
+ * (REFACTORED: Now runs fetches sequentially)
+ */
 async function handleVerify(task, taskId, { db, logger, ...dependencies }, config) {
     const { users, blockId, instrument, userType } = task;
     const batch = db.batch();
     const speculatorUpdates = {}, normalUpdates = {}, bronzeStates = {}, usernameMap = {};
     const specSet = new Set(config.SPECULATOR_INSTRUMENTS_ARRAY);
-    const results = await Promise.allSettled(users.map(u => fetchAndVerifyUser(u, dependencies, { ...config, userType })));
+
+    // --- REFACTOR 1: REMOVE CONCURRENCY ---
+    logger.log('INFO', `[VERIFY/${taskId}] Starting sequential verification for ${users.length} users...`);
+    const results = [];
+    for (const user of users) {
+        // Await each user one by one
+        const result = await fetchAndVerifyUser(user, { db, logger, ...dependencies }, { ...config, userType }, taskId);
+        results.push(result); // Push the actual result (or null)
+    }
+    logger.log('INFO', `[VERIFY/${taskId}] Sequential verification complete.`);
+    // --- END REFACTOR 1 ---
+
     let validUserCount = 0;
     results.forEach(r => {
-        if (r.status === 'fulfilled' && r.value) { const d = r.value; usernameMap[d.userId] = { username: d.username }; bronzeStates[d.userId] = d.isBronze; validUserCount++; if (d.type === 'speculator') speculatorUpdates[`users.${d.userId}`] = d.updateData; else normalUpdates[`users.${d.userId}`] = d.updateData; } });
+        if (r) { // Only process non-null results
+            const d = r; 
+            usernameMap[d.userId] = { username: d.username }; 
+            bronzeStates[d.userId] = d.isBronze; 
+            validUserCount++; 
+            if (d.type === 'speculator') speculatorUpdates[`users.${d.userId}`] = d.updateData; 
+            else normalUpdates[`users.${d.userId}`] = d.updateData; 
+        }
+    });
+
     if (Object.keys(speculatorUpdates).length || Object.keys(normalUpdates).length) {
         const blockRef = db.collection(userType === 'speculator' ? config.FIRESTORE_COLLECTION_SPECULATOR_BLOCKS : config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS).doc(String(blockId));
         batch.set(blockRef, userType === 'speculator' ? speculatorUpdates : normalUpdates, { merge: true });
@@ -35,9 +149,10 @@ async function handleVerify(task, taskId, { db, logger, ...dependencies }, confi
         for (const cid in usernameMap) { const shardId = `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`;
             if (!shardedUpdates[shardId]) { shardedUpdates[shardId] = {}; } shardedUpdates[shardId][cid] = usernameMap[cid]; }
         for (const shardId in shardedUpdates) { const mapRef = db.collection(config.FIRESTORE_COLLECTION_USERNAME_MAP).doc(shardId); batch.set(mapRef, shardedUpdates[shardId], { merge: true }); }
-        logger.log('INFO', `[VERIFY] Staging username updates across ${Object.keys(shardedUpdates).length} shards.`); }
+        logger.log('INFO', `[VERIFY/${taskId}] Staging username updates across ${Object.keys(shardedUpdates).length} shards.`); }
+    
     await batch.commit();
-    if (validUserCount) logger.log('INFO', `[VERIFY] Verified and stored ${validUserCount} new ${userType} users.`);
+    if (validUserCount) logger.log('SUCCESS', `[VERIFY/${taskId}] Verified and stored ${validUserCount} new ${userType} users.`);
 }
 
 module.exports = { handleVerify };

package/functions/task-engine/utils/task_engine_utils.js

@@ -1,9 +1,6 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/utils/task_engine_utils.js
- * (MODIFIED: To pass down a Set to track history fetches)
- * (MODIFIED: To run all update tasks in parallel with a concurrency limit)
- * (MODIFIED: To use a SINGLE parallel work pool for ALL tasks)
- * (MODIFIED: To remove local history cache set)
+ * (REFACTORED: Concurrency limit set to 1 to prevent API throttling)
  */
 
 /**
@@ -15,7 +12,7 @@
 const { handleDiscover } = require('../helpers/discover_helpers');
 const { handleVerify } = require('../helpers/verify_helpers');
 const { handleUpdate, lookupUsernames } = require('../helpers/update_helpers');
-const pLimit = require('p-limit'); // <--- IMPORT p-limit
+const pLimit = require('p-limit'); 
 
 /**
  * Parses Pub/Sub message into task array.
@@ -55,15 +52,16 @@ async function runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config
 
 /**
  * Executes all tasks.
- * (MODIFIED: Runs ALL tasks in a single parallel pool)
+ * (REFACTORED: Concurrency limit set to 1)
  */
 async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId) {
-    const { logger, batchManager } = dependencies; // <--- Get batchManager
+    const { logger, batchManager } = dependencies; 
     
-    // REMOVED: const historyFetchedForUser = new Set();
+    // --- REFACTOR 1: REMOVE CONCURRENCY ---
+    // Set limit to 1 to serialize all tasks and prevent AppScript throttling.
+    const limit = pLimit(1); 
+    // --- END REFACTOR 1 ---
     
-    // Create one unified parallel pool
-    const limit = pLimit(config.TASK_ENGINE_CONCURRENCY || 3); // TODO Work out what the optimal concurrency is
     const allTaskPromises = [];
     let taskCounters = { update: 0, discover: 0, verify: 0, unknown: 0, failed: 0 };
 
@@ -94,8 +92,7 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
         const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId}`;
         allTaskPromises.push(
             limit(() => 
-                // Pass batchManager instead of the local set
-                handleUpdate(task, subTaskId, dependencies, config, username) // <--- REMOVED historyFetchedForUser
+                handleUpdate(task, subTaskId, dependencies, config, username) 
                     .then(() => taskCounters.update++)
                     .catch(err => {
                         logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message });
@@ -105,7 +102,7 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
         );
     }
 
-    // 3. Wait for ALL tasks to complete
+    // 3. Wait for ALL tasks to complete (now sequentially)
     await Promise.all(allTaskPromises);
 
     // 4. Log final summary

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.168",
+  "version": "1.0.169",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [