bulltrackers-module 1.0.178 → 1.0.179

package/functions/task-engine/helpers/update_helpers.js

@@ -3,7 +3,6 @@
  * (REFACTORED: Removed all concurrency from `handleUpdate` and `lookupUsernames`)
  * (REFACTORED: Added node-fetch fallback for all API calls)
  * (REFACTORED: Added verbose, user-centric logging for all operations)
- * (FIXED: Resolved ReferenceError 'instId is not defined' in final timestamp loops)
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
@@ -16,83 +15,54 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
     if (!cids?.length) return [];
     logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
     
-    // --- Set concurrency to 1 to prevent AppScript rate limits. DO NOT CHANGE THIS.
+    // --- Set concurrency to 1 because appscript gets really fucked up with undocumented rate limits if we try spam it concurrently, a shame but that's life. DO NOT CHANGE THIS
     const limit = pLimit(1);
     const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
     const batches = [];
-    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) { 
-        batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number)); 
-    }
-
-    const batchPromises = batches.map((batch, index) => limit(async () => { 
-        const batchId = `batch-${index + 1}`;
+    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) { batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number)); }
+    const batchPromises = batches.map((batch, index) => limit(async () => { const batchId = `batch-${index + 1}`;
         logger.log('INFO', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`);
         const header = await headerManager.selectHeader();
         if (!header) { logger.log('ERROR', `[lookupUsernames/${batchId}] Could not select a header.`); return null; }
-        
         let wasSuccess = false;
         let proxyUsed = true;
         let response;
         const url = `${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
         const options = { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) };
-        
         try {
             logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
             response = await proxyManager.fetch(url, options);
             if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-            wasSuccess = true;
+            wasSuccess = true; // Yay we win
             logger.log('INFO', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`);
-        } catch (proxyError) { 
-            logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
-            proxyUsed = false; 
-            try { 
-                response = await fetch(url, options); 
-                if (!response.ok) { 
-                    const errorText = await response.text(); 
-                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); 
-                }
-                logger.log('INFO', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`);
-            } catch (fallbackError) { 
-                logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, { error: fallbackError.message, source: 'eToro/Network' });
-                return null;
+        } catch (proxyError) { logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' }); // SHIT we failed...
+            proxyUsed = false; // Don't penalize header for proxy failure
+            try { response = await fetch(url, options);  // Ok let's try again with node, using GCP IP pools
+                if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); }
+                logger.log('INFO', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`); // Yay we win
+            } catch (fallbackError) { logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, { error: fallbackError.message, source: 'eToro/Network' }); // SHIT, we failed here too
+                return null; // Give up on this batch
             }
-        } finally { 
-            if (proxyUsed) { headerManager.updatePerformance(header.id, wasSuccess); } 
-        } 
-        
-        try { 
-            const data = await response.json(); 
-            return data;
-        } catch (parseError) { 
-            logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message }); 
-            return null; 
-        } 
-    }));
+        } finally { if (proxyUsed) { headerManager.updatePerformance(header.id, wasSuccess); } } // If we used Appscript IP Pool and not GCP IP Pool, record performance
+        try { const data = await response.json(); return data;
+        } catch (parseError) { logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message }); return null; } }));
 
     const results  = await Promise.allSettled(batchPromises);
-    const allUsers = results
-        .filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value))
-        .flatMap(r => r.value); 
-    
+    const allUsers = results .filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value)) .flatMap(r => r.value); 
     logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
     return allUsers;
 }
 
 
 /**
- * (REFACTORED: Fully sequential, verbose logging, node-fetch fallback, FIXED SCOPING)
+ * (REFACTORED: Fully sequential, verbose logging, node-fetch fallback)
  */
 async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) {
     const { userId, instruments, instrumentId, userType } = task;
-    
-    // For normal users, we pass [undefined] so the loop runs exactly once.
-    // For speculators, we pass the list of instruments to fetch individually.
-    const instrumentsToProcess = userType === 'speculator' ? (instruments || [instrumentId]) : [undefined]; 
-    
+    const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
     let isPrivate = false;
-
     logger.log('INFO', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
 
     // --- 1. Process History Fetch (Sequentially) ---
@@ -104,70 +74,52 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         if (!batchManager.checkAndSetHistoryFetched(userId)) {
             logger.log('INFO', `[handleUpdate/${userId}] Attempting history fetch.`);
             historyHeader = await headerManager.selectHeader();
-            if (!historyHeader) { 
-                logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
+            if (!historyHeader) { logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
             } else {
                 const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
                 const options = { headers: historyHeader.header };
                 let response;
-                
-                try { 
-                    logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
+                try { logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
                     response = await proxyManager.fetch(historyUrl, options);
-                    if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-                    wasHistorySuccess = true;
+                    if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // SHIT we failed here
+                    wasHistorySuccess = true; // Appscript worked, we are very smart
+                    
                 } catch (proxyError) {
-                    logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
+                    logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' }); // SHIT we failed here
                     proxyUsedForHistory = false;
-                    try { 
-                        response = await fetch(historyUrl, options);
-                        if (!response.ok) { 
-                            const errorText = await response.text(); 
-                            throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); 
-                        }
-                        wasHistorySuccess = true;
-                    } catch (fallbackError) { 
-                        logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
-                        wasHistorySuccess = false; 
+                    try { response = await fetch(historyUrl, options);
+                        if (!response.ok) { const errorText = await response.text(); 
+                            throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); } // SHIT we failed here too
+                        wasHistorySuccess = true; // Fallback succeeded, we are so smart
+                        
+                    } catch (fallbackError) { logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' }); // We are dumb, everything failed
+                        wasHistorySuccess = false; // Nope we are dumb....
                     }
                 }
 
-                if (wasHistorySuccess) { 
-                    logger.log('INFO', `[handleUpdate/${userId}] History fetch successful.`);
+                if (wasHistorySuccess) { logger.log('INFO', `[handleUpdate/${userId}] History fetch successful.`); // Some method worked, we are very smart
                     const data = await response.json();
-                    await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);  
-                }
+                    await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);  }
             }
-        } else { 
-            logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched by this instance).`); 
-        }
-    } catch (err) { 
-        logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message }); 
-        wasHistorySuccess = false; 
-    } finally { 
-        if (historyHeader && proxyUsedForHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } 
-    }
+        } else { logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched by this instance).`); }
+    } catch (err) { logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message }); wasHistorySuccess = false; // We fucked up.
+    } finally { if (historyHeader && proxyUsedForHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } } // If we used appscript proxy, record performance, otherwise fuck off.
     
     // --- 2. Process Portfolio Fetches (Sequentially) ---
     logger.log('INFO', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`);
     
-    // Renamed loop variable to currentInstrumentId to prevent 'instId' reference errors later
-    for (const currentInstrumentId of instrumentsToProcess) { 
+    for (const instId of instrumentsToProcess) { 
         if (isPrivate) { 
             logger.log('INFO', `[handleUpdate/${userId}] Skipping remaining instruments because user was marked as private.`); 
             break;  
         }
 
         const portfolioHeader = await headerManager.selectHeader();
-        if (!portfolioHeader) { 
-            logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header. Skipping instrument.`);
+        if (!portfolioHeader) { logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`);
             continue;
         }
 
-        // Only append InstrumentID if it is defined (i.e., for speculators)
-        const portfolioUrl = userType === 'speculator' && currentInstrumentId 
-            ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${currentInstrumentId}` 
-            : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+        const portfolioUrl = userType === 'speculator'  ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
         
         const options = { headers: portfolioHeader.header };
         let response;
@@ -175,54 +127,48 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         let proxyUsedForPortfolio = true;
 
         try {
-            logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch via AppScript proxy...`);
+            // --- REFACTOR 3: ADD FALLBACK ---
+            logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch for instId ${instId} via AppScript proxy...`);
             response = await proxyManager.fetch(portfolioUrl, options);
-            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-            wasPortfolioSuccess = true;
+            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // SHIT we failed here
+            wasPortfolioSuccess = true; // Oh we are smart, worked first time.
 
-        } catch (proxyError) { 
-            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
-            proxyUsedForPortfolio = false; 
+        } catch (proxyError) { // try fallback with local node fetch using GCP IP Pools
+            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
+            proxyUsedForPortfolio = false; // We are not using Appscript proxy here as fallback is GCP based, so false
             
             try {
-                response = await fetch(portfolioUrl, options);
+                response = await fetch(portfolioUrl, options); // Direct node-fetch
                 if (!response.ok) {
                     const errorText = await response.text();
-                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
+                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); // SHIT we failed here
                 }
-                wasPortfolioSuccess = true;
+                wasPortfolioSuccess = true; // Fallback succeeded we are so smart
+                
             } catch (fallbackError) {
-                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
+                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
                 wasPortfolioSuccess = false;
             }
         }
 
+        // --- 4. Process Portfolio Result (with verbose, raw logging) ---
         if (wasPortfolioSuccess) {
             const body = await response.text();
-            if (body.includes("user is PRIVATE")) { 
-                isPrivate = true; 
-                logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`);
-                break; 
+            if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`);
+                break; // Stop processing more portfolios for this private user
             }
 
             try {
                 const portfolioJson = JSON.parse(body);
-                // Pass currentInstrumentId to batchManager. For normal users it's undefined, which is fine.
-                await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, currentInstrumentId);
-                
-                if (userType === 'speculator') { 
-                    logger.log('INFO', `[handleUpdate/${userId}] Successfully processed portfolio for instId ${currentInstrumentId}.`); 
-                } else { 
-                    logger.log('INFO', `[handleUpdate/${userId}] Successfully processed full portfolio (normal user).`); 
-                }
+                await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
+                if (userType === 'speculator') { logger.log('INFO', `[handleUpdate/${userId}] Successfully processed portfolio for instId ${instId}.`); // Only speculators have an instid, so this is conditional
+                } else { logger.log('INFO', `[handleUpdate/${userId}] Successfully processed full portfolio (normal user).`); } // Normal users
 
-            } catch (parseError) { 
-                wasPortfolioSuccess = false; 
-                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, { url: portfolioUrl, parseErrorMessage: parseError.message, rawResponseText: body }); 
+            } catch (parseError) { // Idk why this would happen, but if it does....log.
+                wasPortfolioSuccess = false; // Mark as failure
+                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, { url: portfolioUrl, parseErrorMessage: parseError.message, rawResponseText: body }); // Return full response
             }
-        } else { 
-            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed. No response to process.`); 
-        }
+        } else { logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}. No response to process.`); }
         
         if (proxyUsedForPortfolio) { headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess); }
     } 
@@ -230,32 +176,21 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     // --- 5. Handle Private Users & Timestamps ---
     if (isPrivate) {
         logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
-        // Iterate again using correct scoping
-        for (const currentInstrumentId of instrumentsToProcess) { 
-            await batchManager.deleteFromTimestampBatch(userId, userType, currentInstrumentId); 
-        }
-        
+        for (const instrumentId of instrumentsToProcess) { await batchManager.deleteFromTimestampBatch(userId, userType, instId); }
         const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
-        for (const currentInstrumentId of instrumentsToProcess) {
-            // Only update counts if we have a valid instrument ID (speculators)
-            if (currentInstrumentId) {
-                const incrementField = `counts.${currentInstrumentId}_${Math.floor(userId/1e6)*1e6}`;
-                await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
-            }
+        for (const instrumentId of instrumentsToProcess) {
+            const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
+            await blockCountsRef.set({ [incrementField]: FieldValue.increment(-1) }, { merge: true });
         }
         return; 
     }
 
     // If not private, update all timestamps
-    for (const currentInstrumentId of instrumentsToProcess) { 
-        await batchManager.updateUserTimestamp(userId, userType, currentInstrumentId); 
-    }
-    
-    if (userType === 'speculator') { 
-        await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); 
-    }
+    for (const instrumentId of instrumentsToProcess) { await batchManager.updateUserTimestamp(userId, userType, instId); }
+    if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
     
     logger.log('INFO', `[handleUpdate/${userId}] Update task finished successfully.`);
+    // 'finally' block for header flushing is handled by the main handler_creator.js
 }
 
 module.exports = { handleUpdate, lookupUsernames };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.178",
+  "version": "1.0.179",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [