bulltrackers-module 1.0.166 → 1.0.168

package/functions/computation-system/helpers/computation_manifest_builder.js

@@ -12,26 +12,15 @@
  */
 
 const fs                 = require('fs');
-//Hacky solution to force tmp writes
-// --- TEMP PATCH START ---
+//Hacky solution to force tmp writes TODO : This Tmp write is really dodgy, not ideal but works, consider less hacky solutions to writing to filesystem
 process.env.TMPDIR = '/tmp';
-process.env.TMP = '/tmp';
-process.env.TEMP = '/tmp';
-
-const os = require('os');
+process.env.TMP    = '/tmp';
+process.env.TEMP   = '/tmp';
+const os           = require('os');
 os.tmpdir = () => '/tmp';
-
-try {
-  const temp = require('temp');
-  const path = require('path');
-  const fs = require('fs');
-  const tmp = '/tmp';
-  if (!fs.existsSync(tmp)) fs.mkdirSync(tmp);
-  temp.dir = tmp;
-  temp.path = () => path.join(tmp, 'temp-' + Math.random().toString(36).slice(2));
-} catch {}
-// --- TEMP PATCH END ---
-const Viz = require('graphviz'); // Uses your local module
+try { const temp = require('temp'); const path = require('path'); const fs = require('fs'); const tmp = '/tmp';
+if (!fs.existsSync(tmp)) fs.mkdirSync(tmp); temp.dir = tmp; temp.path = () => path.join(tmp, 'temp-' + Math.random().toString(36).slice(2)); } catch {}
+const Viz = require('graphviz'); 
 
 /* --------------------------------------------------
  * Pretty Console Helpers

package/functions/core/utils/intelligent_proxy_manager.js

@@ -82,30 +82,63 @@ class IntelligentProxyManager {
     }
 
     /**
-     * --- MODIFIED: Makes a fetch request with exponential backoff for rate limits ---
+     * --- CORRECTED LOGIC: Makes a fetch request by trying different proxies ---
      * @param {string} targetUrl - The URL to fetch.
      * @param {object} options - Fetch options (e.g., headers).
      * @returns {Promise<object>} A mock Response object.
      */
     async fetch(targetUrl, options = {}) {
-        let proxy = null;
-        try { proxy = await this._selectProxy(); } catch (error) { return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() }; }
-        let backoff = this.INITIAL_BACKOFF_MS;
         let lastResponse = null;
+
+        // Use MAX_RETRIES to define the number of *different proxies* we will try
+        // before giving up on the request.
         for (let attempt = 1; attempt <= this.MAX_RETRIES; attempt++) {
+            let proxy;
+            try {
+                // 1. Select a new, UNLOCKED proxy *inside* the loop.
+                proxy = await this._selectProxy();
+            } catch (error) {
+                // This fails if all proxies are locked.
+                this.logger.log('ERROR', '[ProxyManager] fetch failed: All proxies are locked.', { url: targetUrl });
+                return { ok: false, status: 503, error: { message: error.message }, headers: new Headers() };
+            }
+
+            // 2. Make a SINGLE attempt with this selected proxy.
             const response = await this._fetchViaAppsScript(proxy.url, targetUrl, options);
-            lastResponse = response; 
-            // 1. Success
-            if (response.ok) { return response; }
-            // 2. Rate Limit Error (Retryable)
-            if (response.isRateLimitError) { this.logger.log('WARN', `[ProxyManager] Rate limit hit on proxy ${proxy.owner} (Attempt ${attempt}/${this.MAX_RETRIES}). Backing off for ${backoff}ms...`, { url: targetUrl }); await sleep(backoff); backoff *= 2; continue;  }
-            // 3. Other Fetch Error (Non-Retryable, Lock Proxy)
-            if (response.isUrlFetchError) { this.logger.log('ERROR', `[ProxyManager] Proxy ${proxy.owner} failed (non-rate-limit). Locking proxy.`, { url: targetUrl, status: response.status }); await this.lockProxy(proxy.owner); return response;  }
-            // 4. Standard Error (e.g., 404, 500 from *target* URL, not proxy)
-            return response; }
-        // If loop finishes, all retries failed (likely all were rate-limit errors)
-        this.logger.log('ERROR', `[ProxyManager] Request failed after ${this.MAX_RETRIES} rate-limit retries.`, { url: targetUrl });
-        return lastResponse;
+            lastResponse = response; // Save this response in case it's the last one
+
+            // 3. Case 1: Success! Return immediately.
+            if (response.ok) {
+                return response;
+            }
+
+            // 4. Case 2: Proxy-level error (Rate limit OR other fetch error)
+            //    This is a *proxy* failure, not a *target* failure.
+            if (response.isUrlFetchError) {
+                this.logger.log('WARN', `[ProxyManager] Proxy ${proxy.owner} failed (Attempt ${attempt}/${this.MAX_RETRIES}). Locking it and trying a new proxy.`, {
+                    reason: response.isRateLimitError ? "Rate Limit" : "Other Fetch Error",
+                    status: response.status,
+                    url: targetUrl
+                });
+
+                // LOCK THE FAILED PROXY so _selectProxy() won't pick it again.
+                await this.lockProxy(proxy.owner);
+
+                // Back off slightly before trying the *next* proxy to avoid a thundering herd.
+                await sleep(this.INITIAL_BACKOFF_MS * attempt);
+
+                continue; // Go to the next loop iteration to select a *new* proxy.
+            }
+
+            // 5. Case 3: Standard error from *target* URL (e.g., 404, 500)
+            //    This means the proxy *worked* but the target URL is bad.
+            //    This is a "successful" fetch. Do not retry, just return the response.
+            return response;
+        }
+
+        // 6. If loop finishes, all (this.MAX_RETRIES) proxy attempts failed.
+        this.logger.log('ERROR', `[ProxyManager] Request failed after ${this.MAX_RETRIES} proxy attempts.`, { url: targetUrl, lastStatus: lastResponse?.status });
+        return lastResponse; // Return the last failed response
     }
 
 

package/functions/fetch-insights/helpers/handler_helpers.js

@@ -25,10 +25,18 @@ exports.fetchAndStoreInsights = async (config, dependencies) => {
         try {
             logger.log('INFO', '[FetchInsightsHelpers] Attempting fetch via proxy manager...');
             response = await proxyManager.fetch(config.etoroInsightsUrl, fetchOptions);
+            if (!response.ok) {
+                const errorText = await response.text();
+                throw new Error(`Proxy request failed with status ${response.status}: ${errorText}`);
+            }
         } catch (proxyError) {
             logger.log('WARNING', `[FetchInsightsHelpers] Proxy manager fetch failed. Attempting fallback with node-fetch. Error: ${proxyError.message}`);
             try {
                 response = await fetch(config.etoroInsightsUrl, fetchOptions);
+                if (!response.ok) {
+                    const errorText = await response.text();
+                    throw new Error(`Fallback node-fetch failed with status ${response.status}: ${errorText}`);
+                }
             } catch (nodeFetchError) {
                 logger.log('ERROR', `[FetchInsightsHelpers] Fallback node-fetch also failed. Error: ${nodeFetchError.message}`);
                 throw nodeFetchError; // Throw the error from the last attempt

package/functions/task-engine/helpers/update_helpers.js

@@ -4,6 +4,8 @@
  * (MODIFIED: `lookupUsernames` runs batches in parallel)
  * (MODIFIED: `handleUpdate` fetches history and all portfolios in parallel)
  * (MODIFIED: `handleUpdate` now uses batchManager for history cache)
+ * (FIXED: Added try/catch around JSON.parse to log raw HTML error pages)
+ * (FIXED: Improved logging for proxy failures)
  */
 
 /**
@@ -149,14 +151,59 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                     isPrivate = true;
                     logger.log('WARN', `User ${userId} is private. Removing from updates.`);
                     break; // Stop processing more portfolios for this private user
-                } else {
+                }
+
+                // --- START OF THE FIX ---
+                try {
+                    // Try to parse the body as JSON
+                    const portfolioJson = JSON.parse(body);
+                    
+                    // If successful, proceed as normal
                     wasPortfolioSuccess = true;
-                    await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, JSON.parse(body), userType, requestInfo.instrumentId);
+                    await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, requestInfo.instrumentId);
+                    logger.log('DEBUG', 'Processing portfolio for user (Success)', { userId, portfolioUrl: requestInfo.url });
+                    logger.log('DEBUG', 'Response returned (parsed OK)', { body } , 'for user' , { userId });
+
+                } catch (parseError) {
+                    // IT FAILED. This means 'body' is NOT JSON. It's the HTML block page.
+                    wasPortfolioSuccess = false; // Mark as failure
+                    logger.log('ERROR', `[handleUpdate] FAILED TO PARSE RESPONSE. RAW BODY:`, {
+                        userId: userId,
+                        url: requestInfo.url,
+                        parseErrorMessage: parseError.message,
+                        rawResponseText: body // <--- THIS WILL LOG THE FULL HTML RESPONSE
+                    });
                 }
-                logger.log('DEBUG', 'Processing portfolio for user', { userId, portfolioUrl: requestInfo.url });
-                logger.log('DEBUG', 'Response returned ', { body } , 'for user' , { userId })
+                // --- END OF THE FIX ---
+
             } else {
-                logger.log('WARN', `Failed to fetch portfolio`, { userId, url: requestInfo.url, error: portfolioRes.reason || `status ${portfolioRes.value?.status}` });
+                // --- IMPROVED ERROR LOGGING FOR PROXY FAILURES ---
+                let errorLog = {};
+                if (portfolioRes.status === 'rejected') {
+                    // Promise.allSettled rejected (e.g., unhandled exception)
+                    errorLog = { 
+                        error: "Promise rejected", 
+                        reason: portfolioRes.reason?.message || portfolioRes.reason, 
+                        stack: portfolioRes.reason?.stack 
+                    };
+                } else {
+                    // Proxy returned ok: false
+                    const responseValue = portfolioRes.value || {};
+                    // Get the detailed error object from the proxy manager (if it exists)
+                    const errorDetails = responseValue.error || { message: `status ${responseValue.status}`, details: "No error object provided." };
+                    errorLog = { 
+                        error: "Proxy fetch failed", 
+                        errorMessage: errorDetails.message, 
+                        errorDetails: errorDetails.details  // This will log the raw error from the proxy
+                    };
+                }
+                
+                logger.log('WARN', `Failed to fetch portfolio (Proxy Error or Rejected)`, { 
+                    userId, 
+                    url: requestInfo.url, 
+                    ...errorLog 
+                });
+                // --- END IMPROVED LOGGING ---
             }
             // Update performance for this specific header
             headerManager.updatePerformance(requestInfo.header.id, wasPortfolioSuccess);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.166",
+  "version": "1.0.168",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [