bulltrackers-module 1.0.170 → 1.0.171

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,6 +1,8 @@
 /**
  * FIXED: computation_pass_runner.js
  * Now calculates earliest date PER CALCULATION, not per pass
+ * --- MODIFIED: Passes 'existingResults' to 'runStandardComputationPass'
+ * to replicate the test harness's 7-argument signature. ---
  */
 
 const { groupByPass, checkRootDataAvailability, fetchExistingResults, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
@@ -52,7 +54,13 @@ async function runComputationPass(config, dependencies, computationManifest) {
             const metaToRun     = metaCalcs.filter(c => checkDeps(c, rootData, existingResults, dateToProcess));
             if (!standardToRun.length && !metaToRun.length) return logger.log('INFO', `[PassRunner] All calcs complete for ${dateStr}. Skipping.`);
             logger.log('INFO', `[PassRunner] Running ${dateStr}: ${standardToRun.length} standard, ${metaToRun.length} meta`);
-            if (standardToRun.length) await runStandardComputationPass(dateToProcess, standardToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
+            
+            // --- THIS IS THE CHANGE ---
+            // 'existingResults' (which is the test harness's 'precomputedDependencies')
+            // is now passed to 'runStandardComputationPass'.
+            if (standardToRun.length) await runStandardComputationPass(dateToProcess, standardToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData, existingResults);
+            // --- END CHANGE ---
+
             if (metaToRun.length) await runMetaComputationPass(dateToProcess, metaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);
         } catch (err) { logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack }); }
     };
@@ -63,4 +71,4 @@ async function runComputationPass(config, dependencies, computationManifest) {
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 
-module.exports = { runComputationPass };
+module.exports = { runComputationPass };

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -248,7 +248,7 @@ function initializeCalculators(calcs, logger) {
 
 /**
  * Stage 7: Load T-1 (yesterday) data needed by historical calculations.
- * --- THIS FUNCTION IS NOW FIXED ---
+ * --- THIS IS THE FULLY CORRECTED FUNCTION (WITH FRIEND'S BUG FIX) ---
  */
 async function loadHistoricalData(date, calcs, config, deps, rootData) {
     const { logger } = deps;
@@ -259,7 +259,7 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
     const needsYesterdayInsights = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
     const needsYesterdaySocial = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
     const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
-    const needsYesterdayHistory = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('history')); // <-- THE FIX
+    const needsYesterdayHistory = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('history')); // <-- The check
     const needsYesterdayDependencies = calcs.some(c => c.isHistorical && c.dependencies && c.dependencies.length > 0);
 
     const prev = new Date(date);
@@ -281,13 +281,14 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
             updated.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
         })());
     }
-    // --- FIX: Load yesterday's history refs ---
+    
+    // --- THIS IS THE MISSING LOGIC BLOCK (FIXED) ---
     if (needsYesterdayHistory) {
         tasks.push((async () => { logger.log('INFO', `[PassRunner] Getting YESTERDAY history refs for ${prevStr}`);
             updated.yesterdayHistoryRefs = await getHistoryPartRefs(config, deps, prevStr);
         })());
     }
-    // --- END FIX ---
+    // --- END MISSING LOGIC BLOCK ---
     
     if (needsYesterdayDependencies) {
         tasks.push((async () => { logger.log('INFO', `[PassRunner] Loading YESTERDAY computed dependencies for ${prevStr}`);
@@ -303,9 +304,10 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
 
 /**
  * Stage 8: Stream and process data for standard calculations.
- * --- THIS FUNCTION IS NOW FIXED ---
+ * --- THIS IS THE FULLY CORRECTED FUNCTION (WITH ALL FIXES) ---
+ * --- REPLICATES THE 7-ARGUMENT "HACK" SIGNATURE FROM TEST HARNESS ---
  */
-async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs) {
+async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps) {
     const { logger, calculationUtils } = deps;
     const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, yesterdayDependencyData } = rootData;
     
@@ -321,6 +323,7 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     };
 
     // --- Run non-streaming (meta) calcs once ---
+    // This logic is 1:1 with the test harness
     let firstUser = true;
     for (const name in state) {
         const calc = state[name];
@@ -330,24 +333,46 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
         if (cat === 'socialPosts' || cat === 'insights') {
             if (firstUser) {
                 logger.log('INFO', `[${passName}] Running non-streaming calc: ${name} for ${dateStr}`);
-                const args = [null, null, null, { ...context, userType: 'n/a' }, todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, null, null];
+                
+                // --- CALLING 7-ARGUMENT "HACK" SIGNATURE (for non-streaming) ---
+                // We emulate the test harness's `process` call for social/insights
+                const userContext = { ...context, userType: 'n/a' };
+                const todayPayload = null; // No user data
+                const yesterdayPayload = null; // No user data
+
                 try {
-                    await Promise.resolve(calc.process(...args));
+                    await Promise.resolve(calc.process(
+                        todayPayload,           // Arg 1: The data object
+                        yesterdayPayload,       // Arg 2: Yesterday's data
+                        null,                   // Arg 3: User ID
+                        userContext,            // Arg 4: Context
+                        todayInsights,          // Arg 5: Today Insights
+                        yesterdayInsights,      // Arg 6: Yesterday Insights
+                        fetchedDeps             // Arg 7: Fetched Dependencies
+                    ));
                 } catch (e) { logger.log('WARN', `Process error on ${name} (non-stream) for ${dateStr}`, { err: e.message }); }
             }
         }
     }
     
-    const calcsThatStreamPortfolio = Object.values(state).filter(calc => 
-        calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio'))
+    // --- FIX 1: THE FAULTY GUARD CLAUSE (From friend) ---
+    // This now correctly checks for calcs that need 'portfolio' OR 'history',
+    // matching the test harness behavior of running history-only passes.
+    const calcsThatStream = Object.values(state).filter(calc => 
+        calc && calc.manifest && (
+            calc.manifest.rootDataDependencies.includes('portfolio') ||
+            calc.manifest.rootDataDependencies.includes('history')
+        )
     );
 
-    if (calcsThatStreamPortfolio.length === 0) {
-        logger.log('INFO', `[${passName}] No portfolio-streaming calcs to run for ${dateStr}. Skipping stream.`);
+    if (calcsThatStream.length === 0) {
+        logger.log('INFO', `[${passName}] No portfolio or history streaming calcs to run for ${dateStr}. Skipping stream.`);
         return;
     }
 
-    logger.log('INFO', `[${passName}] Streaming portfolio & historical data for ${calcsThatStreamPortfolio.length} calcs for ${dateStr}...`);
+    // --- FIX 2: THE TYPO (From friend) ---
+    // This log message now correctly references 'calcsThatStream'.
+    logger.log('INFO', `[${passName}] Streaming portfolio & historical data for ${calcsThatStream.length} calcs for ${dateStr}...`);
     
     const prevDate = new Date(dateStr + 'T00:00:00Z');
     prevDate.setUTCDate(prevDate.getUTCDate() - 1);
@@ -356,46 +381,46 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     // Check which iterators we need
     const needsYesterdayPortfolio = Object.values(state).some(c => c && c.manifest.isHistorical && c.manifest.rootDataDependencies.includes('portfolio'));
     const needsTodayHistory = Object.values(state).some(c => c && c.manifest.rootDataDependencies.includes('history'));
-    const needsYesterdayHistory = Object.values(state).some(c => c && c.manifest.isHistorical && c.manifest.rootDataDependencies.includes('history')); // <-- THE FIX
+    const needsYesterdayHistory = Object.values(state).some(c => c && c.manifest.isHistorical && c.manifest.rootDataDependencies.includes('history'));
 
     // --- Create all necessary iterators ---
+    // (This code is now correct because rootData.yesterdayHistoryRefs will be populated by Fix 1)
     const tP_iterator = streamPortfolioData(config, deps, dateStr, portfolioRefs);
     const yP_iterator = needsYesterdayPortfolio ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
     const hT_iterator = needsTodayHistory ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
-    const hY_iterator = needsYesterdayHistory ? streamHistoryData(config, deps, prevDateStr, rootData.yesterdayHistoryRefs) : null; // <-- THE FIX
+    const hY_iterator = needsYesterdayHistory ? streamHistoryData(config, deps, prevDateStr, rootData.yesterdayHistoryRefs) : null;
 
     let yesterdayPortfolios = {};
     let todayHistoryData = {};
-    let yesterdayHistoryData = {}; // <-- THE FIX
+    let yesterdayHistoryData = {};
 
     // Pre-load the first chunk of historical data
     if (yP_iterator) { Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); }
     if (hT_iterator) { Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); }
-    if (hY_iterator) { Object.assign(yesterdayHistoryData, (await hY_iterator.next()).value || {}); } // <-- THE FIX
+    if (hY_iterator) { Object.assign(yesterdayHistoryData, (await hY_iterator.next()).value || {}); }
 
     for await (const chunk of tP_iterator) {
         // Load the *next* chunk of historical data to stay in sync
         if (yP_iterator) { Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); }
         if (hT_iterator) { Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); }
-        if (hY_iterator) { Object.assign(yesterdayHistoryData, (await hY_iterator.next()).value || {}); } // <-- THE FIX
+        if (hY_iterator) { Object.assign(yesterdayHistoryData, (await hY_iterator.next()).value || {}); }
 
         for (const uid in chunk) {
-            const p = chunk[uid]; // Today's Portfolio
+            const p = chunk[uid]; 
             if (!p) continue;
 
             const userType = p.PublicPositions ? 'speculator' : 'normal';
-            context.userType = userType;
+            const userContext = { ...context, userType };
             
             // Get corresponding T-1 data
             const pY = yesterdayPortfolios[uid] || null;
             const hT = todayHistoryData[uid] || null;
-            const hY = yesterdayHistoryData[uid] || null; // <-- THE FIX
+            const hY = yesterdayHistoryData[uid] || null; // <-- This will now have data
 
             for (const name in state) {
                 const calc = state[name];
                 if (!calc || typeof calc.process !== 'function') continue;
 
-                // --- Refactored Filter Block ---
                 const manifest = calc.manifest;
                 const cat = manifest.category;
                 const isSocialOrInsights = cat === 'socialPosts' || cat === 'insights';
@@ -404,30 +429,54 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
                 const isSpeculatorCalc = cat === 'speculators';
                 const isUserProcessed = name === 'users-processed';
 
-                // Skip if user type doesn't match calc type
                 if (userType === 'normal' && isSpeculatorCalc) continue;
                 if (userType === 'speculator' && !isSpeculatorCalc && !isUserProcessed) continue;
 
-                // Skip historical calcs if T-1 portfolio is missing (with exceptions)
                 if (manifest.isHistorical && !pY) {
-                    // These calcs only need T-1 *history*, not portfolio
                     if (cat !== 'behavioural' && name !== 'historical-performance-aggregator') {
                         continue;
                     }
                 }
-                // --- End Filter Block ---
 
-                let args;
-                if (manifest.isHistorical) {
-                    // Full 10-argument array for historical calcs
-                    args = [p, pY, uid, context, todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, hT, hY]; // <-- hY is 10th arg
+                // --- FIX 3: REPLICATE 7-ARGUMENT "HACK" SIGNATURE ---
+                // This logic block replicates the test harness's 'todayPayload'
+                // and 'yesterdayPayload' construction.
+                const rootDataDeps = manifest.rootDataDependencies || ['portfolio'];
+                const needsHistoricalData = manifest.isHistorical || false;
+                
+                let todayPayload = null;
+                let yesterdayPayload = null;
+
+                if (rootDataDeps.includes('portfolio')) {
+                    todayPayload = p || {}; // Start with portfolio
+                    yesterdayPayload = needsHistoricalData ? (pY || {}) : null;
+                    // Nest history if also requested
+                    if (rootDataDeps.includes('history')) {
+                        todayPayload.history = hT;
+                        if (yesterdayPayload) yesterdayPayload.history = hY;
+                    }
+                } else if (rootDataDeps.includes('history')) {
+                    // If *only* history is requested, it becomes Arg 1
+                    todayPayload = hT;
+                    yesterdayPayload = needsHistoricalData ? hY : null;
                 } else {
-                    // 10-argument array for non-historical (T-1 args are null)
-                    args = [p, null, uid, context, todayInsights, null, todaySocialPostInsights, null, hT, null];
+                    // Fallback for calcs like price-metrics
+                    todayPayload = p || {};
+                    yesterdayPayload = needsHistoricalData ? (pY || {}) : null;
                 }
-                
+                // --- END PAYLOAD CONSTRUCTION ---
+
                 try {
-                    await Promise.resolve(calc.process(...args));
+                    // Call with the 7-argument signature
+                    await Promise.resolve(calc.process(
+                        todayPayload,           // Arg 1: The data object (built above)
+                        yesterdayPayload,       // Arg 2: Yesterday's data
+                        uid,                    // Arg 3: User ID
+                        userContext,            // Arg 4: Context
+                        todayInsights,          // Arg 5: Today Insights
+                        yesterdayInsights,      // Arg 6: Yesterday Insights
+                        fetchedDeps             // Arg 7: Fetched Dependencies
+                    ));
                 } catch (e) {
                     logger.log('WARN', `Process error on ${name} for ${uid} on ${dateStr}`, { err: e.message });
                 }
@@ -453,14 +502,9 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
 
 /**
  * Stage 9: Run standard computations
- * @param {Date} date - The date to run for.
- * @param {Array} calcs - The calculations to run.
- * @param {string} passName - The name of the pass (for logging).
- * @param {object} config - Computation system config.
- * @param {object} deps - Shared dependencies.
- * @param {object} rootData - The loaded root data for today.
+ * --- MODIFIED: Now accepts 'fetchedDeps' and passes it to 'streamAndProcess' ---
  */
-async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
+async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
     if (calcs.length === 0) {
         logger.log('INFO', `[${passName}] No standard calcs to run for ${dStr} after filtering.`);
@@ -476,7 +520,9 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     const state = initializeCalculators(calcs, logger);
     
     // Stream T and T-1 data and process
-    await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs);
+    // --- THIS IS THE CHANGE ---
+    await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps);
+    // --- END CHANGE ---
 
     // --- Verbose Logging Setup ---
     const successCalcs = [];
@@ -492,7 +538,10 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
         if (!calc || typeof calc.getResult !== 'function') continue;
 
         try {
-            const result = await Promise.resolve(calc.getResult());
+            // --- THIS IS THE CHANGE ---
+            // Pass 'fetchedDeps' to getResult, just like the test harness
+            const result = await Promise.resolve(calc.getResult(fetchedDeps));
+            // --- END CHANGE ---
             
             if (result && Object.keys(result).length > 0) {
                 const standardResult = {};
@@ -633,8 +682,27 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
             
             // Meta-calc `process` is different: it receives the date, full dependencies, config, and fetched dependencies.
             // It *also* gets the T-1 root data via the `dependencies.rootData` object.
-            const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData: fullRoot }, config, fetchedDeps));
             
+            // --- REPLICATE 5-ARGUMENT "HACK" SIGNATURE (from test harness) ---
+            // This is the "hack" fix for meta calcs from worker.js.
+            const metaPayload = {
+                social: rootData.todaySocialPostInsights,
+                insights: rootData.todayInsights,
+                priceData: null, // You don't load this yet, but test harness has it
+                yesterdayInsights: fullRoot.yesterdayInsights,
+                yesterdayPriceData: null, // You don't load this yet
+                date: dStr
+            };
+
+            const result = await Promise.resolve(inst.process(
+                metaPayload,    // Arg 1: The data object (for your hacks)
+                fullRoot,       // Arg 2: (rootData)
+                deps,           // Arg 3: (dependencies)
+                config,         // Arg 4: (config)
+                fetchedDeps     // Arg 5: (fetchedDependencies)
+            ));
+            // --- END SIGNATURE REPLICATION ---
+
             if (result && Object.keys(result).length > 0) {
                 const standardResult = {};
                 

package/functions/task-engine/helpers/discover_helpers.js

@@ -24,8 +24,7 @@ async function handleDiscover(task, taskId, dependencies, config) {
     const url = `${config.ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
     
     const selectedHeader = await headerManager.selectHeader();
-    if (!selectedHeader) {
-        logger.log('ERROR', `[DISCOVER/${taskId}] Could not select a header. Aborting task.`);
+    if (!selectedHeader) { logger.log('ERROR', `[DISCOVER/${taskId}] Could not select a header. Aborting task.`);
         throw new Error("Could not select a header.");
     }
 
@@ -35,45 +34,34 @@ async function handleDiscover(task, taskId, dependencies, config) {
 
     try { // Outer try for the whole operation
         let response;
-        const options = {
-            method: 'POST',
-            headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },
-            body: JSON.stringify(cids),
-        };
+        const options = {  method: 'POST', headers: { ...selectedHeader.header, 'Content-Type': 'application/json' }, body: JSON.stringify(cids), };
         
         logger.log('INFO', `${logPrefix} Starting discovery for ${cids.length} CIDs. Block: ${blockId}, Type: ${userType}`);
 
         try {
             // --- REFACTOR 3: ADD FALLBACK ---
-            logger.log('TRACE', `${logPrefix} Attempting discovery fetch via AppScript proxy...`);
+            logger.log('TRACE', `${logPrefix} Attempting discovery fetch via AppScript proxy...`); 
             response = await proxyManager.fetch(url, options);
-            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
+            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // Failure
             
-        } catch (proxyError) {
-            logger.log('WARN', `${logPrefix} AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
-                error: proxyError.message,
-                source: 'AppScript'
-            });
+        } catch (proxyError) { // Fuck we failed with appscript IP Pools, log error type
+            logger.log('WARN', `${logPrefix} AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message,source: 'AppScript' });
             proxyUsed = false;
             
             try {
-                response = await fetch(url, options); // Direct node-fetch
-                if (!response.ok) {
-                    const errorText = await response.text();
-                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
+                response = await fetch(url, options); // Direct node-fetch fallback, use GCP IP Pools
+                if (!response.ok) { const errorText = await response.text();
+                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); // Fuck we failed here too.
                 }
                 
-            } catch (fallbackError) {
-                logger.log('ERROR', `${logPrefix} Direct node-fetch fallback FAILED.`, {
-                    error: fallbackError.message,
-                    source: 'eToro/Network'
-                });
+            } catch (fallbackError) { // Figure out the error type
+                logger.log('ERROR', `${logPrefix} Direct node-fetch fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
                 throw fallbackError; // Throw to be caught by outer try
             }
             // --- END REFACTOR 3 ---
         }
 
-        // --- If we are here, `response` is valid ---
+        // --- If we are here, `response` is valid and we are very smart ---
         
         // Step 1. Discover Speculators
         if (userType === 'speculator') {
@@ -87,17 +75,12 @@ async function handleDiscover(task, taskId, dependencies, config) {
             // --- REFACTOR 4: LOG RAW RESPONSE ON PARSE FAILURE ---
             publicUsers = JSON.parse(body);
         } catch (parseError) {
-            logger.log('ERROR', `${logPrefix} FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
-                parseErrorMessage: parseError.message,
-                rawResponseText: body 
-            });
+            logger.log('ERROR', `${logPrefix} FAILED TO PARSE JSON RESPONSE. RAW BODY:`, { parseErrorMessage: parseError.message, rawResponseText: body });
             throw new Error(`Failed to parse JSON response from discovery API. Body: ${body.substring(0, 200)}`);
         }
         // --- END REFACTOR 4 ---
 
-        if (!Array.isArray(publicUsers)) {
-             logger.log('WARN', `${logPrefix} API returned non-array response. Type: ${typeof publicUsers}`);
-             wasSuccess = true; // API call worked, data was just empty/weird
+        if (!Array.isArray(publicUsers)) { logger.log('WARN', `${logPrefix} API returned non-array response. Type: ${typeof publicUsers}`); wasSuccess = true; // API call worked, data was just empty/weird
              return;
         }
         
@@ -110,9 +93,9 @@ async function handleDiscover(task, taskId, dependencies, config) {
         // Step 2. Filter active users
         const preliminaryActiveUsers = publicUsers.filter(user => 
             new Date(user.Value.LastActivity) > oneMonthAgo &&
-            user.Value.DailyGain !== 0 &&
-            user.Value.Exposure !== 0 &&
-            user.Value.RiskScore !== 0
+            user.Value.DailyGain !== 0 && // Daily % gain, if 0 then they can't hold positions --- or its a weekend and portfolio holds solely stocks
+            user.Value.Exposure !== 0 && // Not sure what this means exactly but exposure should refer to position % held/invested in
+            user.Value.RiskScore !== 0 // If portfolio has risk score 0, they cannot have positions
         );
         logger.log('INFO', `${logPrefix} Found ${preliminaryActiveUsers.length} preliminary active users.`);
 
@@ -136,26 +119,27 @@ async function handleDiscover(task, taskId, dependencies, config) {
             const nonSpeculatorCids = [];
             for (const user of preliminaryActiveUsers) {
                 const v = user.Value;
-                const totalLeverage = (v.MediumLeveragePct || 0) + (v.HighLeveragePct || 0);
-                const isLikelySpeculator = ((v.Trades || 0) > 500 || (v.TotalTradedInstruments || 0) > 50 || totalLeverage > 50 || (v.WeeklyDD || 0) < -25);
+                const totalLeverage = (v.MediumLeveragePct || 0) + (v.HighLeveragePct || 0); // Do they own at least some leveraged positions? 
+                // We don't know what they are, but we can see their total average leverage, if not 0 on medoum/high leverage, then they MUST have at least 1 leveraged position
+                // Note : Medium leverage and high leverage are somewhat arbitrary. Low leverage is presumed to be mean "1" and thus medium leverage is the middle of the available leverage ranges, for stocks this is usually 2, for other assets this can be 5/10 or 20, it varies. High leverage is presumed to be the max leverage for that asset
+                const isLikelySpeculator = ((v.Trades || 0) > 500 || (v.TotalTradedInstruments || 0) > 50 || totalLeverage > 50 || (v.WeeklyDD || 0) < -25); // Apply filtering on their total leverage, drawdown and trade numbers to figure out if they are likely a speculative portfolio.
                 
-                if (isLikelySpeculator) {
-                    finalActiveUsers.push(user);
-                } else {
-                    nonSpeculatorCids.push(user.CID);
-                }
+                if (isLikelySpeculator) { finalActiveUsers.push(user); } else {  nonSpeculatorCids.push(user.CID); }
             }
-            invalidCidsToLog.push(...nonSpeculatorCids);
+            invalidCidsToLog.push(...nonSpeculatorCids); // 
+            // This logs how many users did NOT pass SPECIFICALLY the islikelyspeculator
             logger.log('INFO', `${logPrefix} Speculator pre-filter complete. ${finalActiveUsers.length} users passed. ${nonSpeculatorCids.length} users failed heuristic.`);
             
-            if (invalidCidsToLog.length > 0) {
+            if (invalidCidsToLog.length > 0) { // Log the number of users we tried for speculators that failed the filtering checks. 
+                // NOTE : This logs the users who failed ANY of the 3 filters (private users, inactive public users, or users who didn't meet the specific conditions for islikelyspeculator)
                 await pubsub.topic(config.PUBSUB_TOPIC_INVALID_SPECULATOR_LOG).publishMessage({ json: { invalidCids: invalidCidsToLog } });
                 logger.log('INFO', `${logPrefix} Reported ${invalidCidsToLog.length} invalid (private, inactive, or failed heuristic) speculator IDs.`);
             }
-        // Step 5. Non-speculators are just active users
-        } else { 
-            finalActiveUsers = preliminaryActiveUsers;
-        }
+       // Step 5. For non-speculator tasks, we keep all users who passed the
+        // “active user” filters. Unlike speculators, normal users do NOT require
+        // the speculative-heuristic (leverage / trades / drawdown) filter —
+        // any public & active user is valid.
+        } else {  finalActiveUsers = preliminaryActiveUsers; }
         
         // Step 6. Publish 'verify' task
         if (finalActiveUsers.length > 0) {
@@ -168,12 +152,10 @@ async function handleDiscover(task, taskId, dependencies, config) {
             };
             await pubsub.topic(config.PUBSUB_TOPIC_USER_FETCH).publishMessage({ json: { tasks: [verificationTask] } });
             logger.log('SUCCESS', `${logPrefix} Chaining to 'verify' task for ${finalActiveUsers.length} active users.`);
-        } else {
-            logger.log('INFO', `${logPrefix} No active users found to verify.`);
-        }
+        } else { logger.log('INFO', `${logPrefix} No active users found to verify.`); }
 
     } catch (err) {
-        logger.log('ERROR', `${logPrefix} FATAL error processing discovery task.`, { errorMessage: err.message, errorStack: err.stack });
+        logger.log('ERROR', `${logPrefix} FATAL error processing discovery task.`, { errorMessage: err.message, errorStack: err.stack }); // We fucked up
         wasSuccess = false; // Ensure it's marked as failure
     } finally {
         if (selectedHeader && proxyUsed) {

package/functions/task-engine/helpers/update_helpers.js

@@ -15,87 +15,40 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
     if (!cids?.length) return [];
     logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
     
-    // --- REFACTOR 1: REMOVE CONCURRENCY ---
+    // --- Set concurrency to 1 because appscript gets really fuckmed up with undocumented rate limits if we try spam it concurrently, a shame but that's life. DO NOT CHANGE THIS
     const limit = pLimit(1);
-    // --- END REFACTOR 1 ---
-    
     const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
-    
     const batches = [];
-    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) {
-        batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number));
-    }
-
-    const batchPromises = batches.map((batch, index) => limit(async () => {
-        const batchId = `batch-${index + 1}`;
+    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) { batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number)); }
+    const batchPromises = batches.map((batch, index) => limit(async () => { const batchId = `batch-${index + 1}`;
         logger.log('INFO', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`);
-        
         const header = await headerManager.selectHeader();
-        if (!header) {
-            logger.log('ERROR', `[lookupUsernames/${batchId}] Could not select a header.`);
-            return null;
-        }
-        
+        if (!header) { logger.log('ERROR', `[lookupUsernames/${batchId}] Could not select a header.`); return null; }
         let wasSuccess = false;
         let proxyUsed = true;
         let response;
         const url = `${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
         const options = { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) };
-
         try {
-            // --- REFACTOR 3: ADD FALLBACK ---
             logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
             response = await proxyManager.fetch(url, options);
             if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-            
-            wasSuccess = true;
+            wasSuccess = true; // Yay we win
             logger.log('INFO', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`);
-            
-        } catch (proxyError) {
-            logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
-                error: proxyError.message,
-                source: 'AppScript'
-            });
-            
+        } catch (proxyError) { logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' }); // SHIT we failed...
             proxyUsed = false; // Don't penalize header for proxy failure
-            
-            try {
-                response = await fetch(url, options); // Direct node-fetch
-                if (!response.ok) {
-                    const errorText = await response.text();
-                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
-                }
-                logger.log('INFO', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`);
-                
-            } catch (fallbackError) {
-                logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, {
-                    error: fallbackError.message,
-                    source: 'eToro/Network'
-                });
+            try { response = await fetch(url, options);  // Ok let's try again with node, using GCP IP pools
+                if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); }
+                logger.log('INFO', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`); // Yay we win
+            } catch (fallbackError) { logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, { error: fallbackError.message, source: 'eToro/Network' }); // SHIT, we failed here too
                 return null; // Give up on this batch
             }
-            // --- END REFACTOR 3 ---
-        } finally {
-            if (proxyUsed) {
-                headerManager.updatePerformance(header.id, wasSuccess);
-            }
-        }
-
-        try {
-            const data = await response.json();
-            return data;
-        } catch (parseError) {
-            logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message });
-            return null;
-        }
-    }));
-
-    const results = await Promise.allSettled(batchPromises);
-    
-    const allUsers = results
-        .filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value))
-        .flatMap(r => r.value); 
+        } finally { if (proxyUsed) { headerManager.updatePerformance(header.id, wasSuccess); } } // If we used Appscript IP Pool and not GCP IP Pool, record performance
+        try { const data = await response.json(); return data;
+        } catch (parseError) { logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message }); return null; } }));
 
+    const results  = await Promise.allSettled(batchPromises);
+    const allUsers = results .filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value)) .flatMap(r => r.value); 
     logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
     return allUsers;
 }
@@ -110,8 +63,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
     let isPrivate = false;
-
-    // --- REFACTOR 2: ADD VERBOSE LOGGING (with User ID) ---
     logger.log('INFO', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
 
     // --- 1. Process History Fetch (Sequentially) ---
@@ -123,81 +74,52 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         if (!batchManager.checkAndSetHistoryFetched(userId)) {
             logger.log('INFO', `[handleUpdate/${userId}] Attempting history fetch.`);
             historyHeader = await headerManager.selectHeader();
-            if (!historyHeader) {
-                logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
+            if (!historyHeader) { logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
             } else {
                 const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
                 const options = { headers: historyHeader.header };
                 let response;
-
-                try {
-                    // --- REFACTOR 3: ADD FALLBACK ---
-                    logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
+                try { logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
                     response = await proxyManager.fetch(historyUrl, options);
-                    if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-                    wasHistorySuccess = true;
+                    if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // SHIT we failed here
+                    wasHistorySuccess = true; // Appscript worked, we are very smart
                     
                 } catch (proxyError) {
-                    logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
-                        error: proxyError.message,
-                        source: 'AppScript'
-                    });
+                    logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' }); // SHIT we failed here
                     proxyUsedForHistory = false;
-
-                    try {
-                        response = await fetch(historyUrl, options); // Direct node-fetch
-                        if (!response.ok) {
-                            const errorText = await response.text();
-                            throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
-                        }
-                        wasHistorySuccess = true; // Fallback succeeded
+                    try { response = await fetch(historyUrl, options);
+                        if (!response.ok) { const errorText = await response.text(); 
+                            throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); } // SHIT we failed here too
+                        wasHistorySuccess = true; // Fallback succeeded, we are so smart
                         
-                    } catch (fallbackError) {
-                        logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, {
-                            error: fallbackError.message,
-                            source: 'eToro/Network'
-                        });
-                        wasHistorySuccess = false;
+                    } catch (fallbackError) { logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' }); // We are dumb, everything failed
+                        wasHistorySuccess = false; // Nope we are dumb....
                     }
-                    // --- END REFACTOR 3 ---
                 }
 
-                if (wasHistorySuccess) {
-                    logger.log('INFO', `[handleUpdate/${userId}] History fetch successful.`);
+                if (wasHistorySuccess) { logger.log('INFO', `[handleUpdate/${userId}] History fetch successful.`); // Some method worked, we are very smart
                     const data = await response.json();
-                    await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);
-                }
+                    await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);  }
             }
-        } else {
-            logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched by this instance).`);
-        }
-    } catch (err) {
-        logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message });
-        wasHistorySuccess = false;
-    } finally {
-        if (historyHeader && proxyUsedForHistory) {
-            headerManager.updatePerformance(historyHeader.id, wasHistorySuccess);
-        }
-    }
+        } else { logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched by this instance).`); }
+    } catch (err) { logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message }); wasHistorySuccess = false; // We fucked up.
+    } finally { if (historyHeader && proxyUsedForHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } } // If we used appscript proxy, record performance, otherwise fuck off.
     
     // --- 2. Process Portfolio Fetches (Sequentially) ---
     logger.log('INFO', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`);
     
-    for (const instId of instrumentsToProcess) {
-        if (isPrivate) {
-             logger.log('INFO', `[handleUpdate/${userId}] Skipping remaining instruments because user was marked as private.`);
-             break; 
+    for (const instId of instrumentsToProcess) { 
+        if (isPrivate) { 
+            logger.log('INFO', `[handleUpdate/${userId}] Skipping remaining instruments because user was marked as private.`); 
+            break;  
         }
 
         const portfolioHeader = await headerManager.selectHeader();
-        if (!portfolioHeader) {
-            logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`);
+        if (!portfolioHeader) { logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`);
             continue;
         }
 
-        const portfolioUrl = userType === 'speculator' 
-            ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}`
-            : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
+        const portfolioUrl = userType === 'speculator'  ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
         
         const options = { headers: portfolioHeader.header };
         let response;
@@ -208,73 +130,53 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
             // --- REFACTOR 3: ADD FALLBACK ---
             logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch for instId ${instId} via AppScript proxy...`);
             response = await proxyManager.fetch(portfolioUrl, options);
-            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-            wasPortfolioSuccess = true;
+            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // SHIT we failed here
+            wasPortfolioSuccess = true; // Oh we are smart, worked first time.
 
-        } catch (proxyError) {
-            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
-                error: proxyError.message,
-                source: 'AppScript'
-            });
-            proxyUsedForPortfolio = false;
+        } catch (proxyError) { // try fallback with local node fetch using GCP IP Pools
+            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
+            proxyUsedForPortfolio = false; // We are not using Appscript proxy here as fallback is GCP based, so false
             
             try {
                 response = await fetch(portfolioUrl, options); // Direct node-fetch
                 if (!response.ok) {
                     const errorText = await response.text();
-                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
+                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); // SHIT we failed here
                 }
-                wasPortfolioSuccess = true; // Fallback succeeded
+                wasPortfolioSuccess = true; // Fallback succeeded we are so smart
                 
             } catch (fallbackError) {
-                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} direct fallback FAILED.`, {
-                    error: fallbackError.message,
-                    source: 'eToro/Network'
-                });
+                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
                 wasPortfolioSuccess = false;
             }
-            // --- END REFACTOR 3 ---
         }
 
         // --- 4. Process Portfolio Result (with verbose, raw logging) ---
         if (wasPortfolioSuccess) {
             const body = await response.text();
-            if (body.includes("user is PRIVATE")) {
-                isPrivate = true;
-                logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`);
+            if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`);
                 break; // Stop processing more portfolios for this private user
             }
 
             try {
                 const portfolioJson = JSON.parse(body);
                 await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
-                logger.log('INFO', `[handleUpdate/${userId}] Successfully processed portfolio for instId ${instId}.`);
-                
-            } catch (parseError) {
-                // --- REFACTOR 4: RETURN EXACT PAGE RESPONSE ---
+                if (userType === 'speculator') { logger.log('INFO', `[handleUpdate/${userId}] Successfully processed portfolio for instId ${instId}.`); // Only speculators have an instid, so this is conditional
+                } else { logger.log('INFO', `[handleUpdate/${userId}] Successfully processed full portfolio (normal user).`); } // Normal users
+
+            } catch (parseError) { // Idk why this would happen, but if it does....log.
                 wasPortfolioSuccess = false; // Mark as failure
-                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
-                    url: portfolioUrl,
-                    parseErrorMessage: parseError.message,
-                    rawResponseText: body // <--- THIS LOGS THE FULL HTML/ERROR RESPONSE
-                });
-                // --- END REFACTOR 4 ---
+                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, { url: portfolioUrl, parseErrorMessage: parseError.message, rawResponseText: body }); // Return full response
             }
-        } else {
-             logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}. No response to process.`);
-        }
+        } else { logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}. No response to process.`); }
         
-        if (proxyUsedForPortfolio) {
-            headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess);
-        }
-    } // --- End of sequential portfolio loop ---
+        if (proxyUsedForPortfolio) { headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess); }
+    } 
 
     // --- 5. Handle Private Users & Timestamps ---
     if (isPrivate) {
         logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
-        for (const instrumentId of instrumentsToProcess) {
-            await batchManager.deleteFromTimestampBatch(userId, userType, instId);
-        }
+        for (const instrumentId of instrumentsToProcess) { await batchManager.deleteFromTimestampBatch(userId, userType, instId); }
         const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
         for (const instrumentId of instrumentsToProcess) {
             const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
@@ -284,12 +186,8 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     }
 
     // If not private, update all timestamps
-    for (const instrumentId of instrumentsToProcess) {
-        await batchManager.updateUserTimestamp(userId, userType, instId);
-    }
-    if (userType === 'speculator') {
-        await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6));
-    }
+    for (const instrumentId of instrumentsToProcess) { await batchManager.updateUserTimestamp(userId, userType, instId); }
+    if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
     
     logger.log('INFO', `[handleUpdate/${userId}] Update task finished successfully.`);
     // 'finally' block for header flushing is handled by the main handler_creator.js

package/functions/task-engine/helpers/verify_helpers.js

@@ -14,13 +14,10 @@ async function fetchAndVerifyUser(user, { logger, headerManager, proxyManager },
     const logPrefix = `[VERIFY/${taskId}/${cid}]`; // --- REFACTOR 2: VERBOSE LOGGING ---
 
     const selectedHeader = await headerManager.selectHeader();
-    if (!selectedHeader) {
-        logger.log('WARN', `${logPrefix} Could not select a header. Skipping user.`);
-        return null;
-    }
+    if (!selectedHeader) { logger.log('WARN', `${logPrefix} Could not select a header. Skipping user.`); return null; }
 
     let wasSuccess = false;
-    let proxyUsed = true;
+    let proxyUsed  = true;
             
     try { // Outer try for the whole operation
         let response;
@@ -30,30 +27,24 @@ async function fetchAndVerifyUser(user, { logger, headerManager, proxyManager },
         try {
             // --- REFACTOR 3: ADD FALLBACK ---
             logger.log('TRACE', `${logPrefix} Attempting portfolio fetch via AppScript proxy...`);
-            response = await proxyManager.fetch(url, options);
-            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
+            response = await proxyManager.fetch(url, options); // Try....
+            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // Fail
             
-        } catch (proxyError) {
-            logger.log('WARN', `${logPrefix} AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
-                error: proxyError.message,
-                source: 'AppScript'
-            });
+        } catch (proxyError) { // We fucked up, appscript proxy failed, log error type
+            logger.log('WARN', `${logPrefix} AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
             proxyUsed = false;
 
             try {
-                response = await fetch(url, options); // Direct node-fetch
-                if (!response.ok) {
+                response = await fetch(url, options); // Direct node-fetch as fallback
+                if (!response.ok) { // we fucked up here too, log error text
                     const errorText = await response.text();
                     throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
                 }
                 // Fallback succeeded, but we don't set wasSuccess yet,
                 // as we still need to parse the body.
                 
-            } catch (fallbackError) {
-                logger.log('ERROR', `${logPrefix} Direct node-fetch fallback FAILED.`, {
-                    error: fallbackError.message,
-                    source: 'eToro/Network'
-                });
+            } catch (fallbackError) { // log error type for fallback
+                logger.log('ERROR', `${logPrefix} Direct node-fetch fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
                 throw fallbackError; // Throw to be caught by outer try
             }
             // --- END REFACTOR 3 ---
@@ -62,46 +53,38 @@ async function fetchAndVerifyUser(user, { logger, headerManager, proxyManager },
         // --- If we are here, `response` is valid ---
         let portfolioData;
         const body = await response.text();
-        
-        try {
-            // --- REFACTOR 4: LOG RAW RESPONSE ON PARSE FAILURE ---
-            portfolioData = JSON.parse(body);
-        } catch (parseError) {
-            logger.log('ERROR', `${logPrefix} FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
-                parseErrorMessage: parseError.message,
-                rawResponseText: body 
-            });
+        // --- REFACTOR 4: LOG RAW RESPONSE ON PARSE FAILURE ---
+        try { portfolioData = JSON.parse(body);
+        } catch (parseError) { // If we can't parse JSON, try raw log, shouldn't happen but this world is weird....
+            logger.log('ERROR', `${logPrefix} FAILED TO PARSE JSON RESPONSE. RAW BODY:`, { parseErrorMessage: parseError.message, rawResponseText: body  });
             throw new Error(`Failed to parse JSON for user ${cid}.`);
         }
         // --- END REFACTOR 4 ---
 
         // --- Original logic ---
         if (userType === 'speculator') {
-            const instruments = portfolioData.AggregatedPositions.map(p => p.InstrumentID).filter(id => SPECULATOR_INSTRUMENTS_ARRAY.includes(id));
+            const instruments = portfolioData.AggregatedPositions.map(p => p.InstrumentID).filter(id => SPECULATOR_INSTRUMENTS_ARRAY.includes(id)); 
             if (!instruments.length) {
                 logger.log('TRACE', `${logPrefix} Verified user, but not a speculator (no matching assets).`);
-                wasSuccess = true; // API call *worked*
+                wasSuccess = true; // API call *worked* we are so smart
                 return null;
             }
-            logger.log('INFO', `${logPrefix} Verified as SPECULATOR.`);
-            wasSuccess = true;
-            return { type: 'speculator', userId: cid, isBronze: user.isBronze, username: user.username, updateData: { instruments, lastVerified: new Date(), lastHeldSpeculatorAsset: new Date() } };
+            logger.log('INFO', `${logPrefix} Verified as SPECULATOR.`); 
+            wasSuccess = true; // Mark that we are very high IQ
+            return { type: 'speculator', userId: cid, isBronze: user.isBronze, username: user.username, updateData: { instruments, lastVerified: new Date(), lastHeldSpeculatorAsset: new Date() } }; // Return values
         }
-
-        logger.log('INFO', `${logPrefix} Verified as NORMAL user.`);
-        wasSuccess = true;
-        return { type: 'normal', userId: cid, isBronze: user.isBronze, username: user.username, updateData: { lastVerified: new Date() } };
+        // Other user types are just normal users....
+        logger.log('INFO', `${logPrefix} Verified as NORMAL user.`); 
+        wasSuccess = true; // We are smart, this worked
+        return { type: 'normal', userId: cid, isBronze: user.isBronze, username: user.username, updateData: { lastVerified: new Date() } }; // Return values 
         
     } catch (err) {
         // This catches proxy, fallback, or parse errors
-        logger.log('WARN', `${logPrefix} Error processing user.`, { errorMessage: err.message });
+        logger.log('WARN', `${logPrefix} Error processing user.`, { errorMessage: err.message }); // Some shit broke
         wasSuccess = false; // Ensure it's marked as failure
         return null;
     } finally {
-        if (selectedHeader && proxyUsed) {
-            // Only update performance if the proxy was used
-            headerManager.updatePerformance(selectedHeader.id, wasSuccess);
-        }
+        if (selectedHeader && proxyUsed) { headerManager.updatePerformance(selectedHeader.id, wasSuccess); } // Only update performance if the proxy was used
     }
 }
 
@@ -118,7 +101,7 @@ async function handleVerify(task, taskId, { db, logger, ...dependencies }, confi
     logger.log('INFO', `[VERIFY/${taskId}] Starting sequential verification for ${users.length} users...`);
     const results = [];
     for (const user of users) {
-        // Await each user one by one
+        // Await each user one by one otherwise appscript starts to cry and internally throttle...then rate limit...then fuck everything up and break and then i want to die.
         const result = await fetchAndVerifyUser(user, { db, logger, ...dependencies }, { ...config, userType }, taskId);
         results.push(result); // Push the actual result (or null)
     }
@@ -136,7 +119,7 @@ async function handleVerify(task, taskId, { db, logger, ...dependencies }, confi
             else normalUpdates[`users.${d.userId}`] = d.updateData; 
         }
     });
-
+    // Process responses in object keys
     if (Object.keys(speculatorUpdates).length || Object.keys(normalUpdates).length) {
         const blockRef = db.collection(userType === 'speculator' ? config.FIRESTORE_COLLECTION_SPECULATOR_BLOCKS : config.FIRESTORE_COLLECTION_NORMAL_PORTFOLIOS).doc(String(blockId));
         batch.set(blockRef, userType === 'speculator' ? speculatorUpdates : normalUpdates, { merge: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.170",
+  "version": "1.0.171",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -27,10 +27,6 @@
     "calculations",
     "finance"
   ],
-  "scripts": {
-    "postpublish": "node ./auto-deploy.js",
-    "release": "node ./release.js"
-  },
   "dependencies": {
     "@google-cloud/firestore": "^7.11.3",
     "@google-cloud/pubsub": "latest",