bulltrackers-module 1.0.194 → 1.0.196

package/functions/computation-system/controllers/computation_controller.js

@@ -3,8 +3,8 @@
  * V4.1: Supports Smart Shard Lookup via Wrapper
  */
 
-const { DataExtractor, HistoryExtractor, MathPrimitives, Aggregators, Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics, TimeSeries, priceExtractor } = require('../layers/math_primitives');
-const { loadDailyInsights, loadDailySocialPostInsights, getRelevantShardRefs, getPriceShardRefs } = require('../utils/data_loader');
+const { DataExtractor, HistoryExtractor, MathPrimitives, Aggregators, Validators, SCHEMAS, SignalPrimitives, DistributionAnalytics, TimeSeries, priceExtractor } = require('../layers/math_primitives'); // TODO, SURELY THIS COULD BE DYNAMIC, DETECT WHAT IS EXPORTED? 
+const { loadDailyInsights, loadDailySocialPostInsights, getRelevantShardRefs, getPriceShardRefs } = require('../utils/data_loader');                                                                     // THE IDEA IS THAT WE CAN PRODUCE NEW COMPUTATIONS IN MATH PRIMITIVES, AND NOT NEED TO UPDATE ANY OTHER CODE FILE
 
 class DataLoader {
     constructor(config, dependencies) {
@@ -64,7 +64,7 @@ class DataLoader {
     }
 }
 
-class ContextBuilder {
+class ContextBuilder { //TODO, THE MATH EXTRACT HERE COULD SURELY BE DYNAMIC LISTED, JUST TAKE WHAT IS EXPORTED?
     static buildPerUserContext(options) {
         const { todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory, userId, userType, dateStr, metadata, mappings, insights, socialData, computedDependencies, previousComputedDependencies, config, deps } = options;
         return {
@@ -80,7 +80,7 @@ class ContextBuilder {
         };
     }
 
-    static buildMetaContext(options) {
+    static buildMetaContext(options) { //TODO, THE MATH EXTRACT HERE COULD SURELY BE DYNAMIC LISTED, JUST TAKE WHAT IS EXPORTED?
         const { dateStr, metadata, mappings, insights, socialData, prices, computedDependencies, previousComputedDependencies, config, deps } = options;
         return {
             date: { today: dateStr },

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -13,7 +13,7 @@ const {
     runStandardComputationPass, 
     runMetaComputationPass,
     checkRootDependencies,
-    runBatchPriceComputation // NEW IMPORT
+    runBatchPriceComputation 
 } = require('./orchestration_helpers.js');
 
 const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
@@ -31,10 +31,10 @@ async function runComputationPass(config, dependencies, computationManifest) {
     // Hardcoded earliest dates
     const earliestDates = { 
         portfolio: new Date('2025-09-25T00:00:00Z'), 
-        history: new Date('2025-11-05T00:00:00Z'), 
-        social: new Date('2025-10-30T00:00:00Z'), 
-        insights: new Date('2025-08-26T00:00:00Z'),
-        price: new Date('2025-08-01T00:00:00Z')
+        history:   new Date('2025-11-05T00:00:00Z'), 
+        social:    new Date('2025-10-30T00:00:00Z'), 
+        insights:  new Date('2025-08-26T00:00:00Z'),
+        price:     new Date('2025-08-01T00:00:00Z') // This is slightly arbitrary, the true earliest date is 1 year ago, but there is no need to backfill that far so we just set it to be slightly earlier than the 2nd earliest computation.
     };
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
 
@@ -169,12 +169,12 @@ async function runComputationPass(config, dependencies, computationManifest) {
             const prevDateStr     = prevDate.toISOString().slice(0, 10);
             const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
             
-            // Changed skipstatus write to false to ensure updates are recorded
+            // Changed skipstatus write to false to ensure updates are recorded, allowing for proper tracking and avoiding re-computation in future passes. NOTE : Writing true here introduces significant bugs and should be avoided.
             if (finalStandardToRun.length) {
                 const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, false); 
                 Object.assign(dateUpdates, updates);
             }
-            if (finalMetaToRun.length) {
+            if (finalMetaToRun.length) { // Again, writing true here introduces significant bugs and should be avoided.
                 const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, false); 
                 Object.assign(dateUpdates, updates);
             }

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -13,7 +13,8 @@ const {
     getHistoryPartRefs, streamPortfolioData, streamHistoryData,
     getRelevantShardRefs, loadDataByRefs
 } = require('../utils/data_loader');
-const pLimit = require('p-limit'); // Ensure p-limit is required
+
+const pLimit = require('p-limit'); // TODO, THIS OUGHT TO BE INJECTED.
 
 /**
  * Groups calculations from a manifest by their 'pass' property.
@@ -25,7 +26,7 @@ function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[ca
  * Scans a result set for suspicious patterns (e.g., a field is NULL for 100% of tickers).
  * Logs warnings but DOES NOT block the commit.
  */
-function validateResultPatterns(logger, calcName, results, category) {
+function validateResultPatterns(logger, calcName, results, category) { // TODO, THIS COULD BE MUCH MORE SOPHISTICATED, WE WILL NEVER FORCE FAIL A COMPUTATION REGARDLESS, BUT IT COULD BE A GREAT WARNING SYSTEM, USE GLOUD LOG SINKS TO DETECT PROBLEMS BASED ON THESE LOGS
     // 1. Skip Speculators (Too sparse, nulls are expected)
     if (category === 'speculator' || category === 'speculators') return;
 
@@ -90,16 +91,16 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-    let portfolioRefs = [], historyRefs = [];
-    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
-    let insightsData = null, socialData = null;
+    let portfolioRefs   = [], historyRefs = [];
+    let hasPortfolio    = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
+    let insightsData    = null, socialData = null;
 
     try {
         const tasks = [];
         if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights) tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
-        if (dateToProcess >= earliestDates.social) tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
-        if (dateToProcess >= earliestDates.history) tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
+        if (dateToProcess >= earliestDates.insights)  tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
+        if (dateToProcess >= earliestDates.social)    tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
+        if (dateToProcess >= earliestDates.history)   tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
         
         if (dateToProcess >= earliestDates.price) {
             tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));

package/functions/computation-system/layers/math_primitives.js

@@ -732,7 +732,13 @@ class DistributionAnalytics {
 }
 
 
-// This block is dynamically generated. Do not edit manually.
+// TODO, THIS EXPORT CAN SURELY BE DYNAMICALLY PRODUCED? TAKE WHATEVER CLASSES ARE DEFINED? THE IDEA IS THAT WE CAN SIMPLY PRODUCE NEW CLASSES AND NOT NEED TO MAKE ANY FURTHER CHANGES SO :
+// 1. DYNAMICALLY FINDS THE CLASS, EXPORTS HERE
+// 2. DYNAMICALLY IMPORTS INTO COMPUTATION CONTROLLER
+// 3. DYNAMICALLY POPULATES THE MATH VALUE OF THE COMPUTATION CONTROLLER FOR STANDARD AND META COMPUTATION CONTEXTBUILDERS
+// 4. GO AHEAD AND WRITE YOUR COMPUTATION FILES USING THE NEW MATH PRIMITIVES DEFINED AS NEEDED.
+// THIS THEN MEANS THE MATH PRIMTIVIES CAN BE UPDATED EASILY WITHOUT FURTHER CHANGES TO OTHER SYSTEMS AND AVOID COMPLICATED DEBUGGING OR FORGETFUL INTEGRATIONS.
+
 module.exports = {
     Aggregators,
     DataExtractor,

package/functions/computation-system/utils/data_loader.js

@@ -177,7 +177,7 @@ async function getPriceShardRefs(config, deps) {
 async function ensurePriceShardIndex(config, deps) {
     const { db, logger } = deps;
     const metadataCol = config.metadataCollection || 'system_metadata';
-    const indexDocRef = db.collection(metadataCol).doc('price_shard_index');
+    const indexDocRef = db.collection(metadataCol).doc('price_shard_index'); // TODO. TEST THIS SHARD INDEX SYSTEM, CURRENTLY UNUSED IN COMPUTATIONS BUT IS EXTREMELY EFFICIENT AND GREAT FOR COST REDUCTION
     
     // 1. Try to fetch existing index
     const snap = await indexDocRef.get();

package/functions/computation-system/utils/schema_capture.js

@@ -6,6 +6,7 @@
 /**
  * Batch store schemas for multiple computations.
  * This function now expects a fully-formed schema, not sample output.
+ * It strictly stamps a 'lastUpdated' field to support stale-schema filtering in the API.
  *
  * @param {object} dependencies - Contains db, logger
  * @param {object} config - Configuration object
@@ -13,16 +14,49 @@
  */
 async function batchStoreSchemas(dependencies, config, schemas) {
     const { db, logger } = dependencies;
-    if (config.captureSchemas === false)  { logger.log('INFO', '[SchemaCapture] Schema capture is disabled. Skipping.'); return; }
+    
+    if (config.captureSchemas === false) {
+        logger.log('INFO', '[SchemaCapture] Schema capture is disabled. Skipping.');
+        return;
+    }
+
     const batch = db.batch();
     const schemaCollection = config.schemaCollection || 'computation_schemas';
+    let validCount = 0;
+
     for (const item of schemas) {
-    try { if (!item.schema) { logger.log('WARN', `[SchemaCapture] No schema provided for ${item.name}. Skipping.`); continue; }
-    const docRef = db.collection(schemaCollection).doc(item.name);
-    batch.set(docRef, { computationName: item.name, category: item.category, schema: item.schema, metadata: item.metadata || {}, lastUpdated: new Date() }, { merge: true });
-    } catch (error) { logger.log('WARN', `[SchemaCapture] Failed to add schema to batch for ${item.name}`, { errorMessage: error.message }); } }
-    try { await batch.commit(); logger.log('INFO', `[SchemaCapture] Batch stored ${schemas.length} computation schemas`);
-    } catch (error) { logger.log('ERROR', '[SchemaCapture] Failed to commit schema batch', { errorMessage: error.message }); }
+        try {
+            if (!item.schema) {
+                logger.log('WARN', `[SchemaCapture] No schema provided for ${item.name}. Skipping.`);
+                continue;
+            }
+
+            const docRef = db.collection(schemaCollection).doc(item.name);
+            
+            // Critical: Always overwrite 'lastUpdated' to now
+            batch.set(docRef, {
+                computationName: item.name,
+                category: item.category,
+                schema: item.schema,
+                metadata: item.metadata || {},
+                lastUpdated: new Date() 
+            }, { merge: true });
+            
+            validCount++;
+
+        } catch (error) {
+            logger.log('WARN', `[SchemaCapture] Failed to add schema to batch for ${item.name}`, { errorMessage: error.message });
+        }
+    }
+
+    if (validCount > 0) {
+        try {
+            await batch.commit();
+            logger.log('INFO', `[SchemaCapture] Batch stored ${validCount} computation schemas`);
+        } catch (error) {
+            logger.log('ERROR', '[SchemaCapture] Failed to commit schema batch', { errorMessage: error.message });
+        }
+    }
 }
 
 module.exports = {

package/functions/computation-system/utils/utils.js

@@ -83,7 +83,7 @@ async function getEarliestDataDates(config, deps) {
         speculatorHistoryDate, 
         insightsDate, 
         socialDate,
-        priceDate  // NEW
+        priceDate  
     ] = await Promise.all([
         getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection),
         getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection),
@@ -91,7 +91,7 @@ async function getEarliestDataDates(config, deps) {
         getFirstDateFromCollection(config, deps, config.speculatorHistoryCollection),
         getFirstDateFromSimpleCollection(config, deps, config.insightsCollectionName),
         getFirstDateFromSimpleCollection(config, deps, config.socialInsightsCollectionName),
-        getFirstDateFromPriceCollection(config, deps)  // NEW
+        getFirstDateFromPriceCollection(config, deps)  //TODO, Why no config.pricecollectionname here, looks ugly.
     ]);
     
     const getMinDate = (...dates) => { 
@@ -101,35 +101,35 @@ async function getEarliestDataDates(config, deps) {
     };
     
     const earliestPortfolioDate = getMinDate(investorDate, speculatorDate);
-    const earliestHistoryDate = getMinDate(investorHistoryDate, speculatorHistoryDate);
-    const earliestInsightsDate = getMinDate(insightsDate);
-    const earliestSocialDate = getMinDate(socialDate); 
-    const earliestPriceDate = getMinDate(priceDate);  // NEW
-    const absoluteEarliest = getMinDate(
+    const earliestHistoryDate   = getMinDate(investorHistoryDate, speculatorHistoryDate);
+    const earliestInsightsDate  = getMinDate(insightsDate);
+    const earliestSocialDate    = getMinDate(socialDate); 
+    const earliestPriceDate     = getMinDate(priceDate);  
+    const absoluteEarliest      = getMinDate(
         earliestPortfolioDate, 
         earliestHistoryDate, 
         earliestInsightsDate, 
         earliestSocialDate,
-        earliestPriceDate  // NEW
+        earliestPriceDate  
     );
     
     const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
     
     const result = { 
-        portfolio: earliestPortfolioDate || new Date('2999-12-31'), 
-        history: earliestHistoryDate || new Date('2999-12-31'), 
-        insights: earliestInsightsDate || new Date('2999-12-31'), 
-        social: earliestSocialDate || new Date('2999-12-31'),
-        price: earliestPriceDate || new Date('2999-12-31'),  // NEW
-        absoluteEarliest: absoluteEarliest || fallbackDate 
+        portfolio: earliestPortfolioDate   || new Date('2999-12-31T00:00:00Z'), 
+        history: earliestHistoryDate       || new Date('2999-12-31T00:00:00Z'), 
+        insights: earliestInsightsDate     || new Date('2999-12-31T00:00:00Z'), 
+        social: earliestSocialDate         || new Date('2999-12-31T00:00:00Z'),
+        price: earliestPriceDate           || new Date('2999-12-31T00:00:00Z'),  
+        absoluteEarliest: absoluteEarliest || fallbackDate
     };
     
     logger.log('INFO', 'Earliest data availability map built:', { 
-        portfolio: result.portfolio.toISOString().slice(0, 10), 
-        history: result.history.toISOString().slice(0, 10), 
-        insights: result.insights.toISOString().slice(0, 10), 
-        social: result.social.toISOString().slice(0, 10),
-        price: result.price.toISOString().slice(0, 10),  // NEW
+        portfolio:        result.portfolio.toISOString().slice(0, 10), 
+        history:          result.history.toISOString().slice(0, 10), 
+        insights:         result.insights.toISOString().slice(0, 10), 
+        social:           result.social.toISOString().slice(0, 10),
+        price:            result.price.toISOString().slice(0, 10),  
         absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10) 
     });
     
@@ -143,7 +143,7 @@ async function getEarliestDataDates(config, deps) {
 async function getFirstDateFromPriceCollection(config, deps) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
-    const collection = config.priceCollection || 'asset_prices';
+    const collection = config.priceCollection || 'asset_prices'; // TODO  This hardcode is right, but we should really be passing the config directly as other data sources do.
     
     try {
         logger.log('TRACE', `[getFirstDateFromPriceCollection] Querying ${collection}...`);
@@ -185,7 +185,7 @@ async function getFirstDateFromPriceCollection(config, deps) {
         });
         
         if (earliestDate) {
-            logger.log('TRACE', `[getFirstDateFromPriceCollection] Earliest price date: ${earliestDate.toISOString().slice(0, 10)}`);
+            logger.log('TRACE', `[getFirstDateFromPriceCollection] Earliest price date: ${earliestDate.toISOString().slice(0, 10)}`); // TODO, WTF IS THIS TS ERROR. Property 'toISOString' does not exist on type 'never'.ts(2339)
         }
         
         return earliestDate;

package/functions/generic-api/helpers/api_helpers.js

@@ -2,7 +2,7 @@
  * @fileoverview API sub-pipes.
  * REFACTORED: Now stateless and receive dependencies.
  * NEW: getDynamicSchema now reads static schema.
- * NEW: createManifestHandler added to serve all schemas.
+ * NEW: createManifestHandler filters out STALE schemas (>7 days old).
  */
 
 const { FieldPath } = require('@google-cloud/firestore');
@@ -15,13 +15,18 @@ const validateRequest = (query, config) => {
     if (!query.computations) return "Missing 'computations' parameter.";
     if (!query.startDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.startDate)) return "Missing or invalid 'startDate'.";
     if (!query.endDate || !/^\d{4}-\d{2}-\d{2}$/.test(query.endDate)) return "Missing or invalid 'endDate'.";
+    
     const start = new Date(query.startDate);
     const end = new Date(query.endDate);
+    
     if (end < start) return "'endDate' must be after 'startDate'.";
+    
     const maxDateRange = config.maxDateRange || 100;
     const diffTime = Math.abs(end - start);
     const diffDays = Math.ceil(diffTime / (1000 * 60 * 60 * 24)) + 1;
+    
     if (diffDays > maxDateRange) return `Date range cannot exceed ${maxDateRange} days.`;
+    
     return null;
 };
 
@@ -34,8 +39,19 @@ const validateRequest = (query, config) => {
 const buildCalculationMap = (unifiedCalculations) => {
     const calcMap = {};
     for (const category in unifiedCalculations) {
-        for (const subKey in unifiedCalculations[category]) { const item = unifiedCalculations[category][subKey]; if (subKey === 'historical' && typeof item === 'object') { for (const calcName in item) { calcMap[calcName] = {  category: category, class: item[calcName]  }; } }  
-        else if (typeof item === 'function') { const calcName = subKey; calcMap[calcName] = {  category: category, class: item  }; }}} return calcMap;
+        for (const subKey in unifiedCalculations[category]) {
+            const item = unifiedCalculations[category][subKey];
+            if (subKey === 'historical' && typeof item === 'object') {
+                for (const calcName in item) {
+                    calcMap[calcName] = { category: category, class: item[calcName] };
+                }
+            } else if (typeof item === 'function') {
+                const calcName = subKey;
+                calcMap[calcName] = { category: category, class: item };
+            }
+        }
+    }
+    return calcMap;
 };
 
 /**
@@ -45,7 +61,10 @@ const getDateStringsInRange = (startDate, endDate) => {
     const dates = [];
     const current = new Date(startDate + 'T00:00:00Z');
     const end = new Date(endDate + 'T00:00:00Z');
-    while (current <= end) { dates.push(current.toISOString().slice(0, 10)); current.setUTCDate(current.getUTCDate() + 1); }
+    while (current <= end) {
+        dates.push(current.toISOString().slice(0, 10));
+        current.setUTCDate(current.getUTCDate() + 1);
+    }
     return dates;
 };
 
@@ -58,19 +77,42 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
     const insightsCollection = config.unifiedInsightsCollection || 'unified_insights';
     const resultsSub = config.resultsSubcollection || 'results';
     const compsSub = config.computationsSubcollection || 'computations';
+    
     try {
         for (const date of dateStrings) {
             response[date] = {};
             const docRefs = [];
             const keyPaths = [];
+            
             for (const key of calcKeys) {
                 const pathInfo = calcMap[key];
-                if (pathInfo) { const docRef = db.collection(insightsCollection).doc(date) .collection(resultsSub).doc(pathInfo.category) .collection(compsSub).doc(key); docRefs.push(docRef); keyPaths.push(key);
-                } else { logger.log('WARN', `[${date}] No path info found for computation key: ${key}`); } }
+                if (pathInfo) {
+                    const docRef = db.collection(insightsCollection).doc(date)
+                        .collection(resultsSub).doc(pathInfo.category)
+                        .collection(compsSub).doc(key);
+                    docRefs.push(docRef);
+                    keyPaths.push(key);
+                } else {
+                    logger.log('WARN', `[${date}] No path info found for computation key: ${key}`);
+                }
+            }
+            
             if (docRefs.length === 0) continue;
+            
             const snapshots = await db.getAll(...docRefs);
-            snapshots.forEach((doc, i) => { const key = keyPaths[i]; if (doc.exists) { response[date][key] = doc.data(); } else { response[date][key] = null; } }); }
-    } catch (error) { logger.log('ERROR', 'API: Error fetching data from Firestore.', { errorMessage: error.message }); throw new Error('Failed to retrieve computation data.'); }
+            snapshots.forEach((doc, i) => {
+                const key = keyPaths[i];
+                if (doc.exists) {
+                    response[date][key] = doc.data();
+                } else {
+                    response[date][key] = null;
+                }
+            });
+        }
+    } catch (error) {
+        logger.log('ERROR', 'API: Error fetching data from Firestore.', { errorMessage: error.message });
+        throw new Error('Failed to retrieve computation data.');
+    }
     return response;
 };
 
@@ -79,14 +121,30 @@ const fetchUnifiedData = async (config, dependencies, calcKeys, dateStrings, cal
  */
 const createApiHandler = (config, dependencies, calcMap) => {
     const { logger } = dependencies; 
-    return async (req, res) => { const validationError = validateRequest(req.query, config);
-        if (validationError) { logger.log('WARN', 'API Bad Request', { error: validationError, query: req.query });  return res.status(400).send({ status: 'error', message: validationError }); }
+    return async (req, res) => {
+        const validationError = validateRequest(req.query, config);
+        if (validationError) {
+            logger.log('WARN', 'API Bad Request', { error: validationError, query: req.query });
+            return res.status(400).send({ status: 'error', message: validationError });
+        }
         try {
             const computationKeys = req.query.computations.split(',');
             const dateStrings = getDateStringsInRange(req.query.startDate, req.query.endDate);
             const data = await fetchUnifiedData(config, dependencies, computationKeys, dateStrings, calcMap);
-            res.status(200).send({ status: 'success', metadata: { computations: computationKeys, startDate: req.query.startDate, endDate: req.query.endDate, }, data, });
-        } catch (error) { logger.log('ERROR', 'API processing failed.', { errorMessage: error.message, stack: error.stack }); res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); } };
+            res.status(200).send({
+                status: 'success',
+                metadata: {
+                    computations: computationKeys,
+                    startDate: req.query.startDate,
+                    endDate: req.query.endDate,
+                },
+                data,
+            });
+        } catch (error) {
+            logger.log('ERROR', 'API processing failed.', { errorMessage: error.message, stack: error.stack });
+            res.status(500).send({ status: 'error', message: 'An internal error occurred.' });
+        }
+    };
 };
 
 /**
@@ -99,21 +157,29 @@ function createStructureSnippet(data, maxKeys = 20) {
         if (typeof data === 'boolean') return true;
         return data;
     }
-    if (Array.isArray(data)) { if (data.length === 0) return "<empty array>"; return [ createStructureSnippet(data[0], maxKeys) ]; }
+    if (Array.isArray(data)) {
+        if (data.length === 0) return "<empty array>";
+        return [ createStructureSnippet(data[0], maxKeys) ];
+    }
     const newObj = {};
-    const keys = Object.keys(data)
+    const keys = Object.keys(data);
+    
     if (keys.length > 0 && keys.every(k => k.match(/^[A-Z.]+$/) || k.includes('_') || k.match(/^[0-9]+$/))) {
         const exampleKey = keys[0];
         newObj[exampleKey] = createStructureSnippet(data[exampleKey], maxKeys);
         newObj["... (more items)"] = "...";
         return newObj;
     }
+    
     if (keys.length > maxKeys) {
         const firstKey = keys[0] || "example_key";
         newObj[firstKey] = createStructureSnippet(data[firstKey], maxKeys);
         newObj[`... (${keys.length - 1} more keys)`] = "<object>";
     } else {
-        for (const key of keys) { newObj[key] = createStructureSnippet(data[key], maxKeys); } }
+        for (const key of keys) {
+            newObj[key] = createStructureSnippet(data[key], maxKeys);
+        }
+    }
     return newObj;
 }
 
@@ -130,11 +196,16 @@ async function getComputationStructure(computationName, calcMap, config, depende
         const resultsSub = config.resultsSubcollection || 'results';
         const compsSub = config.computationsSubcollection || 'computations';
         const computationQueryPath = `${category}.${computationName}`;
-        const dateQuery = db.collection(insightsCollection) .where(computationQueryPath, '==', true) .orderBy(FieldPath.documentId(), 'desc') .limit(1);
+        const dateQuery = db.collection(insightsCollection)
+            .where(computationQueryPath, '==', true)
+            .orderBy(FieldPath.documentId(), 'desc')
+            .limit(1);
         const dateSnapshot = await dateQuery.get();
         if (dateSnapshot.empty) { return { status: 'error', computation: computationName, message: `No computed data found. (Query path: ${computationQueryPath})` }; }
         const latestStoredDate = dateSnapshot.docs[0].id;
-        const docRef = db.collection(insightsCollection).doc(latestStoredDate) .collection(resultsSub).doc(category) .collection(compsSub).doc(computationName);
+        const docRef = db.collection(insightsCollection).doc(latestStoredDate)
+            .collection(resultsSub).doc(category)
+            .collection(compsSub).doc(computationName);
         const doc = await docRef.get();
         if (!doc.exists) { return { status: 'error', computation: computationName, message: `Summary flag was present for ${latestStoredDate} but doc is missing.` }; }
         const fullData = doc.data();
@@ -152,30 +223,104 @@ async function getComputationStructure(computationName, calcMap, config, depende
  */
 async function getDynamicSchema(CalcClass, calcName) {
     if (CalcClass && typeof CalcClass.getSchema === 'function') {
-        try { return CalcClass.getSchema();
-        } catch (e) { console.error(`Error running static getSchema() for ${calcName}: ${e.message}`); return { "ERROR": `Failed to get static schema: ${e.message}` };}
-    } else { return { "ERROR": `Computation '${calcName}' does not have a static getSchema() method defined.` }; }
+        try {
+            return CalcClass.getSchema();
+        } catch (e) {
+            console.error(`Error running static getSchema() for ${calcName}: ${e.message}`);
+            return { "ERROR": `Failed to get static schema: ${e.message}` };
+        }
+    } else {
+        return { "ERROR": `Computation '${calcName}' does not have a static getSchema() method defined.` };
+    }
 }
 
 
 /**
- * --- NEW: MANIFEST API HANDLER ---
+ * --- NEW: MANIFEST API HANDLER (With Filtering) ---
  */
 const createManifestHandler = (config, dependencies, calcMap) => {
     const { db, logger } = dependencies;
     const schemaCollection = config.schemaCollection || 'computation_schemas';
+    
     return async (req, res) => {
         try {
             logger.log('INFO', '[API /manifest] Fetching all computation schemas...');
             const snapshot = await db.collection(schemaCollection).get();
-            if (snapshot.empty) { logger.log('WARN', '[API /manifest] No schemas found in collection.'); return res.status(404).send({ status: 'error', message: 'No computation schemas have been generated yet.' }); }
+            
+            if (snapshot.empty) {
+                logger.log('WARN', '[API /manifest] No schemas found in collection.');
+                return res.status(404).send({ status: 'error', message: 'No computation schemas have been generated yet.' });
+            }
+
+            // --- FILTERING LOGIC ---
             const manifest = {};
-            snapshot.forEach(doc => { const data = doc.data(); manifest[doc.id] = { category: data.category, structure: data.schema,  metadata: data.metadata, lastUpdated: data.lastUpdated }; });
-            res.status(200).send({ status: 'success', summary: { source: 'firestore_computation_schemas', totalComputations: snapshot.size, schemasAvailable: snapshot.size, schemasFailed: 0, lastUpdated: Math.max(...Object.values(manifest).map(m =>  m.lastUpdated ? m.lastUpdated.toMillis() : 0 )) }, manifest: manifest });
+            const now = Date.now();
+            const MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
+            
+            let activeCount = 0;
+            let staleCount = 0;
+
+            snapshot.forEach(doc => {
+                const data = doc.data();
+                
+                // Safe Timestamp conversion
+                let lastUpdatedMs = 0;
+                if (data.lastUpdated && typeof data.lastUpdated.toMillis === 'function') {
+                    lastUpdatedMs = data.lastUpdated.toMillis();
+                } else if (data.lastUpdated instanceof Date) {
+                    lastUpdatedMs = data.lastUpdated.getTime();
+                } else {
+                    // Fallback for very old records without a timestamp
+                    lastUpdatedMs = 0; 
+                }
+
+                // Exclude stale records
+                if ((now - lastUpdatedMs) < MAX_AGE_MS) {
+                    manifest[doc.id] = {
+                        category: data.category,
+                        structure: data.schema,
+                        metadata: data.metadata,
+                        lastUpdated: data.lastUpdated
+                    };
+                    activeCount++;
+                } else {
+                    staleCount++;
+                }
+            });
+            
+            // Log filtering results
+            if (staleCount > 0) {
+                logger.log('INFO', `[API /manifest] Filtered out ${staleCount} stale schemas (older than 7 days). Returning ${activeCount} active.`);
+            }
+
+            res.status(200).send({
+                status: 'success',
+                summary: {
+                    source: 'firestore_computation_schemas',
+                    totalComputations: snapshot.size,
+                    schemasAvailable: activeCount,
+                    schemasFiltered: staleCount,
+                    lastUpdated: Math.max(...Object.values(manifest).map(m => 
+                         (m.lastUpdated && m.lastUpdated.toMillis) ? m.lastUpdated.toMillis() : 0 
+                    ), 0)
+                },
+                manifest: manifest
+            });
+
         } catch (error) {
             logger.log('ERROR', 'API /manifest handler failed.', { errorMessage: error.message, stack: error.stack });
-            res.status(500).send({ status: 'error', message: 'An internal error occurred.' }); } };
+            res.status(500).send({ status: 'error', message: 'An internal error occurred.' });
+        }
+    };
 };
 
 
-module.exports = { validateRequest, buildCalculationMap, fetchUnifiedData, createApiHandler, getComputationStructure, getDynamicSchema, createManifestHandler  };
+module.exports = {
+    validateRequest,
+    buildCalculationMap,
+    fetchUnifiedData,
+    createApiHandler,
+    getComputationStructure,
+    getDynamicSchema,
+    createManifestHandler 
+};

package/functions/task-engine/helpers/update_helpers.js

@@ -1,14 +1,41 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
- * (REFACTORED: Removed all concurrency from `handleUpdate` and `lookupUsernames`)
- * (REFACTORED: Added node-fetch fallback for all API calls)
- * (REFACTORED: Added verbose, user-centric logging for all operations)
- * (FIXED: Corrected variable name 'instId' to 'instrumentId' in final timestamp loops)
+ * (OPTIMIZED V2: Added "Circuit Breaker" for Proxy failures)
+ * (OPTIMIZED V2: Downgraded verbose per-user logs to TRACE to save costs)
+ * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
 const pLimit = require('p-limit');
 
+// --- CIRCUIT BREAKER STATE ---
+// Persists across function invocations in the same instance.
+// If the Proxy fails 3 times in a row, we stop trying it to save the 5s timeout cost.
+let _consecutiveProxyFailures = 0;
+const MAX_PROXY_FAILURES = 3;
+
+/**
+ * Helper to check if we should attempt the proxy
+ */
+function shouldTryProxy() {
+    return _consecutiveProxyFailures < MAX_PROXY_FAILURES;
+}
+
+/**
+ * Helper to record proxy result
+ */
+function recordProxyOutcome(success) {
+    if (success) {
+        if (_consecutiveProxyFailures > 0) {
+            // Optional: Only log recovery to reduce noise
+            // console.log('[ProxyCircuit] Proxy recovered.');
+        }
+        _consecutiveProxyFailures = 0;
+    } else {
+        _consecutiveProxyFailures++;
+    }
+}
+
 /**
  * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
  */
@@ -21,35 +48,65 @@ async function lookupUsernames(cids, { logger, headerManager, proxyManager }, co
     const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
     const batches = [];
     for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) { batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number)); }
-    const batchPromises = batches.map((batch, index) => limit(async () => { const batchId = `batch-${index + 1}`;
-        logger.log('INFO', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`);
+    
+    const batchPromises = batches.map((batch, index) => limit(async () => { 
+        const batchId = `batch-${index + 1}`;
+        logger.log('TRACE', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`); // DOWNGRADED TO TRACE
+        
         const header = await headerManager.selectHeader();
         if (!header) { logger.log('ERROR', `[lookupUsernames/${batchId}] Could not select a header.`); return null; }
+        
         let wasSuccess = false;
-        let proxyUsed = true;
+        let proxyUsed = false;
         let response;
         const url = `${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
         const options = { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) };
-        try {
-            logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
-            response = await proxyManager.fetch(url, options);
-            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-            wasSuccess = true; // Yay we win
-            logger.log('INFO', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`);
-        } catch (proxyError) { logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' }); // SHIT we failed...
-            proxyUsed = false; // Don't penalize header for proxy failure
-            try { response = await fetch(url, options);  // Ok let's try again with node, using GCP IP pools
+        
+        // --- 1. Try Proxy (Circuit Breaker Protected) ---
+        if (shouldTryProxy()) {
+            try {
+                logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
+                response = await proxyManager.fetch(url, options);
+                if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
+                
+                wasSuccess = true; 
+                proxyUsed = true;
+                recordProxyOutcome(true); // Reset failure count
+                logger.log('TRACE', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`); // DOWNGRADED TO TRACE
+
+            } catch (proxyError) { 
+                recordProxyOutcome(false); // Increment failure count
+                logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
+                // Fall through to direct...
+            }
+        } else {
+            logger.log('TRACE', `[lookupUsernames/${batchId}] Circuit Breaker Open. Skipping Proxy.`);
+        }
+
+        // --- 2. Direct Fallback ---
+        if (!wasSuccess) {
+            try { 
+                response = await fetch(url, options);  
                 if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); }
-                logger.log('INFO', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`); // Yay we win
-            } catch (fallbackError) { logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, { error: fallbackError.message, source: 'eToro/Network' }); // SHIT, we failed here too
+                logger.log('TRACE', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`); // DOWNGRADED TO TRACE
+                wasSuccess = true; // It worked eventually
+            } catch (fallbackError) { 
+                logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, { error: fallbackError.message, source: 'eToro/Network' }); 
                 return null; // Give up on this batch
             }
-        } finally { if (proxyUsed) { headerManager.updatePerformance(header.id, wasSuccess); } } // If we used Appscript IP Pool and not GCP IP Pool, record performance
-        try { const data = await response.json(); return data;
-        } catch (parseError) { logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message }); return null; } }));
+        } 
+        
+        if (proxyUsed) { headerManager.updatePerformance(header.id, wasSuccess); } 
+        
+        try { 
+            const data = await response.json(); return data;
+        } catch (parseError) { 
+            logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message }); return null; 
+        } 
+    }));
 
     const results  = await Promise.allSettled(batchPromises);
-    const allUsers = results .filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value)) .flatMap(r => r.value); 
+    const allUsers = results.filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value)).flatMap(r => r.value); 
     logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
     return allUsers;
 }
@@ -64,118 +121,140 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
     let isPrivate = false;
-    logger.log('INFO', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
+    
+    // DOWNGRADED TO TRACE
+    logger.log('TRACE', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
 
     // --- 1. Process History Fetch (Sequentially) ---
     let historyHeader = null;
     let wasHistorySuccess = false;
-    let proxyUsedForHistory = true;
+    let proxyUsedForHistory = false;
 
     try {
         if (!batchManager.checkAndSetHistoryFetched(userId)) {
-            logger.log('INFO', `[handleUpdate/${userId}] Attempting history fetch.`);
+            logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch.`);
             historyHeader = await headerManager.selectHeader();
-            if (!historyHeader) { logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
+            if (!historyHeader) { 
+                logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
             } else {
                 const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
                 const options = { headers: historyHeader.header };
                 let response;
-                try { logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
-                    response = await proxyManager.fetch(historyUrl, options);
-                    if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // SHIT we failed here
-                    wasHistorySuccess = true; // Appscript worked, we are very smart
-                    
-                } catch (proxyError) {
-                    logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' }); // SHIT we failed here
-                    proxyUsedForHistory = false;
-                    try { response = await fetch(historyUrl, options);
-                        if (!response.ok) { const errorText = await response.text(); 
-                            throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); } // SHIT we failed here too
-                        wasHistorySuccess = true; // Fallback succeeded, we are so smart
+
+                // --- PROXY ATTEMPT ---
+                if (shouldTryProxy()) {
+                    try { 
+                        logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
+                        response = await proxyManager.fetch(historyUrl, options);
+                        if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
                         
-                    } catch (fallbackError) { logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' }); // We are dumb, everything failed
-                        wasHistorySuccess = false; // Nope we are dumb....
+                        wasHistorySuccess = true; 
+                        proxyUsedForHistory = true;
+                        recordProxyOutcome(true); // Reset
+
+                    } catch (proxyError) {
+                        recordProxyOutcome(false); // Count failure
+                        logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
                     }
                 }
 
-                if (wasHistorySuccess) { logger.log('INFO', `[handleUpdate/${userId}] History fetch successful.`); // Some method worked, we are very smart
+                // --- DIRECT FALLBACK ---
+                if (!wasHistorySuccess) {
+                    try { 
+                        response = await fetch(historyUrl, options);
+                        if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); } 
+                        wasHistorySuccess = true; 
+                        logger.log('TRACE', `[handleUpdate/${userId}] History fetch direct success.`);
+                    } catch (fallbackError) { 
+                        logger.log('ERROR', `[handleUpdate/${userId}] History fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' }); 
+                        wasHistorySuccess = false; 
+                    }
+                }
+
+                if (wasHistorySuccess) { 
                     const data = await response.json();
-                    await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);  }
+                    await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);  
+                }
             }
-        } else { logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched by this instance).`); }
-    } catch (err) { logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message }); wasHistorySuccess = false; // We fucked up.
-    } finally { if (historyHeader && proxyUsedForHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } } // If we used appscript proxy, record performance, otherwise fuck off.
+        } else { 
+            logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched).`); 
+        }
+    } catch (err) { 
+        logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message }); wasHistorySuccess = false; 
+    } finally { 
+        if (historyHeader && proxyUsedForHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } 
+    } 
     
     // --- 2. Process Portfolio Fetches (Sequentially) ---
-    logger.log('INFO', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`);
+    logger.log('TRACE', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`); // DOWNGRADED TO TRACE
     
     for (const instId of instrumentsToProcess) { 
         if (isPrivate) { 
-            logger.log('INFO', `[handleUpdate/${userId}] Skipping remaining instruments because user was marked as private.`); 
+            logger.log('TRACE', `[handleUpdate/${userId}] Skipping remaining instruments (User Private).`); 
             break;  
         }
 
         const portfolioHeader = await headerManager.selectHeader();
-        if (!portfolioHeader) { logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`);
-            continue;
-        }
+        if (!portfolioHeader) { logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`); continue; }
 
         const portfolioUrl = userType === 'speculator'  ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
-        
         const options = { headers: portfolioHeader.header };
         let response;
         let wasPortfolioSuccess = false;
-        let proxyUsedForPortfolio = true;
-
-        try {
-            // --- REFACTOR 3: ADD FALLBACK ---
-            logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch for instId ${instId} via AppScript proxy...`);
-            response = await proxyManager.fetch(portfolioUrl, options);
-            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // SHIT we failed here
-            wasPortfolioSuccess = true; // Oh we are smart, worked first time.
-
-        } catch (proxyError) { // try fallback with local node fetch using GCP IP Pools
-            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} via AppScript proxy FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message, source: 'AppScript' });
-            proxyUsedForPortfolio = false; // We are not using Appscript proxy here as fallback is GCP based, so false
-            
+        let proxyUsedForPortfolio = false;
+
+        // --- PROXY ATTEMPT ---
+        if (shouldTryProxy()) {
             try {
-                response = await fetch(portfolioUrl, options); // Direct node-fetch
-                if (!response.ok) {
-                    const errorText = await response.text();
-                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); // SHIT we failed here
-                }
-                wasPortfolioSuccess = true; // Fallback succeeded we are so smart
+                logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch via AppScript proxy...`);
+                response = await proxyManager.fetch(portfolioUrl, options);
+                if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); 
                 
+                wasPortfolioSuccess = true; 
+                proxyUsedForPortfolio = true;
+                recordProxyOutcome(true); // Reset
+
+            } catch (proxyError) { 
+                recordProxyOutcome(false); // Count failure
+                logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch via Proxy FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
+            }
+        }
+
+        // --- DIRECT FALLBACK ---
+        if (!wasPortfolioSuccess) {
+            try {
+                response = await fetch(portfolioUrl, options); 
+                if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); }
+                wasPortfolioSuccess = true; 
+                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio fetch direct success.`);
             } catch (fallbackError) {
-                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch for instId ${instId} direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
+                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
                 wasPortfolioSuccess = false;
             }
         }
 
-        // --- 4. Process Portfolio Result (with verbose, raw logging) ---
+        // --- 4. Process Portfolio Result ---
         if (wasPortfolioSuccess) {
             const body = await response.text();
-            if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`);
-                break; // Stop processing more portfolios for this private user
-            }
+            if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`); break; }
 
             try {
                 const portfolioJson = JSON.parse(body);
                 await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
-                if (userType === 'speculator') { logger.log('INFO', `[handleUpdate/${userId}] Successfully processed portfolio for instId ${instId}.`); // Only speculators have an instid, so this is conditional
-                } else { logger.log('INFO', `[handleUpdate/${userId}] Successfully processed full portfolio (normal user).`); } // Normal users
+                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio processed successfully.`); // DOWNGRADED TO TRACE
 
-            } catch (parseError) { // Idk why this would happen, but if it does....log.
-                wasPortfolioSuccess = false; // Mark as failure
-                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE. RAW BODY:`, { url: portfolioUrl, parseErrorMessage: parseError.message, rawResponseText: body }); // Return full response
+            } catch (parseError) { 
+                wasPortfolioSuccess = false; 
+                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE.`, { url: portfolioUrl, parseErrorMessage: parseError.message }); 
             }
-        } else { logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}. No response to process.`); }
+        } else { 
+            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}.`); 
+        }
         
         if (proxyUsedForPortfolio) { headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess); }
     } 
 
     // --- 5. Handle Private Users & Timestamps ---
-    // FIXED: Corrected variable naming here from 'instId' to 'instrumentId'
     if (isPrivate) {
         logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
         for (const instrumentId of instrumentsToProcess) { 
@@ -190,15 +269,13 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     }
 
     // If not private, update all timestamps
-    // FIXED: Corrected variable naming here from 'instId' to 'instrumentId'
     for (const instrumentId of instrumentsToProcess) { 
         await batchManager.updateUserTimestamp(userId, userType, instrumentId); 
     }
     
     if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
     
-    logger.log('INFO', `[handleUpdate/${userId}] Update task finished successfully.`);
-    // 'finally' block for header flushing is handled by the main handler_creator.js
+    logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`); // DOWNGRADED TO TRACE
 }
 
 module.exports = { handleUpdate, lookupUsernames };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.194",
+  "version": "1.0.196",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [