bulltrackers-module 1.0.205 → 1.0.207

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -25,7 +25,7 @@ const PARALLEL_BATCH_SIZE = 7;
  */
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
-    const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
+    const passToRun  = String(config.COMPUTATION_PASS_TO_RUN);
     if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
 
     logger.log('INFO', `🚀 Starting PASS ${passToRun} (Legacy Mode)...`);
@@ -33,21 +33,21 @@ async function runComputationPass(config, dependencies, computationManifest) {
     // Hardcoded earliest dates
     const earliestDates = {
         portfolio: new Date('2025-09-25T00:00:00Z'),
-        history: new Date('2025-11-05T00:00:00Z'),
-        social: new Date('2025-10-30T00:00:00Z'),
-        insights: new Date('2025-08-26T00:00:00Z'),
-        price: new Date('2025-08-01T00:00:00Z')
+        history:   new Date('2025-11-05T00:00:00Z'),
+        social:    new Date('2025-10-30T00:00:00Z'),
+        insights:  new Date('2025-08-26T00:00:00Z'),
+        price:     new Date('2025-08-01T00:00:00Z')
     };
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a, b) => a < b ? a : b);
 
-    const passes = groupByPass(computationManifest);
+    const passes          = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
     if (!calcsInThisPass.length)
         return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
 
     const passEarliestDate = earliestDates.absoluteEarliest;
-    const endDateUTC = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
+    const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
     // Legacy Batch Optimization for Price (Only used in legacy loop)
@@ -98,10 +98,10 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
     // 2. Check Root Data Availability
     const earliestDates = {
         portfolio: new Date('2025-09-25T00:00:00Z'),
-        history: new Date('2025-11-05T00:00:00Z'),
-        social: new Date('2025-10-30T00:00:00Z'),
-        insights: new Date('2025-08-26T00:00:00Z'),
-        price: new Date('2025-08-01T00:00:00Z')
+        history:   new Date('2025-11-05T00:00:00Z'),
+        social:    new Date('2025-10-30T00:00:00Z'),
+        insights:  new Date('2025-08-26T00:00:00Z'),
+        price:     new Date('2025-08-01T00:00:00Z')
     };
 
     const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
@@ -129,8 +129,8 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
 
         // Fetch dependencies (results from this day or yesterday)
         const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
-        const prevDate = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-        const prevDateStr = prevDate.toISOString().slice(0, 10);
+        const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+        const prevDateStr     = prevDate.toISOString().slice(0, 10);
         const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
 
         if (standardToRun.length) {

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -5,8 +5,8 @@
  * allowing others in the same pass/date to succeed and be recorded.
  */
 
-const { ComputationController } = require('../controllers/computation_controller');
-const { batchStoreSchemas } = require('../utils/schema_capture');
+const { ComputationController }              = require('../controllers/computation_controller');
+const { batchStoreSchemas }                  = require('../utils/schema_capture');
 const { normalizeName, commitBatchInChunks } = require('../utils/utils');
 const {
     getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights,
@@ -33,7 +33,7 @@ function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[ca
 function validateResultPatterns(logger, calcName, results, category) {
     if (category === 'speculator' || category === 'speculators') return;
 
-    const tickers = Object.keys(results);
+    const tickers    = Object.keys(results);
     const totalItems = tickers.length;
 
     if (totalItems < 5) return;
@@ -46,8 +46,8 @@ function validateResultPatterns(logger, calcName, results, category) {
     keys.forEach(key => {
         if (key.startsWith('_')) return;
 
-        let nullCount = 0;
-        let nanCount = 0;
+        let nullCount      = 0;
+        let nanCount       = 0;
         let undefinedCount = 0;
 
         for (const t of tickers) {
@@ -72,11 +72,11 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
     for (const dep of calcManifest.rootDataDependencies) {
-        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
-        else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
-        else if (dep === 'social' && !rootDataStatus.hasSocial) missing.push('social');
-        else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
-        else if (dep === 'price' && !rootDataStatus.hasPrices) missing.push('price');
+        if (dep === 'portfolio'     && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        else if (dep === 'insights' && !rootDataStatus.hasInsights)  missing.push('insights');
+        else if (dep === 'social'   && !rootDataStatus.hasSocial)    missing.push('social');
+        else if (dep === 'history'  && !rootDataStatus.hasHistory)   missing.push('history');
+        else if (dep === 'price'    && !rootDataStatus.hasPrices)    missing.push('price');
     }
     return { canRun: missing.length === 0, missing };
 }
@@ -90,10 +90,10 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
 
     try {
         const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights) tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
-        if (dateToProcess >= earliestDates.social) tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
-        if (dateToProcess >= earliestDates.history) tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
+        if (dateToProcess >= earliestDates.portfolio) tasks.push(getPortfolioPartRefs        (config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio  = !!r.length; }));
+        if (dateToProcess >= earliestDates.insights)  tasks.push(loadDailyInsights           (config, dependencies, dateStr).then(r => { insightsData  = r; hasInsights   = !!r; }));
+        if (dateToProcess >= earliestDates.social)    tasks.push(loadDailySocialPostInsights (config, dependencies, dateStr).then(r => { socialData    = r; hasSocial     = !!r; }));
+        if (dateToProcess >= earliestDates.history)   tasks.push(getHistoryPartRefs          (config, dependencies, dateStr).then(r => { historyRefs   = r; hasHistory    = !!r.length; }));
 
         if (dateToProcess >= earliestDates.price) {
             tasks.push(checkPriceDataAvailability(config, dependencies).then(r => { hasPrices = r; }));
@@ -142,22 +142,22 @@ async function fetchComputationStatus(dateStr, config, { db }) {
 
 async function fetchGlobalComputationStatus(config, { db }) {
     const collection = config.computationStatusCollection || 'computation_status';
-    const docRef = db.collection(collection).doc('global_status');
-    const snap = await docRef.get();
+    const docRef     = db.collection(collection).doc('global_status');
+    const snap       = await docRef.get();
     return snap.exists ? snap.data() : {};
 }
 
 async function updateComputationStatus(dateStr, updates, config, { db }) {
     if (!updates || Object.keys(updates).length === 0) return;
     const collection = config.computationStatusCollection || 'computation_status';
-    const docRef = db.collection(collection).doc(dateStr);
+    const docRef     = db.collection(collection).doc(dateStr);
     await docRef.set(updates, { merge: true });
 }
 
 async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
     if (!updatesByDate || Object.keys(updatesByDate).length === 0) return;
     const collection = config.computationStatusCollection || 'computation_status';
-    const docRef = db.collection(collection).doc('global_status');
+    const docRef     = db.collection(collection).doc('global_status');
     const flattenUpdates = {};
     for (const [date, statuses] of Object.entries(updatesByDate)) {
         for (const [calc, status] of Object.entries(statuses)) {
@@ -180,7 +180,7 @@ async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
 }
 
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
-    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
+    const manifestMap  = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     const calcsToFetch = new Set();
     for (const calc of calcsInPass) {
         if (calc.dependencies) { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); }
@@ -189,7 +189,7 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
     if (!calcsToFetch.size) return {};
     const fetched = {};
     const docRefs = [];
-    const names = [];
+    const names   = [];
     for (const name of calcsToFetch) {
         const m = manifestMap.get(name);
         if (m) {
@@ -208,8 +208,8 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
 
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
     const { logger } = deps;
-    const controller = new ComputationController(config, deps);
-    const calcs = Object.values(state).filter(c => c && c.manifest);
+    const controller     = new ComputationController(config, deps);
+    const calcs          = Object.values(state).filter(c => c && c.manifest);
     const streamingCalcs = calcs.filter(c =>
         c.manifest.rootDataDependencies.includes('portfolio') ||
         c.manifest.rootDataDependencies.includes('history')
@@ -220,14 +220,14 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
 
     await controller.loader.loadMappings();
-    const prevDate = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+    const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
     const prevDateStr = prevDate.toISOString().slice(0, 10);
 
-    const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
+    const tP_iter                 = streamPortfolioData(config, deps, dateStr, portfolioRefs);
     const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
-    const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-    const tH_iter = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
+    const yP_iter                 = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
+    const needsTradingHistory     = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
+    const tH_iter                 = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
 
     let yP_chunk = {};
     let tH_chunk = {};
@@ -316,9 +316,9 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 continue;
             }
 
-            const standardRes = {};
+            const standardRes   = {};
             const shardedWrites = [];
-            const calcWrites = []; // Accumulate all writes for THIS specific calculation
+            const calcWrites    = []; // Accumulate all writes for THIS specific calculation
 
             // 1. Separate Standard and Sharded Data
             for (const key in result) {

package/functions/computation-system/layers/math_primitives.js

@@ -95,8 +95,8 @@ class DataExtractor { // For generic access of data types
      * Extract Current Equity Value %.
      * Schema: 'Value' is the current value as a % of total portfolio equity.
      */
-    static getPositionValuePct(position) {            // TODO - VERIFY THIS WORKS FOR SPECULATORS,
-        return position ? (position.Value || 0) : 0; //  IS VALUE ACTUALLY THE VALUE OF POSITION AS A % OF TOTAL PORTFOLIO EQUITY? IS IT THE SAME FOR NORMAL USERS?
+    static getPositionValuePct(position) {            
+        return position ? (position.Value || 0) : 0; 
     }
 
     /**
@@ -262,39 +262,108 @@ class priceExtractor {
 }
 
 class HistoryExtractor {
-    // --- Schema Accessor (NEW) ---
+    // --- Schema Accessor (REFACTORED for Granular API) ---
     /**
      * Extracts the daily history snapshot from the User object.
-     * This decouples the computation from knowing 'user.history.today'.
+     * Returns the raw granular positions list.
      */
     static getDailyHistory(user) {
+        // The new API returns { PublicHistoryPositions: [...] }
         return user?.history?.today || null;
     }
 
-    // --- Data Extractors ---
+    // --- Data Extractors (ADAPTER PATTERN) ---
+    /**
+     * Adapts granular trade history into the legacy 'TradedAssets' format.
+     * Groups trades by InstrumentID and calculates average holding time.
+     */
     static getTradedAssets(historyDoc) {
-        if (!historyDoc || !Array.isArray(historyDoc.assets)) return [];
-        return historyDoc.assets;
+        const trades = historyDoc?.PublicHistoryPositions || [];
+        if (!trades.length) return [];
+        
+        // Group by InstrumentID
+        const assetsMap = new Map();
+        
+        for (const t of trades) {
+            const instId = t.InstrumentID;
+            if (!instId) continue;
+            
+            if (!assetsMap.has(instId)) {
+                assetsMap.set(instId, { 
+                    instrumentId: instId, 
+                    totalDuration: 0, 
+                    count: 0 
+                });
+            }
+            
+            const asset = assetsMap.get(instId);
+            
+            // Calculate Duration in Minutes
+            const open = new Date(t.OpenDateTime);
+            const close = new Date(t.CloseDateTime);
+            const durationMins = (close - open) / 60000; // ms -> min
+            
+            if (durationMins > 0) {
+                asset.totalDuration += durationMins;
+                asset.count++;
+            }
+        }
+        
+        // Convert Map to Array format expected by existing calculations
+        // (Returns objects with .instrumentId and .avgHoldingTimeInMinutes)
+        return Array.from(assetsMap.values()).map(a => ({
+            instrumentId: a.instrumentId,
+            avgHoldingTimeInMinutes: a.count > 0 ? (a.totalDuration / a.count) : 0
+        }));
     }
 
     static getInstrumentId(asset) { 
         return asset ? asset.instrumentId : null;
     }
 
-    static getAvgHoldingTimeMinutes(asset) { // Note, in minutes, we could convert values here into hours or days but we leave as-is for now.
+    static getAvgHoldingTimeMinutes(asset) { 
         return asset ? (asset.avgHoldingTimeInMinutes || 0) : 0;
     }
 
-    static getSummary(historyDoc) { // This returns the top-level summary of trade history
-        const all = historyDoc?.all;
-        if (!all) return null;
+    /**
+     * Adapts granular trade history into the legacy 'Summary' format.
+     * Calculates WinRatio, AvgProfit, etc. on the fly from the raw list.
+     */
+    static getSummary(historyDoc) { 
+        const trades = historyDoc?.PublicHistoryPositions || [];
+        if (!trades.length) return null;
+
+        let totalTrades = trades.length;
+        let wins = 0;
+        let totalProf = 0;
+        let totalLoss = 0;
+        let profCount = 0;
+        let lossCount = 0;
+        let totalDur = 0;
+
+        for (const t of trades) {
+            // P&L Stats (NetProfit is %)
+            if (t.NetProfit > 0) {
+                wins++;
+                totalProf += t.NetProfit;
+                profCount++;
+            } else if (t.NetProfit < 0) {
+                totalLoss += t.NetProfit;
+                lossCount++;
+            }
+            
+            // Duration Stats
+            const open = new Date(t.OpenDateTime);
+            const close = new Date(t.CloseDateTime);
+            totalDur += (close - open) / 60000; // ms -> min
+        }
 
-        return { // The all object contains instrumentid of -1 value, we do not include this, it's a junk backend-eToro placeholder.
-            totalTrades:             all.totalTrades             || 0,
-            winRatio:                all.winRatio                || 0,
-            avgProfitPct:            all.avgProfitPct            || 0,
-            avgLossPct:              all.avgLossPct              || 0,
-            avgHoldingTimeInMinutes: all.avgHoldingTimeInMinutes || 0
+        return { 
+            totalTrades:             totalTrades,
+            winRatio:                totalTrades > 0 ? (wins / totalTrades) * 100 : 0,
+            avgProfitPct:            profCount > 0 ? totalProf / profCount : 0,
+            avgLossPct:              lossCount > 0 ? totalLoss / lossCount : 0,
+            avgHoldingTimeInMinutes: totalTrades > 0 ? totalDur / totalTrades : 0
         };
     }
 }
@@ -354,7 +423,7 @@ class SignalPrimitives {
 
     static getPreviousState(previousComputed, calcName, ticker, fieldName = null) { // This is used for either fetching computations listed in getdependencies() OR self-history
         if (!previousComputed || !previousComputed[calcName]) return null;          // Using this for self-history DOES NOT cause a circular dependency because we assign a special rule in orchestration_helpers
-                                                                                    // Which handles the self-reference, see 2. SMART SELF-FETCH in orchestration_helpers
+                                                                                    // Which handles the self-reference.
         const tickerData = previousComputed[calcName][ticker];
         if (!tickerData) return null;
 
@@ -374,7 +443,7 @@ class MathPrimitives {
     static median(values) {
         if (!values || !values.length) return 0;
         const sorted = [...values].sort((a, b) => a - b);
-        const mid = Math.floor(sorted.length / 2);
+        const mid    = Math.floor(sorted.length / 2);
         return sorted.length % 2 === 0
             ? (sorted[mid - 1] + sorted[mid]) / 2
             : sorted[mid];
@@ -382,7 +451,7 @@ class MathPrimitives {
 
     static standardDeviation(values) {
         if (!values || !values.length) return 0;
-        const avg = this.average(values);
+        const avg         = this.average(values);
         const squareDiffs = values.map(val => Math.pow((val || 0) - avg, 2));
         return Math.sqrt(this.average(squareDiffs));
     }
@@ -398,7 +467,7 @@ class MathPrimitives {
      * Where: 
      * b = ln(Barrier/Price)
      * v = drift - 0.5 * volatility^2
-     * * @param {number} currentPrice - The current price of the asset
+     * @param {number} currentPrice - The current price of the asset
      * @param {number} barrierPrice - The target price (SL or TP)
      * @param {number} volatility - Annualized volatility (e.g., 0.40 for 40%)
      * @param {number} days - Number of days to forecast (e.g., 3)
@@ -408,9 +477,9 @@ class MathPrimitives {
     static calculateHitProbability(currentPrice, barrierPrice, volatility, days, drift = 0) { // https://www.ma.ic.ac.uk/~bin06/M3A22/m3f22chVII.pdf
         if (currentPrice <= 0 || barrierPrice <= 0 || volatility <= 0 || days <= 0) return 0;
 
-        const t = days / 365.0; // Convert days to years
+        const t     = days / 365.0; // Convert days to years
         const sigma = volatility;
-        const mu = drift;
+        const mu    = drift;
         
         // The barrier in log-space
         const b = Math.log(barrierPrice / currentPrice);
@@ -418,7 +487,7 @@ class MathPrimitives {
         // Adjusted drift (nu)
         const nu = mu - 0.5 * Math.pow(sigma, 2);
 
-        const sqrtT = Math.sqrt(t);
+        const sqrtT      = Math.sqrt(t);
         const sigmaSqrtT = sigma * sqrtT;
 
         // Helper for Standard Normal CDF (Φ)
@@ -448,8 +517,7 @@ class MathPrimitives {
 
         // Calculate Probability
         // Note: If nu is 0, the second term simplifies significantly, but we keep full form.
-        const probability = normCDF(( -Math.abs(b) - nu * t ) / sigmaSqrtT) + 
-                            Math.exp((2 * nu * Math.abs(b)) / (sigma * sigma)) * normCDF(( -Math.abs(b) + nu * t ) / sigmaSqrtT);
+        const probability = normCDF(( -Math.abs(b) - nu * t ) / sigmaSqrtT) +  Math.exp((2 * nu * Math.abs(b)) / (sigma * sigma)) * normCDF(( -Math.abs(b) + nu * t ) / sigmaSqrtT);
 
         return Math.min(Math.max(probability, 0), 1);
     }
@@ -468,11 +536,11 @@ class MathPrimitives {
     static simulateGBM(currentPrice, volatility, days, simulations = 1000, drift = 0) {
         if (currentPrice <= 0 || volatility <= 0 || days <= 0) return new Float32Array(0);
         
-        const t = days / 365.0;
-        const sigma = volatility;
-        const mu = drift;
+        const t         = days / 365.0;
+        const sigma     = volatility;
+        const mu        = drift;
         const driftTerm = (mu - 0.5 * sigma * sigma) * t;
-        const volTerm = sigma * Math.sqrt(t);
+        const volTerm   = sigma * Math.sqrt(t);
         
         // Use Float32Array for memory efficiency with large simulation counts
         const results = new Float32Array(simulations);
@@ -481,7 +549,7 @@ class MathPrimitives {
             // Box-Muller transform for efficient standard normal distribution generation
             const u1 = Math.random();
             const u2 = Math.random();
-            const z = Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2.0 * Math.PI * u2);
+            const z  = Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2.0 * Math.PI * u2);
             
             // GBM Formula: St = S0 * exp((mu - 0.5*sigma^2)t + sigma*Wt)
             results[i] = currentPrice * Math.exp(driftTerm + volTerm * z);
@@ -643,14 +711,14 @@ class TimeSeries {
         let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0, sumY2 = 0;
 
         for (let i = 0; i < n; i++) {
-            sumX += x[i];
-            sumY += y[i];
+            sumX  += x[i];
+            sumY  += y[i];
             sumXY += x[i] * y[i];
             sumX2 += x[i] * x[i];
             sumY2 += y[i] * y[i];
         }
 
-        const numerator = (n * sumXY) - (sumX * sumY);
+        const numerator   = (n * sumXY) - (sumX * sumY);
         const denominator = Math.sqrt(((n * sumX2) - (sumX * sumX)) * ((n * sumY2) - (sumY * sumY)));
 
         return (denominator === 0) ? 0 : numerator / denominator;
@@ -719,8 +787,8 @@ class DistributionAnalytics {
         
         let sumX = 0, sumY = 0, sumXY = 0, sumXX = 0, sumYY = 0;
         for (let i = 0; i < n; i++) {
-            sumX += xValues[i];
-            sumY += yValues[i];
+            sumX  += xValues[i];
+            sumY  += yValues[i];
             sumXY += xValues[i] * yValues[i];
             sumXX += xValues[i] * xValues[i];
             sumYY += yValues[i] * yValues[i];

package/functions/task-engine/helpers/update_helpers.js

@@ -1,12 +1,13 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
+ * (OPTIMIZED V4: Auto-Speculator Detection via History/Portfolio Intersection)
+ * (OPTIMIZED V3: Removed obsolete username lookup logic)
  * (OPTIMIZED V2: Added "Circuit Breaker" for Proxy failures)
- * (OPTIMIZED V2: Downgraded verbose per-user logs to TRACE to save costs)
  * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
-const pLimit = require('p-limit');
+const crypto = require('crypto'); 
 
 // --- CIRCUIT BREAKER STATE ---
 // Persists across function invocations in the same instance.
@@ -26,10 +27,6 @@ function shouldTryProxy() {
  */
 function recordProxyOutcome(success) {
     if (success) {
-        if (_consecutiveProxyFailures > 0) {
-            // Optional: Only log recovery to reduce noise
-            // console.log('[ProxyCircuit] Proxy recovered.');
-        }
         _consecutiveProxyFailures = 0;
     } else {
         _consecutiveProxyFailures++;
@@ -37,92 +34,47 @@ function recordProxyOutcome(success) {
 }
 
 /**
- * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
+ * --- NEW HELPER: Speculator Detector ---
+ * intersections: (History: Leverage > 1) AND (Portfolio: Currently Owned)
  */
-async function lookupUsernames(cids, { logger, headerManager, proxyManager }, config) {
-    if (!cids?.length) return [];
-    logger.log('INFO', `[lookupUsernames] Looking up usernames for ${cids.length} CIDs.`);
-    
-    // --- Set concurrency to 1 because appscript gets really fucked up with undocumented rate limits if we try spam it concurrently, a shame but that's life. DO NOT CHANGE THIS
-    const limit = pLimit(1);
-    const { USERNAME_LOOKUP_BATCH_SIZE, ETORO_API_RANKINGS_URL } = config;
-    const batches = [];
-    for (let i = 0; i < cids.length; i += USERNAME_LOOKUP_BATCH_SIZE) { batches.push(cids.slice(i, i + USERNAME_LOOKUP_BATCH_SIZE).map(Number)); }
-    
-    const batchPromises = batches.map((batch, index) => limit(async () => { 
-        const batchId = `batch-${index + 1}`;
-        logger.log('TRACE', `[lookupUsernames/${batchId}] Processing batch of ${batch.length} CIDs...`); // DOWNGRADED TO TRACE
-        
-        const header = await headerManager.selectHeader();
-        if (!header) { logger.log('ERROR', `[lookupUsernames/${batchId}] Could not select a header.`); return null; }
-        
-        let wasSuccess = false;
-        let proxyUsed = false;
-        let response;
-        const url = `${ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
-        const options = { method: 'POST', headers: { ...header.header, 'Content-Type': 'application/json' }, body: JSON.stringify(batch) };
-        
-        // --- 1. Try Proxy (Circuit Breaker Protected) ---
-        if (shouldTryProxy()) {
-            try {
-                logger.log('TRACE', `[lookupUsernames/${batchId}] Attempting fetch via AppScript proxy...`);
-                response = await proxyManager.fetch(url, options);
-                if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-                
-                wasSuccess = true; 
-                proxyUsed = true;
-                recordProxyOutcome(true); // Reset failure count
-                logger.log('TRACE', `[lookupUsernames/${batchId}] AppScript proxy fetch successful.`); // DOWNGRADED TO TRACE
+function detectSpeculatorTargets(historyData, portfolioData) {
+    if (!historyData?.PublicHistoryPositions || !portfolioData?.AggregatedPositions) return [];
 
-            } catch (proxyError) { 
-                recordProxyOutcome(false); // Increment failure count
-                logger.log('WARN', `[lookupUsernames/${batchId}] AppScript proxy fetch FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
-                // Fall through to direct...
-            }
-        } else {
-            logger.log('TRACE', `[lookupUsernames/${batchId}] Circuit Breaker Open. Skipping Proxy.`);
+    // 1. Identify assets that have EVER been traded with leverage > 1
+    const leveragedAssets = new Set();
+    for (const pos of historyData.PublicHistoryPositions) {
+        if (pos.Leverage > 1 && pos.InstrumentID) {
+            leveragedAssets.add(pos.InstrumentID);
         }
+    }
 
-        // --- 2. Direct Fallback ---
-        if (!wasSuccess) {
-            try { 
-                response = await fetch(url, options);  
-                if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); }
-                logger.log('TRACE', `[lookupUsernames/${batchId}] Direct node-fetch fallback successful.`); // DOWNGRADED TO TRACE
-                wasSuccess = true; // It worked eventually
-            } catch (fallbackError) { 
-                logger.log('ERROR', `[lookupUsernames/${batchId}] Direct node-fetch fallback FAILED. Giving up on this batch.`, { error: fallbackError.message, source: 'eToro/Network' }); 
-                return null; // Give up on this batch
-            }
-        } 
-        
-        if (proxyUsed) { headerManager.updatePerformance(header.id, wasSuccess); } 
-        
-        try { 
-            const data = await response.json(); return data;
-        } catch (parseError) { 
-            logger.log('ERROR', `[lookupUsernames/${batchId}] Failed to parse JSON response.`, { error: parseError.message }); return null; 
-        } 
-    }));
+    if (leveragedAssets.size === 0) return [];
 
-    const results  = await Promise.allSettled(batchPromises);
-    const allUsers = results.filter(r => r.status === 'fulfilled' && r.value && Array.isArray(r.value)).flatMap(r => r.value); 
-    logger.log('INFO', `[lookupUsernames] Found ${allUsers.length} public users out of ${cids.length}.`);
-    return allUsers;
+    // 2. Check if the user CURRENTLY owns any of these assets
+    const targets = [];
+    for (const pos of portfolioData.AggregatedPositions) {
+        if (leveragedAssets.has(pos.InstrumentID)) {
+            targets.push(pos.InstrumentID);
+        }
+    }
+    
+    return targets;
 }
 
-
 /**
  * (REFACTORED: Fully sequential, verbose logging, node-fetch fallback)
  */
-async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager }, config, username) {
+async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager, pubsub }, config) {
     const { userId, instruments, instrumentId, userType } = task;
     const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
     let isPrivate = false;
     
-    // DOWNGRADED TO TRACE
+    // Captured data for detection logic
+    let capturedHistory = null;
+    let capturedPortfolio = null;
+
     logger.log('TRACE', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
 
     // --- 1. Process History Fetch (Sequentially) ---
@@ -137,7 +89,14 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
             if (!historyHeader) { 
                 logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
             } else {
-                const historyUrl = `${config.ETORO_API_USERSTATS_URL}${username}/trades/oneYearAgo?CopyAsAsset=true`;
+                
+                // --- REFACTOR: New Granular API Logic ---
+                const d = new Date();
+                d.setFullYear(d.getFullYear() - 1);
+                const oneYearAgoStr = d.toISOString();
+                const uuid = crypto.randomUUID ? crypto.randomUUID() : '0205aca7-bd37-4884-8455-f28ce1add2de'; 
+
+                const historyUrl = `https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat?StartTime=${oneYearAgoStr}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${userId}&client_request_id=${uuid}`;
                 const options = { headers: historyHeader.header };
                 let response;
 
@@ -147,13 +106,12 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                         logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
                         response = await proxyManager.fetch(historyUrl, options);
                         if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-                        
                         wasHistorySuccess = true; 
                         proxyUsedForHistory = true;
-                        recordProxyOutcome(true); // Reset
+                        recordProxyOutcome(true); 
 
                     } catch (proxyError) {
-                        recordProxyOutcome(false); // Count failure
+                        recordProxyOutcome(false); 
                         logger.log('WARN', `[handleUpdate/${userId}] History fetch via AppScript proxy FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
                     }
                 }
@@ -173,6 +131,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
 
                 if (wasHistorySuccess) { 
                     const data = await response.json();
+                    capturedHistory = data; // Capture for later
                     await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);  
                 }
             }
@@ -186,7 +145,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     } 
     
     // --- 2. Process Portfolio Fetches (Sequentially) ---
-    logger.log('TRACE', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`); // DOWNGRADED TO TRACE
+    logger.log('TRACE', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`); 
     
     for (const instId of instrumentsToProcess) { 
         if (isPrivate) { 
@@ -209,13 +168,12 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                 logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch via AppScript proxy...`);
                 response = await proxyManager.fetch(portfolioUrl, options);
                 if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); 
-                
                 wasPortfolioSuccess = true; 
                 proxyUsedForPortfolio = true;
-                recordProxyOutcome(true); // Reset
+                recordProxyOutcome(true);
 
             } catch (proxyError) { 
-                recordProxyOutcome(false); // Count failure
+                recordProxyOutcome(false);
                 logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch via Proxy FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
             }
         }
@@ -240,8 +198,9 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
 
             try {
                 const portfolioJson = JSON.parse(body);
+                capturedPortfolio = portfolioJson; // Capture for detection
                 await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
-                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio processed successfully.`); // DOWNGRADED TO TRACE
+                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio processed successfully.`); 
 
             } catch (parseError) { 
                 wasPortfolioSuccess = false; 
@@ -254,7 +213,34 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         if (proxyUsedForPortfolio) { headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess); }
     } 
 
-    // --- 5. Handle Private Users & Timestamps ---
+    // --- 5. SPECULATOR DETECTION & QUEUEING (NEW) ---
+    // Only run detection if:
+    // 1. We are processing a Normal User (userType !== 'speculator')
+    // 2. We successfully fetched both history and portfolio
+    // 3. We have PubSub available to queue new tasks
+    if (userType !== 'speculator' && capturedHistory && capturedPortfolio && pubsub && config.PUBSUB_TOPIC_TASK_ENGINE) {
+        try {
+            const speculatorAssets = detectSpeculatorTargets(capturedHistory, capturedPortfolio);
+            if (speculatorAssets.length > 0) {
+                logger.log('INFO', `[handleUpdate/${userId}] DETECTED SPECULATOR BEHAVIOR. Queuing ${speculatorAssets.length} targeted updates.`);
+                
+                const newTasks = speculatorAssets.map(assetId => ({
+                    type: 'update',
+                    userType: 'speculator',
+                    userId: userId,
+                    instrumentId: assetId
+                }));
+
+                // Publish to Task Engine (Tasks are wrapped in a 'tasks' array payload)
+                const dataBuffer = Buffer.from(JSON.stringify({ tasks: newTasks }));
+                await pubsub.topic(config.PUBSUB_TOPIC_TASK_ENGINE).publishMessage({ data: dataBuffer });
+            }
+        } catch (detectionError) {
+            logger.log('ERROR', `[handleUpdate/${userId}] Error during Speculator Detection.`, { error: detectionError.message });
+        }
+    }
+
+    // --- 6. Handle Private Users & Timestamps ---
     if (isPrivate) {
         logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
         for (const instrumentId of instrumentsToProcess) { 
@@ -275,7 +261,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     
     if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
     
-    logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`); // DOWNGRADED TO TRACE
+    logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`); 
 }
 
-module.exports = { handleUpdate, lookupUsernames };
+module.exports = { handleUpdate };

package/functions/task-engine/utils/task_engine_utils.js

@@ -11,7 +11,7 @@
 
 const { handleDiscover } = require('../helpers/discover_helpers');
 const { handleVerify } = require('../helpers/verify_helpers');
-const { handleUpdate, lookupUsernames } = require('../helpers/update_helpers');
+const { handleUpdate } = require('../helpers/update_helpers'); // Removed lookupUsernames import
 const pLimit = require('p-limit'); 
 
 /**
@@ -27,27 +27,25 @@ function parseTaskPayload(message, logger) {
 }
 
 /**
- * Sorts tasks into update (with username), lookup (missing username), and other (discover/verify).
+ * Sorts tasks into update and other (discover/verify).
+ * REFACTORED: Simplified. No username lookup logic needed.
  */
 async function prepareTaskBatches(tasks, batchManager, logger) {
-    const tasksToRun = [], cidsToLookup = new Map(), otherTasks = [];
-    await batchManager.loadUsernameMap();
+    const tasksToRun = [], otherTasks = [];
+    
     for (const task of tasks) {
-        if (task.type === 'update') { const username = batchManager.getUsername(task.userId); username ? tasksToRun.push({ task, username }) : cidsToLookup.set(String(task.userId), task); } else otherTasks.push(task); }
-    logger.log('INFO', `[TaskEngine] Sorting complete. Known: ${tasksToRun.length}, Lookup: ${cidsToLookup.size}, Other: ${otherTasks.length}`);
-    return { tasksToRun, cidsToLookup, otherTasks };
-}
-
-/**
- * Runs username lookups for missing CIDs and adds to tasksToRun.
- */
-async function runUsernameLookups(tasksToRun, cidsToLookup, dependencies, config, batchManager, logger) {
-    if (!cidsToLookup.size) return;
-    logger.log('INFO', `[TaskEngine] Looking up ${cidsToLookup.size} usernames...`);
-    // Pass config to lookupUsernames
-    const foundUsers = await lookupUsernames([...cidsToLookup.keys()], dependencies, config); // <--- PASS FULL CONFIG
-    for (const u of foundUsers) { const cid = String(u.CID), username = u.Value.UserName; batchManager.addUsernameMapUpdate(cid, username); const task = cidsToLookup.get(cid); if (task) { tasksToRun.push({ task, username }); cidsToLookup.delete(cid); } }
-    if (cidsToLookup.size) logger.log('WARN', `[TaskEngine] Could not find ${cidsToLookup.size} usernames (likely private).`, { skippedCids: [...cidsToLookup.keys()] });
+        if (task.type === 'update') { 
+            // New API uses CID (userId), so we push directly to run.
+            tasksToRun.push(task); 
+        } else {
+            otherTasks.push(task); 
+        }
+    }
+    
+    // We explicitly return empty structures for compatibility if handler_creator expects them,
+    // though ideally handler_creator should also be simplified.
+    // For now, we return compatible object structure.
+    return { tasksToRun, cidsToLookup: new Map(), otherTasks };
 }
 
 /**
@@ -88,11 +86,16 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
     }
 
     // 2. Queue 'update' tasks
-    for (const { task, username } of tasksToRun) {
+    for (const task of tasksToRun) {
+        // We unpack 'task' directly now, no wrapping object {task, username}
+        // However, we must ensure backward compatibility if the array was {task, username} before.
+        // In prepareTaskBatches above, we pushed raw 'task'. 
+        // So we use 'task' directly.
+        
         const subTaskId = `${task.type}-${task.userType || 'unknown'}-${task.userId}`;
         allTaskPromises.push(
             limit(() => 
-                handleUpdate(task, subTaskId, dependencies, config, username) 
+                handleUpdate(task, subTaskId, dependencies, config) 
                     .then(() => taskCounters.update++)
                     .catch(err => {
                         logger.log('ERROR', `[TaskEngine/${taskId}] Error in handleUpdate for ${task.userId}`, { errorMessage: err.message });
@@ -112,4 +115,5 @@ async function executeTasks(tasksToRun, otherTasks, dependencies, config, taskId
     );
 }
 
-module.exports = { parseTaskPayload, prepareTaskBatches, runUsernameLookups, executeTasks };
+// Note: runUsernameLookups removed from exports
+module.exports = { parseTaskPayload, prepareTaskBatches, executeTasks };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.205",
+  "version": "1.0.207",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [

package/functions/computation-system/helpers/reproduce_error.ts

@@ -1,19 +0,0 @@
-
-// Mock types
-namespace Firestore {
-    export class DocumentReference { }
-}
-
-const pLimit = (concurrency: number) => {
-    return (fn: () => Promise<any>) => fn();
-};
-
-const OUTER_CONCURRENCY_LIMIT = 2;
-const outerLimit = pLimit(OUTER_CONCURRENCY_LIMIT);
-
-const shardChunks: Firestore.DocumentReference[][] = [];
-
-// The problematic code
-const chunkPromises = shardChunks.map((shardChunkRefs, index) => outerLimit(async () => {
-    console.log(index);
-}));