bulltrackers-module 1.0.212 → 1.0.214

package/functions/computation-system/helpers/computation_manifest_builder.js

@@ -1,23 +1,19 @@
 /**
  * @fileoverview
- * Dynamic Manifest Builder (v5 - Smart Hashing with Safe Mode)
+ * Dynamic Manifest Builder (v6 - Merkle Tree Dependency Hashing)
  *
- * This script builds the computation manifest and generates a "Smart Hash"
- * for each calculation.
- *
- * KEY FEATURE:
- * It performs static analysis on the calculation code to detect which
- * specific math layers it utilizes. It then combines the calculation's own
- * code hash with the current hashes of those specific layers.
- * * SAFE MODE:
- * If a calculation uses no detectable layer keywords, it defaults to
- * depending on ALL layers to prevent staleness.
+ * KEY FEATURES:
+ * 1. Smart Layer Hashing: Detects used layers (Math, Extractors) to avoid stale helper code.
+ * 2. Cascading Invalidation (Merkle Hashing): 
+ * The final hash of a computation is derived from:
+ * [Own Code] + [Layer States] + [Hashes of all Dependencies]
+ * * This guarantees that if Calculation A is updated, Calculation B (which depends on A)
+ * will automatically generate a new hash, forcing the system to re-run it.
  */
 
 const { generateCodeHash } = require('../utils/utils');
 
 // 1. Import Layers directly to generate their "State Hashes"
-// We import them individually to hash them as distinct domains.
 const MathematicsLayer = require('../layers/mathematics');
 const ExtractorsLayer  = require('../layers/extractors');
 const ProfilingLayer   = require('../layers/profiling');
@@ -27,10 +23,6 @@ const ValidatorsLayer  = require('../layers/validators');
  * 1. Layer Hash Generation
  * -------------------------------------------------- */
 
-/**
- * Generates a single hash representing the entire state of a layer module.
- * Combines all exported classes/functions/objects into one deterministic string.
- */
 function generateLayerHash(layerExports, layerName) {
     const keys = Object.keys(layerExports).sort(); // Sort for determinism
     let combinedSource = `LAYER:${layerName}`;
@@ -57,21 +49,20 @@ const LAYER_HASHES = {
 };
 
 // Map code patterns to Layer dependencies
-// If a calculation's code contains these strings, it depends on that layer.
 const LAYER_TRIGGERS = {
     'mathematics': [
         'math.compute', 'MathPrimitives',
-        'math.signals', 'SignalPrimitives',
+        'math.signals', 'SignalPrimitives', 'signals.',
         'math.aggregate', 'Aggregators',
-        'math.timeseries', 'TimeSeries',
-        'math.distribution', 'DistributionAnalytics',
+        'math.timeseries', 'TimeSeries', 'timeSeries.',
+        'math.distribution', 'DistributionAnalytics', 'distribution.',
         'math.financial', 'FinancialEngineering'
     ],
     'extractors': [
         'math.extract', 'DataExtractor',
         'math.history', 'HistoryExtractor',
         'math.prices', 'priceExtractor',
-        'math.insights', 'InsightsExtractor',
+        'math.insights', 'InsightsExtractor', 'insights.',
         'math.tradeSeries', 'TradeSeriesBuilder'
     ],
     'profiling': [
@@ -121,18 +112,6 @@ function suggestClosest(name, candidates, n = 3) {
     return scores.slice(0, n).map(s => s[0]);
 }
 
-function findCycles(manifestMap, adjacencyList) {
-    const visited = new Set(), stack = new Set(), cycles = [];
-    const dfs = (node, path) => {
-        if (stack.has(node)) { const idx = path.indexOf(node); cycles.push([...path.slice(idx), node]); return; }
-        if (visited.has(node)) return;
-        visited.add(node); stack.add(node);
-        for (const nb of adjacencyList.get(node) || []) dfs(nb, [...path, nb]);
-        stack.delete(node); };
-    for (const name of manifestMap.keys()) dfs(name, [name]);
-    return cycles;
-}
-
 function getDependencySet(endpoints, adjacencyList) {
     const required = new Set(endpoints);
     const queue    = [...endpoints];
@@ -146,24 +125,21 @@ function getDependencySet(endpoints, adjacencyList) {
  * -------------------------------------------------- */
 
 function buildManifest(productLinesToRun = [], calculations) {
-    log.divider('Building Dynamic Manifest (Smart Hashing)');
+    log.divider('Building Dynamic Manifest (Merkle Hashing)');
     log.info(`Target Product Lines: [${productLinesToRun.join(', ')}]`);
-    log.info(`Layer Hashes Initialized: ${Object.keys(LAYER_HASHES).join(', ')}`);
-
+    
     const manifestMap      = new Map();
     const adjacency        = new Map(); 
     const reverseAdjacency = new Map(); 
     const inDegree         = new Map();
     let hasFatalError      = false;
     
-    /* ---------------- 1. Load All Calculations ---------------- */
+    /* ---------------- 1. Load All Calculations (Phase 1: Intrinsic Hash) ---------------- */
     log.step('Loading and validating all calculation classes…');
-    const allCalculationClasses = new Map();
 
     function processCalc(Class, name, folderName) {
         if (!Class || typeof Class !== 'function') return;
         const normalizedName = normalizeName(name);
-        allCalculationClasses.set(normalizedName, Class);
         
         if (typeof Class.getMetadata !== 'function') { log.fatal(`Calculation "${normalizedName}" is missing static getMetadata().`); hasFatalError = true; return; }
         if (typeof Class.getDependencies !== 'function') { log.fatal(`Calculation "${normalizedName}" is missing static getDependencies().`); hasFatalError = true;return; }
@@ -171,46 +147,34 @@ function buildManifest(productLinesToRun = [], calculations) {
         const metadata     = Class.getMetadata();
         const dependencies = Class.getDependencies().map(normalizeName);
         
-        if (metadata.isHistorical === true && !Class.toString().includes('yesterday')) { 
-            log.warn(`Calculation "${normalizedName}" marked 'isHistorical' but no 'yesterday' reference found.`); 
+        const codeStr = Class.toString();
+        if (metadata.isHistorical === true && !codeStr.includes('yesterday') && !codeStr.includes('previousComputed')) { 
+            log.warn(`Calculation "${normalizedName}" marked 'isHistorical' but no 'previousComputed' state reference found.`); 
         }
 
-        let finalCategory = folderName;
-        if (folderName === 'core') {
-            if (metadata.category) finalCategory = metadata.category;
-        } else {
-            finalCategory = folderName;
-        }
+        let finalCategory = folderName === 'core' && metadata.category ? metadata.category : folderName;
 
-        // --- SMART HASH GENERATION ---
-        const classCode = Class.toString();
-        let compositeHashString = generateCodeHash(classCode); // Start with own code
+        // --- PHASE 1: INTRINSIC HASH (Code + Layers) ---
+        // We do NOT include dependencies yet.
+        let compositeHashString = generateCodeHash(codeStr); 
         const usedLayers = [];
 
-        // 1. Check for specific layer usage
+        // Check for specific layer usage
         for (const [layerName, triggers] of Object.entries(LAYER_TRIGGERS)) {
-            // If code contains any trigger for this layer
-            if (triggers.some(trigger => classCode.includes(trigger))) {
-                compositeHashString += LAYER_HASHES[layerName]; // Append Layer Hash
+            if (triggers.some(trigger => codeStr.includes(trigger))) {
+                compositeHashString += LAYER_HASHES[layerName]; 
                 usedLayers.push(layerName);
             }
         }
 
-        // 2. SAFE MODE FALLBACK
-        // If we found NO specific triggers (e.g. legacy code or unique structure), 
-        // assume it might use anything. Depend on ALL layers to be safe.
+        // Safe Mode Fallback
         let isSafeMode = false;
         if (usedLayers.length === 0) {
             isSafeMode = true;
             Object.values(LAYER_HASHES).forEach(h => compositeHashString += h);
         }
 
-        // Generate final composite hash
-        const finalHash = generateCodeHash(compositeHashString);
-
-        if (isSafeMode) {
-            log.warn(`[Hash] ${normalizedName}: No specific dependencies detected. Enforcing SAFE MODE (Linked to ALL Layers).`);
-        } 
+        const baseHash = generateCodeHash(compositeHashString);
 
         const manifestEntry = {
             name: normalizedName,
@@ -223,7 +187,7 @@ function buildManifest(productLinesToRun = [], calculations) {
             userType: metadata.userType,
             dependencies: dependencies,
             pass: 0,
-            hash: finalHash, // The Smart Hash
+            hash: baseHash, // Intrinsic Hash (Updated later to include deps)
             debugUsedLayers: isSafeMode ? ['ALL (Safe Mode)'] : usedLayers
         };
         
@@ -237,7 +201,6 @@ function buildManifest(productLinesToRun = [], calculations) {
         throw new Error('Manifest build failed: Invalid calculations object.');
     }
 
-    // Iterate over folders (folderName becomes the default category)
     for (const folderName in calculations) {
         if (folderName === 'legacy') continue;
         const group = calculations[folderName];
@@ -308,6 +271,31 @@ function buildManifest(productLinesToRun = [], calculations) {
     if (sortedManifest.length !== filteredManifestMap.size) {
         throw new Error('Circular dependency detected. Manifest build failed.'); }
 
+    /* ---------------- 5. Phase 2: Cascading Dependency Hashing ---------------- */
+    // Now that we have a topological order (Dependencies come BEFORE Consumers),
+    // we can update hashes sequentially.
+    log.step('Computing Cascading Merkle Hashes...');
+
+    for (const entry of sortedManifest) {
+        // Start with the intrinsic hash (Code + Layers)
+        let dependencySignature = entry.hash;
+
+        // Append the hashes of all dependencies
+        // Since we are iterating in topo order, dependencies are guaranteed to be processed/updated already.
+        if (entry.dependencies && entry.dependencies.length > 0) {
+            const depHashes = entry.dependencies.map(depName => {
+                const depEntry = filteredManifestMap.get(depName);
+                if (!depEntry) return ''; // Should not happen given validation
+                return depEntry.hash;
+            }).join('|'); // Use separator to prevent collisions
+            
+            dependencySignature += `|DEPS:${depHashes}`;
+        }
+
+        // Generate the Final Smart Hash
+        entry.hash = generateCodeHash(dependencySignature);
+    }
+
     log.success(`Total passes required: ${maxPass}`);
     return sortedManifest;
 }

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,7 +1,6 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_pass_runner.js
- * FIXED: 'runDateComputation' now executes ALL calculation types (Standard, Meta, AND Price).
- * UPDATED: Uses Code Hash for smart versioning and invalidation.
+ * FIXED: 'storedStatus.substring' crash and 'missing dependency' log clarity.
  */
 
 const {
@@ -20,10 +19,6 @@ const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
 
 const PARALLEL_BATCH_SIZE = 7;
 
-/**
- * LEGACY / MANUAL RUNNER
- * (Kept for backward compatibility if you run the old HTTP endpoint directly)
- */
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
     const passToRun  = String(config.COMPUTATION_PASS_TO_RUN);
@@ -31,7 +26,6 @@ async function runComputationPass(config, dependencies, computationManifest) {
 
     logger.log('INFO', `🚀 Starting PASS ${passToRun} (Legacy Mode)...`);
 
-    // Hardcoded earliest dates
     const earliestDates = {
         portfolio: new Date('2025-09-25T00:00:00Z'),
         history:   new Date('2025-11-05T00:00:00Z'),
@@ -51,13 +45,12 @@ async function runComputationPass(config, dependencies, computationManifest) {
     const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
-    // Legacy Batch Optimization for Price (Only used in legacy loop)
     const priceBatchCalcs = calcsInThisPass.filter(c => c.type === 'meta' && c.rootDataDependencies?.includes('price'));
     const standardAndOtherMetaCalcs = calcsInThisPass.filter(c => !priceBatchCalcs.includes(c));
 
     if (priceBatchCalcs.length > 0) {
         try {
-            await runBatchPriceComputation(config, dependencies, allExpectedDates, priceBatchCalcs); // Simplified for legacy
+            await runBatchPriceComputation(config, dependencies, allExpectedDates, priceBatchCalcs); 
         } catch (e) { logger.log('ERROR', 'Legacy Batch Price failed', e); }
     }
 
@@ -69,61 +62,56 @@ async function runComputationPass(config, dependencies, computationManifest) {
     }
 }
 
-/**
- * UPDATED: Isolated function to run computations for a single date.
- * Uses Code Hash to determine if a re-run is necessary.
- */
 async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) {
     const { logger } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
 
-    // 1. Fetch Status for THIS specific date only
     const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
 
-    // Helper: Check status using HASH comparison
-    const shouldRun = (calc) => {
+    // Filter AND Log reason for skipping
+    const calcsToAttempt = [];
+    
+    for (const calc of calcsInThisPass) {
         const cName = normalizeName(calc.name);
         const storedStatus = dailyStatus[cName];
         const currentHash = calc.hash;
 
-        // If dependency logic is needed, check dependencies are 'complete'
-        // 'Complete' means truthy (hash or true)
+        // 1. Dependency Check
         if (calc.dependencies && calc.dependencies.length > 0) {
-            const missing = calc.dependencies.filter(depName => {
-                const depStatus = dailyStatus[normalizeName(depName)];
-                return !depStatus; // Run if dependency is missing entirely
-            });
-            if (missing.length > 0) return false; // Wait for dependency
+            const missing = calc.dependencies.filter(depName => !dailyStatus[normalizeName(depName)]);
+            if (missing.length > 0) {
+                // Too noisy to log every skip, but useful for debugging if needed. 
+                // Only logging if it's NOT a bulk skip.
+                // logger.log('TRACE', `[Skip] ${cName} missing deps: ${missing.join(', ')}`);
+                continue; 
+            }
         }
 
-        // Logic A: No previous run
+        // 2. Logic A: No previous run
         if (!storedStatus) {
             logger.log('INFO', `[Versioning] ${cName}: New run needed (No prior status).`);
-            return true;
+            calcsToAttempt.push(calc);
+            continue;
         }
 
-        // Logic B: Hash Mismatch (Code Changed or Layer Changed)
-        if (currentHash && storedStatus !== currentHash) {
-            logger.log('INFO', `[Versioning] ${cName}: Code/Layer Changed. Re-running. (Old: ${storedStatus.substring(0,6)}... New: ${currentHash.substring(0,6)}...)`);
-            return true;
+        // 3. Logic B: Hash Mismatch
+        // FIX: Ensure storedStatus is a string before calling substring
+        if (typeof storedStatus === 'string' && currentHash && storedStatus !== currentHash) {
+            logger.log('INFO', `[Versioning] ${cName}: Code Changed. (Old: ${storedStatus.substring(0,6)}... New: ${currentHash.substring(0,6)}...)`);
+            calcsToAttempt.push(calc);
+            continue;
         }
 
-        // Logic C: Legacy boolean check (Stored=true) vs New Hash
-        // If stored is strictly boolean true, but we have a hash, we upgrade (re-run) to stamp the hash.
+        // 4. Logic C: Upgrade Legacy Boolean -> Hash
         if (storedStatus === true && currentHash) {
-            logger.log('INFO', `[Versioning] ${cName}: Upgrading legacy status to Hash. Re-running.`);
-            return true;
+            logger.log('INFO', `[Versioning] ${cName}: Upgrading legacy status to Hash.`);
+            calcsToAttempt.push(calc);
+            continue;
         }
-
-        return false; // Skip
-    };
-
-    // --- FIX: Run ALL calc types (Standard, Meta, Price) ---
-    const calcsToAttempt = calcsInThisPass.filter(shouldRun);
+    }
 
     if (!calcsToAttempt.length) return null;
 
-    // 2. Check Root Data Availability
     const earliestDates = {
         portfolio: new Date('2025-09-25T00:00:00Z'),
         history:   new Date('2025-11-05T00:00:00Z'),
@@ -138,14 +126,11 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
         return null;
     }
 
-    // 3. Filter again based on Root Data availability
     const runnableCalcs = calcsToAttempt.filter(c => checkRootDependencies(c, rootData.status).canRun);
 
     if (!runnableCalcs.length) return null;
 
-    // Split into Standard (Streaming) and Meta (Once-Per-Day/Price)
     const standardToRun = runnableCalcs.filter(c => c.type === 'standard');
-    // Note: Meta includes Price calcs in this flow
     const metaToRun = runnableCalcs.filter(c => c.type === 'meta');
 
     logger.log('INFO', `[DateRunner] Running ${dateStr}: ${standardToRun.length} std, ${metaToRun.length} meta`);
@@ -155,7 +140,6 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
     try {
         const calcsRunning = [...standardToRun, ...metaToRun];
 
-        // Fetch dependencies (results from this day or yesterday)
         const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
         const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
         const prevDateStr     = prevDate.toISOString().slice(0, 10);
@@ -166,14 +150,13 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
             Object.assign(dateUpdates, updates);
         }
         if (metaToRun.length) {
-            // runMetaComputationPass uses the Controller, which handles Price Sharding logic internally for single dates.
             const updates = await runMetaComputationPass(dateToProcess, metaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, false);
             Object.assign(dateUpdates, updates);
         }
     } catch (err) {
         logger.log('ERROR', `[DateRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
         [...standardToRun, ...metaToRun].forEach(c => dateUpdates[normalizeName(c.name)] = false);
-        throw err; // Re-throw to trigger Pub/Sub retry
+        throw err; 
     }
 
     if (Object.keys(dateUpdates).length > 0) {

package/functions/computation-system/layers/extractors.js

@@ -1,45 +1,23 @@
 /**
  * @fileoverview Extractors Layer
- * Core access methods to raw data (Portfolio, History, Prices, Insights).
+ * Core access methods to raw data.
+ * FIX: HistoryExtractor handles both Legacy and Modern (Granular) schemas.
  */
 
 const { SCHEMAS } = require('./profiling');
 
 class TradeSeriesBuilder {
-    /**
-     * Converts raw trade history into a time-series of Strategy Returns.
-     * Assumes a "Flat Bet" model (equal sizing) because absolute position size is not available in history.
-     * This creates a normalized performance curve for the user's *decisions*.
-     * @param {Array} historyTrades - PublicHistoryPositions array.
-     * @returns {Array<number>} Array of NetProfit% values sorted by close date.
-     */
     static buildReturnSeries(historyTrades) {
         if (!historyTrades || !Array.isArray(historyTrades)) return [];
-        
-        // 1. Filter valid closed trades
         const closedTrades = historyTrades.filter(t => t.CloseDateTime && typeof t.NetProfit === 'number');
-        
-        // 2. Sort by Close Date (Ascending)
         closedTrades.sort((a, b) => new Date(a.CloseDateTime) - new Date(b.CloseDateTime));
-        
-        // 3. Extract the PnL sequence
         return closedTrades.map(t => t.NetProfit);
     }
 
-    /**
-     * Builds a cumulative equity curve (starting at 100) based on compounding trade returns.
-     * Useful for visualising the trajectory of the strategy.
-     */
     static buildCumulativeCurve(returnSeries, startValue = 100) {
         const curve = [startValue];
         let current = startValue;
-        
         for (const ret of returnSeries) {
-            // Apply return (e.g. 5% profit -> * 1.05)
-            // Note: NetProfit in eToro history is usually percentage (e.g. 5.4954).
-            // We treat this as the return on *that specific position*.
-            // In a flat-bet model, we assume that position was X% of the portfolio.
-            // Simplified: We just accumulate the "points" captured.
             current = current * (1 + (ret / 100));
             curve.push(current);
         }
@@ -48,177 +26,84 @@ class TradeSeriesBuilder {
 }
 
 class DataExtractor { 
-    // ========================================================================
-    // 1. COLLECTION ACCESSORS
-    // ========================================================================
-    
-    /**
-     * Extract positions array based on User Type.
-     * - Normal: Uses 'AggregatedPositions' (Grouped by Asset + Direction)
-     * - Speculator: Uses 'PublicPositions' (Individual Trades)
-     */
     static getPositions(portfolio, userType) {
-        if (!portfolio) return []; // Handle empty portfolio
-        
+        if (!portfolio) return []; 
         if (userType === SCHEMAS.USER_TYPES.SPECULATOR) { 
-            return portfolio.PublicPositions || []; // SPECULATOR SCHEMA
+            return portfolio.PublicPositions || []; 
         }
-        
-        // Default to Normal User Schema
         return portfolio.AggregatedPositions || [];
     }
 
-    // ========================================================================
-    // 2. IDENTITY & KEYS
-    // ========================================================================
-
-    /**
-     * Extract standardized Instrument ID.
-     */
     static getInstrumentId(position) {
-        if (!position) return null; // Handle empty position data
-        // Handle string or number variations safely
+        if (!position) return null; 
         return position.InstrumentID || position.instrumentId || null;
     }
 
-    /**
-     * Extract a unique Identifier for the position.
-     * - Speculator: Uses 'PositionID'.
-     * - Normal: Generates Composite Key (InstrumentID_Direction) since they lack unique Trade IDs.
-     */
     static getPositionId(position) {
-        if (!position) return null; // Handle empty position data
-        
-        // 1. Try Explicit ID (Speculators)
+        if (!position) return null; 
         if (position.PositionID) return String(position.PositionID);
         if (position.PositionId) return String(position.PositionId);
-
-        // 2. Fallback to Composite Key (Normal Users)
         const instId = this.getInstrumentId(position);
         const dir    = this.getDirection(position);
         if (instId) return `${instId}_${dir}`;
-
         return null;
     }
 
-    // ========================================================================
-    // 3. FINANCIAL METRICS (WEIGHTS & P&L)
-    // ========================================================================
-
-    /**
-     * Extract Net Profit %.
-     * Schema: 'NetProfit' is the percentage profit relative to invested capital.
-     */
     static getNetProfit(position) { 
         return position ? (position.NetProfit || 0) : 0;
     }
 
-    /**
-     * Extract Position Weight (Allocation %).
-     * Schema: 
-     * - Normal: 'Invested' is % of initial capital.
-     * - Speculator: 'Invested' (or 'Amount' in some contexts) is % of initial capital.
-     */
-    static getPositionWeight(position, userType) { // Agnostic on user type, unused.
+    static getPositionWeight(position, userType) { 
         if (!position) return 0;
-        
-        // Both schemas use 'Invested' to represent the allocation percentage.
-        // Speculators might optionally have 'Amount', we prioritize 'Invested' for consistency.
         return position.Invested || position.Amount || 0;
     }
 
-    /**
-     * Extract Current Equity Value %.
-     * Schema: 'Value' is the current value as a % of total portfolio equity.
-     */
     static getPositionValuePct(position) {            
         return position ? (position.Value || 0) : 0; 
     }
 
-    /**
-     * --- NEW PRIMITIVE ---
-     * Derives the approximate Entry Price of a position based on Current Price and Net Profit %.
-     * Formula: Entry = Current / (1 + (NetProfit / 100))
-     * @param {number} currentPrice - The current market price of the asset.
-     * @param {number} netProfitPct - The Net Profit percentage (e.g., -20.5).
-     * @returns {number} Estimated Entry Price.
-     */
     static deriveEntryPrice(currentPrice, netProfitPct) {
         if (!currentPrice || currentPrice <= 0) return 0;
-        // Avoid division by zero if P&L is -100% (unlikely but possible in crypto/options)
-        if (netProfitPct <= -100) return Number.MAX_SAFE_INTEGER; // Effectively infinite entry price (lost everything)
+        if (netProfitPct <= -100) return Number.MAX_SAFE_INTEGER; 
         return currentPrice / (1 + (netProfitPct / 100.0));
     }
 
-    // ========================================================================
-    // 4. PORTFOLIO LEVEL SUMMARY
-    // ========================================================================
-
-    /**
-     * Calculate/Extract Daily Portfolio P&L %.
-     */
     static getPortfolioDailyPnl(portfolio, userType) {
         if (!portfolio) return 0;
-        
-        // 1. Speculator (Explicit 'NetProfit' field on root)
         if (userType === SCHEMAS.USER_TYPES.SPECULATOR) {       
             return portfolio.NetProfit || 0;                    
         }
-        
-        // 2. Normal (Aggregated Calculation)
         if (portfolio.AggregatedPositionsByInstrumentTypeID) {
             return portfolio.AggregatedPositionsByInstrumentTypeID.reduce((sum, agg) => {
                 return sum + ((agg.Value || 0) - (agg.Invested || 0));
             }, 0);
         }
-        
         return 0;
     }
 
-    // ========================================================================
-    // 5. TRADE DETAILS (SPECULATOR SPECIFIC)
-    // ========================================================================
-
     static getDirection(position) {
         if (!position) return "Buy";
         if (position.Direction) return position.Direction;
         if (typeof position.IsBuy === 'boolean') return position.IsBuy ? "Buy" : "Sell";
-        return "Buy"; // Default
-    }
-
-    static getLeverage(position) {
-        return position ? (position.Leverage || 1) : 1; // Default 1 IF NOT FOUND
-    }
-
-    static getOpenRate(position) {
-        return position ? (position.OpenRate || 0) : 0; // Default 0 IF NOT FOUND
-    }
-
-    static getCurrentRate(position) {
-        return position ? (position.CurrentRate || 0) : 0; // Default 0 IF NOT FOUND
+        return "Buy"; 
     }
 
+    static getLeverage(position) { return position ? (position.Leverage || 1) : 1; }
+    static getOpenRate(position) { return position ? (position.OpenRate || 0) : 0; }
+    static getCurrentRate(position) { return position ? (position.CurrentRate || 0) : 0; }
     static getStopLossRate(position) {
         const rate = position ? (position.StopLossRate || 0) : 0; 
-        if (rate > 0 && rate <= 0.01) return 0; // Normalizes bug value to 0
+        if (rate > 0 && rate <= 0.01) return 0; 
         if (rate < 0) return 0;
         return rate;
     }                                                       
-
     static getTakeProfitRate(position) {
         const rate = position ? (position.TakeProfitRate || 0) : 0;
-        if (rate > 0 && rate <= 0.01) return 0; // Normalizes bug value to 0
+        if (rate > 0 && rate <= 0.01) return 0; 
         return rate;
     }
-
-    static getHasTSL(position) {
-        return position ? (position.HasTrailingStopLoss === true) : false; // Default false IF NOT FOUND
-    }
-
-    static getOpenDateTime(position) {
-        if (!position || !position.OpenDateTime) return null;        
-        return new Date(position.OpenDateTime);
-    }
+    static getHasTSL(position) { return position ? (position.HasTrailingStopLoss === true) : false; }
+    static getOpenDateTime(position) { return (!position || !position.OpenDateTime) ? null : new Date(position.OpenDateTime); }
 }
 
 class priceExtractor {
@@ -235,7 +120,6 @@ class priceExtractor {
         }
 
         if (!assetData || !assetData.prices) return [];
-
         const priceMap = assetData.prices;
         const sortedDates = Object.keys(priceMap).sort((a, b) => a.localeCompare(b));
         
@@ -247,14 +131,11 @@ class priceExtractor {
 
     static getAllHistories(pricesContext) {
         if (!pricesContext || !pricesContext.history) return new Map();
-        
         const results = new Map();
         for (const [id, data] of Object.entries(pricesContext.history)) {
             const ticker = data.ticker || id;
             const history = this.getHistory(pricesContext, id);
-            if (history.length > 0) {
-                results.set(ticker, history);
-            }
+            if (history.length > 0) results.set(ticker, history);
         }
         return results;
     }
@@ -266,38 +147,45 @@ class HistoryExtractor {
     }
 
     static getTradedAssets(historyDoc) {
-        const trades = historyDoc?.PublicHistoryPositions || [];
-        if (!trades.length) return [];
-        
-        const assetsMap = new Map();
-        
-        for (const t of trades) {
-            const instId = t.InstrumentID;
-            if (!instId) continue;
-            
-            if (!assetsMap.has(instId)) {
-                assetsMap.set(instId, { 
-                    instrumentId: instId, 
-                    totalDuration: 0, 
-                    count: 0 
-                });
-            }
+        // 1. Try Modern Granular Data (Derive Assets from Trades)
+        if (historyDoc?.PublicHistoryPositions?.length) {
+            const trades = historyDoc.PublicHistoryPositions;
+            const assetsMap = new Map();
             
-            const asset = assetsMap.get(instId);
-            const open = new Date(t.OpenDateTime);
-            const close = new Date(t.CloseDateTime);
-            const durationMins = (close - open) / 60000;
-            
-            if (durationMins > 0) {
-                asset.totalDuration += durationMins;
-                asset.count++;
+            for (const t of trades) {
+                const instId = t.InstrumentID;
+                if (!instId) continue;
+                
+                if (!assetsMap.has(instId)) {
+                    assetsMap.set(instId, { 
+                        instrumentId: instId, 
+                        totalDuration: 0, 
+                        count: 0 
+                    });
+                }
+                
+                const asset = assetsMap.get(instId);
+                const open = new Date(t.OpenDateTime);
+                const close = new Date(t.CloseDateTime);
+                const durationMins = (close - open) / 60000;
+                
+                if (durationMins > 0) {
+                    asset.totalDuration += durationMins;
+                    asset.count++;
+                }
             }
+            return Array.from(assetsMap.values()).map(a => ({
+                instrumentId: a.instrumentId,
+                avgHoldingTimeInMinutes: a.count > 0 ? (a.totalDuration / a.count) : 0
+            }));
         }
-        
-        return Array.from(assetsMap.values()).map(a => ({
-            instrumentId: a.instrumentId,
-            avgHoldingTimeInMinutes: a.count > 0 ? (a.totalDuration / a.count) : 0
-        }));
+
+        // 2. Fallback to Legacy 'assets' array
+        if (historyDoc?.assets && Array.isArray(historyDoc.assets)) {
+            return historyDoc.assets;
+        }
+
+        return [];
     }
 
     static getInstrumentId(asset) { 
@@ -309,79 +197,63 @@ class HistoryExtractor {
     }
 
     static getSummary(historyDoc) { 
-        const trades = historyDoc?.PublicHistoryPositions || [];
-        if (!trades.length) return null;
-
-        let totalTrades = trades.length;
-        let wins = 0;
-        let totalProf = 0;
-        let totalLoss = 0;
-        let profCount = 0;
-        let lossCount = 0;
-        let totalDur = 0;
-
-        for (const t of trades) {
-            if (t.NetProfit > 0) {
-                wins++;
-                totalProf += t.NetProfit;
-                profCount++;
-            } else if (t.NetProfit < 0) {
-                totalLoss += t.NetProfit;
-                lossCount++;
+        // 1. Try Modern Granular Data (Derive Summary)
+        if (historyDoc?.PublicHistoryPositions?.length) {
+            const trades = historyDoc.PublicHistoryPositions;
+            let totalTrades = trades.length;
+            let wins = 0;
+            let totalProf = 0;
+            let totalLoss = 0;
+            let profCount = 0;
+            let lossCount = 0;
+            let totalDur = 0;
+
+            for (const t of trades) {
+                if (t.NetProfit > 0) {
+                    wins++;
+                    totalProf += t.NetProfit;
+                    profCount++;
+                } else if (t.NetProfit < 0) {
+                    totalLoss += t.NetProfit;
+                    lossCount++;
+                }
+                const open = new Date(t.OpenDateTime);
+                const close = new Date(t.CloseDateTime);
+                totalDur += (close - open) / 60000;
             }
-            
-            const open = new Date(t.OpenDateTime);
-            const close = new Date(t.CloseDateTime);
-            totalDur += (close - open) / 60000;
+
+            return { 
+                totalTrades:             totalTrades,
+                winRatio:                totalTrades > 0 ? (wins / totalTrades) * 100 : 0,
+                avgProfitPct:            profCount > 0 ? totalProf / profCount : 0,
+                avgLossPct:              lossCount > 0 ? totalLoss / lossCount : 0,
+                avgHoldingTimeInMinutes: totalTrades > 0 ? totalDur / totalTrades : 0
+            };
         }
 
-        return { 
-            totalTrades:             totalTrades,
-            winRatio:                totalTrades > 0 ? (wins / totalTrades) * 100 : 0,
-            avgProfitPct:            profCount > 0 ? totalProf / profCount : 0,
-            avgLossPct:              lossCount > 0 ? totalLoss / lossCount : 0,
-            avgHoldingTimeInMinutes: totalTrades > 0 ? totalDur / totalTrades : 0
-        };
+        // 2. Fallback to Legacy 'all' object
+        if (historyDoc?.all) {
+            return historyDoc.all;
+        }
+
+        return null;
     }
 }
 
 class InsightsExtractor {
-    /**
-     * Extracts the raw array of insight objects from the context.
-     * Checks for standard context injection paths.
-     */
     static getInsights(context) {
-        // Support multiple potential injection paths depending on controller version
         return context.insights || context.daily_instrument_insights || [];
     }
 
-    /**
-     * returns the specific insight object for a given instrument ID.
-     */
     static getInsightForInstrument(insights, instrumentId) {
         if (!insights || !Array.isArray(insights)) return null;
         return insights.find(i => i.instrumentId === instrumentId) || null;
     }
 
-    // --- Standard Metrics ---
-
-    static getTotalOwners(insight) {
-        return insight ? (insight.total || 0) : 0;
-    }
-
-    static getLongPercent(insight) {
-        return insight ? (insight.buy || 0) : 0;
-    }
-
-    static getShortPercent(insight) {
-        return insight ? (insight.sell || 0) : 0;
-    }
-
-    static getGrowthPercent(insight) {
-        return insight ? (insight.growth || 0) : 0;
-    }
-
-    // --- Derived Counts (Estimated) ---
+    static getTotalOwners(insight) { return insight ? (insight.total || 0) : 0; }
+    static getLongPercent(insight) { return insight ? (insight.buy || 0) : 0; }
+    static getShortPercent(insight) { return insight ? (insight.sell || 0) : 0; }
+    static getGrowthPercent(insight) { return insight ? (insight.growth || 0) : 0; }
 
     static getLongCount(insight) {
         const total = this.getTotalOwners(insight);
@@ -395,19 +267,11 @@ class InsightsExtractor {
         return Math.floor(total * (sellPct / 100));
     }
 
-    /**
-     * Calculates the net change in users from yesterday based on growth %.
-     * Formula: NetChange = Total - (Total / (1 + Growth/100))
-     */
     static getNetOwnershipChange(insight) {
         const total = this.getTotalOwners(insight);
         const growth = this.getGrowthPercent(insight);
         if (total === 0) return 0;
-        
-        // Reverse engineer yesterday's count
-        // Today = Yesterday * (1 + growth)
-        // Yesterday = Today / (1 + growth)
-        const prevTotal = total / (1 + (growth / 100)); // TODO: Check precision issues
+        const prevTotal = total / (1 + (growth / 100)); 
         return Math.round(total - prevTotal);
     }
 }

package/functions/task-engine/helpers/update_helpers.js

@@ -1,32 +1,20 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
- * (OPTIMIZED V4: Auto-Speculator Detection via History/Portfolio Intersection)
- * (OPTIMIZED V3: Removed obsolete username lookup logic)
- * (OPTIMIZED V2: Added "Circuit Breaker" for Proxy failures)
- * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
- * (FIXED: Improved logging clarity for Normal vs Speculator users)
- * (FIXED: Final log now accurately reflects failure state)
+ * (OPTIMIZED V5: Filter Copy Trash from History)
+ * FIX: Filters PublicHistoryPositions to keep only valid close reasons (0, 1, 5).
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
 const crypto = require('crypto'); 
 
 // --- CIRCUIT BREAKER STATE ---
-// Persists across function invocations in the same instance.
-// If the Proxy fails 3 times in a row, we stop trying it to save the 5s timeout cost.
 let _consecutiveProxyFailures = 0;
 const MAX_PROXY_FAILURES = 3;
 
-/**
- * Helper to check if we should attempt the proxy
- */
 function shouldTryProxy() {
     return _consecutiveProxyFailures < MAX_PROXY_FAILURES;
 }
 
-/**
- * Helper to record proxy result
- */
 function recordProxyOutcome(success) {
     if (success) {
         _consecutiveProxyFailures = 0;
@@ -35,14 +23,9 @@ function recordProxyOutcome(success) {
     }
 }
 
-/**
- * --- NEW HELPER: Speculator Detector ---
- * intersections: (History: Leverage > 1) AND (Portfolio: Currently Owned)
- */
 function detectSpeculatorTargets(historyData, portfolioData) {
     if (!historyData?.PublicHistoryPositions || !portfolioData?.AggregatedPositions) return [];
 
-    // 1. Identify assets that have EVER been traded with leverage > 1
     const leveragedAssets = new Set();
     for (const pos of historyData.PublicHistoryPositions) {
         if (pos.Leverage > 1 && pos.InstrumentID) {
@@ -52,7 +35,6 @@ function detectSpeculatorTargets(historyData, portfolioData) {
 
     if (leveragedAssets.size === 0) return [];
 
-    // 2. Check if the user CURRENTLY owns any of these assets
     const targets = [];
     for (const pos of portfolioData.AggregatedPositions) {
         if (leveragedAssets.has(pos.InstrumentID)) {
@@ -63,26 +45,18 @@ function detectSpeculatorTargets(historyData, portfolioData) {
     return targets;
 }
 
-/**
- * (REFACTORED: Fully sequential, verbose logging, node-fetch fallback)
- */
 async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager, pubsub }, config) {
     const { userId, instruments, instrumentId, userType } = task;
     
-    // Normalize the loop: Speculators get specific IDs, Normal users get [undefined] to trigger one pass.
     const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
     let isPrivate = false;
     
-    // Captured data for detection logic
     let capturedHistory = null;
     let capturedPortfolio = null;
-
-    // Track overall success for the final log
     let hasPortfolioErrors = false;
 
-    // FIX 1: Better Start Log
     const scopeLog = userType === 'speculator' ? `Instruments: [${instrumentsToProcess.join(', ')}]` : 'Scope: Full Portfolio';
     logger.log('TRACE', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. ${scopeLog}`);
 
@@ -99,7 +73,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                 logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
             } else {
                 
-                // --- REFACTOR: New Granular API Logic ---
                 const d = new Date();
                 d.setFullYear(d.getFullYear() - 1);
                 const oneYearAgoStr = d.toISOString();
@@ -140,7 +113,18 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
 
                 if (wasHistorySuccess) { 
                     const data = await response.json();
-                    capturedHistory = data; // Capture for later
+                    
+                    // --- FILTER LOGIC FOR GRANULAR API ---
+                    // 0 = Manual, 1 = Stop Loss, 5 = Take Profit.
+                    const VALID_REASONS = [0, 1, 5];
+                    if (data.PublicHistoryPositions && Array.isArray(data.PublicHistoryPositions)) {
+                        const originalCount = data.PublicHistoryPositions.length;
+                        data.PublicHistoryPositions = data.PublicHistoryPositions.filter(p => VALID_REASONS.includes(p.CloseReason));
+                        const filteredCount = data.PublicHistoryPositions.length;
+                        logger.log('INFO', `[handleUpdate/${userId}] History Filter: Reduced ${originalCount} -> ${filteredCount} positions.`);
+                    }
+
+                    capturedHistory = data; 
                     await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);  
                 }
             }
@@ -157,7 +141,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     logger.log('TRACE', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`); 
     
     for (const instId of instrumentsToProcess) { 
-        // FIX 2: Define a clear scope name for logging
         const scopeName = instId ? `Instrument ${instId}` : 'Full Portfolio';
 
         if (isPrivate) { 
@@ -174,7 +157,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         let wasPortfolioSuccess = false;
         let proxyUsedForPortfolio = false;
 
-        // --- PROXY ATTEMPT ---
         if (shouldTryProxy()) {
             try {
                 logger.log('TRACE', `[handleUpdate/${userId}] Attempting fetch for ${scopeName} via AppScript proxy...`);
@@ -190,7 +172,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
             }
         }
 
-        // --- DIRECT FALLBACK ---
         if (!wasPortfolioSuccess) {
             try {
                 response = await fetch(portfolioUrl, options); 
@@ -203,35 +184,29 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
             }
         }
 
-        // --- 4. Process Portfolio Result ---
         if (wasPortfolioSuccess) {
             const body = await response.text();
             if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`); break; }
 
             try {
                 const portfolioJson = JSON.parse(body);
-                capturedPortfolio = portfolioJson; // Capture for detection
+                capturedPortfolio = portfolioJson; 
                 await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
                 logger.log('TRACE', `[handleUpdate/${userId}] Portfolio for ${scopeName} processed successfully.`); 
 
             } catch (parseError) { 
                 wasPortfolioSuccess = false; 
-                hasPortfolioErrors = true; // Mark error state
+                hasPortfolioErrors = true; 
                 logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE for ${scopeName}.`, { url: portfolioUrl, parseErrorMessage: parseError.message }); 
             }
         } else { 
-            hasPortfolioErrors = true; // Mark error state
+            hasPortfolioErrors = true; 
             logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch FAILED for ${scopeName}.`); 
         }
         
         if (proxyUsedForPortfolio) { headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess); }
     } 
 
-    // --- 5. SPECULATOR DETECTION & QUEUEING (NEW) ---
-    // Only run detection if:
-    // 1. We are processing a Normal User (userType !== 'speculator')
-    // 2. We successfully fetched both history and portfolio
-    // 3. We have PubSub available to queue new tasks
     if (userType !== 'speculator' && capturedHistory && capturedPortfolio && pubsub && config.PUBSUB_TOPIC_TASK_ENGINE) {
         try {
             const speculatorAssets = detectSpeculatorTargets(capturedHistory, capturedPortfolio);
@@ -245,7 +220,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                     instrumentId: assetId
                 }));
 
-                // Publish to Task Engine (Tasks are wrapped in a 'tasks' array payload)
                 const dataBuffer = Buffer.from(JSON.stringify({ tasks: newTasks }));
                 await pubsub.topic(config.PUBSUB_TOPIC_TASK_ENGINE).publishMessage({ data: dataBuffer });
             }
@@ -254,7 +228,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         }
     }
 
-    // --- 6. Handle Private Users & Timestamps ---
     if (isPrivate) {
         logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
         for (const instrumentId of instrumentsToProcess) { 
@@ -268,17 +241,12 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         return; 
     }
 
-    // If not private AND no critical errors, update timestamps
-    // (We update timestamps even on partial failures for speculators to avoid infinite retry loops immediately, 
-    // relying on the next scheduled run, but for Normal users, a failure usually means we should retry later. 
-    // Current logic: Update timestamp to prevent immediate re-queueing.)
     for (const instrumentId of instrumentsToProcess) { 
         await batchManager.updateUserTimestamp(userId, userType, instrumentId); 
     }
     
     if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
     
-    // FIX 3: Honest Final Log
     if (hasPortfolioErrors) {
         logger.log('WARN', `[handleUpdate/${userId}] Update task finished with ERRORS. See logs above.`); 
     } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.212",
+  "version": "1.0.214",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [