bulltrackers-module 1.0.170 → 1.0.172

package/functions/computation-system/layers/math_primitives.js

@@ -0,0 +1,346 @@
+/**
+ * @fileoverview Math Layer - Single Source of Truth (V3 Final)
+ * Encapsulates Schema Knowledge, Mathematical Primitives, and Signal Extractors.
+ * * STRICT COMPLIANCE:
+ * - Adheres to 'schema.md' definitions for Normal vs Speculator portfolios.
+ * - standardizes access to P&L, Weights, and Rates.
+ * - Provides safe fallbacks for all fields.
+ */
+
+const SCHEMAS = {
+    USER_TYPES: { NORMAL: 'normal', SPECULATOR: 'speculator' }
+};
+
+class DataExtractor {
+    // ========================================================================
+    // 1. COLLECTION ACCESSORS
+    // ========================================================================
+    
+    /**
+     * Extract positions array based on User Type.
+     * - Normal: Uses 'AggregatedPositions' (Grouped by Asset + Direction)
+     * - Speculator: Uses 'PublicPositions' (Individual Trades)
+     */
+    static getPositions(portfolio, userType) {
+        if (!portfolio) return [];
+        
+        if (userType === SCHEMAS.USER_TYPES.SPECULATOR) {
+            return portfolio.PublicPositions || [];
+        }
+        
+        // Default to Normal User Schema
+        return portfolio.AggregatedPositions || [];
+    }
+
+    // ========================================================================
+    // 2. IDENTITY & KEYS
+    // ========================================================================
+
+    /**
+     * Extract standardized Instrument ID.
+     */
+    static getInstrumentId(position) {
+        if (!position) return null;
+        // Handle string or number variations safely
+        return position.InstrumentID || position.instrumentId || null;
+    }
+
+    /**
+     * Extract a unique Identifier for the position.
+     * - Speculator: Uses 'PositionID'.
+     * - Normal: Generates Composite Key (InstrumentID_Direction) since they lack unique Trade IDs.
+     */
+    static getPositionId(position) {
+        if (!position) return null;
+        
+        // 1. Try Explicit ID (Speculators)
+        if (position.PositionID) return String(position.PositionID);
+        if (position.PositionId) return String(position.PositionId);
+
+        // 2. Fallback to Composite Key (Normal Users)
+        const instId = this.getInstrumentId(position);
+        const dir = this.getDirection(position);
+        if (instId) return `${instId}_${dir}`;
+
+        return null;
+    }
+
+    // ========================================================================
+    // 3. FINANCIAL METRICS (WEIGHTS & P&L)
+    // ========================================================================
+
+    /**
+     * Extract Net Profit %.
+     * Schema: 'NetProfit' is the percentage profit relative to invested capital.
+     */
+    static getNetProfit(position) {
+        return position ? (position.NetProfit || 0) : 0;
+    }
+
+    /**
+     * Extract Position Weight (Allocation %).
+     * Schema: 
+     * - Normal: 'Invested' is % of initial capital.
+     * - Speculator: 'Invested' (or 'Amount' in some contexts) is % of initial capital.
+     */
+    static getPositionWeight(position, userType) {
+        if (!position) return 0;
+        
+        // Both schemas use 'Invested' to represent the allocation percentage.
+        // Speculators might optionally have 'Amount', we prioritize 'Invested' for consistency.
+        return position.Invested || position.Amount || 0;
+    }
+
+    /**
+     * Extract Current Equity Value %.
+     * Schema: 'Value' is the current value as a % of total portfolio equity.
+     */
+    static getPositionValuePct(position) {
+        return position ? (position.Value || 0) : 0;
+    }
+
+    // ========================================================================
+    // 4. PORTFOLIO LEVEL SUMMARY
+    // ========================================================================
+
+    /**
+     * Calculate/Extract Daily Portfolio P&L %.
+     */
+    static getPortfolioDailyPnl(portfolio, userType) {
+        if (!portfolio) return 0;
+        
+        // 1. Speculator (Explicit 'NetProfit' field on root)
+        if (userType === SCHEMAS.USER_TYPES.SPECULATOR) {
+            return portfolio.NetProfit || 0;
+        }
+        
+        // 2. Normal (Aggregated Calculation)
+        // Logic: Sum(Value - Invested) across the 'InstrumentType' breakdown
+        // This gives the net performance change for the day relative to the portfolio.
+        if (portfolio.AggregatedPositionsByInstrumentTypeID) {
+            return portfolio.AggregatedPositionsByInstrumentTypeID.reduce((sum, agg) => {
+                return sum + ((agg.Value || 0) - (agg.Invested || 0));
+            }, 0);
+        }
+        
+        return 0;
+    }
+
+    // ========================================================================
+    // 5. TRADE DETAILS (SPECULATOR SPECIFIC)
+    // ========================================================================
+
+    static getDirection(position) {
+        if (!position) return "Buy";
+        if (position.Direction) return position.Direction;
+        if (typeof position.IsBuy === 'boolean') return position.IsBuy ? "Buy" : "Sell";
+        return "Buy"; // Default
+    }
+
+    static getLeverage(position) {
+        return position ? (position.Leverage || 1) : 1;
+    }
+
+    static getOpenRate(position) {
+        return position ? (position.OpenRate || 0) : 0;
+    }
+
+    static getCurrentRate(position) {
+        return position ? (position.CurrentRate || 0) : 0;
+    }
+
+    static getStopLossRate(position) {
+        return position ? (position.StopLossRate || 0) : 0;
+    }
+
+    static getTakeProfitRate(position) {
+        return position ? (position.TakeProfitRate || 0) : 0;
+    }
+
+    static getHasTSL(position) {
+        return position ? (position.HasTrailingStopLoss === true) : false;
+    }
+
+    /**
+     * Extract Open Date Time.
+     * Used for bagholder calculations.
+     */
+    static getOpenDateTime(position) {
+        if (!position || !position.OpenDateTime) return null;
+        return new Date(position.OpenDateTime);
+    }
+}
+
+class HistoryExtractor {
+    // --- Schema Accessor (NEW) ---
+    /**
+     * Extracts the daily history snapshot from the User object.
+     * This decouples the computation from knowing 'user.history.today'.
+     */
+    static getDailyHistory(user) {
+        return user?.history?.today || null;
+    }
+
+    // --- Data Extractors ---
+    static getTradedAssets(historyDoc) {
+        if (!historyDoc || !Array.isArray(historyDoc.assets)) return [];
+        return historyDoc.assets;
+    }
+
+    static getInstrumentId(asset) {
+        return asset ? asset.instrumentId : null;
+    }
+
+    static getAvgHoldingTimeMinutes(asset) {
+        return asset ? (asset.avgHoldingTimeInMinutes || 0) : 0;
+    }
+
+    static getSummary(historyDoc) {
+        const all = historyDoc?.all;
+        if (!all) return null;
+
+        return {
+            totalTrades: all.totalTrades || 0,
+            winRatio: all.winRatio || 0,
+            avgProfitPct: all.avgProfitPct || 0,
+            avgLossPct: all.avgLossPct || 0,
+            avgHoldingTimeInMinutes: all.avgHoldingTimeInMinutes || 0
+        };
+    }
+}
+
+class SignalPrimitives {
+    /**
+     * Safely extracts a specific numeric field for a specific ticker from a dependency.
+     */
+    static getMetric(dependencies, calcName, ticker, fieldName, fallback = 0) {
+        if (!dependencies || !dependencies[calcName]) return fallback;
+        const tickerData = dependencies[calcName][ticker];
+        if (!tickerData) return fallback;
+        
+        const val = tickerData[fieldName];
+        return (typeof val === 'number') ? val : fallback;
+    }
+
+    /**
+     * Creates a unified Set of all keys (tickers) present across multiple dependency results.
+     */
+    static getUnionKeys(dependencies, calcNames) {
+        const keys = new Set();
+        if (!dependencies) return [];
+        
+        for (const name of calcNames) {
+            const resultObj = dependencies[name];
+            if (resultObj && typeof resultObj === 'object') {
+                Object.keys(resultObj).forEach(k => keys.add(k));
+            }
+        }
+        return Array.from(keys);
+    }
+
+    /**
+     * Hyperbolic Tangent Normalization.
+     * Maps inputs to a strict -Scale to +Scale range.
+     */
+    static normalizeTanh(value, scale = 10, sensitivity = 10.0) {
+        if (value === 0) return 0;
+        return Math.tanh(value / sensitivity) * scale;
+    }
+
+    /**
+     * Standard Z-Score normalization.
+     */
+    static normalizeZScore(value, mean, stdDev) {
+        if (!stdDev || stdDev === 0) return 0;
+        return (value - mean) / stdDev;
+    }
+
+    /**
+     * Simple Divergence (A - B).
+     */
+    static divergence(valueA, valueB) {
+        return (valueA || 0) - (valueB || 0);
+    }
+}
+
+class MathPrimitives {
+    static average(values) {
+        if (!values || !values.length) return 0;
+        return values.reduce((a, b) => a + b, 0) / values.length;
+    }
+    
+    static median(values) {
+        if (!values || !values.length) return 0;
+        const sorted = [...values].sort((a, b) => a - b);
+        const mid = Math.floor(sorted.length / 2);
+        return sorted.length % 2 === 0
+            ? (sorted[mid - 1] + sorted[mid]) / 2
+            : sorted[mid];
+    }
+
+    static standardDeviation(values) {
+        if (!values || !values.length) return 0;
+        const avg = this.average(values);
+        const squareDiffs = values.map(val => Math.pow((val || 0) - avg, 2));
+        return Math.sqrt(this.average(squareDiffs));
+    }
+
+    static bucketBinary(value, threshold = 0) {
+        return value > threshold ? 'winner' : 'loser';
+    }
+}
+
+class Aggregators {
+    /**
+     * Helper to bucket users by P&L Status.
+     * Used by legacy systems or specific aggregators.
+     */
+    static bucketUsersByPnlPerAsset(usersData, tickerMap) {
+        const buckets = new Map();
+        for (const [userId, portfolio] of Object.entries(usersData)) {
+            // Auto-detect type if not provided (Legacy compatibility)
+            const userType = portfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
+            const positions = DataExtractor.getPositions(portfolio, userType);
+            
+            for (const pos of positions) {
+                const id = DataExtractor.getInstrumentId(pos);
+                const pnl = DataExtractor.getNetProfit(pos);
+                if (!id || pnl === 0) continue;
+                
+                const ticker = tickerMap[id];
+                if (!ticker) continue;
+
+                if (!buckets.has(ticker)) buckets.set(ticker, { winners: [], losers: [] });
+                const b = buckets.get(ticker);
+                
+                if (pnl > 0) b.winners.push(userId);
+                else b.losers.push(userId);
+            }
+        }
+        return Object.fromEntries(buckets);
+    }
+}
+
+class Validators {
+    static validatePortfolio(portfolio, userType) {
+        if (!portfolio) return { valid: false, errors: ['Portfolio is null'] };
+        
+        if (userType === SCHEMAS.USER_TYPES.SPECULATOR) {
+             if (!portfolio.PublicPositions) return { valid: false, errors: ['Missing PublicPositions'] };
+        } else {
+             if (!portfolio.AggregatedPositions) return { valid: false, errors: ['Missing AggregatedPositions'] };
+        }
+        
+        return { valid: true, errors: [] };
+    }
+}
+
+module.exports = { 
+    SCHEMAS, 
+    DataExtractor, 
+    HistoryExtractor, 
+    MathPrimitives, 
+    Aggregators, 
+    Validators, 
+    SignalPrimitives 
+};

package/functions/task-engine/helpers/discover_helpers.js

@@ -24,8 +24,7 @@ async function handleDiscover(task, taskId, dependencies, config) {
     const url = `${config.ETORO_API_RANKINGS_URL}?Period=LastTwoYears`;
     
     const selectedHeader = await headerManager.selectHeader();
-    if (!selectedHeader) {
-        logger.log('ERROR', `[DISCOVER/${taskId}] Could not select a header. Aborting task.`);
+    if (!selectedHeader) { logger.log('ERROR', `[DISCOVER/${taskId}] Could not select a header. Aborting task.`);
         throw new Error("Could not select a header.");
     }
 
@@ -35,45 +34,34 @@ async function handleDiscover(task, taskId, dependencies, config) {
 
     try { // Outer try for the whole operation
         let response;
-        const options = {
-            method: 'POST',
-            headers: { ...selectedHeader.header, 'Content-Type': 'application/json' },
-            body: JSON.stringify(cids),
-        };
+        const options = {  method: 'POST', headers: { ...selectedHeader.header, 'Content-Type': 'application/json' }, body: JSON.stringify(cids), };
         
         logger.log('INFO', `${logPrefix} Starting discovery for ${cids.length} CIDs. Block: ${blockId}, Type: ${userType}`);
 
         try {
             // --- REFACTOR 3: ADD FALLBACK ---
-            logger.log('TRACE', `${logPrefix} Attempting discovery fetch via AppScript proxy...`);
+            logger.log('TRACE', `${logPrefix} Attempting discovery fetch via AppScript proxy...`); 
             response = await proxyManager.fetch(url, options);
-            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
+            if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); // Failure
             
-        } catch (proxyError) {
-            logger.log('WARN', `${logPrefix} AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, {
-                error: proxyError.message,
-                source: 'AppScript'
-            });
+        } catch (proxyError) { // Fuck we failed with appscript IP Pools, log error type
+            logger.log('WARN', `${logPrefix} AppScript proxy fetch FAILED. Error: ${proxyError.message}. Attempting direct node-fetch fallback.`, { error: proxyError.message,source: 'AppScript' });
             proxyUsed = false;
             
             try {
-                response = await fetch(url, options); // Direct node-fetch
-                if (!response.ok) {
-                    const errorText = await response.text();
-                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`);
+                response = await fetch(url, options); // Direct node-fetch fallback, use GCP IP Pools
+                if (!response.ok) { const errorText = await response.text();
+                    throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); // Fuck we failed here too.
                 }
                 
-            } catch (fallbackError) {
-                logger.log('ERROR', `${logPrefix} Direct node-fetch fallback FAILED.`, {
-                    error: fallbackError.message,
-                    source: 'eToro/Network'
-                });
+            } catch (fallbackError) { // Figure out the error type
+                logger.log('ERROR', `${logPrefix} Direct node-fetch fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
                 throw fallbackError; // Throw to be caught by outer try
             }
             // --- END REFACTOR 3 ---
         }
 
-        // --- If we are here, `response` is valid ---
+        // --- If we are here, `response` is valid and we are very smart ---
         
         // Step 1. Discover Speculators
         if (userType === 'speculator') {
@@ -87,17 +75,12 @@ async function handleDiscover(task, taskId, dependencies, config) {
             // --- REFACTOR 4: LOG RAW RESPONSE ON PARSE FAILURE ---
             publicUsers = JSON.parse(body);
         } catch (parseError) {
-            logger.log('ERROR', `${logPrefix} FAILED TO PARSE JSON RESPONSE. RAW BODY:`, {
-                parseErrorMessage: parseError.message,
-                rawResponseText: body 
-            });
+            logger.log('ERROR', `${logPrefix} FAILED TO PARSE JSON RESPONSE. RAW BODY:`, { parseErrorMessage: parseError.message, rawResponseText: body });
             throw new Error(`Failed to parse JSON response from discovery API. Body: ${body.substring(0, 200)}`);
         }
         // --- END REFACTOR 4 ---
 
-        if (!Array.isArray(publicUsers)) {
-             logger.log('WARN', `${logPrefix} API returned non-array response. Type: ${typeof publicUsers}`);
-             wasSuccess = true; // API call worked, data was just empty/weird
+        if (!Array.isArray(publicUsers)) { logger.log('WARN', `${logPrefix} API returned non-array response. Type: ${typeof publicUsers}`); wasSuccess = true; // API call worked, data was just empty/weird
              return;
         }
         
@@ -110,9 +93,9 @@ async function handleDiscover(task, taskId, dependencies, config) {
         // Step 2. Filter active users
         const preliminaryActiveUsers = publicUsers.filter(user => 
             new Date(user.Value.LastActivity) > oneMonthAgo &&
-            user.Value.DailyGain !== 0 &&
-            user.Value.Exposure !== 0 &&
-            user.Value.RiskScore !== 0
+            user.Value.DailyGain !== 0 && // Daily % gain, if 0 then they can't hold positions --- or its a weekend and portfolio holds solely stocks
+            user.Value.Exposure !== 0 && // Not sure what this means exactly but exposure should refer to position % held/invested in
+            user.Value.RiskScore !== 0 // If portfolio has risk score 0, they cannot have positions
         );
         logger.log('INFO', `${logPrefix} Found ${preliminaryActiveUsers.length} preliminary active users.`);
 
@@ -136,26 +119,27 @@ async function handleDiscover(task, taskId, dependencies, config) {
             const nonSpeculatorCids = [];
             for (const user of preliminaryActiveUsers) {
                 const v = user.Value;
-                const totalLeverage = (v.MediumLeveragePct || 0) + (v.HighLeveragePct || 0);
-                const isLikelySpeculator = ((v.Trades || 0) > 500 || (v.TotalTradedInstruments || 0) > 50 || totalLeverage > 50 || (v.WeeklyDD || 0) < -25);
+                const totalLeverage = (v.MediumLeveragePct || 0) + (v.HighLeveragePct || 0); // Do they own at least some leveraged positions? 
+                // We don't know what they are, but we can see their total average leverage, if not 0 on medoum/high leverage, then they MUST have at least 1 leveraged position
+                // Note : Medium leverage and high leverage are somewhat arbitrary. Low leverage is presumed to be mean "1" and thus medium leverage is the middle of the available leverage ranges, for stocks this is usually 2, for other assets this can be 5/10 or 20, it varies. High leverage is presumed to be the max leverage for that asset
+                const isLikelySpeculator = ((v.Trades || 0) > 500 || (v.TotalTradedInstruments || 0) > 50 || totalLeverage > 50 || (v.WeeklyDD || 0) < -25); // Apply filtering on their total leverage, drawdown and trade numbers to figure out if they are likely a speculative portfolio.
                 
-                if (isLikelySpeculator) {
-                    finalActiveUsers.push(user);
-                } else {
-                    nonSpeculatorCids.push(user.CID);
-                }
+                if (isLikelySpeculator) { finalActiveUsers.push(user); } else {  nonSpeculatorCids.push(user.CID); }
             }
-            invalidCidsToLog.push(...nonSpeculatorCids);
+            invalidCidsToLog.push(...nonSpeculatorCids); // 
+            // This logs how many users did NOT pass SPECIFICALLY the islikelyspeculator
             logger.log('INFO', `${logPrefix} Speculator pre-filter complete. ${finalActiveUsers.length} users passed. ${nonSpeculatorCids.length} users failed heuristic.`);
             
-            if (invalidCidsToLog.length > 0) {
+            if (invalidCidsToLog.length > 0) { // Log the number of users we tried for speculators that failed the filtering checks. 
+                // NOTE : This logs the users who failed ANY of the 3 filters (private users, inactive public users, or users who didn't meet the specific conditions for islikelyspeculator)
                 await pubsub.topic(config.PUBSUB_TOPIC_INVALID_SPECULATOR_LOG).publishMessage({ json: { invalidCids: invalidCidsToLog } });
                 logger.log('INFO', `${logPrefix} Reported ${invalidCidsToLog.length} invalid (private, inactive, or failed heuristic) speculator IDs.`);
             }
-        // Step 5. Non-speculators are just active users
-        } else { 
-            finalActiveUsers = preliminaryActiveUsers;
-        }
+       // Step 5. For non-speculator tasks, we keep all users who passed the
+        // “active user” filters. Unlike speculators, normal users do NOT require
+        // the speculative-heuristic (leverage / trades / drawdown) filter —
+        // any public & active user is valid.
+        } else {  finalActiveUsers = preliminaryActiveUsers; }
         
         // Step 6. Publish 'verify' task
         if (finalActiveUsers.length > 0) {
@@ -168,12 +152,10 @@ async function handleDiscover(task, taskId, dependencies, config) {
             };
             await pubsub.topic(config.PUBSUB_TOPIC_USER_FETCH).publishMessage({ json: { tasks: [verificationTask] } });
             logger.log('SUCCESS', `${logPrefix} Chaining to 'verify' task for ${finalActiveUsers.length} active users.`);
-        } else {
-            logger.log('INFO', `${logPrefix} No active users found to verify.`);
-        }
+        } else { logger.log('INFO', `${logPrefix} No active users found to verify.`); }
 
     } catch (err) {
-        logger.log('ERROR', `${logPrefix} FATAL error processing discovery task.`, { errorMessage: err.message, errorStack: err.stack });
+        logger.log('ERROR', `${logPrefix} FATAL error processing discovery task.`, { errorMessage: err.message, errorStack: err.stack }); // We fucked up
         wasSuccess = false; // Ensure it's marked as failure
     } finally {
         if (selectedHeader && proxyUsed) {