bulltrackers-module 1.0.211 → 1.0.213

package/functions/computation-system/layers/validators.js

@@ -0,0 +1,170 @@
+/**
+ * @fileoverview Validators Layer
+ * Schema validation logic adhering to schema.md production definitions.
+ */
+
+const { SCHEMAS } = require('./profiling');
+
+class Validators {
+
+    /**
+     * Validates a User Portfolio based on User Type.
+     * @param {Object} portfolio - The portfolio object.
+     * @param {string} userType - 'normal' or 'speculator'.
+     * @returns {Object} { valid: boolean, errors: string[] }
+     */
+    static validatePortfolio(portfolio, userType) {
+        const errors = [];
+        if (!portfolio || typeof portfolio !== 'object') {
+            return { valid: false, errors: ['Portfolio is null or invalid object'] };
+        }
+
+        // 1. SPECULATOR PORTFOLIO VALIDATION
+        if (userType === SCHEMAS.USER_TYPES.SPECULATOR) {
+            // Must have PublicPositions array
+            if (!Array.isArray(portfolio.PublicPositions)) {
+                errors.push('Speculator portfolio missing "PublicPositions" array');
+            } else {
+                // Validate a sample position structure (checking the first one for performance)
+                if (portfolio.PublicPositions.length > 0) {
+                    const pos = portfolio.PublicPositions[0];
+                    if (typeof pos.InstrumentID !== 'number') errors.push('Speculator Position missing numeric "InstrumentID"');
+                    if (typeof pos.PositionID !== 'number') errors.push('Speculator Position missing numeric "PositionID"');
+                    if (typeof pos.NetProfit !== 'number') errors.push('Speculator Position missing numeric "NetProfit"');
+                    if (typeof pos.Leverage !== 'number') errors.push('Speculator Position missing numeric "Leverage"');
+                    if (typeof pos.OpenRate !== 'number') errors.push('Speculator Position missing numeric "OpenRate"');
+                    if (typeof pos.IsBuy !== 'boolean') errors.push('Speculator Position missing boolean "IsBuy"');
+                }
+            }
+            
+            // Root level metrics (Speculator specific)
+            if (typeof portfolio.NetProfit !== 'number') errors.push('Speculator portfolio missing root "NetProfit"');
+            if (typeof portfolio.Invested !== 'number') errors.push('Speculator portfolio missing root "Invested"');
+        } 
+        
+        // 2. NORMAL USER PORTFOLIO VALIDATION
+        else {
+            // Must have AggregatedPositions array
+            if (!Array.isArray(portfolio.AggregatedPositions)) {
+                errors.push('Normal portfolio missing "AggregatedPositions" array');
+            } else {
+                // Validate a sample position structure
+                if (portfolio.AggregatedPositions.length > 0) {
+                    const pos = portfolio.AggregatedPositions[0];
+                    if (typeof pos.InstrumentID !== 'number') errors.push('Normal Position missing numeric "InstrumentID"');
+                    if (typeof pos.NetProfit !== 'number') errors.push('Normal Position missing numeric "NetProfit"');
+                    if (typeof pos.Invested !== 'number') errors.push('Normal Position missing numeric "Invested"');
+                    if (typeof pos.Value !== 'number') errors.push('Normal Position missing numeric "Value"');
+                    if (!pos.Direction) errors.push('Normal Position missing "Direction" string');
+                }
+            }
+        }
+
+        return { valid: errors.length === 0, errors };
+    }
+
+    /**
+     * Validates Trade History Data.
+     * @param {Object} historyDoc - The history document object.
+     * @returns {Object} { valid: boolean, errors: string[] }
+     */
+    static validateTradeHistory(historyDoc) {
+        const errors = [];
+        if (!historyDoc || typeof historyDoc !== 'object') {
+            return { valid: false, errors: ['History document is null'] };
+        }
+
+        if (!Array.isArray(historyDoc.PublicHistoryPositions)) {
+            errors.push('History missing "PublicHistoryPositions" array');
+        } else if (historyDoc.PublicHistoryPositions.length > 0) {
+            // Validate first item schema
+            const trade = historyDoc.PublicHistoryPositions[0];
+            if (typeof trade.PositionID !== 'number') errors.push('History trade missing numeric "PositionID"');
+            if (typeof trade.CID !== 'number') errors.push('History trade missing numeric "CID"');
+            if (typeof trade.InstrumentID !== 'number') errors.push('History trade missing numeric "InstrumentID"');
+            if (typeof trade.CloseRate !== 'number') errors.push('History trade missing numeric "CloseRate"');
+            if (typeof trade.CloseReason !== 'number') errors.push('History trade missing numeric "CloseReason"');
+            if (typeof trade.NetProfit !== 'number') errors.push('History trade missing numeric "NetProfit"');
+            if (!trade.OpenDateTime) errors.push('History trade missing "OpenDateTime"');
+            if (!trade.CloseDateTime) errors.push('History trade missing "CloseDateTime"');
+        }
+
+        return { valid: errors.length === 0, errors };
+    }
+
+    /**
+     * Validates Social Post Data.
+     * @param {Object} post - The social post object.
+     * @returns {Object} { valid: boolean, errors: string[] }
+     */
+    static validateSocialPost(post) {
+        const errors = [];
+        if (!post) return { valid: false, errors: ['Post is null'] };
+
+        if (!post.postOwnerId) errors.push('Post missing "postOwnerId"');
+        if (!post.fullText) errors.push('Post missing "fullText"');
+        if (!post.createdAt) errors.push('Post missing "createdAt"');
+        if (typeof post.likeCount !== 'number') errors.push('Post missing numeric "likeCount"');
+        
+        // Sentiment Map
+        if (!post.sentiment || typeof post.sentiment.overallSentiment !== 'string') {
+            errors.push('Post missing valid "sentiment.overallSentiment"');
+        }
+
+        return { valid: errors.length === 0, errors };
+    }
+
+    /**
+     * Validates Insights Data (Platform Ownership).
+     * @param {Object} insight - A single insight item from the array.
+     * @returns {Object} { valid: boolean, errors: string[] }
+     */
+    static validateInsight(insight) {
+        const errors = [];
+        if (!insight) return { valid: false, errors: ['Insight object is null'] };
+
+        if (typeof insight.instrumentId !== 'number') errors.push('Insight missing numeric "instrumentId"');
+        if (typeof insight.total !== 'number') errors.push('Insight missing numeric "total" (Total Owners)');
+        if (typeof insight.percentage !== 'number') errors.push('Insight missing numeric "percentage" (Global Ownership)');
+        if (typeof insight.growth !== 'number') errors.push('Insight missing numeric "growth"');
+        
+        // Ownership split check
+        if (typeof insight.buy !== 'number') errors.push('Insight missing numeric "buy" %');
+        if (typeof insight.sell !== 'number') errors.push('Insight missing numeric "sell" %');
+
+        return { valid: errors.length === 0, errors };
+    }
+
+    /**
+     * Validates Asset Price Shard Data.
+     * @param {Object} instrumentData - The price data map for a specific instrument ID.
+     * @returns {Object} { valid: boolean, errors: string[] }
+     */
+    static validatePriceData(instrumentData) {
+        const errors = [];
+        if (!instrumentData) return { valid: false, errors: ['Price data is null'] };
+
+        // Must have 'prices' map
+        if (!instrumentData.prices || typeof instrumentData.prices !== 'object') {
+            errors.push('Instrument price data missing "prices" map');
+        } else {
+            // Check at least one price key matches date format YYYY-MM-DD
+            const keys = Object.keys(instrumentData.prices);
+            if (keys.length > 0) {
+                const sampleKey = keys[0];
+                const sampleVal = instrumentData.prices[sampleKey];
+                
+                if (!/^\d{4}-\d{2}-\d{2}$/.test(sampleKey)) {
+                    errors.push(`Price map key "${sampleKey}" does not match YYYY-MM-DD format`);
+                }
+                if (typeof sampleVal !== 'number') {
+                    errors.push('Price value is not a number');
+                }
+            }
+        }
+
+        return { valid: errors.length === 0, errors };
+    }
+}
+
+module.exports = { Validators };

package/functions/computation-system/utils/schema_capture.js

@@ -1,64 +1,64 @@
-/**
- * @fileoverview Schema capture utility for computation outputs
- * This module batches and stores pre-defined static schemas in Firestore.
- */
-
-/**
- * Batch store schemas for multiple computations.
- * This function now expects a fully-formed schema, not sample output.
- * It strictly stamps a 'lastUpdated' field to support stale-schema filtering in the API.
- *
- * @param {object} dependencies - Contains db, logger
- * @param {object} config - Configuration object
- * @param {Array} schemas - Array of {name, category, schema, metadata} objects
- */
-async function batchStoreSchemas(dependencies, config, schemas) {
-    const { db, logger } = dependencies;
-    
-    if (config.captureSchemas === false) {
-        logger.log('INFO', '[SchemaCapture] Schema capture is disabled. Skipping.');
-        return;
-    }
-
-    const batch = db.batch();
-    const schemaCollection = config.schemaCollection || 'computation_schemas';
-    let validCount = 0;
-
-    for (const item of schemas) {
-        try {
-            if (!item.schema) {
-                logger.log('WARN', `[SchemaCapture] No schema provided for ${item.name}. Skipping.`);
-                continue;
-            }
-
-            const docRef = db.collection(schemaCollection).doc(item.name);
-            
-            // Critical: Always overwrite 'lastUpdated' to now
-            batch.set(docRef, {
-                computationName: item.name,
-                category: item.category,
-                schema: item.schema,
-                metadata: item.metadata || {},
-                lastUpdated: new Date() 
-            }, { merge: true });
-            
-            validCount++;
-
-        } catch (error) {
-            logger.log('WARN', `[SchemaCapture] Failed to add schema to batch for ${item.name}`, { errorMessage: error.message });
-        }
-    }
-
-    if (validCount > 0) {
-        try {
-            await batch.commit();
-            logger.log('INFO', `[SchemaCapture] Batch stored ${validCount} computation schemas`);
-        } catch (error) {
-            logger.log('ERROR', '[SchemaCapture] Failed to commit schema batch', { errorMessage: error.message });
-        }
-    }
-}
-
-module.exports = {
-    batchStoreSchemas
+/**
+ * @fileoverview Schema capture utility for computation outputs
+ * This module batches and stores pre-defined static schemas in Firestore.
+ */
+
+/**
+ * Batch store schemas for multiple computations.
+ * This function now expects a fully-formed schema, not sample output.
+ * It strictly stamps a 'lastUpdated' field to support stale-schema filtering in the API.
+ *
+ * @param {object} dependencies - Contains db, logger
+ * @param {object} config - Configuration object
+ * @param {Array} schemas - Array of {name, category, schema, metadata} objects
+ */
+async function batchStoreSchemas(dependencies, config, schemas) {
+    const { db, logger } = dependencies;
+    
+    if (config.captureSchemas === false) {
+        logger.log('INFO', '[SchemaCapture] Schema capture is disabled. Skipping.');
+        return;
+    }
+
+    const batch = db.batch();
+    const schemaCollection = config.schemaCollection || 'computation_schemas';
+    let validCount = 0;
+
+    for (const item of schemas) {
+        try {
+            if (!item.schema) {
+                logger.log('WARN', `[SchemaCapture] No schema provided for ${item.name}. Skipping.`);
+                continue;
+            }
+
+            const docRef = db.collection(schemaCollection).doc(item.name);
+            
+            // Critical: Always overwrite 'lastUpdated' to now
+            batch.set(docRef, {
+                computationName: item.name,
+                category: item.category,
+                schema: item.schema,
+                metadata: item.metadata || {},
+                lastUpdated: new Date() 
+            }, { merge: true });
+            
+            validCount++;
+
+        } catch (error) {
+            logger.log('WARN', `[SchemaCapture] Failed to add schema to batch for ${item.name}`, { errorMessage: error.message });
+        }
+    }
+
+    if (validCount > 0) {
+        try {
+            await batch.commit();
+            logger.log('INFO', `[SchemaCapture] Batch stored ${validCount} computation schemas`);
+        } catch (error) {
+            logger.log('ERROR', '[SchemaCapture] Failed to commit schema batch', { errorMessage: error.message });
+        }
+    }
+}
+
+module.exports = {
+    batchStoreSchemas
 };

package/functions/computation-system/utils/utils.js

@@ -2,13 +2,34 @@
  * @fileoverview Computation system sub-pipes and utils.
  * REFACTORED: Now stateless and receive dependencies where needed.
  * FIXED: 'commitBatchInChunks' now respects Firestore 10MB size limit.
+ * NEW: Added 'generateCodeHash' for version control.
  */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
+const crypto = require('crypto');
 
 /** Stage 1: Normalize a calculation name to kebab-case */
 function normalizeName(name) { return name.replace(/_/g, '-'); }
 
+/**
+ * Generates a SHA-256 hash of a code string, ignoring comments and whitespace.
+ * This effectively versions the logic.
+ * @param {string} codeString - The source code of the function/class.
+ * @returns {string} The hex hash.
+ */
+function generateCodeHash(codeString) {
+    if (!codeString) return 'unknown';
+    
+    // 1. Remove single-line comments (//...)
+    let clean = codeString.replace(/\/\/.*$/gm, '');
+    // 2. Remove multi-line comments (/*...*/)
+    clean = clean.replace(/\/\*[\s\S]*?\*\//g, '');
+    // 3. Remove all whitespace (spaces, tabs, newlines)
+    clean = clean.replace(/\s+/g, '');
+    
+    return crypto.createHash('sha256').update(clean).digest('hex');
+}
+
 /** * Stage 2: Commit a batch of writes in chunks 
  * FIXED: Now splits batches by SIZE (9MB limit) and COUNT (450 docs) 
  * to prevent "Request payload size exceeds the limit" errors.
@@ -249,4 +270,4 @@ async function getFirstDateFromPriceCollection(config, deps) {
     }
 }
 
-module.exports = { FieldValue, FieldPath, normalizeName, commitBatchInChunks, getExpectedDateStrings, getEarliestDataDates };
+module.exports = { FieldValue, FieldPath, normalizeName, commitBatchInChunks, getExpectedDateStrings, getEarliestDataDates, generateCodeHash };

package/functions/task-engine/helpers/update_helpers.js

@@ -1,32 +1,20 @@
 /*
  * FILENAME: CloudFunctions/NpmWrappers/bulltrackers-module/functions/task-engine/helpers/update_helpers.js
- * (OPTIMIZED V4: Auto-Speculator Detection via History/Portfolio Intersection)
- * (OPTIMIZED V3: Removed obsolete username lookup logic)
- * (OPTIMIZED V2: Added "Circuit Breaker" for Proxy failures)
- * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
- * (FIXED: Improved logging clarity for Normal vs Speculator users)
- * (FIXED: Final log now accurately reflects failure state)
+ * (OPTIMIZED V5: Filter Copy Trash from History)
+ * FIX: Filters PublicHistoryPositions to keep only valid close reasons (0, 1, 5).
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
 const crypto = require('crypto'); 
 
 // --- CIRCUIT BREAKER STATE ---
-// Persists across function invocations in the same instance.
-// If the Proxy fails 3 times in a row, we stop trying it to save the 5s timeout cost.
 let _consecutiveProxyFailures = 0;
 const MAX_PROXY_FAILURES = 3;
 
-/**
- * Helper to check if we should attempt the proxy
- */
 function shouldTryProxy() {
     return _consecutiveProxyFailures < MAX_PROXY_FAILURES;
 }
 
-/**
- * Helper to record proxy result
- */
 function recordProxyOutcome(success) {
     if (success) {
         _consecutiveProxyFailures = 0;
@@ -35,14 +23,9 @@ function recordProxyOutcome(success) {
     }
 }
 
-/**
- * --- NEW HELPER: Speculator Detector ---
- * intersections: (History: Leverage > 1) AND (Portfolio: Currently Owned)
- */
 function detectSpeculatorTargets(historyData, portfolioData) {
     if (!historyData?.PublicHistoryPositions || !portfolioData?.AggregatedPositions) return [];
 
-    // 1. Identify assets that have EVER been traded with leverage > 1
     const leveragedAssets = new Set();
     for (const pos of historyData.PublicHistoryPositions) {
         if (pos.Leverage > 1 && pos.InstrumentID) {
@@ -52,7 +35,6 @@ function detectSpeculatorTargets(historyData, portfolioData) {
 
     if (leveragedAssets.size === 0) return [];
 
-    // 2. Check if the user CURRENTLY owns any of these assets
     const targets = [];
     for (const pos of portfolioData.AggregatedPositions) {
         if (leveragedAssets.has(pos.InstrumentID)) {
@@ -63,26 +45,18 @@ function detectSpeculatorTargets(historyData, portfolioData) {
     return targets;
 }
 
-/**
- * (REFACTORED: Fully sequential, verbose logging, node-fetch fallback)
- */
 async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager, pubsub }, config) {
     const { userId, instruments, instrumentId, userType } = task;
     
-    // Normalize the loop: Speculators get specific IDs, Normal users get [undefined] to trigger one pass.
     const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
     let isPrivate = false;
     
-    // Captured data for detection logic
     let capturedHistory = null;
     let capturedPortfolio = null;
-
-    // Track overall success for the final log
     let hasPortfolioErrors = false;
 
-    // FIX 1: Better Start Log
     const scopeLog = userType === 'speculator' ? `Instruments: [${instrumentsToProcess.join(', ')}]` : 'Scope: Full Portfolio';
     logger.log('TRACE', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. ${scopeLog}`);
 
@@ -99,7 +73,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                 logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
             } else {
                 
-                // --- REFACTOR: New Granular API Logic ---
                 const d = new Date();
                 d.setFullYear(d.getFullYear() - 1);
                 const oneYearAgoStr = d.toISOString();
@@ -140,7 +113,18 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
 
                 if (wasHistorySuccess) { 
                     const data = await response.json();
-                    capturedHistory = data; // Capture for later
+                    
+                    // --- FILTER LOGIC FOR GRANULAR API ---
+                    // 0 = Manual, 1 = Stop Loss, 5 = Take Profit.
+                    const VALID_REASONS = [0, 1, 5];
+                    if (data.PublicHistoryPositions && Array.isArray(data.PublicHistoryPositions)) {
+                        const originalCount = data.PublicHistoryPositions.length;
+                        data.PublicHistoryPositions = data.PublicHistoryPositions.filter(p => VALID_REASONS.includes(p.CloseReason));
+                        const filteredCount = data.PublicHistoryPositions.length;
+                        logger.log('INFO', `[handleUpdate/${userId}] History Filter: Reduced ${originalCount} -> ${filteredCount} positions.`);
+                    }
+
+                    capturedHistory = data; 
                     await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);  
                 }
             }
@@ -157,7 +141,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     logger.log('TRACE', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`); 
     
     for (const instId of instrumentsToProcess) { 
-        // FIX 2: Define a clear scope name for logging
         const scopeName = instId ? `Instrument ${instId}` : 'Full Portfolio';
 
         if (isPrivate) { 
@@ -174,7 +157,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         let wasPortfolioSuccess = false;
         let proxyUsedForPortfolio = false;
 
-        // --- PROXY ATTEMPT ---
         if (shouldTryProxy()) {
             try {
                 logger.log('TRACE', `[handleUpdate/${userId}] Attempting fetch for ${scopeName} via AppScript proxy...`);
@@ -190,7 +172,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
             }
         }
 
-        // --- DIRECT FALLBACK ---
         if (!wasPortfolioSuccess) {
             try {
                 response = await fetch(portfolioUrl, options); 
@@ -203,35 +184,29 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
             }
         }
 
-        // --- 4. Process Portfolio Result ---
         if (wasPortfolioSuccess) {
             const body = await response.text();
             if (body.includes("user is PRIVATE")) { isPrivate = true; logger.log('WARN', `[handleUpdate/${userId}] User is PRIVATE. Marking for removal.`); break; }
 
             try {
                 const portfolioJson = JSON.parse(body);
-                capturedPortfolio = portfolioJson; // Capture for detection
+                capturedPortfolio = portfolioJson; 
                 await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
                 logger.log('TRACE', `[handleUpdate/${userId}] Portfolio for ${scopeName} processed successfully.`); 
 
             } catch (parseError) { 
                 wasPortfolioSuccess = false; 
-                hasPortfolioErrors = true; // Mark error state
+                hasPortfolioErrors = true; 
                 logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE for ${scopeName}.`, { url: portfolioUrl, parseErrorMessage: parseError.message }); 
             }
         } else { 
-            hasPortfolioErrors = true; // Mark error state
+            hasPortfolioErrors = true; 
             logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch FAILED for ${scopeName}.`); 
         }
         
         if (proxyUsedForPortfolio) { headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess); }
     } 
 
-    // --- 5. SPECULATOR DETECTION & QUEUEING (NEW) ---
-    // Only run detection if:
-    // 1. We are processing a Normal User (userType !== 'speculator')
-    // 2. We successfully fetched both history and portfolio
-    // 3. We have PubSub available to queue new tasks
     if (userType !== 'speculator' && capturedHistory && capturedPortfolio && pubsub && config.PUBSUB_TOPIC_TASK_ENGINE) {
         try {
             const speculatorAssets = detectSpeculatorTargets(capturedHistory, capturedPortfolio);
@@ -245,7 +220,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                     instrumentId: assetId
                 }));
 
-                // Publish to Task Engine (Tasks are wrapped in a 'tasks' array payload)
                 const dataBuffer = Buffer.from(JSON.stringify({ tasks: newTasks }));
                 await pubsub.topic(config.PUBSUB_TOPIC_TASK_ENGINE).publishMessage({ data: dataBuffer });
             }
@@ -254,7 +228,6 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         }
     }
 
-    // --- 6. Handle Private Users & Timestamps ---
     if (isPrivate) {
         logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
         for (const instrumentId of instrumentsToProcess) { 
@@ -268,17 +241,12 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         return; 
     }
 
-    // If not private AND no critical errors, update timestamps
-    // (We update timestamps even on partial failures for speculators to avoid infinite retry loops immediately, 
-    // relying on the next scheduled run, but for Normal users, a failure usually means we should retry later. 
-    // Current logic: Update timestamp to prevent immediate re-queueing.)
     for (const instrumentId of instrumentsToProcess) { 
         await batchManager.updateUserTimestamp(userId, userType, instrumentId); 
     }
     
     if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
     
-    // FIX 3: Honest Final Log
     if (hasPortfolioErrors) {
         logger.log('WARN', `[handleUpdate/${userId}] Update task finished with ERRORS. See logs above.`); 
     } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.211",
+  "version": "1.0.213",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [