bulltrackers-module 1.0.210 → 1.0.212

package/functions/computation-system/layers/validators.js

@@ -0,0 +1,170 @@
+/**
+ * @fileoverview Validators Layer
+ * Schema validation logic adhering to schema.md production definitions.
+ */
+
+const { SCHEMAS } = require('./profiling');
+
+class Validators {
+
+    /**
+     * Validates a User Portfolio based on User Type.
+     * @param {Object} portfolio - The portfolio object.
+     * @param {string} userType - 'normal' or 'speculator'.
+     * @returns {Object} { valid: boolean, errors: string[] }
+     */
+    static validatePortfolio(portfolio, userType) {
+        const errors = [];
+        if (!portfolio || typeof portfolio !== 'object') {
+            return { valid: false, errors: ['Portfolio is null or invalid object'] };
+        }
+
+        // 1. SPECULATOR PORTFOLIO VALIDATION
+        if (userType === SCHEMAS.USER_TYPES.SPECULATOR) {
+            // Must have PublicPositions array
+            if (!Array.isArray(portfolio.PublicPositions)) {
+                errors.push('Speculator portfolio missing "PublicPositions" array');
+            } else {
+                // Validate a sample position structure (checking the first one for performance)
+                if (portfolio.PublicPositions.length > 0) {
+                    const pos = portfolio.PublicPositions[0];
+                    if (typeof pos.InstrumentID !== 'number') errors.push('Speculator Position missing numeric "InstrumentID"');
+                    if (typeof pos.PositionID !== 'number') errors.push('Speculator Position missing numeric "PositionID"');
+                    if (typeof pos.NetProfit !== 'number') errors.push('Speculator Position missing numeric "NetProfit"');
+                    if (typeof pos.Leverage !== 'number') errors.push('Speculator Position missing numeric "Leverage"');
+                    if (typeof pos.OpenRate !== 'number') errors.push('Speculator Position missing numeric "OpenRate"');
+                    if (typeof pos.IsBuy !== 'boolean') errors.push('Speculator Position missing boolean "IsBuy"');
+                }
+            }
+            
+            // Root level metrics (Speculator specific)
+            if (typeof portfolio.NetProfit !== 'number') errors.push('Speculator portfolio missing root "NetProfit"');
+            if (typeof portfolio.Invested !== 'number') errors.push('Speculator portfolio missing root "Invested"');
+        } 
+        
+        // 2. NORMAL USER PORTFOLIO VALIDATION
+        else {
+            // Must have AggregatedPositions array
+            if (!Array.isArray(portfolio.AggregatedPositions)) {
+                errors.push('Normal portfolio missing "AggregatedPositions" array');
+            } else {
+                // Validate a sample position structure
+                if (portfolio.AggregatedPositions.length > 0) {
+                    const pos = portfolio.AggregatedPositions[0];
+                    if (typeof pos.InstrumentID !== 'number') errors.push('Normal Position missing numeric "InstrumentID"');
+                    if (typeof pos.NetProfit !== 'number') errors.push('Normal Position missing numeric "NetProfit"');
+                    if (typeof pos.Invested !== 'number') errors.push('Normal Position missing numeric "Invested"');
+                    if (typeof pos.Value !== 'number') errors.push('Normal Position missing numeric "Value"');
+                    if (!pos.Direction) errors.push('Normal Position missing "Direction" string');
+                }
+            }
+        }
+
+        return { valid: errors.length === 0, errors };
+    }
+
+    /**
+     * Validates Trade History Data.
+     * @param {Object} historyDoc - The history document object.
+     * @returns {Object} { valid: boolean, errors: string[] }
+     */
+    static validateTradeHistory(historyDoc) {
+        const errors = [];
+        if (!historyDoc || typeof historyDoc !== 'object') {
+            return { valid: false, errors: ['History document is null'] };
+        }
+
+        if (!Array.isArray(historyDoc.PublicHistoryPositions)) {
+            errors.push('History missing "PublicHistoryPositions" array');
+        } else if (historyDoc.PublicHistoryPositions.length > 0) {
+            // Validate first item schema
+            const trade = historyDoc.PublicHistoryPositions[0];
+            if (typeof trade.PositionID !== 'number') errors.push('History trade missing numeric "PositionID"');
+            if (typeof trade.CID !== 'number') errors.push('History trade missing numeric "CID"');
+            if (typeof trade.InstrumentID !== 'number') errors.push('History trade missing numeric "InstrumentID"');
+            if (typeof trade.CloseRate !== 'number') errors.push('History trade missing numeric "CloseRate"');
+            if (typeof trade.CloseReason !== 'number') errors.push('History trade missing numeric "CloseReason"');
+            if (typeof trade.NetProfit !== 'number') errors.push('History trade missing numeric "NetProfit"');
+            if (!trade.OpenDateTime) errors.push('History trade missing "OpenDateTime"');
+            if (!trade.CloseDateTime) errors.push('History trade missing "CloseDateTime"');
+        }
+
+        return { valid: errors.length === 0, errors };
+    }
+
+    /**
+     * Validates Social Post Data.
+     * @param {Object} post - The social post object.
+     * @returns {Object} { valid: boolean, errors: string[] }
+     */
+    static validateSocialPost(post) {
+        const errors = [];
+        if (!post) return { valid: false, errors: ['Post is null'] };
+
+        if (!post.postOwnerId) errors.push('Post missing "postOwnerId"');
+        if (!post.fullText) errors.push('Post missing "fullText"');
+        if (!post.createdAt) errors.push('Post missing "createdAt"');
+        if (typeof post.likeCount !== 'number') errors.push('Post missing numeric "likeCount"');
+        
+        // Sentiment Map
+        if (!post.sentiment || typeof post.sentiment.overallSentiment !== 'string') {
+            errors.push('Post missing valid "sentiment.overallSentiment"');
+        }
+
+        return { valid: errors.length === 0, errors };
+    }
+
+    /**
+     * Validates Insights Data (Platform Ownership).
+     * @param {Object} insight - A single insight item from the array.
+     * @returns {Object} { valid: boolean, errors: string[] }
+     */
+    static validateInsight(insight) {
+        const errors = [];
+        if (!insight) return { valid: false, errors: ['Insight object is null'] };
+
+        if (typeof insight.instrumentId !== 'number') errors.push('Insight missing numeric "instrumentId"');
+        if (typeof insight.total !== 'number') errors.push('Insight missing numeric "total" (Total Owners)');
+        if (typeof insight.percentage !== 'number') errors.push('Insight missing numeric "percentage" (Global Ownership)');
+        if (typeof insight.growth !== 'number') errors.push('Insight missing numeric "growth"');
+        
+        // Ownership split check
+        if (typeof insight.buy !== 'number') errors.push('Insight missing numeric "buy" %');
+        if (typeof insight.sell !== 'number') errors.push('Insight missing numeric "sell" %');
+
+        return { valid: errors.length === 0, errors };
+    }
+
+    /**
+     * Validates Asset Price Shard Data.
+     * @param {Object} instrumentData - The price data map for a specific instrument ID.
+     * @returns {Object} { valid: boolean, errors: string[] }
+     */
+    static validatePriceData(instrumentData) {
+        const errors = [];
+        if (!instrumentData) return { valid: false, errors: ['Price data is null'] };
+
+        // Must have 'prices' map
+        if (!instrumentData.prices || typeof instrumentData.prices !== 'object') {
+            errors.push('Instrument price data missing "prices" map');
+        } else {
+            // Check at least one price key matches date format YYYY-MM-DD
+            const keys = Object.keys(instrumentData.prices);
+            if (keys.length > 0) {
+                const sampleKey = keys[0];
+                const sampleVal = instrumentData.prices[sampleKey];
+                
+                if (!/^\d{4}-\d{2}-\d{2}$/.test(sampleKey)) {
+                    errors.push(`Price map key "${sampleKey}" does not match YYYY-MM-DD format`);
+                }
+                if (typeof sampleVal !== 'number') {
+                    errors.push('Price value is not a number');
+                }
+            }
+        }
+
+        return { valid: errors.length === 0, errors };
+    }
+}
+
+module.exports = { Validators };

package/functions/computation-system/utils/schema_capture.js

@@ -1,64 +1,64 @@
-/**
- * @fileoverview Schema capture utility for computation outputs
- * This module batches and stores pre-defined static schemas in Firestore.
- */
-
-/**
- * Batch store schemas for multiple computations.
- * This function now expects a fully-formed schema, not sample output.
- * It strictly stamps a 'lastUpdated' field to support stale-schema filtering in the API.
- *
- * @param {object} dependencies - Contains db, logger
- * @param {object} config - Configuration object
- * @param {Array} schemas - Array of {name, category, schema, metadata} objects
- */
-async function batchStoreSchemas(dependencies, config, schemas) {
-    const { db, logger } = dependencies;
-    
-    if (config.captureSchemas === false) {
-        logger.log('INFO', '[SchemaCapture] Schema capture is disabled. Skipping.');
-        return;
-    }
-
-    const batch = db.batch();
-    const schemaCollection = config.schemaCollection || 'computation_schemas';
-    let validCount = 0;
-
-    for (const item of schemas) {
-        try {
-            if (!item.schema) {
-                logger.log('WARN', `[SchemaCapture] No schema provided for ${item.name}. Skipping.`);
-                continue;
-            }
-
-            const docRef = db.collection(schemaCollection).doc(item.name);
-            
-            // Critical: Always overwrite 'lastUpdated' to now
-            batch.set(docRef, {
-                computationName: item.name,
-                category: item.category,
-                schema: item.schema,
-                metadata: item.metadata || {},
-                lastUpdated: new Date() 
-            }, { merge: true });
-            
-            validCount++;
-
-        } catch (error) {
-            logger.log('WARN', `[SchemaCapture] Failed to add schema to batch for ${item.name}`, { errorMessage: error.message });
-        }
-    }
-
-    if (validCount > 0) {
-        try {
-            await batch.commit();
-            logger.log('INFO', `[SchemaCapture] Batch stored ${validCount} computation schemas`);
-        } catch (error) {
-            logger.log('ERROR', '[SchemaCapture] Failed to commit schema batch', { errorMessage: error.message });
-        }
-    }
-}
-
-module.exports = {
-    batchStoreSchemas
+/**
+ * @fileoverview Schema capture utility for computation outputs
+ * This module batches and stores pre-defined static schemas in Firestore.
+ */
+
+/**
+ * Batch store schemas for multiple computations.
+ * This function now expects a fully-formed schema, not sample output.
+ * It strictly stamps a 'lastUpdated' field to support stale-schema filtering in the API.
+ *
+ * @param {object} dependencies - Contains db, logger
+ * @param {object} config - Configuration object
+ * @param {Array} schemas - Array of {name, category, schema, metadata} objects
+ */
+async function batchStoreSchemas(dependencies, config, schemas) {
+    const { db, logger } = dependencies;
+    
+    if (config.captureSchemas === false) {
+        logger.log('INFO', '[SchemaCapture] Schema capture is disabled. Skipping.');
+        return;
+    }
+
+    const batch = db.batch();
+    const schemaCollection = config.schemaCollection || 'computation_schemas';
+    let validCount = 0;
+
+    for (const item of schemas) {
+        try {
+            if (!item.schema) {
+                logger.log('WARN', `[SchemaCapture] No schema provided for ${item.name}. Skipping.`);
+                continue;
+            }
+
+            const docRef = db.collection(schemaCollection).doc(item.name);
+            
+            // Critical: Always overwrite 'lastUpdated' to now
+            batch.set(docRef, {
+                computationName: item.name,
+                category: item.category,
+                schema: item.schema,
+                metadata: item.metadata || {},
+                lastUpdated: new Date() 
+            }, { merge: true });
+            
+            validCount++;
+
+        } catch (error) {
+            logger.log('WARN', `[SchemaCapture] Failed to add schema to batch for ${item.name}`, { errorMessage: error.message });
+        }
+    }
+
+    if (validCount > 0) {
+        try {
+            await batch.commit();
+            logger.log('INFO', `[SchemaCapture] Batch stored ${validCount} computation schemas`);
+        } catch (error) {
+            logger.log('ERROR', '[SchemaCapture] Failed to commit schema batch', { errorMessage: error.message });
+        }
+    }
+}
+
+module.exports = {
+    batchStoreSchemas
 };

package/functions/computation-system/utils/utils.js

@@ -2,13 +2,34 @@
  * @fileoverview Computation system sub-pipes and utils.
  * REFACTORED: Now stateless and receive dependencies where needed.
  * FIXED: 'commitBatchInChunks' now respects Firestore 10MB size limit.
+ * NEW: Added 'generateCodeHash' for version control.
  */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
+const crypto = require('crypto');
 
 /** Stage 1: Normalize a calculation name to kebab-case */
 function normalizeName(name) { return name.replace(/_/g, '-'); }
 
+/**
+ * Generates a SHA-256 hash of a code string, ignoring comments and whitespace.
+ * This effectively versions the logic.
+ * @param {string} codeString - The source code of the function/class.
+ * @returns {string} The hex hash.
+ */
+function generateCodeHash(codeString) {
+    if (!codeString) return 'unknown';
+    
+    // 1. Remove single-line comments (//...)
+    let clean = codeString.replace(/\/\/.*$/gm, '');
+    // 2. Remove multi-line comments (/*...*/)
+    clean = clean.replace(/\/\*[\s\S]*?\*\//g, '');
+    // 3. Remove all whitespace (spaces, tabs, newlines)
+    clean = clean.replace(/\s+/g, '');
+    
+    return crypto.createHash('sha256').update(clean).digest('hex');
+}
+
 /** * Stage 2: Commit a batch of writes in chunks 
  * FIXED: Now splits batches by SIZE (9MB limit) and COUNT (450 docs) 
  * to prevent "Request payload size exceeds the limit" errors.
@@ -249,4 +270,4 @@ async function getFirstDateFromPriceCollection(config, deps) {
     }
 }
 
-module.exports = { FieldValue, FieldPath, normalizeName, commitBatchInChunks, getExpectedDateStrings, getEarliestDataDates };
+module.exports = { FieldValue, FieldPath, normalizeName, commitBatchInChunks, getExpectedDateStrings, getEarliestDataDates, generateCodeHash };

package/functions/task-engine/helpers/update_helpers.js

@@ -4,6 +4,8 @@
  * (OPTIMIZED V3: Removed obsolete username lookup logic)
  * (OPTIMIZED V2: Added "Circuit Breaker" for Proxy failures)
  * (REFACTORED: Concurrency set to 1, added fallback and verbose logging)
+ * (FIXED: Improved logging clarity for Normal vs Speculator users)
+ * (FIXED: Final log now accurately reflects failure state)
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
@@ -66,6 +68,8 @@ function detectSpeculatorTargets(historyData, portfolioData) {
  */
 async function handleUpdate(task, taskId, { logger, headerManager, proxyManager, db, batchManager, pubsub }, config) {
     const { userId, instruments, instrumentId, userType } = task;
+    
+    // Normalize the loop: Speculators get specific IDs, Normal users get [undefined] to trigger one pass.
     const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
     const today = new Date().toISOString().slice(0, 10);
     const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
@@ -75,7 +79,12 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     let capturedHistory = null;
     let capturedPortfolio = null;
 
-    logger.log('TRACE', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. Instruments: ${instrumentsToProcess.join(', ')}`);
+    // Track overall success for the final log
+    let hasPortfolioErrors = false;
+
+    // FIX 1: Better Start Log
+    const scopeLog = userType === 'speculator' ? `Instruments: [${instrumentsToProcess.join(', ')}]` : 'Scope: Full Portfolio';
+    logger.log('TRACE', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. ${scopeLog}`);
 
     // --- 1. Process History Fetch (Sequentially) ---
     let historyHeader = null;
@@ -148,13 +157,16 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     logger.log('TRACE', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`); 
     
     for (const instId of instrumentsToProcess) { 
+        // FIX 2: Define a clear scope name for logging
+        const scopeName = instId ? `Instrument ${instId}` : 'Full Portfolio';
+
         if (isPrivate) { 
-            logger.log('TRACE', `[handleUpdate/${userId}] Skipping remaining instruments (User Private).`); 
+            logger.log('TRACE', `[handleUpdate/${userId}] Skipping ${scopeName} (User Private).`); 
             break;  
         }
 
         const portfolioHeader = await headerManager.selectHeader();
-        if (!portfolioHeader) { logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for instId ${instId}. Skipping this instrument.`); continue; }
+        if (!portfolioHeader) { logger.log('ERROR', `[handleUpdate/${userId}] Could not select portfolio header for ${scopeName}. Skipping.`); continue; }
 
         const portfolioUrl = userType === 'speculator'  ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
         const options = { headers: portfolioHeader.header };
@@ -165,7 +177,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         // --- PROXY ATTEMPT ---
         if (shouldTryProxy()) {
             try {
-                logger.log('TRACE', `[handleUpdate/${userId}] Attempting portfolio fetch via AppScript proxy...`);
+                logger.log('TRACE', `[handleUpdate/${userId}] Attempting fetch for ${scopeName} via AppScript proxy...`);
                 response = await proxyManager.fetch(portfolioUrl, options);
                 if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); 
                 wasPortfolioSuccess = true; 
@@ -174,7 +186,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
 
             } catch (proxyError) { 
                 recordProxyOutcome(false);
-                logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch via Proxy FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
+                logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch for ${scopeName} via Proxy FAILED. Error: ${proxyError.message}. Failures: ${_consecutiveProxyFailures}/${MAX_PROXY_FAILURES}.`, { error: proxyError.message, source: 'AppScript' });
             }
         }
 
@@ -184,9 +196,9 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                 response = await fetch(portfolioUrl, options); 
                 if (!response.ok) { const errorText = await response.text(); throw new Error(`Direct fetch failed with status ${response.status}. Response: ${errorText.substring(0, 200)}`); }
                 wasPortfolioSuccess = true; 
-                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio fetch direct success.`);
+                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio fetch for ${scopeName} direct success.`);
             } catch (fallbackError) {
-                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
+                logger.log('ERROR', `[handleUpdate/${userId}] Portfolio fetch for ${scopeName} direct fallback FAILED.`, { error: fallbackError.message, source: 'eToro/Network' });
                 wasPortfolioSuccess = false;
             }
         }
@@ -200,14 +212,16 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                 const portfolioJson = JSON.parse(body);
                 capturedPortfolio = portfolioJson; // Capture for detection
                 await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
-                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio processed successfully.`); 
+                logger.log('TRACE', `[handleUpdate/${userId}] Portfolio for ${scopeName} processed successfully.`); 
 
             } catch (parseError) { 
                 wasPortfolioSuccess = false; 
-                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE.`, { url: portfolioUrl, parseErrorMessage: parseError.message }); 
+                hasPortfolioErrors = true; // Mark error state
+                logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE for ${scopeName}.`, { url: portfolioUrl, parseErrorMessage: parseError.message }); 
             }
         } else { 
-            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch failed for instId ${instId}.`); 
+            hasPortfolioErrors = true; // Mark error state
+            logger.log('WARN', `[handleUpdate/${userId}] Portfolio fetch FAILED for ${scopeName}.`); 
         }
         
         if (proxyUsedForPortfolio) { headerManager.updatePerformance(portfolioHeader.id, wasPortfolioSuccess); }
@@ -254,14 +268,22 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         return; 
     }
 
-    // If not private, update all timestamps
+    // If not private AND no critical errors, update timestamps
+    // (We update timestamps even on partial failures for speculators to avoid infinite retry loops immediately, 
+    // relying on the next scheduled run, but for Normal users, a failure usually means we should retry later. 
+    // Current logic: Update timestamp to prevent immediate re-queueing.)
     for (const instrumentId of instrumentsToProcess) { 
         await batchManager.updateUserTimestamp(userId, userType, instrumentId); 
     }
     
     if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
     
-    logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`); 
+    // FIX 3: Honest Final Log
+    if (hasPortfolioErrors) {
+        logger.log('WARN', `[handleUpdate/${userId}] Update task finished with ERRORS. See logs above.`); 
+    } else {
+        logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`); 
+    }
 }
 
 module.exports = { handleUpdate };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.210",
+  "version": "1.0.212",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [