bulltrackers-module 1.0.214 → 1.0.216

package/functions/computation-system/utils/utils.js

@@ -1,8 +1,5 @@
 /**
- * @fileoverview Computation system sub-pipes and utils.
- * REFACTORED: Now stateless and receive dependencies where needed.
- * FIXED: 'commitBatchInChunks' now respects Firestore 10MB size limit.
- * NEW: Added 'generateCodeHash' for version control.
+ * FILENAME: computation-system/utils/utils.js
  */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
@@ -12,27 +9,18 @@ const crypto = require('crypto');
 function normalizeName(name) { return name.replace(/_/g, '-'); }
 
 /**
- * Generates a SHA-256 hash of a code string, ignoring comments and whitespace.
- * This effectively versions the logic.
- * @param {string} codeString - The source code of the function/class.
- * @returns {string} The hex hash.
+ * Generates a SHA-256 hash of a code string.
  */
 function generateCodeHash(codeString) {
     if (!codeString) return 'unknown';
-    
-    // 1. Remove single-line comments (//...)
     let clean = codeString.replace(/\/\/.*$/gm, '');
-    // 2. Remove multi-line comments (/*...*/)
     clean = clean.replace(/\/\*[\s\S]*?\*\//g, '');
-    // 3. Remove all whitespace (spaces, tabs, newlines)
     clean = clean.replace(/\s+/g, '');
-    
     return crypto.createHash('sha256').update(clean).digest('hex');
 }
 
 /** * Stage 2: Commit a batch of writes in chunks 
- * FIXED: Now splits batches by SIZE (9MB limit) and COUNT (450 docs) 
- * to prevent "Request payload size exceeds the limit" errors.
+ * FIXED: Now respects write.options (e.g. { merge: false }) to allow overwrites/deletes.
  */
 async function commitBatchInChunks(config, deps, writes, operationName) {
     const { db, logger, calculationUtils } = deps;
@@ -43,17 +31,14 @@ async function commitBatchInChunks(config, deps, writes, operationName) {
         return;
     }
 
-    // Firestore Constraints
-    const MAX_BATCH_OPS = 300;          // Safety limit (Max 500)
-    const MAX_BATCH_BYTES = 9 * 1024 * 1024; // 9MB Safety limit (Max 10MB)
+    const MAX_BATCH_OPS = 300;
+    const MAX_BATCH_BYTES = 9 * 1024 * 1024; 
 
     let currentBatch = db.batch();
     let currentOpsCount = 0;
     let currentBytesEst = 0;
     let batchIndex = 1;
-    let totalChunks = 0; // We don't know total chunks in advance now due to dynamic sizing
 
-    // Helper to commit the current batch and reset
     const commitAndReset = async () => {
         if (currentOpsCount > 0) {
             try {
@@ -74,30 +59,25 @@ async function commitBatchInChunks(config, deps, writes, operationName) {
     };
 
     for (const write of writes) {
-        // 1. Estimate Size: JSON stringify is a decent proxy for Firestore payload size
-        // We handle potential circular refs or failures gracefully by assuming a minimum size
         let docSize = 100;
-        try {
-            if (write.data) docSize = JSON.stringify(write.data).length;
-        } catch (e) { /* ignore size check error */ }
+        try { if (write.data) docSize = JSON.stringify(write.data).length; } catch (e) { }
 
-        // 2. Warn if a SINGLE document is approaching the 1MB limit
         if (docSize > 900 * 1024) {
-            logger.log('WARN', `[${operationName}] Large document detected (~${(docSize / 1024).toFixed(2)} KB). This allows few ops per batch.`);
+            logger.log('WARN', `[${operationName}] Large document detected (~${(docSize / 1024).toFixed(2)} KB).`);
         }
 
-        // 3. Check if adding this write would overflow the batch
         if ((currentOpsCount + 1 > MAX_BATCH_OPS) || (currentBytesEst + docSize > MAX_BATCH_BYTES)) {
             await commitAndReset();
         }
 
-        // 4. Add to batch
-        currentBatch.set(write.ref, write.data, { merge: true });
+        // USE PROVIDED OPTIONS OR DEFAULT TO MERGE: TRUE
+        const options = write.options || { merge: true };
+        currentBatch.set(write.ref, write.data, options);
+        
         currentOpsCount++;
         currentBytesEst += docSize;
     }
 
-    // 5. Commit remaining
     await commitAndReset();
 }
 
@@ -112,10 +92,7 @@ function getExpectedDateStrings(startDate, endDate) {
     return dateStrings;
 }
 
-/**
- * --- NEW HELPER ---
- * Stage 4: Get the earliest date in a *flat* collection where doc IDs are dates.
- */
+/** Stage 4: Get the earliest date in a *flat* collection where doc IDs are dates. */
 async function getFirstDateFromSimpleCollection(config, deps, collectionName) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
@@ -149,22 +126,10 @@ async function getFirstDateFromCollection(config, deps, collectionName) {
     return earliestDate;
 }
 
-/** * --- MODIFIED FUNCTION ---
- * Stage 5: Determine the earliest date from *all* source data.
- */
+/** Stage 5: Determine the earliest date from *all* source data. */
 async function getEarliestDataDates(config, deps) {
     const { logger } = deps;
-    logger.log('INFO', 'Querying for earliest date from ALL source data collections...');
-
-    const [
-        investorDate,
-        speculatorDate,
-        investorHistoryDate,
-        speculatorHistoryDate,
-        insightsDate,
-        socialDate,
-        priceDate
-    ] = await Promise.all([
+    const [ investorDate, speculatorDate, investorHistoryDate, speculatorHistoryDate, insightsDate, socialDate, priceDate ] = await Promise.all([
         getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection),
         getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection),
         getFirstDateFromCollection(config, deps, config.normalUserHistoryCollection),
@@ -185,17 +150,11 @@ async function getEarliestDataDates(config, deps) {
     const earliestInsightsDate = getMinDate(insightsDate);
     const earliestSocialDate = getMinDate(socialDate);
     const earliestPriceDate = getMinDate(priceDate);
-    const absoluteEarliest = getMinDate(
-        earliestPortfolioDate,
-        earliestHistoryDate,
-        earliestInsightsDate,
-        earliestSocialDate,
-        earliestPriceDate
-    );
+    const absoluteEarliest = getMinDate(earliestPortfolioDate, earliestHistoryDate, earliestInsightsDate, earliestSocialDate, earliestPriceDate);
 
     const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
 
-    const result = {
+    return {
         portfolio: earliestPortfolioDate || new Date('2999-12-31T00:00:00Z'),
         history: earliestHistoryDate || new Date('2999-12-31T00:00:00Z'),
         insights: earliestInsightsDate || new Date('2999-12-31T00:00:00Z'),
@@ -203,71 +162,26 @@ async function getEarliestDataDates(config, deps) {
         price: earliestPriceDate || new Date('2999-12-31T00:00:00Z'),
         absoluteEarliest: absoluteEarliest || fallbackDate
     };
-
-    logger.log('INFO', 'Earliest data availability map built:', {
-        portfolio: result.portfolio.toISOString().slice(0, 10),
-        history: result.history.toISOString().slice(0, 10),
-        insights: result.insights.toISOString().slice(0, 10),
-        social: result.social.toISOString().slice(0, 10),
-        price: result.price.toISOString().slice(0, 10),
-        absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10)
-    });
-
-    return result;
 }
 
-/**
- * NEW HELPER: Get the earliest date from price collection
- */
 async function getFirstDateFromPriceCollection(config, deps) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
     const collection = config.priceCollection || 'asset_prices';
-
     try {
-        logger.log('TRACE', `[getFirstDateFromPriceCollection] Querying ${collection}...`);
-
-        const snapshot = await withRetry(
-            () => db.collection(collection).limit(10).get(),
-            `GetPriceShards(${collection})`
-        );
-
-        if (snapshot.empty) {
-            logger.log('WARN', `No price shards found in ${collection}`);
-            return null;
-        }
-
+        const snapshot = await withRetry(() => db.collection(collection).limit(10).get(), `GetPriceShards(${collection})`);
         let earliestDate = null;
-
         snapshot.forEach(doc => {
             const shardData = doc.data();
             for (const instrumentId in shardData) {
                 const instrumentData = shardData[instrumentId];
                 if (!instrumentData.prices) continue;
-
-                const dates = Object.keys(instrumentData.prices)
-                    .filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d))
-                    .sort();
-
-                if (dates.length > 0) {
-                    const firstDate = new Date(dates[0] + 'T00:00:00Z');
-                    if (!earliestDate || firstDate < earliestDate) {
-                        earliestDate = firstDate;
-                    }
-                }
+                const dates = Object.keys(instrumentData.prices).filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d)).sort();
+                if (dates.length > 0) { const firstDate = new Date(dates[0] + 'T00:00:00Z'); if (!earliestDate || firstDate < earliestDate) earliestDate = firstDate; }
             }
         });
-
-        if (earliestDate) {
-            logger.log('TRACE', `[getFirstDateFromPriceCollection] Earliest price date: ${earliestDate.toISOString().slice(0, 10)}`);
-        }
-
         return earliestDate;
-
-    } catch (e) {
-        logger.log('ERROR', `Failed to get earliest price date from ${collection}`, { errorMessage: e.message });
-        return null;
-    }
+    } catch (e) { logger.log('ERROR', `Failed to get earliest price date from ${collection}`, { errorMessage: e.message }); return null; }
 }
 
 module.exports = { FieldValue, FieldPath, normalizeName, commitBatchInChunks, getExpectedDateStrings, getEarliestDataDates, generateCodeHash };

package/functions/task-engine/helpers/update_helpers.js

@@ -5,43 +5,26 @@
  */
 
 const { FieldValue } = require('@google-cloud/firestore');
-const crypto = require('crypto'); 
+const crypto         = require('crypto'); 
 
 // --- CIRCUIT BREAKER STATE ---
 let _consecutiveProxyFailures = 0;
-const MAX_PROXY_FAILURES = 3;
+const MAX_PROXY_FAILURES      = 3;
 
-function shouldTryProxy() {
-    return _consecutiveProxyFailures < MAX_PROXY_FAILURES;
-}
+function shouldTryProxy() { return _consecutiveProxyFailures < MAX_PROXY_FAILURES; }
 
 function recordProxyOutcome(success) {
-    if (success) {
-        _consecutiveProxyFailures = 0;
-    } else {
-        _consecutiveProxyFailures++;
-    }
+    if (success) { _consecutiveProxyFailures = 0;
+    } else       { _consecutiveProxyFailures++; }
 }
 
 function detectSpeculatorTargets(historyData, portfolioData) {
     if (!historyData?.PublicHistoryPositions || !portfolioData?.AggregatedPositions) return [];
-
     const leveragedAssets = new Set();
-    for (const pos of historyData.PublicHistoryPositions) {
-        if (pos.Leverage > 1 && pos.InstrumentID) {
-            leveragedAssets.add(pos.InstrumentID);
-        }
-    }
-
+    for (const pos of historyData.PublicHistoryPositions) { if (pos.Leverage > 1 && pos.InstrumentID) { leveragedAssets.add(pos.InstrumentID); } }
     if (leveragedAssets.size === 0) return [];
-
     const targets = [];
-    for (const pos of portfolioData.AggregatedPositions) {
-        if (leveragedAssets.has(pos.InstrumentID)) {
-            targets.push(pos.InstrumentID);
-        }
-    }
-    
+    for (const pos of portfolioData.AggregatedPositions) { if (leveragedAssets.has(pos.InstrumentID)) { targets.push(pos.InstrumentID); } }
     return targets;
 }
 
@@ -49,37 +32,35 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     const { userId, instruments, instrumentId, userType } = task;
     
     const instrumentsToProcess = userType === 'speculator'  ? (instruments || [instrumentId]) : [undefined]; 
-    const today = new Date().toISOString().slice(0, 10);
-    const portfolioBlockId = `${Math.floor(parseInt(userId) / 1000000)}M`;
-    let isPrivate = false;
-    
-    let capturedHistory = null;
-    let capturedPortfolio = null;
-    let hasPortfolioErrors = false;
+    const today                = new Date().toISOString().slice(0, 10);
+    const portfolioBlockId     = `${Math.floor(parseInt(userId) / 1000000)}M`;
+    let isPrivate              = false;
+    let capturedHistory        = null;
+    let capturedPortfolio      = null;
+    let hasPortfolioErrors     = false;
 
     const scopeLog = userType === 'speculator' ? `Instruments: [${instrumentsToProcess.join(', ')}]` : 'Scope: Full Portfolio';
     logger.log('TRACE', `[handleUpdate/${userId}] Starting update task. Type: ${userType}. ${scopeLog}`);
 
     // --- 1. Process History Fetch (Sequentially) ---
-    let historyHeader = null;
-    let wasHistorySuccess = false;
+    let historyHeader       = null;
+    let wasHistorySuccess   = false;
     let proxyUsedForHistory = false;
 
     try {
         if (!batchManager.checkAndSetHistoryFetched(userId)) {
             logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch.`);
             historyHeader = await headerManager.selectHeader();
-            if (!historyHeader) { 
-                logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
+            if (!historyHeader) {  logger.log('WARN', `[handleUpdate/${userId}] Could not select history header. Skipping history.`);
             } else {
                 
                 const d = new Date();
                 d.setFullYear(d.getFullYear() - 1);
                 const oneYearAgoStr = d.toISOString();
-                const uuid = crypto.randomUUID ? crypto.randomUUID() : '0205aca7-bd37-4884-8455-f28ce1add2de'; 
+                const uuid          = crypto.randomUUID ? crypto.randomUUID() : '0205aca7-bd37-4884-8455-f28ce1add2de'; 
 
                 const historyUrl = `https://www.etoro.com/sapi/trade-data-real/history/public/credit/flat?StartTime=${oneYearAgoStr}&PageNumber=1&ItemsPerPage=30000&PublicHistoryPortfolioFilter=&CID=${userId}&client_request_id=${uuid}`;
-                const options = { headers: historyHeader.header };
+                const options    = { headers: historyHeader.header };
                 let response;
 
                 // --- PROXY ATTEMPT ---
@@ -88,7 +69,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                         logger.log('TRACE', `[handleUpdate/${userId}] Attempting history fetch via AppScript proxy...`);
                         response = await proxyManager.fetch(historyUrl, options);
                         if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`);
-                        wasHistorySuccess = true; 
+                        wasHistorySuccess   = true; 
                         proxyUsedForHistory = true;
                         recordProxyOutcome(true); 
 
@@ -113,9 +94,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
 
                 if (wasHistorySuccess) { 
                     const data = await response.json();
-                    
-                    // --- FILTER LOGIC FOR GRANULAR API ---
-                    // 0 = Manual, 1 = Stop Loss, 5 = Take Profit.
+
                     const VALID_REASONS = [0, 1, 5];
                     if (data.PublicHistoryPositions && Array.isArray(data.PublicHistoryPositions)) {
                         const originalCount = data.PublicHistoryPositions.length;
@@ -128,14 +107,9 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                     await batchManager.addToTradingHistoryBatch(userId, portfolioBlockId, today, data, userType);  
                 }
             }
-        } else { 
-            logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched).`); 
-        }
-    } catch (err) { 
-        logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message }); wasHistorySuccess = false; 
-    } finally { 
-        if (historyHeader && proxyUsedForHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } 
-    } 
+        } else { logger.log('TRACE', `[handleUpdate/${userId}] History fetch skipped (already fetched).`); }
+    } catch (err) { logger.log('ERROR', `[handleUpdate/${userId}] Unhandled error during history processing.`, { error: err.message }); wasHistorySuccess = false; 
+    } finally { if (historyHeader && proxyUsedForHistory) { headerManager.updatePerformance(historyHeader.id, wasHistorySuccess); } } 
     
     // --- 2. Process Portfolio Fetches (Sequentially) ---
     logger.log('TRACE', `[handleUpdate/${userId}] Starting ${instrumentsToProcess.length} sequential portfolio fetches.`); 
@@ -154,7 +128,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         const portfolioUrl = userType === 'speculator'  ? `${config.ETORO_API_POSITIONS_URL}?cid=${userId}&InstrumentID=${instId}` : `${config.ETORO_API_PORTFOLIO_URL}?cid=${userId}`;
         const options = { headers: portfolioHeader.header };
         let response;
-        let wasPortfolioSuccess = false;
+        let wasPortfolioSuccess   = false;
         let proxyUsedForPortfolio = false;
 
         if (shouldTryProxy()) {
@@ -162,7 +136,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                 logger.log('TRACE', `[handleUpdate/${userId}] Attempting fetch for ${scopeName} via AppScript proxy...`);
                 response = await proxyManager.fetch(portfolioUrl, options);
                 if (!response.ok) throw new Error(`AppScript proxy failed with status ${response.status}`); 
-                wasPortfolioSuccess = true; 
+                wasPortfolioSuccess   = true; 
                 proxyUsedForPortfolio = true;
                 recordProxyOutcome(true);
 
@@ -190,13 +164,13 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
 
             try {
                 const portfolioJson = JSON.parse(body);
-                capturedPortfolio = portfolioJson; 
+                capturedPortfolio   = portfolioJson; 
                 await batchManager.addToPortfolioBatch(userId, portfolioBlockId, today, portfolioJson, userType, instId);
                 logger.log('TRACE', `[handleUpdate/${userId}] Portfolio for ${scopeName} processed successfully.`); 
 
             } catch (parseError) { 
                 wasPortfolioSuccess = false; 
-                hasPortfolioErrors = true; 
+                hasPortfolioErrors  = true; 
                 logger.log('ERROR', `[handleUpdate/${userId}] FAILED TO PARSE JSON RESPONSE for ${scopeName}.`, { url: portfolioUrl, parseErrorMessage: parseError.message }); 
             }
         } else { 
@@ -210,8 +184,7 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
     if (userType !== 'speculator' && capturedHistory && capturedPortfolio && pubsub && config.PUBSUB_TOPIC_TASK_ENGINE) {
         try {
             const speculatorAssets = detectSpeculatorTargets(capturedHistory, capturedPortfolio);
-            if (speculatorAssets.length > 0) {
-                logger.log('INFO', `[handleUpdate/${userId}] DETECTED SPECULATOR BEHAVIOR. Queuing ${speculatorAssets.length} targeted updates.`);
+            if (speculatorAssets.length > 0) { logger.log('INFO', `[handleUpdate/${userId}] DETECTED SPECULATOR BEHAVIOR. Queuing ${speculatorAssets.length} targeted updates.`);
                 
                 const newTasks = speculatorAssets.map(assetId => ({
                     type: 'update',
@@ -223,16 +196,12 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
                 const dataBuffer = Buffer.from(JSON.stringify({ tasks: newTasks }));
                 await pubsub.topic(config.PUBSUB_TOPIC_TASK_ENGINE).publishMessage({ data: dataBuffer });
             }
-        } catch (detectionError) {
-            logger.log('ERROR', `[handleUpdate/${userId}] Error during Speculator Detection.`, { error: detectionError.message });
-        }
+        } catch (detectionError) { logger.log('ERROR', `[handleUpdate/${userId}] Error during Speculator Detection.`, { error: detectionError.message }); }
     }
 
     if (isPrivate) {
         logger.log('WARN', `[handleUpdate/${userId}] Removing private user from updates.`);
-        for (const instrumentId of instrumentsToProcess) { 
-            await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId); 
-        }
+        for (const instrumentId of instrumentsToProcess) { await batchManager.deleteFromTimestampBatch(userId, userType, instrumentId); }
         const blockCountsRef = db.doc(config.FIRESTORE_DOC_SPECULATOR_BLOCK_COUNTS);
         for (const instrumentId of instrumentsToProcess) {
             const incrementField = `counts.${instrumentId}_${Math.floor(userId/1e6)*1e6}`;
@@ -241,17 +210,12 @@ async function handleUpdate(task, taskId, { logger, headerManager, proxyManager,
         return; 
     }
 
-    for (const instrumentId of instrumentsToProcess) { 
-        await batchManager.updateUserTimestamp(userId, userType, instrumentId); 
-    }
+    for (const instrumentId of instrumentsToProcess) {  await batchManager.updateUserTimestamp(userId, userType, instrumentId); }
     
     if (userType === 'speculator') { await batchManager.addSpeculatorTimestampFix(userId, String(Math.floor(userId/1e6)*1e6)); }
-    
-    if (hasPortfolioErrors) {
-        logger.log('WARN', `[handleUpdate/${userId}] Update task finished with ERRORS. See logs above.`); 
-    } else {
-        logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`); 
-    }
+
+    if (hasPortfolioErrors) { logger.log('WARN', `[handleUpdate/${userId}] Update task finished with ERRORS. See logs above.`); 
+    } else                  { logger.log('TRACE', `[handleUpdate/${userId}] Update task finished successfully.`);  }
 }
 
 module.exports = { handleUpdate };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.214",
+  "version": "1.0.216",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [