bulltrackers-module 1.0.181 → 1.0.182

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -6,13 +6,15 @@ const {
     groupByPass, 
     checkRootDataAvailability, 
     fetchExistingResults, 
-    fetchGlobalComputationStatus,  // <--- New Import
-    updateGlobalComputationStatus, // <--- New Import
+    fetchComputationStatus,   
+    updateComputationStatus,  
     runStandardComputationPass, 
     runMetaComputationPass,
     checkRootDependencies
 } = require('./orchestration_helpers.js');
-const { getExpectedDateStrings } = require('../utils/utils.js');
+
+const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
+
 const PARALLEL_BATCH_SIZE = 7;
 
 async function runComputationPass(config, dependencies, computationManifest) {
@@ -21,7 +23,7 @@ async function runComputationPass(config, dependencies, computationManifest) {
     if (!passToRun) 
         return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
     
-    logger.log('INFO', `🚀 Starting PASS ${passToRun} with Global Status Check...`);
+    logger.log('INFO', `🚀 Starting PASS ${passToRun} (Targeting /computation_status/{YYYY-MM-DD})...`);
 
     // Hardcoded earliest dates
     const earliestDates = { portfolio: new Date('2025-09-25T00:00:00Z'), history: new Date('2025-11-05T00:00:00Z'), social: new Date('2025-10-30T00:00:00Z'), insights: new Date('2025-08-26T00:00:00Z') };
@@ -40,36 +42,47 @@ async function runComputationPass(config, dependencies, computationManifest) {
     const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
     const metaCalcs     = calcsInThisPass.filter(c => c.type === 'meta');
 
-    // 1. Fetch Global Status ONCE (Memory Cache)
-    // Returns { "2023-10-27": { calcA: true, calcB: false }, ... }
-    const globalStatusData = await fetchGlobalComputationStatus(config, dependencies);
-
-    // Helper: Check status using in-memory data
-    const shouldRun = (calc, dateStr) => {
-        const dailyStatus = globalStatusData[dateStr] || {};
-        
-        // 1. If explicitly TRUE, ignore.
-        if (dailyStatus[calc.name] === true) return false;
-
-        // 2. Check dependencies (using same in-memory status)
-        if (calc.dependencies && calc.dependencies.length > 0) {
-            const depsMet = calc.dependencies.every(depName => dailyStatus[depName] === true);
-            if (!depsMet) return false; 
-        }
-        return true;
-    };
-
-    // Process a single date and RETURN updates (do not write)
+    // Process a single date
     const processDate = async (dateStr) => {
         const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-        const standardToRun = standardCalcs.filter(c => shouldRun(c, dateStr));
-        const metaToRun     = metaCalcs.filter(c => shouldRun(c, dateStr));
+
+        // 1. Fetch Status for THIS specific date only
+        // This ensures Pass 2 sees exactly what Pass 1 wrote for this date.
+        const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+
+        // Helper: Check status using the fetched daily data
+        const shouldRun = (calc) => {
+            const cName = normalizeName(calc.name);
+            
+            // A. If recorded as TRUE -> Ignore (already ran)
+            if (dailyStatus[cName] === true) return false;
+
+            // B. If recorded as FALSE or UNDEFINED -> Run it (retry or new)
+            // But first, check if we have the necessary data dependencies.
+            
+            if (calc.dependencies && calc.dependencies.length > 0) {
+                // Check if prerequisites (from previous passes on THIS date) are complete
+                const missing = calc.dependencies.filter(depName => dailyStatus[normalizeName(depName)] !== true);
+                if (missing.length > 0) {
+                    // Dependency missing: cannot run yet.
+                    return false;
+                }
+            }
+            
+            // If we are here, status is false/undefined AND dependencies are met.
+            return true;
+        };
+
+        const standardToRun = standardCalcs.filter(shouldRun);
+        const metaToRun     = metaCalcs.filter(shouldRun);
         
-        if (!standardToRun.length && !metaToRun.length) return null; // No work
+        if (!standardToRun.length && !metaToRun.length) return null; // No work for this date
 
+        // 2. Check Root Data Availability (Portfolio, History, etc.)
         const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
         if (!rootData) return null;
 
+        // 3. Filter again based on Root Data availability
         const finalStandardToRun = standardToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
         const finalMetaToRun     = metaToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
 
@@ -85,52 +98,35 @@ async function runComputationPass(config, dependencies, computationManifest) {
             const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
             const prevDateStr     = prevDate.toISOString().slice(0, 10);
             const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
+            
+            // Note: We use skipStatusWrite=true because we want to batch write the status at the end of this function
             if (finalStandardToRun.length) {
-                const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, true); // skipStatusWrite=true
+                const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, true); 
                 Object.assign(dateUpdates, updates);
             }
             if (finalMetaToRun.length) {
-                const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, true); // skipStatusWrite=true
+                const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, true); 
                 Object.assign(dateUpdates, updates);
             }
         } catch (err) { 
             logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
-            // Mark failures
-            [...finalStandardToRun, ...finalMetaToRun].forEach(c => dateUpdates[c.name] = false);
+            [...finalStandardToRun, ...finalMetaToRun].forEach(c => dateUpdates[normalizeName(c.name)] = false);
+        }
+
+        // 4. Write "true" or "false" results for THIS specific date immediately
+        if (Object.keys(dateUpdates).length > 0) {
+            await updateComputationStatus(dateStr, dateUpdates, config, dependencies);
         }
 
-        // Return the updates for this date
         return { date: dateStr, updates: dateUpdates };
     };
 
     // Batch process dates
     for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) { 
         const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
-        
-        // Run batch in parallel
-        const results = await Promise.all(batch.map(processDate));
-        
-        // Aggregate updates from the batch
-        const batchUpdates = {};
-        let hasUpdates = false;
-        
-        results.forEach(res => {
-            if (res && res.updates && Object.keys(res.updates).length > 0) {
-                batchUpdates[res.date] = res.updates;
-                hasUpdates = true;
-                
-                // Also update our local in-memory copy so subsequent logic in this run sees it (though passes usually rely on prev days)
-                if (!globalStatusData[res.date]) globalStatusData[res.date] = {};
-                Object.assign(globalStatusData[res.date], res.updates);
-            }
-        });
-
-        // Write status ONCE per batch
-        if (hasUpdates) {
-            await updateGlobalComputationStatus(batchUpdates, config, dependencies);
-            logger.log('INFO', `[PassRunner] Batched status update for ${Object.keys(batchUpdates).length} dates.`);
-        }
+        await Promise.all(batch.map(processDate));
     }
+    
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 

package/functions/task-engine/utils/firestore_batch_manager.js

@@ -10,30 +10,31 @@ const { FieldValue } = require('@google-cloud/firestore');
 
 class FirestoreBatchManager { 
     constructor(db, headerManager, logger, config) { 
-        this.db = db; 
+        this.db            = db; 
         this.headerManager = headerManager; 
-        this.logger = logger; 
-        this.config = config; 
-        this.portfolioBatch = {}; 
-        this.timestampBatch = {}; 
-        this.tradingHistoryBatch = {}; 
+        this.logger        = logger; 
+        this.config        = config; 
+        this.portfolioBatch              = {}; 
+        this.timestampBatch              = {}; 
+        this.tradingHistoryBatch         = {}; 
         this.speculatorTimestampFixBatch = {};
 
         // Username map cache
-        this.usernameMap = new Map();
-        this.usernameMapUpdates = {};
+        this.usernameMap           = new Map();
+        this.usernameMapUpdates    = {};
         this.usernameMapLastLoaded = 0;
         
         // History fetch cache (NEW)
         this.historyFetchedUserIds = new Set();
         this.historyCacheTimestamp = Date.now();
-        this.HISTORY_CACHE_TTL_MS = config.HISTORY_CACHE_TTL_MS || 600000; 
+        this.HISTORY_CACHE_TTL_MS  = config.HISTORY_CACHE_TTL_MS || 600000; 
 
-        this.processedSpeculatorCids = new Set();
-        this.usernameMapCollectionName = config.FIRESTORE_COLLECTION_USERNAME_MAP;
-        this.normalHistoryCollectionName = config.FIRESTORE_COLLECTION_NORMAL_HISTORY;
+        this.processedSpeculatorCids         = new Set();
+        this.usernameMapCollectionName       = config.FIRESTORE_COLLECTION_USERNAME_MAP;
+        this.normalHistoryCollectionName     = config.FIRESTORE_COLLECTION_NORMAL_HISTORY;
         this.speculatorHistoryCollectionName = config.FIRESTORE_COLLECTION_SPECULATOR_HISTORY;
-        this.batchTimeout = null;
+        this.batchTimeout                    = null;
+
         logger.log('INFO', 'FirestoreBatchManager initialized.');
     }
 
@@ -50,16 +51,10 @@ class FirestoreBatchManager {
 
     _getUsernameShardId(cid) { return `cid_map_shard_${Math.floor(parseInt(cid) / 10000) % 10}`; }
 
-    // --- CRITICAL FIX: Removed aggressive timeout flush ---
-    // With sequential processing, the timer was firing too often, causing 1 write per user (expensive).
-    // Now we only flush if we hit the memory limit (MAX_BATCH_SIZE) or when explicitly called at the end.
     _scheduleFlush() {
         const maxBatch = this.config.TASK_ENGINE_MAX_BATCH_SIZE ? Number(this.config.TASK_ENGINE_MAX_BATCH_SIZE) : 400;
         const totalOps = this._estimateBatchSize();
-        if (totalOps >= maxBatch) { 
-            this.flushBatches(); 
-            return;
-        }
+        if (totalOps >= maxBatch) {  this.flushBatches();  return; }
     }
 
     _estimateBatchSize() {
@@ -144,54 +139,33 @@ class FirestoreBatchManager {
      */
     _flushDataBatch(batchData, firestoreBatch, logName) {
         let count = 0;
-        
-        // 1. Determine Shard Strategy
-        // If we expect ~1500 users in a block and want 200 users per shard:
-        // We need ceil(1500 / 200) = 8 shards total (part_0 to part_7).
-        // Any ID, no matter how random, will map to one of these 8 buckets.
         const TARGET_USERS = this.config.DISCOVERY_ORCHESTRATOR_TARGET_USERS_PER_BLOCK ? Number(this.config.DISCOVERY_ORCHESTRATOR_TARGET_USERS_PER_BLOCK) : 1500;
         const SHARD_CAPACITY = this.config.TASK_ENGINE_MAX_USERS_PER_SHARD ? Number(this.config.TASK_ENGINE_MAX_USERS_PER_SHARD) : 200;
-        
-        // Ensure at least 1 shard exists
         const TOTAL_SHARDS = Math.max(1, Math.ceil(TARGET_USERS / SHARD_CAPACITY));
-
         for (const basePath in batchData) {
             const users = batchData[basePath];
             const userIds = Object.keys(users);
             if (!userIds.length) continue;
-
             const updatesByShard = {};
-
             for (const userId of userIds) {
                 const cid = parseInt(userId, 10);
                 let shardId;
 
                 if (!isNaN(cid)) {
-                    // --- MODULO SHARDING ---
-                    // Even if IDs are 10, 1000000, 500... they will round-robin into 
-                    // the fixed set of shards (e.g. 8 shards), ensuring density.
                     const shardIndex = cid % TOTAL_SHARDS;
                     shardId = `part_${shardIndex}`;
-                } else {
-                    shardId = 'part_misc';
-                }
+                } else { shardId = 'part_misc'; }
 
-                if (!updatesByShard[shardId]) {
-                    updatesByShard[shardId] = {};
-                }
+                if (!updatesByShard[shardId]) { updatesByShard[shardId] = {}; }
                 updatesByShard[shardId][userId] = users[userId];
             }
-
             for (const shardId in updatesByShard) {
                 const chunkData = updatesByShard[shardId];
                 const docRef = this.db.collection(`${basePath}/parts`).doc(shardId);
-                // merge: true ensures we append to the doc if it was started in a previous batch
                 firestoreBatch.set(docRef, chunkData, { merge: true });
                 count++;
             }
-
             this.logger.log('INFO', `[BATCH] Staged ${userIds.length} ${logName} users into ${Object.keys(updatesByShard).length} buckets (Modulo ${TOTAL_SHARDS}) for ${basePath}.`);
-            
             delete batchData[basePath];
         }
         return count;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.181",
+  "version": "1.0.182",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [