bulltrackers-module 1.0.241 → 1.0.242

package/functions/computation-system/tools/BuildReporter.js

@@ -2,12 +2,14 @@
  * @fileoverview Build Reporter & Auto-Runner.
  * Generates a "Pre-Flight" report of what the computation system WILL do.
  * Simulates execution logic (Hash Mismatches) respecting DEFINITIVE start dates.
+ * UPDATED: Implements Parallel Execution to prevent DEADLINE_EXCEEDED on 90-day scans.
  */
 
 const { analyzeDateExecution } = require('../WorkflowOrchestrator');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
 const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
 const { FieldValue } = require('@google-cloud/firestore');
+const pLimit = require('p-limit');
 
 // Attempt to load package.json to get version. Path depends on where this is invoked.
 let packageVersion = '1.0.300';
@@ -60,8 +62,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const startDate = new Date();
     startDate.setDate(today.getDate() - daysBack);
     
-    // We check UP TO yesterday usually, as today might be partial. 
-    // But let's check today too to see immediate effects.
     const datesToCheck = getExpectedDateStrings(startDate, today);
     const manifestMap = new Map(manifest.map(c => [normalizeName(c.name), c]));
     
@@ -76,86 +76,106 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     let totalReRuns = 0;
     let totalNew = 0;
 
-    // 2. Iterate Dates & Simulate
-    for (const dateStr of datesToCheck) {
-        // A. Fetch REAL status from DB (What ran previously?)
-        const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
-
-        // B. SMART MOCK Root Data
-        // We only mock "True" if the date is logically valid for that data type.
-        // This prevents impossible calculations (e.g. 2025-09-11 requiring History which starts 2025-11-05)
-        // from showing as "WillRun".
-        const dateObj = new Date(dateStr + 'T00:00:00Z');
-        
-        const mockRootDataStatus = {
-            hasPortfolio: dateObj >= DEFINITIVE_EARLIEST_DATES.portfolio,
-            hasHistory:   dateObj >= DEFINITIVE_EARLIEST_DATES.history,
-            hasSocial:    dateObj >= DEFINITIVE_EARLIEST_DATES.social,
-            hasInsights:  dateObj >= DEFINITIVE_EARLIEST_DATES.insights,
-            hasPrices:    dateObj >= DEFINITIVE_EARLIEST_DATES.price
-        };
-
-        // C. Run Logic Analysis
-        // Pass ENTIRE manifest to see global state
-        const analysis = analyzeDateExecution(dateStr, manifest, mockRootDataStatus, dailyStatus, manifestMap);
-
-        // D. Format Findings
-        const dateSummary = {
-            willRun: [],
-            willReRun: [],
-            blocked: [],
-            impossible: [] // New explicit category
-        };
-
-        // -- Runnable (New) --
-        analysis.runnable.forEach(item => {
-            dateSummary.willRun.push({ name: item.name, reason: "New / No Previous Record" });
-        });
-
-        // -- Re-Runs (Hash Mismatch / Migration) --
-        analysis.reRuns.forEach(item => {
-            let reason = "Hash Mismatch";
-            let details = `Old: ${item.oldHash?.substring(0,6)}... New: ${item.newHash?.substring(0,6)}...`;
+    // 2. PARALLEL PROCESSING (Fix for DEADLINE_EXCEEDED)
+    // Run 20 reads in parallel. 
+    // 90 days / 20 concurrent = ~5 batches = ~2-3 seconds total vs 45+ seconds sequentially.
+    const limit = pLimit(20);
+
+    const processingPromises = datesToCheck.map(dateStr => limit(async () => {
+        try {
+            // A. Fetch REAL status from DB (What ran previously?)
+            const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+
+            // B. SMART MOCK Root Data
+            const dateObj = new Date(dateStr + 'T00:00:00Z');
+            const mockRootDataStatus = {
+                hasPortfolio: dateObj >= DEFINITIVE_EARLIEST_DATES.portfolio,
+                hasHistory:   dateObj >= DEFINITIVE_EARLIEST_DATES.history,
+                hasSocial:    dateObj >= DEFINITIVE_EARLIEST_DATES.social,
+                hasInsights:  dateObj >= DEFINITIVE_EARLIEST_DATES.insights,
+                hasPrices:    dateObj >= DEFINITIVE_EARLIEST_DATES.price
+            };
+
+            // C. Run Logic Analysis
+            const analysis = analyzeDateExecution(dateStr, manifest, mockRootDataStatus, dailyStatus, manifestMap);
+
+            // D. Format Findings
+            const dateSummary = {
+                willRun: [],
+                willReRun: [],
+                blocked: [],
+                impossible: [] 
+            };
+
+            // -- Runnable (New) --
+            analysis.runnable.forEach(item => {
+                dateSummary.willRun.push({ name: item.name, reason: "New / No Previous Record" });
+            });
+
+            // -- Re-Runs (Hash Mismatch / Migration) --
+            analysis.reRuns.forEach(item => {
+                let reason = "Hash Mismatch";
+                let details = `Old: ${item.oldHash?.substring(0,6)}... New: ${item.newHash?.substring(0,6)}...`;
+                
+                if (item.previousCategory) {
+                    reason = "Migration";
+                    details = `Moving ${item.previousCategory} -> ${item.newCategory}`;
+                }
+
+                dateSummary.willReRun.push({ name: item.name, reason, details });
+            });
+
+            // -- Impossible (Permanent) --
+            analysis.impossible.forEach(item => {
+                dateSummary.impossible.push({ name: item.name, reason: item.reason });
+            });
+
+            // -- Blocked (Retriable) --
+            analysis.blocked.forEach(item => {
+                dateSummary.blocked.push({ name: item.name, reason: item.reason });
+            });
+            analysis.failedDependency.forEach(item => {
+                dateSummary.blocked.push({ name: item.name, reason: `Dependency Missing: ${item.missing.join(', ')}` });
+            });
+
+            // Return result for aggregation
+            const hasUpdates = dateSummary.willRun.length || dateSummary.willReRun.length || dateSummary.blocked.length || dateSummary.impossible.length;
             
-            if (item.previousCategory) {
-                reason = "Migration";
-                details = `Moving ${item.previousCategory} -> ${item.newCategory}`;
-            }
-
-            dateSummary.willReRun.push({ name: item.name, reason, details });
-        });
-
-        // -- Impossible (Permanent) --
-        // Mapped from analysis.impossible (Missing Root Data or Explicit Impossible)
-        analysis.impossible.forEach(item => {
-             dateSummary.impossible.push({ name: item.name, reason: item.reason }); // e.g. "Permanently Impossible"
-        });
-
-        // -- Blocked (Retriable) --
-        // Mapped from analysis.blocked (Waiting for data) AND failedDependency
-        analysis.blocked.forEach(item => {
-             dateSummary.blocked.push({ name: item.name, reason: item.reason });
-        });
-        analysis.failedDependency.forEach(item => {
-             // If a dependency failed, it's blocked, not necessarily impossible (unless dependency is impossible)
-             dateSummary.blocked.push({ name: item.name, reason: `Dependency Missing: ${item.missing.join(', ')}` });
-        });
-
-        // Only add date to report if something interesting is happening
-        if (dateSummary.willRun.length || dateSummary.willReRun.length || dateSummary.blocked.length || dateSummary.impossible.length) {
-            reportData.dates[dateStr] = dateSummary;
-            totalNew += dateSummary.willRun.length;
-            totalReRuns += dateSummary.willReRun.length;
+            return {
+                dateStr,
+                dateSummary,
+                hasUpdates,
+                stats: {
+                    new: dateSummary.willRun.length,
+                    rerun: dateSummary.willReRun.length
+                }
+            };
+
+        } catch (err) {
+            logger.log('ERROR', `[BuildReporter] Error analyzing date ${dateStr}: ${err.message}`);
+            return null;
         }
-    }
+    }));
+
+    // Wait for all dates to process
+    const results = await Promise.all(processingPromises);
+
+    // 3. Aggregate Results
+    results.forEach(res => {
+        if (res && res.hasUpdates) {
+            reportData.dates[res.dateStr] = res.dateSummary;
+            totalNew += res.stats.new;
+            totalReRuns += res.stats.rerun;
+        }
+    });
 
     reportData.summary = { totalReRuns, totalNew, scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` };
 
-    // 3. Store Report
+    // 4. Store Report
     const reportRef = db.collection('computation_build_records').doc(buildId);
     await reportRef.set(reportData);
 
-    // 4. Update 'latest' pointer
+    // 5. Update 'latest' pointer
     await db.collection('computation_build_records').doc('latest').set({
         ...reportData,
         note: "Latest build report pointer."

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.241",
+  "version": "1.0.242",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [