bulltrackers-module 1.0.241 → 1.0.243

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,7 +1,6 @@
 /**
  * @fileoverview Main Orchestrator. Coordinates the topological execution.
- * UPDATED: Exports analyzeDateExecution for Build Reporting tools.
- * UPDATED: Uses centralized DEFINITIVE_EARLIEST_DATES.
+ * UPDATED: Enforces Strict Historical Hash Consistency to prevent recursive data corruption.
  */
 const { normalizeName, DEFINITIVE_EARLIEST_DATES }         = require('./utils/utils');
 const { checkRootDataAvailability }                        = require('./data/AvailabilityChecker');
@@ -20,7 +19,11 @@ function groupByPass(manifest) {
     }, {}); 
 }
 
-function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap) {
+/**
+ * Analyzes whether calculations should run, be skipped, or are blocked.
+ * Now supports checking yesterday's status for chronological integrity.
+ */
+function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
     const report = {
         runnable: [],
         blocked: [],          
@@ -39,7 +42,7 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
 
         if (!stored) return false;
         
-        // Handle IMPOSSIBLE flag (stored as object property or legacy string check)
+        // Handle IMPOSSIBLE flag
         if (stored.hash === STATUS_IMPOSSIBLE) return false;
 
         if (!depManifest) return false;
@@ -56,7 +59,7 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         const storedCategory = stored ? stored.category : null;
         const currentHash = calc.hash;
 
-        // [SMART MIGRATION] Detect if category changed, independent of hash check
+        // [SMART MIGRATION] Detect if category changed
         let migrationOldCategory = null;
         if (storedCategory && storedCategory !== calc.category) {
             migrationOldCategory = storedCategory;
@@ -119,12 +122,33 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             continue;
         }
 
-        // 4. Hash & Category Check (Smart Migration Logic)
+        // 4. [NEW] Strict Historical Consistency (The Fix)
+        // If a calculation depends on history, Yesterday MUST exist AND match the current hash.
+        if (calc.isHistorical && prevDailyStatus) {
+            const yesterday = new Date(dateStr + 'T00:00:00Z');
+            yesterday.setUTCDate(yesterday.getUTCDate() - 1);
+            
+            // Only enforce check if yesterday is a valid computation date (after Start of Time)
+            if (yesterday >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
+                const prevStored = prevDailyStatus[cName];
+                
+                // BLOCK IF:
+                // 1. Yesterday doesn't exist yet (Wavefront propagation)
+                // 2. Yesterday exists but has an OLD hash (We must wait for yesterday to re-run first)
+                if (!prevStored || prevStored.hash !== currentHash) {
+                    report.blocked.push({ 
+                        name: cName, 
+                        reason: `Waiting for historical continuity (Yesterday ${!prevStored ? 'Missing' : 'Hash Mismatch'})` 
+                    });
+                    continue;
+                }
+            }
+        }
+
+        // 5. Hash & Category Check (Runnable Decision)
         if (!storedHash) { 
             report.runnable.push(calc);
         } else if (storedHash !== currentHash) {
-            // Hash Mismatch (Code Changed).
-            // Pass migration info here too, in case category ALSO changed.
             report.reRuns.push({ 
                 name: cName, 
                 oldHash: storedHash, 
@@ -132,7 +156,6 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
                 previousCategory: migrationOldCategory 
             });
         } else if (migrationOldCategory) {
-            // Hash Matches, BUT category changed. Force Re-run.
             report.reRuns.push({ 
                 name: cName, 
                 reason: 'Category Migration', 
@@ -140,7 +163,6 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
                 newCategory: calc.category 
             });
         } else {
-            // Stored Hash === Current Hash AND Category matches
             report.skipped.push({ name: cName });
         }
     }
@@ -153,19 +175,41 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
     const orchestratorPid = generateProcessId(PROCESS_TYPES.ORCHESTRATOR, passToRun, dateStr);
     const dateToProcess   = new Date(dateStr + 'T00:00:00Z');
     
-    // 1. Fetch State
+    // 1. Fetch State (Today)
     const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+
+    // 2. [NEW] Fetch State (Yesterday) if needed
+    // This allows us to perform the integrity check in the analyzer
+    let prevDailyStatus = null;
+    const needsHistory = calcsInThisPass.some(c => c.isHistorical);
     
-    // 2. Check Data Availability
-    // [UPDATE] Using centralized dates to ensure consistency with BuildReporter
+    if (needsHistory) {
+        const prevDate = new Date(dateToProcess);
+        prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+        
+        // Only fetch if yesterday is a valid computation date
+        if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
+            const prevDateStr = prevDate.toISOString().slice(0, 10);
+            try {
+                prevDailyStatus = await fetchComputationStatus(prevDateStr, config, dependencies);
+            } catch (e) {
+                logger.log('WARN', `[Orchestrator] Failed to fetch yesterday's status (${prevDateStr}). Assuming empty.`);
+                prevDailyStatus = {};
+            }
+        }
+    }
+    
+    // 3. Check Data Availability
     const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
     const rootStatus = rootData ? rootData.status : { hasPortfolio: false, hasPrices: false, hasInsights: false, hasSocial: false, hasHistory: false };
 
-    // 3. ANALYZE EXECUTION
+    // 4. ANALYZE EXECUTION
     const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
-    const analysisReport = analyzeDateExecution(dateStr, calcsInThisPass, rootStatus, dailyStatus, manifestMap);
     
-    // 4. LOG ANALYSIS
+    // Pass prevDailyStatus to the analyzer
+    const analysisReport = analyzeDateExecution(dateStr, calcsInThisPass, rootStatus, dailyStatus, manifestMap, prevDailyStatus);
+    
+    // 5. LOG ANALYSIS
     if (logger && typeof logger.logDateAnalysis === 'function') {
         logger.logDateAnalysis(dateStr, analysisReport);
     } else {
@@ -174,7 +218,7 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
         else console.log(logMsg);
     }
 
-    // 5. UPDATE STATUS FOR NON-RUNNABLE ITEMS
+    // 6. UPDATE STATUS FOR NON-RUNNABLE ITEMS
     const statusUpdates = {};
     
     analysisReport.blocked.forEach(item => statusUpdates[item.name] = { hash: false, category: 'unknown' });
@@ -185,9 +229,7 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
         await updateComputationStatus(dateStr, statusUpdates, config, dependencies);
     }
 
-    // 6. EXECUTE RUNNABLES
-    
-    // [SMART MIGRATION] Build map of items needing cleanup
+    // 7. EXECUTE RUNNABLES
     const migrationMap = {};
     analysisReport.reRuns.forEach(item => {
         if (item.previousCategory) {
@@ -200,12 +242,10 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
         ...analysisReport.reRuns.map(c => c.name) 
     ]);
 
-    // [SMART MIGRATION] Create Safe Copies with previousCategory attached
-    // We clone the manifest object so we don't pollute the global cache with run-specific flags
     const finalRunList = calcsInThisPass
         .filter(c => calcsToRunNames.has(normalizeName(c.name)))
         .map(c => {
-            const clone = { ...c }; // Shallow copy
+            const clone = { ...c }; 
             const prevCat = migrationMap[normalizeName(c.name)];
             if (prevCat) {
                 clone.previousCategory = prevCat;

package/functions/computation-system/tools/BuildReporter.js

@@ -1,16 +1,18 @@
 /**
  * @fileoverview Build Reporter & Auto-Runner.
  * Generates a "Pre-Flight" report of what the computation system WILL do.
- * Simulates execution logic (Hash Mismatches) respecting DEFINITIVE start dates.
+ * UPDATED: Removed "Smart Mocking" in favor of REAL data availability checks to detect gaps/impossible dates.
  */
 
 const { analyzeDateExecution } = require('../WorkflowOrchestrator');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
 const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
+const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
 const { FieldValue } = require('@google-cloud/firestore');
+const pLimit = require('p-limit');
 
 // Attempt to load package.json to get version. Path depends on where this is invoked.
-let packageVersion = '1.0.300';
+let packageVersion = '1.0.301'; // Bumped version to reflect logic change
 
 
 /**
@@ -60,8 +62,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const startDate = new Date();
     startDate.setDate(today.getDate() - daysBack);
     
-    // We check UP TO yesterday usually, as today might be partial. 
-    // But let's check today too to see immediate effects.
     const datesToCheck = getExpectedDateStrings(startDate, today);
     const manifestMap = new Map(manifest.map(c => [normalizeName(c.name), c]));
     
@@ -76,86 +76,109 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     let totalReRuns = 0;
     let totalNew = 0;
 
-    // 2. Iterate Dates & Simulate
-    for (const dateStr of datesToCheck) {
-        // A. Fetch REAL status from DB (What ran previously?)
-        const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+    // 2. PARALLEL PROCESSING (Fix for DEADLINE_EXCEEDED)
+    // Run 20 reads in parallel. 
+    // This is now slightly heavier because we verify root data existence, but necessary for accuracy.
+    const limit = pLimit(20);
 
-        // B. SMART MOCK Root Data
-        // We only mock "True" if the date is logically valid for that data type.
-        // This prevents impossible calculations (e.g. 2025-09-11 requiring History which starts 2025-11-05)
-        // from showing as "WillRun".
-        const dateObj = new Date(dateStr + 'T00:00:00Z');
-        
-        const mockRootDataStatus = {
-            hasPortfolio: dateObj >= DEFINITIVE_EARLIEST_DATES.portfolio,
-            hasHistory:   dateObj >= DEFINITIVE_EARLIEST_DATES.history,
-            hasSocial:    dateObj >= DEFINITIVE_EARLIEST_DATES.social,
-            hasInsights:  dateObj >= DEFINITIVE_EARLIEST_DATES.insights,
-            hasPrices:    dateObj >= DEFINITIVE_EARLIEST_DATES.price
-        };
-
-        // C. Run Logic Analysis
-        // Pass ENTIRE manifest to see global state
-        const analysis = analyzeDateExecution(dateStr, manifest, mockRootDataStatus, dailyStatus, manifestMap);
-
-        // D. Format Findings
-        const dateSummary = {
-            willRun: [],
-            willReRun: [],
-            blocked: [],
-            impossible: [] // New explicit category
-        };
-
-        // -- Runnable (New) --
-        analysis.runnable.forEach(item => {
-            dateSummary.willRun.push({ name: item.name, reason: "New / No Previous Record" });
-        });
-
-        // -- Re-Runs (Hash Mismatch / Migration) --
-        analysis.reRuns.forEach(item => {
-            let reason = "Hash Mismatch";
-            let details = `Old: ${item.oldHash?.substring(0,6)}... New: ${item.newHash?.substring(0,6)}...`;
+    const processingPromises = datesToCheck.map(dateStr => limit(async () => {
+        try {
+            // A. Fetch REAL status from DB (What ran previously?)
+            const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+
+            // B. REAL Root Data Check [FIXED]
+            // Previously we mocked this based on dates. Now we check if the data ACTUALLY exists.
+            // This ensures missing social data (even if after the start date) is flagged as IMPOSSIBLE.
+            const availability = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
             
-            if (item.previousCategory) {
-                reason = "Migration";
-                details = `Moving ${item.previousCategory} -> ${item.newCategory}`;
-            }
-
-            dateSummary.willReRun.push({ name: item.name, reason, details });
-        });
-
-        // -- Impossible (Permanent) --
-        // Mapped from analysis.impossible (Missing Root Data or Explicit Impossible)
-        analysis.impossible.forEach(item => {
-             dateSummary.impossible.push({ name: item.name, reason: item.reason }); // e.g. "Permanently Impossible"
-        });
-
-        // -- Blocked (Retriable) --
-        // Mapped from analysis.blocked (Waiting for data) AND failedDependency
-        analysis.blocked.forEach(item => {
-             dateSummary.blocked.push({ name: item.name, reason: item.reason });
-        });
-        analysis.failedDependency.forEach(item => {
-             // If a dependency failed, it's blocked, not necessarily impossible (unless dependency is impossible)
-             dateSummary.blocked.push({ name: item.name, reason: `Dependency Missing: ${item.missing.join(', ')}` });
-        });
-
-        // Only add date to report if something interesting is happening
-        if (dateSummary.willRun.length || dateSummary.willReRun.length || dateSummary.blocked.length || dateSummary.impossible.length) {
-            reportData.dates[dateStr] = dateSummary;
-            totalNew += dateSummary.willRun.length;
-            totalReRuns += dateSummary.willReRun.length;
+            const rootDataStatus = availability ? availability.status : {
+                hasPortfolio: false,
+                hasHistory:   false,
+                hasSocial:    false,
+                hasInsights:  false,
+                hasPrices:    false
+            };
+
+            // C. Run Logic Analysis
+            const analysis = analyzeDateExecution(dateStr, manifest, rootDataStatus, dailyStatus, manifestMap);
+
+            // D. Format Findings
+            const dateSummary = {
+                willRun: [],
+                willReRun: [],
+                blocked: [],
+                impossible: [] 
+            };
+
+            // -- Runnable (New) --
+            analysis.runnable.forEach(item => {
+                dateSummary.willRun.push({ name: item.name, reason: "New / No Previous Record" });
+            });
+
+            // -- Re-Runs (Hash Mismatch / Migration) --
+            analysis.reRuns.forEach(item => {
+                let reason = "Hash Mismatch";
+                let details = `Old: ${item.oldHash?.substring(0,6)}... New: ${item.newHash?.substring(0,6)}...`;
+                
+                if (item.previousCategory) {
+                    reason = "Migration";
+                    details = `Moving ${item.previousCategory} -> ${item.newCategory}`;
+                }
+
+                dateSummary.willReRun.push({ name: item.name, reason, details });
+            });
+
+            // -- Impossible (Permanent) --
+            analysis.impossible.forEach(item => {
+                dateSummary.impossible.push({ name: item.name, reason: item.reason });
+            });
+
+            // -- Blocked (Retriable) --
+            analysis.blocked.forEach(item => {
+                dateSummary.blocked.push({ name: item.name, reason: item.reason });
+            });
+            analysis.failedDependency.forEach(item => {
+                dateSummary.blocked.push({ name: item.name, reason: `Dependency Missing: ${item.missing.join(', ')}` });
+            });
+
+            // Return result for aggregation
+            const hasUpdates = dateSummary.willRun.length || dateSummary.willReRun.length || dateSummary.blocked.length || dateSummary.impossible.length;
+            
+            return {
+                dateStr,
+                dateSummary,
+                hasUpdates,
+                stats: {
+                    new: dateSummary.willRun.length,
+                    rerun: dateSummary.willReRun.length
+                }
+            };
+
+        } catch (err) {
+            logger.log('ERROR', `[BuildReporter] Error analyzing date ${dateStr}: ${err.message}`);
+            return null;
         }
-    }
+    }));
+
+    // Wait for all dates to process
+    const results = await Promise.all(processingPromises);
+
+    // 3. Aggregate Results
+    results.forEach(res => {
+        if (res && res.hasUpdates) {
+            reportData.dates[res.dateStr] = res.dateSummary;
+            totalNew += res.stats.new;
+            totalReRuns += res.stats.rerun;
+        }
+    });
 
     reportData.summary = { totalReRuns, totalNew, scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` };
 
-    // 3. Store Report
+    // 4. Store Report
     const reportRef = db.collection('computation_build_records').doc(buildId);
     await reportRef.set(reportData);
 
-    // 4. Update 'latest' pointer
+    // 5. Update 'latest' pointer
     await db.collection('computation_build_records').doc('latest').set({
         ...reportData,
         note: "Latest build report pointer."

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.241",
+  "version": "1.0.243",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [