bulltrackers-module 1.0.240 → 1.0.242

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,8 +1,9 @@
 /**
  * @fileoverview Main Orchestrator. Coordinates the topological execution.
  * UPDATED: Exports analyzeDateExecution for Build Reporting tools.
+ * UPDATED: Uses centralized DEFINITIVE_EARLIEST_DATES.
  */
-const { normalizeName }                                    = require('./utils/utils');
+const { normalizeName, DEFINITIVE_EARLIEST_DATES }         = require('./utils/utils');
 const { checkRootDataAvailability }                        = require('./data/AvailabilityChecker');
 const { fetchExistingResults }                             = require('./data/DependencyFetcher');
 const { fetchComputationStatus, updateComputationStatus }  = require('./persistence/StatusRepository');
@@ -123,6 +124,7 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             report.runnable.push(calc);
         } else if (storedHash !== currentHash) {
             // Hash Mismatch (Code Changed).
+            // Pass migration info here too, in case category ALSO changed.
             report.reRuns.push({ 
                 name: cName, 
                 oldHash: storedHash, 
@@ -155,15 +157,8 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
     const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
     
     // 2. Check Data Availability
-    const earliestDates = {
-        portfolio: new Date('2025-09-25T00:00:00Z'),
-        history:   new Date('2025-11-05T00:00:00Z'), 
-        social:    new Date('2025-10-30T00:00:00Z'),
-        insights:  new Date('2025-08-26T00:00:00Z'),
-        price:     new Date('2025-08-01T00:00:00Z')
-    };
-    
-    const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
+    // [UPDATE] Using centralized dates to ensure consistency with BuildReporter
+    const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
     const rootStatus = rootData ? rootData.status : { hasPortfolio: false, hasPrices: false, hasInsights: false, hasSocial: false, hasHistory: false };
 
     // 3. ANALYZE EXECUTION

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -1,10 +1,10 @@
 /**
  * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_dispatcher.js
  * PURPOSE: Dispatches granular computation tasks (1 task = 1 calculation/date).
- * UPDATED: Implements "Atomic Task Dispatch" for maximum reliability.
+ * UPDATED: Implements "Atomic Task Dispatch" and uses DEFINITIVE dates to prevent waste.
  */
 
-const { getExpectedDateStrings, normalizeName } = require('../utils/utils.js');
+const { getExpectedDateStrings, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
 const { groupByPass }                           = require('../WorkflowOrchestrator.js');
 const { PubSubUtils }                           = require('../../core/utils/pubsub_utils');
 
@@ -32,16 +32,9 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     logger.log('INFO', `[Dispatcher] Target Calculations: [${calcNames.join(', ')}]`);
 
     // 2. Determine Date Range
-    // (You can make this dynamic or config-driven)
-    const earliestDates = {
-        portfolio: new Date('2025-09-25T00:00:00Z'),
-        history:   new Date('2025-11-05T00:00:00Z'),
-        social:    new Date('2025-10-30T00:00:00Z'),
-        insights:  new Date('2025-08-26T00:00:00Z'),
-        price:     new Date('2025-08-01T00:00:00Z')
-    };
-
-    const passEarliestDate = Object.values(earliestDates).reduce((a, b) => a < b ? a : b);
+    // [UPDATE] Using DEFINITIVE_EARLIEST_DATES ensures we don't dispatch tasks
+    // for years before data existed (e.g. 2023), saving massive Pub/Sub costs.
+    const passEarliestDate = Object.values(DEFINITIVE_EARLIEST_DATES).reduce((a, b) => a < b ? a : b);
     const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 

package/functions/computation-system/tools/BuildReporter.js

@@ -1,16 +1,18 @@
 /**
  * @fileoverview Build Reporter & Auto-Runner.
  * Generates a "Pre-Flight" report of what the computation system WILL do.
- * Simulates execution logic (Hash Mismatches) assuming perfect data availability.
+ * Simulates execution logic (Hash Mismatches) respecting DEFINITIVE start dates.
+ * UPDATED: Implements Parallel Execution to prevent DEADLINE_EXCEEDED on 90-day scans.
  */
 
 const { analyzeDateExecution } = require('../WorkflowOrchestrator');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
-const { normalizeName, getExpectedDateStrings } = require('../utils/utils');
+const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
 const { FieldValue } = require('@google-cloud/firestore');
+const pLimit = require('p-limit');
 
 // Attempt to load package.json to get version. Path depends on where this is invoked.
-let packageVersion = '1.0.237';
+let packageVersion = '1.0.300';
 
 
 /**
@@ -33,6 +35,7 @@ async function ensureBuildReport(config, dependencies, manifest) {
         logger.log('INFO', `[BuildReporter] 🚀 New Version Detected (${buildId}). Auto-running Pre-flight Report...`);
         // We do NOT await this in the main thread if called during init, 
         // but here we are in an async function so we proceed.
+        // Defaulting to 90 days for the auto-run to capture full historical impact
         await generateBuildReport(config, dependencies, manifest, 90, buildId);
         
     } catch (e) {
@@ -45,7 +48,7 @@ async function ensureBuildReport(config, dependencies, manifest) {
  * @param {object} config 
  * @param {object} dependencies 
  * @param {Array} manifest 
- * @param {number} daysBack - Days to simulate (default 7)
+ * @param {number} daysBack - Days to simulate (Default 90)
  * @param {string} customBuildId - Optional ID override
  */
 async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null) {
@@ -59,8 +62,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const startDate = new Date();
     startDate.setDate(today.getDate() - daysBack);
     
-    // We check UP TO yesterday usually, as today might be partial. 
-    // But let's check today too to see immediate effects.
     const datesToCheck = getExpectedDateStrings(startDate, today);
     const manifestMap = new Map(manifest.map(c => [normalizeName(c.name), c]));
     
@@ -75,75 +76,106 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     let totalReRuns = 0;
     let totalNew = 0;
 
-    // 2. Iterate Dates & Simulate
-    for (const dateStr of datesToCheck) {
-        // A. Fetch REAL status from DB (What ran previously?)
-        const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
-
-        // B. MOCK Root Data Availability
-        // We force this to TRUE to prove logic changes.
-        // We want to know: "IF data exists, would this run?"
-        const mockRootDataStatus = {
-            hasPortfolio: true,
-            hasInsights: true,
-            hasSocial: true,
-            hasHistory: true,
-            hasPrices: true
-        };
-
-        // C. Run Logic Analysis
-        // Pass ENTIRE manifest to see global state
-        const analysis = analyzeDateExecution(dateStr, manifest, mockRootDataStatus, dailyStatus, manifestMap);
-
-        // D. Format Findings
-        const dateSummary = {
-            willRun: [],
-            willReRun: [],
-            blocked: []
-            // We omit 'skipped' to save space unless explicitly IMPOSSIBLE
-        };
-
-        // -- Runnable (New) --
-        analysis.runnable.forEach(item => {
-            dateSummary.willRun.push({ name: item.name, reason: "New / No Previous Record" });
-        });
-
-        // -- Re-Runs (Hash Mismatch / Migration) --
-        analysis.reRuns.forEach(item => {
-            let reason = "Hash Mismatch";
-            let details = `Old: ${item.oldHash?.substring(0,6)}... New: ${item.newHash?.substring(0,6)}...`;
+    // 2. PARALLEL PROCESSING (Fix for DEADLINE_EXCEEDED)
+    // Run 20 reads in parallel. 
+    // 90 days / 20 concurrent = ~5 batches = ~2-3 seconds total vs 45+ seconds sequentially.
+    const limit = pLimit(20);
+
+    const processingPromises = datesToCheck.map(dateStr => limit(async () => {
+        try {
+            // A. Fetch REAL status from DB (What ran previously?)
+            const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
+
+            // B. SMART MOCK Root Data
+            const dateObj = new Date(dateStr + 'T00:00:00Z');
+            const mockRootDataStatus = {
+                hasPortfolio: dateObj >= DEFINITIVE_EARLIEST_DATES.portfolio,
+                hasHistory:   dateObj >= DEFINITIVE_EARLIEST_DATES.history,
+                hasSocial:    dateObj >= DEFINITIVE_EARLIEST_DATES.social,
+                hasInsights:  dateObj >= DEFINITIVE_EARLIEST_DATES.insights,
+                hasPrices:    dateObj >= DEFINITIVE_EARLIEST_DATES.price
+            };
+
+            // C. Run Logic Analysis
+            const analysis = analyzeDateExecution(dateStr, manifest, mockRootDataStatus, dailyStatus, manifestMap);
+
+            // D. Format Findings
+            const dateSummary = {
+                willRun: [],
+                willReRun: [],
+                blocked: [],
+                impossible: [] 
+            };
+
+            // -- Runnable (New) --
+            analysis.runnable.forEach(item => {
+                dateSummary.willRun.push({ name: item.name, reason: "New / No Previous Record" });
+            });
+
+            // -- Re-Runs (Hash Mismatch / Migration) --
+            analysis.reRuns.forEach(item => {
+                let reason = "Hash Mismatch";
+                let details = `Old: ${item.oldHash?.substring(0,6)}... New: ${item.newHash?.substring(0,6)}...`;
+                
+                if (item.previousCategory) {
+                    reason = "Migration";
+                    details = `Moving ${item.previousCategory} -> ${item.newCategory}`;
+                }
+
+                dateSummary.willReRun.push({ name: item.name, reason, details });
+            });
+
+            // -- Impossible (Permanent) --
+            analysis.impossible.forEach(item => {
+                dateSummary.impossible.push({ name: item.name, reason: item.reason });
+            });
+
+            // -- Blocked (Retriable) --
+            analysis.blocked.forEach(item => {
+                dateSummary.blocked.push({ name: item.name, reason: item.reason });
+            });
+            analysis.failedDependency.forEach(item => {
+                dateSummary.blocked.push({ name: item.name, reason: `Dependency Missing: ${item.missing.join(', ')}` });
+            });
+
+            // Return result for aggregation
+            const hasUpdates = dateSummary.willRun.length || dateSummary.willReRun.length || dateSummary.blocked.length || dateSummary.impossible.length;
             
-            if (item.previousCategory) {
-                reason = "Migration";
-                details = `Moving ${item.previousCategory} -> ${item.newCategory}`;
-            }
-
-            dateSummary.willReRun.push({ name: item.name, reason, details });
-        });
-
-        // -- Blocked / Impossible --
-        analysis.impossible.forEach(item => {
-             dateSummary.blocked.push({ name: item.name, reason: item.reason }); // e.g. "Permanently Impossible"
-        });
-        analysis.failedDependency.forEach(item => {
-             dateSummary.blocked.push({ name: item.name, reason: `Dependency Missing: ${item.missing.join(', ')}` });
-        });
-
-        // Only add date to report if something interesting is happening
-        if (dateSummary.willRun.length || dateSummary.willReRun.length || dateSummary.blocked.length) {
-            reportData.dates[dateStr] = dateSummary;
-            totalNew += dateSummary.willRun.length;
-            totalReRuns += dateSummary.willReRun.length;
+            return {
+                dateStr,
+                dateSummary,
+                hasUpdates,
+                stats: {
+                    new: dateSummary.willRun.length,
+                    rerun: dateSummary.willReRun.length
+                }
+            };
+
+        } catch (err) {
+            logger.log('ERROR', `[BuildReporter] Error analyzing date ${dateStr}: ${err.message}`);
+            return null;
         }
-    }
+    }));
+
+    // Wait for all dates to process
+    const results = await Promise.all(processingPromises);
+
+    // 3. Aggregate Results
+    results.forEach(res => {
+        if (res && res.hasUpdates) {
+            reportData.dates[res.dateStr] = res.dateSummary;
+            totalNew += res.stats.new;
+            totalReRuns += res.stats.rerun;
+        }
+    });
 
     reportData.summary = { totalReRuns, totalNew, scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` };
 
-    // 3. Store Report
+    // 4. Store Report
     const reportRef = db.collection('computation_build_records').doc(buildId);
     await reportRef.set(reportData);
 
-    // 4. Update 'latest' pointer
+    // 5. Update 'latest' pointer
     await db.collection('computation_build_records').doc('latest').set({
         ...reportData,
         note: "Latest build report pointer."

package/functions/computation-system/utils/utils.js

@@ -5,6 +5,15 @@
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 const crypto                    = require('crypto');
 
+// [NEW] Single Source of Truth for Data Availability
+const DEFINITIVE_EARLIEST_DATES = {
+    portfolio: new Date('2025-09-25T00:00:00Z'),
+    history:   new Date('2025-11-05T00:00:00Z'),
+    social:    new Date('2025-10-30T00:00:00Z'),
+    insights:  new Date('2025-08-26T00:00:00Z'),
+    price:     new Date('2025-08-01T00:00:00Z')
+};
+
 /** Stage 1: Normalize a calculation name to kebab-case */
 function normalizeName(name) { return name.replace(/_/g, '-'); }
 
@@ -211,5 +220,6 @@ module.exports = {
     getExpectedDateStrings, 
     getEarliestDataDates, 
     generateCodeHash,
-    withRetry 
+    withRetry,
+    DEFINITIVE_EARLIEST_DATES // [NEW EXPORT]
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.240",
+  "version": "1.0.242",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [