bulltrackers-module 1.0.266 → 1.0.268

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,6 +1,7 @@
 /**
  * @fileoverview Main Orchestrator. Coordinates the topological execution.
  * UPDATED: Implements Smart Audit logic to detect WHY a hash mismatch occurred.
+ * FIX: Added 'Audit Upgrade' check to force re-run if composition metadata is missing.
  */
 const { normalizeName, DEFINITIVE_EARLIEST_DATES }         = require('./utils/utils');
 const { checkRootDataAvailability, checkRootDependencies } = require('./data/AvailabilityChecker');
@@ -50,6 +51,7 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         const markRunnable = (isReRun = false, reRunDetails = null) => {
             if (isReRun) report.reRuns.push(reRunDetails);
             else report.runnable.push({ name: cName, ...reRunDetails });
+            // Simulate success so dependents can pass their check
             simulationStatus[cName] = { hash: currentHash, category: calc.category, composition: calc.composition };
         };
 
@@ -116,22 +118,17 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             const newComp = calc.composition;
 
             if (oldComp && newComp) {
-                // 1. Check Code
                 if (oldComp.code !== newComp.code) {
                     changeReason = "Code Changed";
                 }
-                // 2. Check Layers
                 else if (JSON.stringify(oldComp.layers) !== JSON.stringify(newComp.layers)) {
-                    // Find specific layer
                     const changedLayers = [];
                     for(const lKey in newComp.layers) {
                         if (newComp.layers[lKey] !== oldComp.layers[lKey]) changedLayers.push(lKey);
                     }
                     changeReason = `Layer Update: [${changedLayers.join(', ')}]`;
                 }
-                // 3. Check Dependencies
                 else if (JSON.stringify(oldComp.deps) !== JSON.stringify(newComp.deps)) {
-                    // Find specific dep
                     const changedDeps = [];
                     for(const dKey in newComp.deps) {
                         if (newComp.deps[dKey] !== oldComp.deps[dKey]) changedDeps.push(dKey);
@@ -150,12 +147,21 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
                 oldHash: storedHash, 
                 newHash: currentHash, 
                 previousCategory: migrationOldCategory,
-                reason: changeReason // <--- Passed to Reporter
+                reason: changeReason 
             }); 
         } 
         else if (migrationOldCategory) {       
             markRunnable(true, { name: cName, reason: 'Category Migration', previousCategory: migrationOldCategory, newCategory: calc.category }); 
         } 
+        // [CRITICAL FIX] Audit Upgrade Check: Force re-run if hash matches but composition is missing (Legacy Record)
+        else if (!stored.composition) {
+            markRunnable(true, { 
+                name: cName, 
+                oldHash: storedHash, 
+                newHash: currentHash, 
+                reason: 'Audit Upgrade (Populating Composition Metadata)' 
+            });
+        }
         else { 
             report.skipped.push({ name: cName }); 
             simulationStatus[cName] = { hash: currentHash, category: calc.category, composition: calc.composition }; 

package/functions/computation-system/tools/BuildReporter.js

@@ -1,14 +1,14 @@
 /**
  * @fileoverview Build Reporter & Auto-Runner.
  * Generates a "Pre-Flight" report of what the computation system WILL do.
- * UPDATED: Fixed 'latest' document overwrite bug.
- * UPDATED: Now reports specific reasons for Re-Runs.
+ * UPDATED: Shards report details to subcollections to bypass 1MB limit on 'Invalidate All' scenarios.
  */
 
 const { analyzeDateExecution }                                             = require('../WorkflowOrchestrator');
 const { fetchComputationStatus }                                           = require('../persistence/StatusRepository');
 const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
 const { checkRootDataAvailability }                                        = require('../data/AvailabilityChecker');
+const { commitBatchInChunks }                                              = require('../persistence/FirestoreUtils'); // Reuse chunker
 const pLimit                                                               = require('p-limit');
 const path                                                                 = require('path');
 const packageJson                                                          = require(path.join(__dirname, '..', '..', '..', 'package.json'));
@@ -21,13 +21,11 @@ const packageVersion                                                       = pac
 async function ensureBuildReport(config, dependencies, manifest) {
     const { db, logger } = dependencies;
     const now = new Date();
-    // Create a standardized build ID
     const buildId = `v${packageVersion}_${now.getFullYear()}-${String(now.getMonth()+1).padStart(2,'0')}-${String(now.getDate()).padStart(2,'0')}_${String(now.getHours()).padStart(2,'0')}-${String(now.getMinutes()).padStart(2,'0')}-${String(now.getSeconds()).padStart(2,'0')}`;
     const latestRef = db.collection('computation_build_records').doc('latest');
 
     try {
         const latestDoc = await latestRef.get();
-        // Check using 'packageVersion' key to match what we store
         const priorVersion = latestDoc.exists ? latestDoc.data().packageVersion : null;
 
         if (priorVersion === packageVersion) {
@@ -37,7 +35,7 @@ async function ensureBuildReport(config, dependencies, manifest) {
 
         logger.log('INFO', `[BuildReporter] 🚀 New Version Detected (${packageVersion}). Auto-running Pre-flight Report...`);
         
-        // Run generation. This function handles writing the 'latest' document with FULL data.
+        // Scope: 90 days is fine now that we shard the output
         await generateBuildReport(config, dependencies, manifest, 90, buildId);
 
     } catch (e) {
@@ -46,7 +44,7 @@ async function ensureBuildReport(config, dependencies, manifest) {
 }
 
 /**
- * Generates the report and saves to Firestore.
+ * Generates the report and saves to Firestore (Sharded).
  */
 async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null) {
     const { db, logger } = dependencies;
@@ -54,7 +52,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     
     logger.log('INFO', `[BuildReporter] Generating Build Report: ${buildId} (Scope: ${daysBack} days)...`);
 
-    // 1. Determine Date Range
     const today     = new Date();
     const startDate = new Date();
     startDate.setDate(today.getDate() - daysBack);
@@ -62,37 +59,33 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const datesToCheck = getExpectedDateStrings(startDate, today);
     const manifestMap  = new Map(manifest.map(c => [normalizeName(c.name), c]));
     
-    const reportData = {
+    // Main Report Header (Summary Only)
+    const reportHeader = {
         buildId,
         packageVersion: packageVersion, 
         generatedAt: new Date().toISOString(),
         summary: {},
-        dates: {}
+        _sharded: true // Flag to tell UI/Tools to look in subcollection
     };
 
     let totalReRuns = 0;
     let totalNew    = 0;
+    const detailWrites = []; // Accumulate writes for batching
 
-    // 2. PARALLEL PROCESSING
     const limit = pLimit(20);
 
     const processingPromises = datesToCheck.map(dateStr => limit(async () => {
         try {
-            // [IMPROVED] Fetch all statuses in parallel
             const fetchPromises = [
-                // A. Real status
                 fetchComputationStatus(dateStr, config, dependencies),
-                // C. Real Root Data
                 checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES)
             ];
 
-            // B. Yesterday's Status (only if needed)
             let prevDateStr = null;
             if (manifest.some(c => c.isHistorical)) {
                 const prevDate = new Date(dateStr + 'T00:00:00Z');
                 prevDate.setUTCDate(prevDate.getUTCDate() - 1);
                 prevDateStr = prevDate.toISOString().slice(0, 10);
-                
                 if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
                     fetchPromises.push(fetchComputationStatus(prevDateStr, config, dependencies));
                 }
@@ -101,18 +94,13 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             const results = await Promise.all(fetchPromises);
             const dailyStatus = results[0];
             const availability = results[1];
-            // If we fetched prevStatus, it's at index 2
             const prevDailyStatus = (prevDateStr && results[2]) ? results[2] : (prevDateStr ? {} : null);
-
             const rootDataStatus = availability ? availability.status : { hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false };
 
-            // D. Run Logic Analysis
             const analysis = analyzeDateExecution(dateStr, manifest, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
 
-            // E. Format Findings
             const dateSummary = { willRun: [], willReRun: [], blocked: [], impossible: [] };
 
-            // Pass the generated "Reason" string through to the report
             analysis.runnable.forEach   (item => dateSummary.willRun.push    ({ name: item.name, reason: "New / No Previous Record" }));
             analysis.reRuns.forEach     (item => dateSummary.willReRun.push  ({ name: item.name, reason: item.reason || "Hash Mismatch" }));
             analysis.impossible.forEach (item => dateSummary.impossible.push ({ name: item.name, reason: item.reason }));
@@ -120,12 +108,19 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
 
             const hasUpdates = dateSummary.willRun.length || dateSummary.willReRun.length || dateSummary.blocked.length || dateSummary.impossible.length;
             
-            return {
-                dateStr,
-                dateSummary,
-                hasUpdates,
-                stats: { new: dateSummary.willRun.length, rerun: dateSummary.willReRun.length }
-            };
+            if (hasUpdates) {
+                // Prepare Write for Subcollection
+                const detailRef = db.collection('computation_build_records').doc(buildId).collection('details').doc(dateStr);
+                detailWrites.push({
+                    ref: detailRef,
+                    data: dateSummary
+                });
+
+                return {
+                    stats: { new: dateSummary.willRun.length, rerun: dateSummary.willReRun.length }
+                };
+            }
+            return null;
 
         } catch (err) {
             logger.log('ERROR', `[BuildReporter] Error analyzing date ${dateStr}: ${err.message}`);
@@ -135,30 +130,34 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
 
     const results = await Promise.all(processingPromises);
 
-    // 3. Aggregate Results
     results.forEach(res => {
-        if (res && res.hasUpdates) {
-            reportData.dates[res.dateStr] = res.dateSummary;
+        if (res) {
             totalNew += res.stats.new;
             totalReRuns += res.stats.rerun;
         }
     });
 
-    reportData.summary = { totalReRuns, totalNew, scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` };
+    reportHeader.summary = { totalReRuns, totalNew, scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` };
 
-    // 4. Store Report
+    // 1. Write Header
     const reportRef = db.collection('computation_build_records').doc(buildId);
-    await reportRef.set(reportData);
+    await reportRef.set(reportHeader);
+
+    // 2. Batch Write Details (Using FirestoreUtils to handle batching constraints)
+    if (detailWrites.length > 0) {
+        logger.log('INFO', `[BuildReporter] Writing ${detailWrites.length} detail records...`);
+        await commitBatchInChunks(config, dependencies, detailWrites, 'BuildReportDetails');
+    }
 
-    // 5. Update 'latest' pointer
-    await db.collection('computation_build_records').doc('latest').set({ ...reportData, note: "Latest build report pointer." });
+    // 3. Update 'latest' pointer (Summary only)
+    await db.collection('computation_build_records').doc('latest').set({ ...reportHeader, note: "Latest build report pointer (See subcollection for details)." });
 
     logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRuns}, New: ${totalNew}.`);
     
     return {
         success: true,
         reportId: buildId,
-        summary: reportData.summary
+        summary: reportHeader.summary
     };
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.266",
+  "version": "1.0.268",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [