bulltrackers-module 1.0.259 → 1.0.261

package/functions/computation-system/tools/BuildReporter.js

@@ -1,14 +1,13 @@
 /**
  * @fileoverview Build Reporter & Auto-Runner.
  * Generates a "Pre-Flight" report of what the computation system WILL do.
- * UPDATED: Now fetches 'prevDailyStatus' to enforce strict historical continuity checks.
+ * UPDATED: Optimized with Parallel Status Fetches inside the date loop.
  */
 
 const { analyzeDateExecution } = require('../WorkflowOrchestrator');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
 const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
 const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
-const { FieldValue } = require('@google-cloud/firestore');
 const pLimit = require('p-limit');
 const path = require('path');
 const packageJson = require(path.join(__dirname, '..', '..', '..', 'package.json'));
@@ -21,11 +20,7 @@ const packageVersion = packageJson.version;
 async function ensureBuildReport(config, dependencies, manifest) {
     const { db, logger } = dependencies;
     const now = new Date();
-
-    // BuildId still includes timestamp for uniqueness
     const buildId = `v${packageVersion}_${now.getFullYear()}-${String(now.getMonth()+1).padStart(2,'0')}-${String(now.getDate()).padStart(2,'0')}_${String(now.getHours()).padStart(2,'0')}-${String(now.getMinutes()).padStart(2,'0')}-${String(now.getSeconds()).padStart(2,'0')}`;
-
-    // Reference to "latest" doc
     const latestRef = db.collection('computation_build_records').doc('latest');
 
     try {
@@ -38,10 +33,8 @@ async function ensureBuildReport(config, dependencies, manifest) {
         }
 
         logger.log('INFO', `[BuildReporter] 🚀 New Version Detected (${packageVersion}). Auto-running Pre-flight Report...`);
-
         await generateBuildReport(config, dependencies, manifest, 90, buildId);
-
-        // Update "latest" pointer
+        
         await latestRef.set({
             packageVersion,
             buildId,
@@ -55,11 +48,6 @@ async function ensureBuildReport(config, dependencies, manifest) {
 
 /**
  * Generates the report and saves to Firestore.
- * @param {object} config 
- * @param {object} dependencies 
- * @param {Array} manifest 
- * @param {number} daysBack - Days to simulate (Default 90)
- * @param {string} customBuildId - Optional ID override
  */
 async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null) {
     const { db, logger } = dependencies;
@@ -86,99 +74,61 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     let totalReRuns = 0;
     let totalNew = 0;
 
-    // 2. PARALLEL PROCESSING (Fix for DEADLINE_EXCEEDED)
-    // Run 20 reads in parallel. 
+    // 2. PARALLEL PROCESSING
     const limit = pLimit(20);
 
     const processingPromises = datesToCheck.map(dateStr => limit(async () => {
         try {
-            // A. Fetch REAL status from DB (What ran previously?)
-            const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
-
-            // [NEW] B. Fetch YESTERDAY'S Status (For strict continuity checks)
-            let prevDailyStatus = null;
-            
-            // Only fetch if ANY calculation in the manifest is historical
-            // (Optimization: In BuildReporter we pass the full manifest, so we check if any exist generally)
+            // [IMPROVED] Fetch all statuses in parallel
+            const fetchPromises = [
+                // A. Real status
+                fetchComputationStatus(dateStr, config, dependencies),
+                // C. Real Root Data
+                checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES)
+            ];
+
+            // B. Yesterday's Status (only if needed)
+            let prevDateStr = null;
             if (manifest.some(c => c.isHistorical)) {
                 const prevDate = new Date(dateStr + 'T00:00:00Z');
                 prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-                const prevDateStr = prevDate.toISOString().slice(0, 10);
+                prevDateStr = prevDate.toISOString().slice(0, 10);
                 
-                // Ensure we don't look before the dawn of time
                 if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
-                    prevDailyStatus = await fetchComputationStatus(prevDateStr, config, dependencies);
-                } else {
-                    prevDailyStatus = {}; // Pre-epoch is valid empty context
+                    fetchPromises.push(fetchComputationStatus(prevDateStr, config, dependencies));
                 }
             }
 
-            // C. REAL Root Data Check
-            // Uses the new Indexer logic (returns null refs, but valid status flags)
-            const availability = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
-            
+            const results = await Promise.all(fetchPromises);
+            const dailyStatus = results[0];
+            const availability = results[1];
+            // If we fetched prevStatus, it's at index 2
+            const prevDailyStatus = (prevDateStr && results[2]) ? results[2] : (prevDateStr ? {} : null);
+
             const rootDataStatus = availability ? availability.status : {
-                hasPortfolio: false,
-                hasHistory:   false,
-                hasSocial:    false,
-                hasInsights:  false,
-                hasPrices:    false
+                hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false
             };
 
             // D. Run Logic Analysis
-            // Now passes prevDailyStatus to enable the "Blocked if yesterday missing" logic
             const analysis = analyzeDateExecution(dateStr, manifest, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
 
             // E. Format Findings
             const dateSummary = {
-                willRun: [],
-                willReRun: [],
-                blocked: [],
-                impossible: [] 
+                willRun: [], willReRun: [], blocked: [], impossible: [] 
             };
 
-            // -- Runnable (New) --
-            analysis.runnable.forEach(item => {
-                dateSummary.willRun.push({ name: item.name, reason: "New / No Previous Record" });
-            });
+            analysis.runnable.forEach(item => dateSummary.willRun.push({ name: item.name, reason: "New / No Previous Record" }));
+            analysis.reRuns.forEach(item => dateSummary.willReRun.push({ name: item.name, reason: item.previousCategory ? "Migration" : "Hash Mismatch" }));
+            analysis.impossible.forEach(item => dateSummary.impossible.push({ name: item.name, reason: item.reason }));
+            [...analysis.blocked, ...analysis.failedDependency].forEach(item => dateSummary.blocked.push({ name: item.name, reason: item.reason || 'Dependency' }));
 
-            // -- Re-Runs (Hash Mismatch / Migration) --
-            analysis.reRuns.forEach(item => {
-                let reason = "Hash Mismatch";
-                let details = `Old: ${item.oldHash?.substring(0,6)}... New: ${item.newHash?.substring(0,6)}...`;
-                
-                if (item.previousCategory) {
-                    reason = "Migration";
-                    details = `Moving ${item.previousCategory} -> ${item.newCategory}`;
-                }
-
-                dateSummary.willReRun.push({ name: item.name, reason, details });
-            });
-
-            // -- Impossible (Permanent) --
-            analysis.impossible.forEach(item => {
-                dateSummary.impossible.push({ name: item.name, reason: item.reason });
-            });
-
-            // -- Blocked (Retriable) --
-            analysis.blocked.forEach(item => {
-                dateSummary.blocked.push({ name: item.name, reason: item.reason });
-            });
-            analysis.failedDependency.forEach(item => {
-                dateSummary.blocked.push({ name: item.name, reason: `Dependency Missing: ${item.missing.join(', ')}` });
-            });
-
-            // Return result for aggregation
             const hasUpdates = dateSummary.willRun.length || dateSummary.willReRun.length || dateSummary.blocked.length || dateSummary.impossible.length;
             
             return {
                 dateStr,
                 dateSummary,
                 hasUpdates,
-                stats: {
-                    new: dateSummary.willRun.length,
-                    rerun: dateSummary.willReRun.length
-                }
+                stats: { new: dateSummary.willRun.length, rerun: dateSummary.willReRun.length }
             };
 
         } catch (err) {
@@ -187,7 +137,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
         }
     }));
 
-    // Wait for all dates to process
     const results = await Promise.all(processingPromises);
 
     // 3. Aggregate Results

package/functions/computation-system/utils/schema_capture.js

@@ -1,12 +1,34 @@
 /**
  * @fileoverview Schema capture utility for computation outputs
  * This module batches and stores pre-defined static schemas in Firestore.
+ * UPDATED: Added schema validation to prevent silent batch failures.
  */
 
+/**
+ * Validates a schema object before storage.
+ * Checks for circular references and size limits.
+ * @param {object} schema 
+ * @returns {object} { valid: boolean, reason: string }
+ */
+function validateSchema(schema) {
+    try {
+        // 1. Detect circular references
+        const jsonStr = JSON.stringify(schema);
+        
+        // 2. Ensure it's not too large (Firestore limit: 1MB, reserve 100KB for metadata)
+        const size = Buffer.byteLength(jsonStr);
+        if (size > 900 * 1024) {
+            return { valid: false, reason: `Schema exceeds 900KB limit (${(size/1024).toFixed(2)} KB)` };
+        }
+        
+        return { valid: true };
+    } catch (e) {
+        return { valid: false, reason: `Serialization failed: ${e.message}` };
+    }
+}
+
 /**
  * Batch store schemas for multiple computations.
- * This function now expects a fully-formed schema, not sample output.
- * It strictly stamps a 'lastUpdated' field to support stale-schema filtering in the API.
  *
  * @param {object} dependencies - Contains db, logger
  * @param {object} config - Configuration object
@@ -31,6 +53,13 @@ async function batchStoreSchemas(dependencies, config, schemas) {
                 continue;
             }
 
+            // [IMPROVED] Validate before adding to batch
+            const validation = validateSchema(item.schema);
+            if (!validation.valid) {
+                logger.log('WARN', `[SchemaCapture] Invalid schema for ${item.name}: ${validation.reason}`);
+                continue;
+            }
+
             const docRef = db.collection(schemaCollection).doc(item.name);
             
             // Critical: Always overwrite 'lastUpdated' to now

package/index.js

@@ -27,7 +27,6 @@ const { handleUpdate }                                    = require('./functions
 
 // Computation System
 const { build: buildManifest }                            = require('./functions/computation-system/context/ManifestBuilder');
-// const { runDateComputation: runComputationPass }          = require('./functions/computation-system/WorkflowOrchestrator'); Depreciated
 const { dispatchComputationPass }                         = require('./functions/computation-system/helpers/computation_dispatcher');
 const { handleComputationTask }                           = require('./functions/computation-system/helpers/computation_worker');
 // [NEW] Import Report Tools
@@ -54,7 +53,7 @@ const { handlePost }                                      = require('./functions
 
 // NEW
 
-const { runRootDataIndexer } = require('./functions/root-data-indexer/index'); // <--- IMPORT
+const { runRootDataIndexer } = require('./functions/root-data-indexer/index'); 
 
 const core = {
   IntelligentHeaderManager,
@@ -88,7 +87,6 @@ const taskEngine = {
 };
 
 const computationSystem = {
-  // runComputationPass, Depreciated
   dispatchComputationPass,
   handleComputationTask,
   dataLoader,
@@ -112,7 +110,7 @@ const maintenance = {
   runSocialOrchestrator,
   handleSocialTask,
   runBackfillAssetPrices,
-  runRootDataIndexer, // <--- EXPORT
+  runRootDataIndexer,
 };
 
 const proxy = { handlePost };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.259",
+  "version": "1.0.261",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [