bulltrackers-module 1.0.277 → 1.0.279

package/functions/computation-system/persistence/ResultCommitter.js

@@ -2,6 +2,7 @@
  * @fileoverview Handles saving computation results with observability and Smart Cleanup.
  * UPDATED: Implements GZIP Compression for efficient storage.
  * UPDATED: Implements Content-Based Hashing (ResultHash) for dependency short-circuiting.
+ * UPDATED: Auto-enforces Weekend Mode validation for 'Price-Only' computations.
  */
 const { commitBatchInChunks, generateDataHash } = require('../utils/utils');
 const { updateComputationStatus }          = require('./StatusRepository');
@@ -10,7 +11,7 @@ const { generateProcessId, PROCESS_TYPES } = require('../logger/logger');
 const { HeuristicValidator }               = require('./ResultsValidator');
 const validationOverrides                  = require('../config/validation_overrides'); 
 const pLimit                               = require('p-limit'); 
-const zlib                                 = require('zlib'); // [NEW] Compression Lib
+const zlib                                 = require('zlib'); 
 
 const NON_RETRYABLE_ERRORS = [
     'PERMISSION_DENIED',     'DATA_LOSS',             'FAILED_PRECONDITION'
@@ -28,30 +29,50 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
     const pid            = generateProcessId(PROCESS_TYPES.STORAGE, passName, dStr);
     
     // Options defaults
-    const flushMode    = options.flushMode || 'STANDARD';
-    const shardIndexes = options.shardIndexes || {};
+    const flushMode        = options.flushMode || 'STANDARD';
+    const shardIndexes     = options.shardIndexes || {};
     const nextShardIndexes = {};
 
     const fanOutLimit = pLimit(10); 
 
     for (const name in stateObj) {
-        const calc = stateObj[name];
+        const calc      = stateObj[name];
         const execStats = calc._executionStats || { processedUsers: 0, skippedUsers: 0 };
         const currentShardIndex = shardIndexes[name] || 0;
 
         const runMetrics = {
-            storage: { sizeBytes: 0, isSharded: false, shardCount: 1, keys: 0 },
+            storage:    { sizeBytes: 0, isSharded: false, shardCount: 1, keys: 0 },
             validation: { isValid: true, anomalies: [] },
             execution: execStats
         };
 
         try {
             const result = await calc.getResult();
-            const overrides   = validationOverrides[calc.manifest.name] || {};
+            const configOverrides = validationOverrides[calc.manifest.name] || {};
             
+            // --- [NEW] AUTO-ENFORCE WEEKEND MODE FOR PRICE-ONLY CALCS ---
+            // If a calculation depends SOLELY on 'price', we assume market closures
+            // will cause 0s/Flatlines on weekends, so we enforce lenient validation.
+            const dataDeps = calc.manifest.rootDataDependencies || [];
+            const isPriceOnly = (dataDeps.length === 1 && dataDeps[0] === 'price');
+            
+            let effectiveOverrides = { ...configOverrides };
+            
+            if (isPriceOnly && !effectiveOverrides.weekend) {
+                // Apply strict leniency for weekend/holiday price actions
+                effectiveOverrides.weekend = {
+                    maxZeroPct: 100,
+                    maxFlatlinePct: 100,
+                    maxNullPct: 100 
+                };
+            }
+            // -----------------------------------------------------------
+
             // Validation
             if (result && Object.keys(result).length > 0) {
-                const healthCheck = HeuristicValidator.analyze(calc.manifest.name, result, overrides);
+                // [FIX] Added 'dStr' as 3rd argument to match HeuristicValidator signature
+                const healthCheck = HeuristicValidator.analyze(calc.manifest.name, result, dStr, effectiveOverrides);
+                
                 if (!healthCheck.valid) {
                     runMetrics.validation.isValid = false;
                     runMetrics.validation.anomalies.push(healthCheck.reason);
@@ -91,7 +112,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             const isMultiDate = resultKeys.length > 0 && resultKeys.every(k => /^\d{4}-\d{2}-\d{2}$/.test(k));
 
             if (isMultiDate) {
-                const datePromises = resultKeys.map((historicalDate) => fanOutLimit(async () => {
+                const datePromises  = resultKeys.map((historicalDate) => fanOutLimit(async () => {
                     const dailyData = result[historicalDate];
                     if (!dailyData || Object.keys(dailyData).length === 0) return;
 
@@ -173,17 +194,14 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
 async function writeSingleResult(result, docRef, name, dateContext, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD') {
     
-    // --- [NEW] COMPRESSION STRATEGY ---
-    // Try to compress before falling back to complex sharding
+    // --- COMPRESSION STRATEGY ---
     try {
         const jsonString = JSON.stringify(result);
         const rawBuffer = Buffer.from(jsonString);
 
-        // Only attempt if meaningful size (> 50KB)
         if (rawBuffer.length > 50 * 1024) { 
             const compressedBuffer = zlib.gzipSync(rawBuffer);
             
-            // If compressed fits in one document (< 900KB safety limit)
             if (compressedBuffer.length < 900 * 1024) {
                 logger.log('INFO', `[Compression] ${name}: Compressed ${(rawBuffer.length/1024).toFixed(0)}KB -> ${(compressedBuffer.length/1024).toFixed(0)}KB. Saved as Blob.`);
                 
@@ -194,7 +212,6 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
                     payload: compressedBuffer
                 };
                 
-                // Write immediately
                 await docRef.set(compressedPayload, { merge: true });
                 
                 return { 

package/functions/computation-system/persistence/ResultsValidator.js

@@ -24,11 +24,11 @@ class HeuristicValidator {
         const sampleSize = Math.min(totalItems, 100);
         const step = Math.floor(totalItems / sampleSize);
         
-        let zeroCount = 0;
-        let nullCount = 0;
-        let nanCount = 0; 
+        let zeroCount        = 0;
+        let nullCount        = 0;
+        let nanCount         = 0; 
         let emptyVectorCount = 0; 
-        let analyzedCount = 0;
+        let analyzedCount    = 0;
         
         const numericValues = []; 
 
@@ -82,9 +82,9 @@ class HeuristicValidator {
 
         // Default Thresholds
         let thresholds = {
-            maxZeroPct: overrides.maxZeroPct ?? 99, 
-            maxNullPct: overrides.maxNullPct ?? 90,
-            maxNanPct:  overrides.maxNanPct  ?? 0,   
+            maxZeroPct:     overrides.maxZeroPct     ?? 99, 
+            maxNullPct:     overrides.maxNullPct     ?? 90,
+            maxNanPct:      overrides.maxNanPct      ?? 0,   
             maxFlatlinePct: overrides.maxFlatlinePct ?? 95 
         };
 

package/functions/computation-system/tools/BuildReporter.js

@@ -2,15 +2,14 @@
  * @fileoverview Build Reporter & Auto-Runner.
  * Generates a "Pre-Flight" report of what the computation system WILL do.
  * REFACTORED: Strict 5-category reporting with date-based exclusion logic.
- * UPDATED: Added transactional locking to prevent duplicate reports on concurrent cold starts.
- * UPDATED: Adds 'pass' number to detail records for better waterfall visibility.
+ * UPDATED: Replaced Batch Writes with Parallel Writes to prevent DEADLINE_EXCEEDED timeouts.
+ * FIXED: Ensures 'latest' pointer updates even if detail writes fail.
  */
 
 const { analyzeDateExecution }                                             = require('../WorkflowOrchestrator');
 const { fetchComputationStatus }                                           = require('../persistence/StatusRepository');
 const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
 const { checkRootDataAvailability }                                        = require('../data/AvailabilityChecker');
-const { commitBatchInChunks }                                              = require('../persistence/FirestoreUtils');
 const pLimit                                                               = require('p-limit');
 const path                                                                 = require('path');
 const packageJson                                                          = require(path.join(__dirname, '..', '..', '..', 'package.json'));
@@ -44,8 +43,7 @@ function isDateBeforeAvailability(dateStr, calcManifest) {
 
 /**
  * AUTO-RUN ENTRY POINT
- * UPDATED: Uses transactional locking to prevent race conditions.
- * If we deploy multiple computation pass nodes simultaneously, only one should run the report.
+ * Uses transactional locking to prevent race conditions.
  */
 async function ensureBuildReport(config, dependencies, manifest) {
     const { db, logger } = dependencies;
@@ -87,7 +85,7 @@ async function ensureBuildReport(config, dependencies, manifest) {
 }
 
 /**
- * Generates the report and saves to Firestore (Sharded).
+ * Generates the report and saves to Firestore.
  */
 async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null) {
     const { db, logger } = dependencies;
@@ -152,7 +150,7 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
                 impossible: [], // Missing Data (Historical) / Impossible Dependency
                 uptodate:   [], // Hash Match (Previously "Skipped")
                 
-                // [NEW] Metadata for Verification
+                // Metadata for Verification
                 meta: {
                     totalIncluded: 0,
                     totalExpected: 0,
@@ -165,7 +163,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             dateSummary.meta.totalExpected = expectedCount;
 
             // Helper to push only if date is valid for this specific calc
-            // [UPDATED] Adds 'pass' number to the record
             const pushIfValid = (targetArray, item, extraReason = null) => {
                 const calcManifest = manifestMap.get(item.name);
                 if (calcManifest && isDateBeforeAvailability(dateStr, calcManifest)) {
@@ -186,7 +183,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             analysis.reRuns.forEach(item => pushIfValid(dateSummary.rerun, item, "Hash Mismatch"));
 
             // 3. BLOCKED (Temporary Issues)
-            // Merging 'blocked' and 'failedDependency' as both are temporary blocks
             analysis.blocked.forEach(item => pushIfValid(dateSummary.blocked, item));
             analysis.failedDependency.forEach(item => pushIfValid(dateSummary.blocked, item, "Dependency Missing"));
 
@@ -210,7 +206,7 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
                 logger.log('WARN', `[BuildReporter] ⚠️ Mismatch on ${dateStr}: Expected ${expectedCount} but got ${includedCount}.`);
             }
 
-            // ALWAYS WRITE THE REPORT (No filtering based on activity)
+            // QUEUE THE WRITE (Don't write yet)
             const detailRef = db.collection('computation_build_records').doc(buildId).collection('details').doc(dateStr);
             detailWrites.push({
                 ref: detailRef,
@@ -242,17 +238,50 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
         scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` 
     };
 
+    // 1. Write the main report header (Specific Version)
     const reportRef = db.collection('computation_build_records').doc(buildId);
     await reportRef.set(reportHeader);
 
+    // 2. Write Details (Protected & Parallelized)
+    // FIX: Using parallel individual writes instead of Batch to avoid DEADLINE_EXCEEDED
+    let detailsSuccess = true;
     if (detailWrites.length > 0) {
-        logger.log('INFO', `[BuildReporter] Writing ${detailWrites.length} detail records...`);
-        await commitBatchInChunks(config, dependencies, detailWrites, 'BuildReportDetails');
-    }
+        logger.log('INFO', `[BuildReporter] Writing ${detailWrites.length} detail records (Parallel Strategy)...`);
+        
+        try {
+            // Concurrency limit of 15 to be safe
+            const writeLimit = pLimit(15);
+            const writePromises = detailWrites.map(w => writeLimit(() => 
+                w.ref.set(w.data).catch(e => {
+                    logger.log('WARN', `[BuildReporter] Failed to write detail for ${w.ref.path}: ${e.message}`);
+                    throw e;
+                })
+            ));
+            
+            await Promise.all(writePromises);
+            logger.log('INFO', `[BuildReporter] Successfully wrote all detail records.`);
 
-    await db.collection('computation_build_records').doc('latest').set({ ...reportHeader, note: "Latest build report pointer (See subcollection for details)." });
+        } catch (detailErr) {
+            detailsSuccess = false;
+            logger.log('ERROR', `[BuildReporter] ⚠️ Failed to write some details. Report Header is preserved.`, detailErr);
+        }
+    }
 
-    logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, New: ${totalRun}.`);
+    // 3. Update 'latest' pointer
+    // This runs regardless of detail write success/failure
+    const latestMetadata = { 
+        ...reportHeader, 
+        note: detailsSuccess 
+            ? "Latest build report pointer (See subcollection for details)." 
+            : "Latest build report pointer (WARNING: Partial detail records due to write error)."
+    };
+    
+    try {
+        await db.collection('computation_build_records').doc('latest').set(latestMetadata);
+        logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, New: ${totalRun}. Pointer Updated.`);
+    } catch (pointerErr) {
+        logger.log('FATAL', `[BuildReporter] Failed to update 'latest' pointer!`, pointerErr);
+    }
     
     return {
         success: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.277",
+  "version": "1.0.279",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [