bulltrackers-module 1.0.278 → 1.0.279

package/functions/computation-system/persistence/ResultCommitter.js

@@ -11,7 +11,7 @@ const { generateProcessId, PROCESS_TYPES } = require('../logger/logger');
 const { HeuristicValidator }               = require('./ResultsValidator');
 const validationOverrides                  = require('../config/validation_overrides'); 
 const pLimit                               = require('p-limit'); 
-const zlib                                 = require('zlib'); // [NEW] Compression Lib
+const zlib                                 = require('zlib'); 
 
 const NON_RETRYABLE_ERRORS = [
     'PERMISSION_DENIED',     'DATA_LOSS',             'FAILED_PRECONDITION'
@@ -59,10 +59,11 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             let effectiveOverrides = { ...configOverrides };
             
             if (isPriceOnly && !effectiveOverrides.weekend) {
+                // Apply strict leniency for weekend/holiday price actions
                 effectiveOverrides.weekend = {
                     maxZeroPct: 100,
                     maxFlatlinePct: 100,
-                    maxNullPct: 100 // Allow full nulls (e.g. holidays)
+                    maxNullPct: 100 
                 };
             }
             // -----------------------------------------------------------
@@ -193,17 +194,14 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
 async function writeSingleResult(result, docRef, name, dateContext, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD') {
     
-    // --- [NEW] COMPRESSION STRATEGY ---
-    // Try to compress before falling back to complex sharding
+    // --- COMPRESSION STRATEGY ---
     try {
         const jsonString = JSON.stringify(result);
         const rawBuffer = Buffer.from(jsonString);
 
-        // Only attempt if meaningful size (> 50KB)
         if (rawBuffer.length > 50 * 1024) { 
             const compressedBuffer = zlib.gzipSync(rawBuffer);
             
-            // If compressed fits in one document (< 900KB safety limit)
             if (compressedBuffer.length < 900 * 1024) {
                 logger.log('INFO', `[Compression] ${name}: Compressed ${(rawBuffer.length/1024).toFixed(0)}KB -> ${(compressedBuffer.length/1024).toFixed(0)}KB. Saved as Blob.`);
                 
@@ -214,7 +212,6 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
                     payload: compressedBuffer
                 };
                 
-                // Write immediately
                 await docRef.set(compressedPayload, { merge: true });
                 
                 return { 

package/functions/computation-system/tools/BuildReporter.js

@@ -2,8 +2,7 @@
  * @fileoverview Build Reporter & Auto-Runner.
  * Generates a "Pre-Flight" report of what the computation system WILL do.
  * REFACTORED: Strict 5-category reporting with date-based exclusion logic.
- * UPDATED: Added transactional locking to prevent duplicate reports on concurrent cold starts.
- * UPDATED: Adds 'pass' number to detail records for better waterfall visibility.
+ * UPDATED: Replaced Batch Writes with Parallel Writes to prevent DEADLINE_EXCEEDED timeouts.
  * FIXED: Ensures 'latest' pointer updates even if detail writes fail.
  */
 
@@ -11,7 +10,6 @@ const { analyzeDateExecution }                                             = req
 const { fetchComputationStatus }                                           = require('../persistence/StatusRepository');
 const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
 const { checkRootDataAvailability }                                        = require('../data/AvailabilityChecker');
-const { commitBatchInChunks }                                              = require('../persistence/FirestoreUtils');
 const pLimit                                                               = require('p-limit');
 const path                                                                 = require('path');
 const packageJson                                                          = require(path.join(__dirname, '..', '..', '..', 'package.json'));
@@ -45,8 +43,7 @@ function isDateBeforeAvailability(dateStr, calcManifest) {
 
 /**
  * AUTO-RUN ENTRY POINT
- * UPDATED: Uses transactional locking to prevent race conditions.
- * If we deploy multiple computation pass nodes simultaneously, only one should run the report.
+ * Uses transactional locking to prevent race conditions.
  */
 async function ensureBuildReport(config, dependencies, manifest) {
     const { db, logger } = dependencies;
@@ -88,7 +85,7 @@ async function ensureBuildReport(config, dependencies, manifest) {
 }
 
 /**
- * Generates the report and saves to Firestore (Sharded).
+ * Generates the report and saves to Firestore.
  */
 async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null) {
     const { db, logger } = dependencies;
@@ -153,7 +150,7 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
                 impossible: [], // Missing Data (Historical) / Impossible Dependency
                 uptodate:   [], // Hash Match (Previously "Skipped")
                 
-                // [NEW] Metadata for Verification
+                // Metadata for Verification
                 meta: {
                     totalIncluded: 0,
                     totalExpected: 0,
@@ -166,7 +163,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             dateSummary.meta.totalExpected = expectedCount;
 
             // Helper to push only if date is valid for this specific calc
-            // [UPDATED] Adds 'pass' number to the record
             const pushIfValid = (targetArray, item, extraReason = null) => {
                 const calcManifest = manifestMap.get(item.name);
                 if (calcManifest && isDateBeforeAvailability(dateStr, calcManifest)) {
@@ -187,7 +183,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             analysis.reRuns.forEach(item => pushIfValid(dateSummary.rerun, item, "Hash Mismatch"));
 
             // 3. BLOCKED (Temporary Issues)
-            // Merging 'blocked' and 'failedDependency' as both are temporary blocks
             analysis.blocked.forEach(item => pushIfValid(dateSummary.blocked, item));
             analysis.failedDependency.forEach(item => pushIfValid(dateSummary.blocked, item, "Dependency Missing"));
 
@@ -211,7 +206,7 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
                 logger.log('WARN', `[BuildReporter] ⚠️ Mismatch on ${dateStr}: Expected ${expectedCount} but got ${includedCount}.`);
             }
 
-            // ALWAYS WRITE THE REPORT (No filtering based on activity)
+            // QUEUE THE WRITE (Don't write yet)
             const detailRef = db.collection('computation_build_records').doc(buildId).collection('details').doc(dateStr);
             detailWrites.push({
                 ref: detailRef,
@@ -247,22 +242,33 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const reportRef = db.collection('computation_build_records').doc(buildId);
     await reportRef.set(reportHeader);
 
-    // 2. Write Details (Protected)
-    // [FIX] We wrap this in try-catch so that if the massive detail write fails,
-    // we still update the 'latest' pointer to the new version.
+    // 2. Write Details (Protected & Parallelized)
+    // FIX: Using parallel individual writes instead of Batch to avoid DEADLINE_EXCEEDED
     let detailsSuccess = true;
     if (detailWrites.length > 0) {
-        logger.log('INFO', `[BuildReporter] Writing ${detailWrites.length} detail records...`);
+        logger.log('INFO', `[BuildReporter] Writing ${detailWrites.length} detail records (Parallel Strategy)...`);
+        
         try {
-            await commitBatchInChunks(config, dependencies, detailWrites, 'BuildReportDetails');
+            // Concurrency limit of 15 to be safe
+            const writeLimit = pLimit(15);
+            const writePromises = detailWrites.map(w => writeLimit(() => 
+                w.ref.set(w.data).catch(e => {
+                    logger.log('WARN', `[BuildReporter] Failed to write detail for ${w.ref.path}: ${e.message}`);
+                    throw e;
+                })
+            ));
+            
+            await Promise.all(writePromises);
+            logger.log('INFO', `[BuildReporter] Successfully wrote all detail records.`);
+
         } catch (detailErr) {
             detailsSuccess = false;
-            logger.log('ERROR', `[BuildReporter] ⚠️ Failed to write all details, but Report Header is saved.`, detailErr);
+            logger.log('ERROR', `[BuildReporter] ⚠️ Failed to write some details. Report Header is preserved.`, detailErr);
         }
     }
 
     // 3. Update 'latest' pointer
-    // This now runs even if details failed, preventing the version mismatch bug.
+    // This runs regardless of detail write success/failure
     const latestMetadata = { 
         ...reportHeader, 
         note: detailsSuccess 
@@ -270,9 +276,12 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             : "Latest build report pointer (WARNING: Partial detail records due to write error)."
     };
     
-    await db.collection('computation_build_records').doc('latest').set(latestMetadata);
-
-    logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, New: ${totalRun}.`);
+    try {
+        await db.collection('computation_build_records').doc('latest').set(latestMetadata);
+        logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, New: ${totalRun}. Pointer Updated.`);
+    } catch (pointerErr) {
+        logger.log('FATAL', `[BuildReporter] Failed to update 'latest' pointer!`, pointerErr);
+    }
     
     return {
         success: true,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.278",
+  "version": "1.0.279",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [