bulltrackers-module 1.0.256 → 1.0.257

package/functions/computation-system/helpers/computation_worker.js

@@ -1,12 +1,13 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes computation tasks from Pub/Sub and executes them.
- * UPDATED: Simplified "Dumb Worker" - Trusts Dispatcher validation.
+ * UPDATED: Integrated Run Ledger for per-run/per-date success/failure tracking.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
 const { getManifest }         = require('../topology/ManifestLoader');
 const { StructuredLogger }    = require('../logger/logger'); 
+const { recordRunAttempt }    = require('../persistence/RunRecorder'); // [NEW IMPORT]
 
 // 1. IMPORT CALCULATIONS
 let calculationPackage;
@@ -21,7 +22,6 @@ const calculations = calculationPackage.calculations;
 
 /**
  * Handles a single Pub/Sub message.
- * Assumes the message contains a VALID, RUNNABLE task from the Smart Dispatcher.
  */
 async function handleComputationTask(message, config, dependencies) {
     
@@ -33,7 +33,7 @@ async function handleComputationTask(message, config, dependencies) {
     });
 
     const runDependencies = { ...dependencies, logger: systemLogger };
-    const { logger } = runDependencies;
+    const { logger, db } = runDependencies;
 
     // 3. PARSE PAYLOAD
     let data;
@@ -67,14 +67,19 @@ async function handleComputationTask(message, config, dependencies) {
         computationManifest = getManifest(config.activeProductLines || [], calculations, runDependencies);
     } catch (manifestError) {
         logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
+        // Record Fatal Manifest Error
+        await recordRunAttempt(db, { date, computation, pass }, 'CRASH', { 
+            message: manifestError.message, 
+            stage: 'MANIFEST_LOAD' 
+        });
         return; 
     }
 
-    // 5. EXECUTE (TRUSTED MODE)
-    // We do not check DB status or analyze feasibility. We assume Dispatcher did its job.
+    // 5. EXECUTE (With Run Ledger)
     try {
         logger.log('INFO', `[Worker] 📥 Received: ${computation} for ${date}`);
         
+        const startTime = Date.now();
         const result = await executeDispatchTask(
             date, 
             pass, 
@@ -83,13 +88,44 @@ async function handleComputationTask(message, config, dependencies) {
             runDependencies, 
             computationManifest
         );
-        
-        if (result && result.updates) {
-             logger.log('INFO', `[Worker] ✅ Stored: ${computation} for ${date}`);
+        const duration = Date.now() - startTime;
+
+        // CHECK FOR INTERNAL FAILURES (Trapped by ResultCommitter)
+        const failureReport = result?.updates?.failureReport || [];
+        const successUpdates = result?.updates?.successUpdates || {};
+
+        if (failureReport.length > 0) {
+            // Task ran, but logic or storage failed (e.g., Sharding Limit)
+            const failReason = failureReport[0]; // Assuming 1 calc per task
+            
+            logger.log('ERROR', `[Worker] ❌ Failed logic/storage for ${computation}`, failReason.error);
+            
+            await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', failReason.error, { durationMs: duration });
+            
+            // Throw error to ensure Pub/Sub retry (if transient) or Visibility (if permanent)
+            throw new Error(failReason.error.message || 'Computation Logic Failed');
+        } 
+        else if (Object.keys(successUpdates).length > 0) {
+            // Success
+            logger.log('INFO', `[Worker] ✅ Stored: ${computation} for ${date}`);
+            await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, { durationMs: duration });
+        } 
+        else {
+            // No updates, but no error (e.g. Empty Result) - Log as Success/Skipped
+            logger.log('WARN', `[Worker] ⚠️ No results produced for ${computation} (Empty?)`);
+            await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', { message: 'Empty Result' }, { durationMs: duration });
         }
 
     } catch (err) { 
-        logger.log('ERROR', `[Worker] ❌ Failed: ${computation} for ${date}: ${err.message}`); 
+        // Catch System Crashes (OOM, Timeout, Unhandled Exception)
+        logger.log('ERROR', `[Worker] ❌ Crash: ${computation} for ${date}: ${err.message}`); 
+        
+        await recordRunAttempt(db, { date, computation, pass }, 'CRASH', { 
+            message: err.message, 
+            stack: err.stack, 
+            stage: 'SYSTEM_CRASH' 
+        });
+        
         throw err; // Trigger Pub/Sub retry
     }
 }

package/functions/computation-system/persistence/ResultCommitter.js

@@ -1,22 +1,20 @@
 /**
  * @fileoverview Handles saving computation results with observability and Smart Cleanup.
- * UPDATED: Implements Audit Ledger completion logic ("Closing the Ledger").
+ * UPDATED: Returns detailed failure reports for the Run Ledger.
  */
 const { commitBatchInChunks }     = require('./FirestoreUtils');
 const { updateComputationStatus } = require('./StatusRepository');
 const { batchStoreSchemas }       = require('../utils/schema_capture');
 const { generateProcessId, PROCESS_TYPES } = require('../logger/logger');
-// Note: normalizeName is typically needed for doc IDs, but keys in stateObj are usually already normalized.
-// If not, ensure it is imported. Based on StandardExecutor, keys are normalized.
 
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
     const successUpdates = {};
+    const failureReport  = []; // [NEW] Track failures per calculation
     const schemas        = [];
-    const cleanupTasks   = []; // Tasks to delete old data
+    const cleanupTasks   = []; 
     const { logger, db } = deps;
     const pid            = generateProcessId(PROCESS_TYPES.STORAGE, passName, dStr);
     
-    // [NEW] Extract numeric pass ID from string (e.g., "Pass 1" -> "1")
     const passNum = passName.replace(/[^0-9]/g, '');
 
     for (const name in stateObj) {
@@ -24,20 +22,18 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
         try {
             const result = await calc.getResult();
             
-            // Validate Result: Check for Null, Empty Object, or Zero
+            // Validate Result
             const isEmpty = !result || 
                             (typeof result === 'object' && Object.keys(result).length === 0) ||
                             (typeof result === 'number' && result === 0);
 
             if (isEmpty) {
-                // Mark status as FALSE (Failed/Empty) so it re-runs or is flagged
                 if (calc.manifest.hash) {
                     successUpdates[name] = { 
                         hash: false, 
                         category: calc.manifest.category 
                     };
                 }
-                // Do not store empty results
                 continue; 
             }
 
@@ -48,9 +44,16 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 .collection(config.computationsSubcollection)
                 .doc(name);
 
-            const updates = await prepareAutoShardedWrites(result, mainDocRef, logger);
+            // [CRITICAL UPDATE] Catch errors specifically during Sharding/Prep
+            let updates;
+            try {
+                updates = await prepareAutoShardedWrites(result, mainDocRef, logger);
+            } catch (prepError) {
+                // If this fails, it's likely a memory or logic issue before DB commit
+                throw { message: prepError.message, stack: prepError.stack, stage: 'PREPARE_SHARDS' };
+            }
 
-            // --- [NEW] ADD AUDIT LEDGER COMPLETION TO BATCH ---
+            // Audit Ledger
             if (passNum && calc.manifest) {
                 const ledgerRef = db.collection(`computation_audit_ledger/${dStr}/passes/${passNum}/tasks`).doc(name);
                 updates.push({
@@ -64,7 +67,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     options: { merge: true }
                 });
             }
-            // --------------------------------------------------
 
             // Capture Schema
             if (calc.manifest.class.getSchema) {
@@ -81,14 +83,25 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 const totalSize = updates.reduce((acc, u) => acc + (u.data ? JSON.stringify(u.data).length : 0), 0);
                 const isSharded = updates.some(u => u.data._sharded === true);
                 
-                await commitBatchInChunks(config, deps, updates, `${name} Results`);
+                try {
+                    await commitBatchInChunks(config, deps, updates, `${name} Results`);
+                } catch (commitErr) {
+                    // Check for Firestore specific limits
+                    let stage = 'COMMIT_BATCH';
+                    let msg = commitErr.message;
+                    if (msg.includes('Transaction too big') || msg.includes('payload is too large')) {
+                        stage = 'SHARDING_LIMIT_EXCEEDED';
+                        msg = `Firestore Limit Exceeded: ${msg}`;
+                    }
+                    throw { message: msg, stack: commitErr.stack, stage };
+                }
                 
-                // Structured Storage Log
+                // Log Storage
                 if (logger && logger.logStorage) {
                     logger.logStorage(pid, name, dStr, mainDocRef.path, totalSize, isSharded);
                 }
                 
-                // Update success tracking (Include Category)
+                // Mark Success
                 if (calc.manifest.hash) {
                     successUpdates[name] = { 
                         hash: calc.manifest.hash, 
@@ -96,22 +109,29 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     };
                 }
 
-                // CHECK FOR MIGRATION CLEANUP
+                // Cleanup Migration
                 if (calc.manifest.previousCategory && calc.manifest.previousCategory !== calc.manifest.category) {
-                    logger.log('INFO', `[Migration] Scheduled cleanup for ${name} from '${calc.manifest.previousCategory}'`);
                     cleanupTasks.push(deleteOldCalculationData(dStr, calc.manifest.previousCategory, name, config, deps));
                 }
             }
         } catch (e) {
+            // [NEW] Intelligent Failure Reporting
+            const stage = e.stage || 'EXECUTION';
+            const msg   = e.message || 'Unknown error';
+            
             if (logger && logger.log) {
-                 logger.log('ERROR', `Commit failed for ${name}`, { processId: pid, error: e.message });
+                 logger.log('ERROR', `Commit failed for ${name} [${stage}]`, { processId: pid, error: msg });
             }
+            
+            failureReport.push({
+                name,
+                error: { message: msg, stack: e.stack, stage }
+            });
         }
     }
 
     if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(() => {});
 
-    // Execute Cleanup Tasks (orphaned data from category changes)
     if (cleanupTasks.length > 0) {
         await Promise.allSettled(cleanupTasks);
     }
@@ -119,12 +139,13 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
     if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
         await updateComputationStatus(dStr, successUpdates, config, deps);
     }
-    return successUpdates;
+    
+    // [UPDATE] Return both success and failures so the Worker can log them
+    return { successUpdates, failureReport };
 }
 
 /**
  * Deletes result documents from a previous category location.
- * Must handle standard docs AND sharded docs (subcollections).
  */
 async function deleteOldCalculationData(dateStr, oldCategory, calcName, config, deps) {
     const { db, logger, calculationUtils } = deps;
@@ -138,20 +159,16 @@ async function deleteOldCalculationData(dateStr, oldCategory, calcName, config,
             .collection(config.computationsSubcollection)
             .doc(calcName);
 
-        // 1. Check for Shards Subcollection
         const shardsCol = oldDocRef.collection('_shards');
         const shardsSnap = await withRetry(() => shardsCol.listDocuments(), 'ListOldShards');
         
         const batch = db.batch();
         let ops = 0;
 
-        // Delete shards
         for (const shardDoc of shardsSnap) {
             batch.delete(shardDoc);
             ops++;
         }
-
-        // Delete main doc
         batch.delete(oldDocRef);
         ops++;
 
@@ -188,7 +205,6 @@ async function prepareAutoShardedWrites(result, docRef, logger) {
     let currentChunkSize         = 0;
     let shardIndex               = 0;
 
-    // [UPDATE] Add _lastUpdated to non-sharded writes
     if ((totalSize + docPathSize) < CHUNK_LIMIT) {  
         const data = { 
             ...result, 

package/functions/computation-system/persistence/RunRecorder.js

@@ -0,0 +1,54 @@
+/**
+ * @fileoverview Utility for recording computation run attempts (The Run Ledger).
+ * Tracks success, failure, and error contexts for every execution pass.
+ */
+const { generateProcessId } = require('../logger/logger');
+
+/**
+ * Records a run attempt to the computation_run_history collection.
+ * * @param {Firestore} db - Firestore instance
+ * @param {Object} context - { date, computation, pass }
+ * @param {string} status - 'SUCCESS', 'FAILURE', or 'CRASH'
+ * @param {Object|null} error - Error object or null
+ * @param {Object} metrics - { durationMs, ... }
+ */
+async function recordRunAttempt(db, context, status, error = null, metrics = {}) {
+    if (!db || !context) return;
+
+    const { date, computation, pass } = context;
+    // Generate a unique ID for this specific run attempt
+    const runId = `${Date.now()}_${generateProcessId('run', computation, date)}`;
+    
+    const docRef = db.collection('computation_run_history')
+        .doc(date)
+        .collection('runs')
+        .doc(runId);
+
+    const entry = {
+        computationName: computation,
+        date: date,
+        pass: String(pass),
+        timestamp: new Date().toISOString(),
+        status: status,
+        metrics: metrics
+    };
+
+    if (error) {
+        entry.error = {
+            message: error.message || 'Unknown Error',
+            // Capture specific sharding/firestore stages if available
+            stage: error.stage || 'UNKNOWN',
+            code: error.code || null,
+            stack: error.stack || null
+        };
+    }
+
+    // Fire and forget (await but catch to ensure logging doesn't crash the worker)
+    try {
+        await docRef.set(entry);
+    } catch (e) {
+        console.error(`[RunRecorder] Failed to save history for ${computation}:`, e.message);
+    }
+}
+
+module.exports = { recordRunAttempt };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.256",
+  "version": "1.0.257",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [