bulltrackers-module 1.0.260 → 1.0.261

package/functions/computation-system/helpers/computation_worker.js

@@ -81,19 +81,28 @@ async function handleComputationTask(message, config, dependencies) {
         const successUpdates = result?.updates?.successUpdates || {};
 
         if (failureReport.length > 0) {
-            // Task ran, but logic or storage failed (e.g., Sharding Limit)
+            // Task ran, but logic or storage failed
             const failReason = failureReport[0]; // Assuming 1 calc per task
             logger.log('ERROR', `[Worker] ❌ Failed logic/storage for ${computation}`, failReason.error);
-            await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', failReason.error, { durationMs: duration });
+            
+            // Extract any metrics gathered before failure (e.g., anomalies)
+            const metrics = failReason.metrics || {};
+            metrics.durationMs = duration;
+
+            await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', failReason.error, metrics);
             throw new Error(failReason.error.message || 'Computation Logic Failed');
         } 
         else if (Object.keys(successUpdates).length > 0) {
             // Success
-            logger.log('INFO', `[Worker] ✅ Stored: ${computation} for ${date}`);
-            await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, { durationMs: duration });
+            const successData = successUpdates[computation]; // Extract specific calc data
+            const metrics = successData.metrics || {};
+            metrics.durationMs = duration;
+
+            logger.log('INFO', `[Worker] ✅ Stored: ${computation} for ${date} (${metrics.storage?.sizeBytes} bytes)`);
+            await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, metrics);
         } 
         else {
-            // No updates, but no error (e.g. Empty Result) - Log as Success/Skipped
+            // No updates, but no error (e.g. Empty Result)
             logger.log('WARN', `[Worker] ⚠️ No results produced for ${computation} (Empty?)`);
             await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', { message: 'Empty Result' }, { durationMs: duration });
         }

package/functions/computation-system/persistence/ResultCommitter.js

@@ -1,7 +1,7 @@
 /**
  * @fileoverview Handles saving computation results with observability and Smart Cleanup.
- * UPDATED: Returns detailed failure reports for the Run Ledger.
- * UPDATED: Stops retrying on non-transient errors (Permissions, Invalid Args).
+ * UPDATED: Returns detailed failure reports AND metrics for the Audit Logger.
+ * UPDATED: Stops retrying on non-transient errors.
  */
 const { commitBatchInChunks }              = require('./FirestoreUtils');
 const { updateComputationStatus }          = require('./StatusRepository');
@@ -30,6 +30,13 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
     for (const name in stateObj) {
         const calc = stateObj[name];
+        
+        // Prep metrics container
+        const runMetrics = {
+            storage: { sizeBytes: 0, isSharded: false, shardCount: 1, keys: 0 },
+            validation: { isValid: true, anomalies: [] }
+        };
+
         try {
             const result = await calc.getResult();
 
@@ -37,14 +44,30 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             const healthCheck = HeuristicValidator.analyze(calc.manifest.name, result, overrides);
 
             if (!healthCheck.valid) {
+                // If validation failed, we consider it an anomaly but we BLOCK the write (throw error)
+                runMetrics.validation.isValid = false;
+                runMetrics.validation.anomalies.push(healthCheck.reason);
                 throw { message: healthCheck.reason, stage: 'QUALITY_CIRCUIT_BREAKER' };
             }
             
+            // Check for minor anomalies (validation warnings that didn't fail) - optional implementation
+            // For now, we assume if valid=true, anomalies are empty unless we add warning logic later.
+
             const isEmpty = !result || (typeof result === 'object' && Object.keys(result).length === 0) || (typeof result === 'number' && result === 0);
             if (isEmpty) { 
-                if (calc.manifest.hash) { successUpdates[name] = {  hash: false,  category: calc.manifest.category  }; } 
+                // Log empty success
+                if (calc.manifest.hash) { 
+                    successUpdates[name] = {  
+                        hash: false,  
+                        category: calc.manifest.category,
+                        metrics: runMetrics // Return empty metrics
+                    }; 
+                } 
                 continue; 
             }
+            
+            // Calculate Key Count rough estimate
+            if (typeof result === 'object') runMetrics.storage.keys = Object.keys(result).length;
 
             const mainDocRef = db.collection(config.resultsCollection)
                 .doc(dStr)
@@ -71,7 +94,18 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 
                 try {
                     // 1. Prepare Shards with current constraints
+                    // This function now needs to help us determine sharding info
                     const updates = await prepareAutoShardedWrites(result, mainDocRef, logger, constraints.bytes, constraints.keys);
+                    
+                    // METRICS CALCULATION
+                    const pointer = updates.find(u => u.data._completed === true);
+                    const isSharded = pointer && pointer.data._sharded === true;
+                    const shardCount = isSharded ? (pointer.data._shardCount || 1) : 1;
+                    const totalSize = updates.reduce((acc, u) => acc + (u.data ? JSON.stringify(u.data).length : 0), 0);
+                    
+                    runMetrics.storage.sizeBytes = totalSize;
+                    runMetrics.storage.isSharded = isSharded;
+                    runMetrics.storage.shardCount = shardCount;
 
                     // 2. Audit Ledger (Only add to the first update batch)
                     if (passNum && calc.manifest) {
@@ -83,16 +117,13 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                                 completedAt: new Date(),
                                 actualHash: calc.manifest.hash,
                                 _verified: true,
-                                _shardingStrategy: attempt + 1 // Track which strategy worked
+                                _shardingStrategy: attempt + 1
                             },
                             options: { merge: true }
                         });
                     }
 
                     // 3. Attempt Commit
-                    const totalSize = updates.reduce((acc, u) => acc + (u.data ? JSON.stringify(u.data).length : 0), 0);
-                    const isSharded = updates.some(u => u.data._sharded === true);
-
                     await commitBatchInChunks(config, deps, updates, `${name} Results (Att ${attempt+1})`);
                     
                     // Log Success
@@ -106,23 +137,19 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     lastError = commitErr;
                     const msg = commitErr.message || '';
                     
-                    // [IMPROVED] Check for non-retryable errors
                     const isNonRetryable = NON_RETRYABLE_ERRORS.some(code => msg.includes(code));
                     if (isNonRetryable) {
                         logger.log('ERROR', `[SelfHealing] ${name} encountered FATAL error (Attempt ${attempt + 1}): ${msg}. Aborting.`);
-                        throw commitErr; // Stop immediately
+                        throw commitErr; 
                     }
 
-                    // Check if error is related to size/indexes
                     const isSizeError  = msg.includes('Transaction too big')    || msg.includes('payload is too large');
-                    const isIndexError = msg.includes('too many index entries') || msg.includes('INVALID_ARGUMENT'); // Note: InvalidArg can be ambiguous, but usually index related in FS
+                    const isIndexError = msg.includes('too many index entries') || msg.includes('INVALID_ARGUMENT'); 
 
                     if (isSizeError || isIndexError) {
                         logger.log('WARN', `[SelfHealing] ${name} failed write attempt ${attempt + 1}. Retrying with tighter constraints...`, { error: msg });
                         continue; // Try next strategy
                     } else {
-                        // If it's a network error or unknown, re-throw or handle based on policy. 
-                        // For now, we allow retrying loop if it wasn't explicitly fatal.
                         logger.log('WARN', `[SelfHealing] ${name} unknown error (Attempt ${attempt + 1}). Retrying...`, { error: msg });
                     }
                 }
@@ -137,8 +164,14 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             }
             // ----------------------------------
 
-            // Mark Success
-            if (calc.manifest.hash) { successUpdates[name] = {  hash: calc.manifest.hash, category: calc.manifest.category  }; }
+            // Mark Success & Pass Metrics
+            if (calc.manifest.hash) { 
+                successUpdates[name] = {  
+                    hash: calc.manifest.hash, 
+                    category: calc.manifest.category,
+                    metrics: runMetrics // Pass metrics up
+                }; 
+            }
 
             // Capture Schema
             if (calc.manifest.class.getSchema) {
@@ -164,7 +197,8 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             
             failureReport.push({
                 name,
-                error: { message: msg, stack: e.stack, stage }
+                error: { message: msg, stack: e.stack, stage },
+                metrics: runMetrics // Pass incomplete metrics for debugging
             });
         }
     }
@@ -180,7 +214,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
  * Deletes result documents from a previous category location.
  */
 async function deleteOldCalculationData(dateStr, oldCategory, calcName, config, deps) {
-
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils || { withRetry: (fn) => fn() };
     
@@ -227,8 +260,6 @@ async function prepareAutoShardedWrites(result, docRef, logger, maxBytes = 900 *
     const OVERHEAD_ALLOWANCE     = 20 * 1024;
     const CHUNK_LIMIT            = maxBytes - OVERHEAD_ALLOWANCE;
     
-    // We only calculate totalSize loosely here for the "skip sharding" check.
-    // The loop below enforces the real limits.
     const totalSize              = calculateFirestoreBytes(result);
     const docPathSize            = Buffer.byteLength(docRef.path, 'utf8') + 16;
     

package/functions/computation-system/persistence/RunRecorder.js

@@ -1,53 +1,148 @@
 /**
- * @fileoverview Utility for recording computation run attempts (The Run Ledger).
- * Tracks success, failure, and error contexts for every execution pass.
+ * @fileoverview Utility for recording computation run attempts (The Audit Logger).
+ * REFACTORED: Organizes logs by Computation Name -> History.
+ * Implements aggregated error stats and advanced performance metrics.
  */
-const { generateProcessId } = require('../logger/logger');
+
+const { FieldValue } = require('../utils/utils'); 
+const os = require('os');
+
+// Root collection for the new audit system
+const AUDIT_COLLECTION = 'computation_audit_logs';
+
+/**
+ * Sanitizes error messages to be used as Firestore Map keys.
+ * Replaces invalid characters (. / [ ] *) with underscores.
+ */
+function sanitizeErrorKey(message) {
+    if (!message) return 'Unknown_Error';
+    // Take first 100 chars to avoid key limit issues
+    const shortMsg = message.toString().substring(0, 100); 
+    return shortMsg.replace(/[./\[\]*`]/g, '_').trim();
+}
 
 /**
- * Records a run attempt to the computation_run_history collection.
- * * @param {Firestore} db - Firestore instance
- * @param {Object} context - { date, computation, pass }
- * @param {string} status - 'SUCCESS', 'FAILURE', or 'CRASH'
- * @param {Object|null} error - Error object or null
- * @param {Object} metrics - { durationMs, ... }
+ * Records a run attempt with detailed metrics and aggregated stats.
+ * @param {Firestore} db - Firestore instance
+ * @param {Object} context - Context object
+ * @param {string} context.date - The "Target Date" of the computation
+ * @param {string} context.computation - The name of the calculation
+ * @param {string} context.pass - The topology pass number
+ * @param {string} status - 'SUCCESS', 'FAILURE', 'CRASH', or 'SKIPPED'
+ * @param {Object|null} error - Error object if failed
+ * @param {Object} detailedMetrics - Expanded metrics object (Optional, defaults provided)
+ * @param {number} [detailedMetrics.durationMs] - Execution time
+ * @param {Object} [detailedMetrics.storage] - { sizeBytes, isSharded, shardCount }
+ * @param {Object} [detailedMetrics.validation] - { isValid, anomalies: [] }
  */
-async function recordRunAttempt(db, context, status, error = null, metrics = {}) {
+async function recordRunAttempt(db, context, status, error = null, detailedMetrics = { durationMs: 0 }) {
     if (!db || !context) return;
 
-    const { date, computation, pass } = context;
-    // Generate a unique ID for this specific run attempt
-    const runId = `${Date.now()}_${generateProcessId('run', computation, date)}`;
+    const { date: targetDate, computation, pass } = context;
+    const now = new Date();
+    const triggerTimestamp = now.getTime();
+    
+    // 1. Construct Paths
+    // Parent Doc: Stores global aggregates for this computation
+    const computationDocRef = db.collection(AUDIT_COLLECTION).doc(computation);
     
-    const docRef = db.collection('computation_run_history')
-        .doc(date)
-        .collection('runs')
-        .doc(runId);
+    // History Doc: Stores this specific run
+    // ID Format: targetDate_triggerTimestamp (Sortable by data date, then execution time)
+    const runId = `${targetDate}_${triggerTimestamp}`;
+    const runDocRef = computationDocRef.collection('history').doc(runId);
+
+    // 2. Prepare Metrics & Environment Info
+    const workerId = process.env.FUNCTION_TARGET || process.env.K_REVISION || os.hostname();
+    
+    // Calculate size in MB
+    let sizeMB = 0;
+    if (detailedMetrics.storage && detailedMetrics.storage.sizeBytes) {
+        sizeMB = Number((detailedMetrics.storage.sizeBytes / (1024 * 1024)).toFixed(4));
+    }
 
-    const entry = {
+    // Extract Validation Anomalies (Unusual Keys/Values)
+    const anomalies = detailedMetrics.validation?.anomalies || [];
+    if (error && error.message && error.message.includes('Data Integrity')) {
+        // If the error itself was a validation failure, add it to anomalies
+        anomalies.push(error.message);
+    }
+
+    // 3. Construct the Run Log Entry
+    const runEntry = {
+        // Identity
+        runId: runId,
         computationName: computation,
-        date: date,
         pass: String(pass),
-        timestamp: new Date().toISOString(),
+        workerId: workerId,
+        
+        // Timing
+        targetDate: targetDate,        // The date the data belongs to
+        triggerTime: now.toISOString(), // The date the code ran
+        durationMs: detailedMetrics.durationMs || 0,
+
+        // Status
         status: status,
-        metrics: metrics
+        
+        // Data Metrics
+        outputStats: {
+            sizeMB: sizeMB,
+            isSharded: !!detailedMetrics.storage?.isSharded,
+            shardCount: detailedMetrics.storage?.shardCount || 1,
+            keysWritten: detailedMetrics.storage?.keys || 0 // If available
+        },
+
+        // Health & Diagnostics
+        anomalies: anomalies, // Logs "Consistent 0s", "N/As" etc.
+        
+        // Metadata
+        _schemaVersion: '2.0'
     };
 
+    // Attach Error Details if present
     if (error) {
-        entry.error = {
+        runEntry.error = {
             message: error.message || 'Unknown Error',
-            // Capture specific sharding/firestore stages if available
             stage: error.stage || 'UNKNOWN',
-            code: error.code || null,
-            stack: error.stack || null
+            stack: error.stack ? error.stack.substring(0, 1000) : null, // Truncate stack
+            code: error.code || null
         };
     }
 
-    // Fire and forget (await but catch to ensure logging doesn't crash the worker)
+    // 4. Prepare Aggregation Update (Atomic Increments)
+    const statsUpdate = {
+        lastRunAt: now,
+        lastRunStatus: status,
+        totalRuns: FieldValue.increment(1)
+    };
+
+    if (status === 'SUCCESS') {
+        statsUpdate.successCount = FieldValue.increment(1);
+    } else {
+        statsUpdate.failureCount = FieldValue.increment(1);
+        // Increment specific error type counter
+        if (error) {
+            const safeKey = sanitizeErrorKey(error.message);
+            statsUpdate[`errorCounts.${safeKey}`] = FieldValue.increment(1);
+        }
+    }
+
+    // 5. Execute as Batch
     try {
-        await docRef.set(entry);
+        const batch = db.batch();
+        
+        // Set the specific run log
+        batch.set(runDocRef, runEntry);
+        
+        // Merge updates into the parent computation document
+        // We use { merge: true } implicitly with set or explicit update. 
+        // Using set({ merge: true }) ensures doc creation if it doesn't exist.
+        batch.set(computationDocRef, statsUpdate, { merge: true });
+
+        await batch.commit();
+
     } catch (e) {
-        console.error(`[RunRecorder] Failed to save history for ${computation}:`, e.message);
+        // Fallback logging if Firestore fails (prevents infinite loop crashing)
+        console.error(`[RunRecorder] ❌ CRITICAL: Failed to write audit log for ${computation}`, e);
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.260",
+  "version": "1.0.261",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [