bulltrackers-module 1.0.260 → 1.0.262

package/functions/computation-system/WorkflowOrchestrator.js

@@ -10,7 +10,8 @@ const { StandardExecutor }                                 = require('./executor
 const { MetaExecutor }                                     = require('./executors/MetaExecutor');
 const { generateProcessId, PROCESS_TYPES }                 = require('./logger/logger');
 
-const STATUS_IMPOSSIBLE = 'IMPOSSIBLE';
+// [FIX] Split IMPOSSIBLE into semantic categories
+const STATUS_IMPOSSIBLE_PREFIX = 'IMPOSSIBLE';
 
 function groupByPass(manifest) {  return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc);  return acc;  }, {}); }
 
@@ -27,7 +28,8 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         const stored      = currentStatusMap[norm]; 
         const depManifest = manifestMap.get(norm);
         if (!stored)                           return false;
-        if (stored.hash === STATUS_IMPOSSIBLE) return false;
+        // [FIX] Check for any IMPOSSIBLE variant
+        if (typeof stored.hash === 'string' && stored.hash.startsWith(STATUS_IMPOSSIBLE_PREFIX)) return false;
         if (!depManifest)                      return false;
         if (stored.hash !== depManifest.hash)  return false;
         return true;
@@ -40,7 +42,12 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         const storedCategory = stored ? stored.category : null;
         const currentHash    = calc.hash;
 
-        const markImpossible = (reason) => { report.impossible.push({ name: cName, reason }); simulationStatus[cName] = { hash: STATUS_IMPOSSIBLE, category: calc.category }; };
+        // [FIX] Granular impossible marking
+        const markImpossible = (reason, type = 'GENERIC') => { 
+            report.impossible.push({ name: cName, reason }); 
+            const statusHash = `${STATUS_IMPOSSIBLE_PREFIX}:${type}`;
+            simulationStatus[cName] = { hash: statusHash, category: calc.category }; 
+        };
 
         const markRunnable = (isReRun = false, reRunDetails = null) => {
             if (isReRun) report.reRuns.push(reRunDetails);
@@ -50,49 +57,48 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
 
         let migrationOldCategory = null;
         if (storedCategory && storedCategory !== calc.category) { migrationOldCategory = storedCategory; }
-        if (storedHash === STATUS_IMPOSSIBLE) { report.skipped.push({ name: cName, reason: 'Permanently Impossible' }); continue; }
-        const rootCheck = checkRootDependencies(calc, rootDataStatus);
         
-        // Check Root Data Availability 
-            // LOGIC : Root data is essential for any calculation
-            // Therefore if a computation has a dependency on rootdata that does not exist for the dates the computation requires, then the computation is impossible to run.
-            // However, to handle edge cases where we might test trigger the computation system early, we do not mark impossible if the computation requires data for today, it might arrive later, we just block and skip.
+        // [FIX] Check for any IMPOSSIBLE variant in storage
+        if (typeof storedHash === 'string' && storedHash.startsWith(STATUS_IMPOSSIBLE_PREFIX)) { 
+            report.skipped.push({ name: cName, reason: `Permanently Impossible (${storedHash})` }); 
+            continue; 
+        }
 
+        const rootCheck = checkRootDependencies(calc, rootDataStatus);
+        
         if (!rootCheck.canRun) {
             const missingStr = rootCheck.missing.join(', ');
             if (!isTargetToday) {
-                markImpossible(`Missing Root Data: ${missingStr} (Historical)`);
+                // [FIX] Mark specifically as NO_DATA
+                markImpossible(`Missing Root Data: ${missingStr} (Historical)`, 'NO_DATA');
             } else {
                 report.blocked.push({ name: cName, reason: `Missing Root Data: ${missingStr} (Waiting)` });
             }
             continue;
         }
 
-        // Check Calculation Dependencies
-            // LOGIC : If a calc B depends on calc A, and calc A is impossible, then calc B is always impossible
-            // This has a cascading effect, if calc C depends on calc B and calc B depends on calc A and calc A is impossible, then calc B and calc C are also impossible.
-
         let dependencyIsImpossible = false;
         const missingDeps = [];
         if (calc.dependencies) {
             for (const dep of calc.dependencies) {
                 const normDep   = normalizeName(dep);
                 const depStored = simulationStatus[normDep];
-                if (depStored && depStored.hash === STATUS_IMPOSSIBLE) { dependencyIsImpossible = true; break; }
+                // [FIX] Check for any IMPOSSIBLE variant in dependencies
+                if (depStored && typeof depStored.hash === 'string' && depStored.hash.startsWith(STATUS_IMPOSSIBLE_PREFIX)) { 
+                    dependencyIsImpossible = true; 
+                    break; 
+                }
                 if (!isDepSatisfied(dep, simulationStatus, manifestMap)) { missingDeps.push(dep); }
             }
         }
 
-        if (dependencyIsImpossible) { markImpossible('Dependency is Impossible'); continue; }
+        if (dependencyIsImpossible) { 
+            // [FIX] Mark specifically as UPSTREAM failure
+            markImpossible('Dependency is Impossible', 'UPSTREAM'); 
+            continue; 
+        }
         if (missingDeps.length > 0) { report.failedDependency.push({ name: cName, missing: missingDeps }); continue; }
 
-        // Historical Continuity Check
-            // LOGIC : For computations that require historical data, we process them chronologically
-            // This is to handle the edge case where calc B runs for Tuesday data, but requires Mondays results from calc B. 
-            // If we triggered a hash mismatch through updating the code of calc B, it would overwrite the results for Tuesday and Monday but without this, 
-            // it would never be guaranteed that Monday runs before Tuesday, and so Tuesday would run with the old Monday hash data, or no data.
-            // This fixes this edge case by ensuring that historical computations only run if the previous day's computation has run with the latest hash, if not, it blocks and waits.
-
         if (calc.isHistorical && prevDailyStatus) {
             const yesterday = new Date(dateStr + 'T00:00:00Z');
             yesterday.setUTCDate(yesterday.getUTCDate() - 1);
@@ -104,9 +110,6 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
                 }
             }
         }
-        // Final Hash Comparison
-            // LOGIC : If the stored hash matches the current hash, we don't need to run the computation again, unless the category stored does not match the current computation category
-            // This is to handle the edge case where a developer changes the category of a computation, the stored results need to be moved into the new location so we trigger a re-run to move the data and also delete the old category stored data.
         
         if (!storedHash) { markRunnable(); } 
         else if (storedHash !== currentHash) { markRunnable(true, { name: cName, oldHash: storedHash, newHash: currentHash, previousCategory: migrationOldCategory }); } 

package/functions/computation-system/executors/StandardExecutor.js

@@ -71,14 +71,22 @@ class StandardExecutor {
 
         let yP_chunk = {}, tH_chunk = {};
         
-        for await (const tP_chunk of tP_iter) {
-            if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
-            if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
-            
-            // Execute chunk for all calcs
-            const promises = streamingCalcs.map(calc =>  StandardExecutor.executePerUser(calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk, fetchedDeps, previousFetchedDeps, config, deps, cachedLoader) );
-            await Promise.all(promises);
+        // [FIX] Ensure manual iterators are closed if loop fails
+        try {
+            for await (const tP_chunk of tP_iter) {
+                if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
+                if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
+                
+                // Execute chunk for all calcs
+                const promises = streamingCalcs.map(calc =>  StandardExecutor.executePerUser(calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk, fetchedDeps, previousFetchedDeps, config, deps, cachedLoader) );
+                await Promise.all(promises);
+            }
+        } finally {
+            // Close manual iterators to release resources
+            if (yP_iter && yP_iter.return) await yP_iter.return();
+            if (tH_iter && tH_iter.return) await tH_iter.return();
         }
+        
         logger.log('INFO', `[${passName}] Streaming complete.`);
     }
 

package/functions/computation-system/helpers/computation_worker.js

@@ -10,31 +10,22 @@ const { getManifest }         = require('../topology/ManifestLoader');
 const { StructuredLogger }    = require('../logger/logger'); 
 const { recordRunAttempt }    = require('../persistence/RunRecorder'); 
 
-// 1. IMPORT CALCULATIONS
 let calculationPackage;
-try {
-    calculationPackage = require('aiden-shared-calculations-unified');
-} catch (e) {
-    console.error("FATAL: Could not load 'aiden-shared-calculations-unified'.");
-    throw e;
-}
-
+try { calculationPackage = require('aiden-shared-calculations-unified');
+} catch (e) {console.error("FATAL: Could not load 'aiden-shared-calculations-unified'."); throw e; }
 const calculations = calculationPackage.calculations;
-const MAX_RETRIES = 3; // [NEW] Poison Pill Threshold
+const MAX_RETRIES = 3; 
 
 /**
  * Handles a single Pub/Sub message.
  */
 async function handleComputationTask(message, config, dependencies) {
-    
-    // 2. INITIALIZE SYSTEM LOGGER
     const systemLogger = new StructuredLogger({ minLevel: config.minLevel || 'INFO', enableStructured: true, ...config });
-
     const runDependencies = { ...dependencies, logger: systemLogger };
     const { logger, db }  = runDependencies;
-
-    // 3. PARSE PAYLOAD
     let data;
+
+    // ----------------------------------- Parse message -----------------------------------
     try {
         if (message.data && message.data.message && message.data.message.data) { data = JSON.parse(Buffer.from(message.data.message.data, 'base64').toString()); 
         } else if (message.data && typeof message.data === 'string')           { data = JSON.parse(Buffer.from(message.data, 'base64').toString()); 
@@ -42,30 +33,22 @@ async function handleComputationTask(message, config, dependencies) {
         } else { data = message; }
     } catch (parseError) { logger.log('ERROR', `[Worker] Failed to parse Pub/Sub payload.`, { error: parseError.message }); return; }
 
+    // ----------------------------------- Validate & Execute -----------------------------------
     if (!data || data.action !== 'RUN_COMPUTATION_DATE') { return; }
-    
-    // [UPDATED] Destructure previousCategory from payload
     const { date, pass, computation, previousCategory } = data;
-    
     if (!date || !pass || !computation) { logger.log('ERROR', `[Worker] Invalid payload: Missing date, pass, or computation.`, data); return; }
-
-    // 4. LOAD MANIFEST
     let computationManifest;
-    try {
-        computationManifest = getManifest(config.activeProductLines || [], calculations, runDependencies);
-    } catch (manifestError) {
-        logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
+    try { computationManifest = getManifest(config.activeProductLines || [], calculations, runDependencies);
+    } catch (manifestError) { logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
         await recordRunAttempt(db, { date, computation, pass }, 'CRASH', { message: manifestError.message, stage: 'MANIFEST_LOAD' });
         return; 
     }
 
-    // 5. EXECUTE (With Run Ledger)
     try {
         logger.log('INFO', `[Worker] 📥 Received: ${computation} for ${date}`);
         
         const startTime = Date.now();
-        // [UPDATED] Pass previousCategory to executor
-        const result = await executeDispatchTask(
+        const result    = await executeDispatchTask(
             date, 
             pass, 
             computation, 
@@ -76,36 +59,32 @@ async function handleComputationTask(message, config, dependencies) {
         );
         const duration = Date.now() - startTime;
 
-        // CHECK FOR INTERNAL FAILURES (Trapped by ResultCommitter)
         const failureReport  = result?.updates?.failureReport  || [];
         const successUpdates = result?.updates?.successUpdates || {};
 
         if (failureReport.length > 0) {
-            // Task ran, but logic or storage failed (e.g., Sharding Limit)
-            const failReason = failureReport[0]; // Assuming 1 calc per task
+            const failReason = failureReport[0]; 
             logger.log('ERROR', `[Worker] ❌ Failed logic/storage for ${computation}`, failReason.error);
-            await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', failReason.error, { durationMs: duration });
+            const metrics      = failReason.metrics || {};
+            metrics.durationMs = duration;
+            await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', failReason.error, metrics);
             throw new Error(failReason.error.message || 'Computation Logic Failed');
         } 
         else if (Object.keys(successUpdates).length > 0) {
-            // Success
-            logger.log('INFO', `[Worker] ✅ Stored: ${computation} for ${date}`);
-            await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, { durationMs: duration });
+            const successData  = successUpdates[computation]; 
+            const metrics      = successData.metrics || {};
+            metrics.durationMs = duration;
+            logger.log('INFO', `[Worker] ✅ Stored: ${computation} for ${date} (${metrics.storage?.sizeBytes} bytes)`);
+            await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, metrics);
         } 
         else {
-            // No updates, but no error (e.g. Empty Result) - Log as Success/Skipped
             logger.log('WARN', `[Worker] ⚠️ No results produced for ${computation} (Empty?)`);
             await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', { message: 'Empty Result' }, { durationMs: duration });
         }
-
     } catch (err) { 
-        // [NEW] POISON PILL LOGIC
-        // Check retry count from Pub/Sub message if available
         const retryCount = message.deliveryAttempt || 0;
-
         if (retryCount >= MAX_RETRIES) {
              logger.log('ERROR', `[Worker] ☠️ Task POISONED. Moved to DLQ: ${computation} ${date} (Attempt ${retryCount})`);
-             
              try {
                  await db.collection('computation_dead_letter_queue').add({
                      originalData: data,
@@ -113,19 +92,12 @@ async function handleComputationTask(message, config, dependencies) {
                      finalAttemptAt: new Date(),
                      failureReason: 'MAX_RETRIES_EXCEEDED'
                  });
-                 // Return normally to ACK the message and remove from subscription
                  return;
-             } catch (dlqErr) {
-                 logger.log('FATAL', `[Worker] Failed to write to DLQ`, dlqErr);
-             }
+             } catch (dlqErr) { logger.log('FATAL', `[Worker] Failed to write to DLQ`, dlqErr); }
         }
-
-        // Catch System Crashes (OOM, Timeout, Unhandled Exception)
         logger.log('ERROR', `[Worker] ❌ Crash: ${computation} for ${date}: ${err.message}`); 
-        
         await recordRunAttempt(db, { date, computation, pass }, 'CRASH', {  message: err.message, stack: err.stack, stage: 'SYSTEM_CRASH' });
-        
-        throw err; // Trigger Pub/Sub retry
+        throw err; 
     }
 }
 

package/functions/computation-system/persistence/ResultCommitter.js

@@ -1,7 +1,7 @@
 /**
  * @fileoverview Handles saving computation results with observability and Smart Cleanup.
- * UPDATED: Returns detailed failure reports for the Run Ledger.
- * UPDATED: Stops retrying on non-transient errors (Permissions, Invalid Args).
+ * UPDATED: Returns detailed failure reports AND metrics for the Audit Logger.
+ * UPDATED: Stops retrying on non-transient errors.
  */
 const { commitBatchInChunks }              = require('./FirestoreUtils');
 const { updateComputationStatus }          = require('./StatusRepository');
@@ -30,6 +30,13 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
     for (const name in stateObj) {
         const calc = stateObj[name];
+        
+        // Prep metrics container
+        const runMetrics = {
+            storage: { sizeBytes: 0, isSharded: false, shardCount: 1, keys: 0 },
+            validation: { isValid: true, anomalies: [] }
+        };
+
         try {
             const result = await calc.getResult();
 
@@ -37,14 +44,30 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             const healthCheck = HeuristicValidator.analyze(calc.manifest.name, result, overrides);
 
             if (!healthCheck.valid) {
+                // If validation failed, we consider it an anomaly but we BLOCK the write (throw error)
+                runMetrics.validation.isValid = false;
+                runMetrics.validation.anomalies.push(healthCheck.reason);
                 throw { message: healthCheck.reason, stage: 'QUALITY_CIRCUIT_BREAKER' };
             }
             
+            // Check for minor anomalies (validation warnings that didn't fail) - optional implementation
+            // For now, we assume if valid=true, anomalies are empty unless we add warning logic later.
+
             const isEmpty = !result || (typeof result === 'object' && Object.keys(result).length === 0) || (typeof result === 'number' && result === 0);
             if (isEmpty) { 
-                if (calc.manifest.hash) { successUpdates[name] = {  hash: false,  category: calc.manifest.category  }; } 
+                // Log empty success
+                if (calc.manifest.hash) { 
+                    successUpdates[name] = {  
+                        hash: false,  
+                        category: calc.manifest.category,
+                        metrics: runMetrics // Return empty metrics
+                    }; 
+                } 
                 continue; 
             }
+            
+            // Calculate Key Count rough estimate
+            if (typeof result === 'object') runMetrics.storage.keys = Object.keys(result).length;
 
             const mainDocRef = db.collection(config.resultsCollection)
                 .doc(dStr)
@@ -71,7 +94,18 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 
                 try {
                     // 1. Prepare Shards with current constraints
+                    // This function now needs to help us determine sharding info
                     const updates = await prepareAutoShardedWrites(result, mainDocRef, logger, constraints.bytes, constraints.keys);
+                    
+                    // METRICS CALCULATION
+                    const pointer = updates.find(u => u.data._completed === true);
+                    const isSharded = pointer && pointer.data._sharded === true;
+                    const shardCount = isSharded ? (pointer.data._shardCount || 1) : 1;
+                    const totalSize = updates.reduce((acc, u) => acc + (u.data ? JSON.stringify(u.data).length : 0), 0);
+                    
+                    runMetrics.storage.sizeBytes = totalSize;
+                    runMetrics.storage.isSharded = isSharded;
+                    runMetrics.storage.shardCount = shardCount;
 
                     // 2. Audit Ledger (Only add to the first update batch)
                     if (passNum && calc.manifest) {
@@ -83,16 +117,13 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                                 completedAt: new Date(),
                                 actualHash: calc.manifest.hash,
                                 _verified: true,
-                                _shardingStrategy: attempt + 1 // Track which strategy worked
+                                _shardingStrategy: attempt + 1
                             },
                             options: { merge: true }
                         });
                     }
 
                     // 3. Attempt Commit
-                    const totalSize = updates.reduce((acc, u) => acc + (u.data ? JSON.stringify(u.data).length : 0), 0);
-                    const isSharded = updates.some(u => u.data._sharded === true);
-
                     await commitBatchInChunks(config, deps, updates, `${name} Results (Att ${attempt+1})`);
                     
                     // Log Success
@@ -106,23 +137,19 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     lastError = commitErr;
                     const msg = commitErr.message || '';
                     
-                    // [IMPROVED] Check for non-retryable errors
-                    const isNonRetryable = NON_RETRYABLE_ERRORS.some(code => msg.includes(code));
+                    const isNonRetryable = NON_RETRYABLE_ERRORS.includes(commitErr.code);
                     if (isNonRetryable) {
                         logger.log('ERROR', `[SelfHealing] ${name} encountered FATAL error (Attempt ${attempt + 1}): ${msg}. Aborting.`);
-                        throw commitErr; // Stop immediately
+                        throw commitErr; 
                     }
 
-                    // Check if error is related to size/indexes
                     const isSizeError  = msg.includes('Transaction too big')    || msg.includes('payload is too large');
-                    const isIndexError = msg.includes('too many index entries') || msg.includes('INVALID_ARGUMENT'); // Note: InvalidArg can be ambiguous, but usually index related in FS
+                    const isIndexError = msg.includes('too many index entries') || msg.includes('INVALID_ARGUMENT'); 
 
                     if (isSizeError || isIndexError) {
                         logger.log('WARN', `[SelfHealing] ${name} failed write attempt ${attempt + 1}. Retrying with tighter constraints...`, { error: msg });
                         continue; // Try next strategy
                     } else {
-                        // If it's a network error or unknown, re-throw or handle based on policy. 
-                        // For now, we allow retrying loop if it wasn't explicitly fatal.
                         logger.log('WARN', `[SelfHealing] ${name} unknown error (Attempt ${attempt + 1}). Retrying...`, { error: msg });
                     }
                 }
@@ -137,8 +164,14 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             }
             // ----------------------------------
 
-            // Mark Success
-            if (calc.manifest.hash) { successUpdates[name] = {  hash: calc.manifest.hash, category: calc.manifest.category  }; }
+            // Mark Success & Pass Metrics
+            if (calc.manifest.hash) { 
+                successUpdates[name] = {  
+                    hash: calc.manifest.hash, 
+                    category: calc.manifest.category,
+                    metrics: runMetrics // Pass metrics up
+                }; 
+            }
 
             // Capture Schema
             if (calc.manifest.class.getSchema) {
@@ -164,7 +197,8 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             
             failureReport.push({
                 name,
-                error: { message: msg, stack: e.stack, stage }
+                error: { message: msg, stack: e.stack, stage },
+                metrics: runMetrics // Pass incomplete metrics for debugging
             });
         }
     }
@@ -180,7 +214,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
  * Deletes result documents from a previous category location.
  */
 async function deleteOldCalculationData(dateStr, oldCategory, calcName, config, deps) {
-
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils || { withRetry: (fn) => fn() };
     
@@ -227,8 +260,6 @@ async function prepareAutoShardedWrites(result, docRef, logger, maxBytes = 900 *
     const OVERHEAD_ALLOWANCE     = 20 * 1024;
     const CHUNK_LIMIT            = maxBytes - OVERHEAD_ALLOWANCE;
     
-    // We only calculate totalSize loosely here for the "skip sharding" check.
-    // The loop below enforces the real limits.
     const totalSize              = calculateFirestoreBytes(result);
     const docPathSize            = Buffer.byteLength(docRef.path, 'utf8') + 16;
     

package/functions/computation-system/persistence/ResultsValidator.js

@@ -70,10 +70,13 @@ class HeuristicValidator {
                 if (numericProp !== undefined) numericValues.push(numericProp);
             } 
             // --- TYPE B: Scalar / Primitive Result ---
-            else if (typeof val === 'number') {
-                if (val === 0) zeroCount++;
-                if (isNaN(val) || !isFinite(val)) nanCount++;
-                else numericValues.push(val);
+            if (typeof val === 'number') {
+                if (isNaN(val) || !isFinite(val)) {
+                    nanCount++;
+                } else {
+                    numericValues.push(val); // Include zeros
+                    if (val === 0) zeroCount++;
+                }
             }
         }
 

package/functions/computation-system/persistence/RunRecorder.js

@@ -1,53 +1,148 @@
 /**
- * @fileoverview Utility for recording computation run attempts (The Run Ledger).
- * Tracks success, failure, and error contexts for every execution pass.
+ * @fileoverview Utility for recording computation run attempts (The Audit Logger).
+ * REFACTORED: Organizes logs by Computation Name -> History.
+ * Implements aggregated error stats and advanced performance metrics.
  */
-const { generateProcessId } = require('../logger/logger');
+
+const { FieldValue } = require('../utils/utils'); 
+const os = require('os');
+
+// Root collection for the new audit system
+const AUDIT_COLLECTION = 'computation_audit_logs';
+
+/**
+ * Sanitizes error messages to be used as Firestore Map keys.
+ * Replaces invalid characters (. / [ ] *) with underscores.
+ */
+function sanitizeErrorKey(message) {
+    if (!message) return 'Unknown_Error';
+    // Take first 100 chars to avoid key limit issues
+    const shortMsg = message.toString().substring(0, 100); 
+    return shortMsg.replace(/[./\[\]*`]/g, '_').trim();
+}
 
 /**
- * Records a run attempt to the computation_run_history collection.
- * * @param {Firestore} db - Firestore instance
- * @param {Object} context - { date, computation, pass }
- * @param {string} status - 'SUCCESS', 'FAILURE', or 'CRASH'
- * @param {Object|null} error - Error object or null
- * @param {Object} metrics - { durationMs, ... }
+ * Records a run attempt with detailed metrics and aggregated stats.
+ * @param {Firestore} db - Firestore instance
+ * @param {Object} context - Context object
+ * @param {string} context.date - The "Target Date" of the computation
+ * @param {string} context.computation - The name of the calculation
+ * @param {string} context.pass - The topology pass number
+ * @param {string} status - 'SUCCESS', 'FAILURE', 'CRASH', or 'SKIPPED'
+ * @param {Object|null} error - Error object if failed
+ * @param {Object} detailedMetrics - Expanded metrics object (Optional, defaults provided)
+ * @param {number} [detailedMetrics.durationMs] - Execution time
+ * @param {Object} [detailedMetrics.storage] - { sizeBytes, isSharded, shardCount }
+ * @param {Object} [detailedMetrics.validation] - { isValid, anomalies: [] }
  */
-async function recordRunAttempt(db, context, status, error = null, metrics = {}) {
+async function recordRunAttempt(db, context, status, error = null, detailedMetrics = { durationMs: 0 }) {
     if (!db || !context) return;
 
-    const { date, computation, pass } = context;
-    // Generate a unique ID for this specific run attempt
-    const runId = `${Date.now()}_${generateProcessId('run', computation, date)}`;
+    const { date: targetDate, computation, pass } = context;
+    const now = new Date();
+    const triggerTimestamp = now.getTime();
+    
+    // 1. Construct Paths
+    // Parent Doc: Stores global aggregates for this computation
+    const computationDocRef = db.collection(AUDIT_COLLECTION).doc(computation);
     
-    const docRef = db.collection('computation_run_history')
-        .doc(date)
-        .collection('runs')
-        .doc(runId);
+    // History Doc: Stores this specific run
+    // ID Format: targetDate_triggerTimestamp (Sortable by data date, then execution time)
+    const runId = `${targetDate}_${triggerTimestamp}`;
+    const runDocRef = computationDocRef.collection('history').doc(runId);
+
+    // 2. Prepare Metrics & Environment Info
+    const workerId = process.env.FUNCTION_TARGET || process.env.K_REVISION || os.hostname();
+    
+    // Calculate size in MB
+    let sizeMB = 0;
+    if (detailedMetrics.storage && detailedMetrics.storage.sizeBytes) {
+        sizeMB = Number((detailedMetrics.storage.sizeBytes / (1024 * 1024)).toFixed(4));
+    }
 
-    const entry = {
+    // Extract Validation Anomalies (Unusual Keys/Values)
+    const anomalies = detailedMetrics.validation?.anomalies || [];
+    if (error && error.message && error.message.includes('Data Integrity')) {
+        // If the error itself was a validation failure, add it to anomalies
+        anomalies.push(error.message);
+    }
+
+    // 3. Construct the Run Log Entry
+    const runEntry = {
+        // Identity
+        runId: runId,
         computationName: computation,
-        date: date,
         pass: String(pass),
-        timestamp: new Date().toISOString(),
+        workerId: workerId,
+        
+        // Timing
+        targetDate: targetDate,        // The date the data belongs to
+        triggerTime: now.toISOString(), // The date the code ran
+        durationMs: detailedMetrics.durationMs || 0,
+
+        // Status
         status: status,
-        metrics: metrics
+        
+        // Data Metrics
+        outputStats: {
+            sizeMB: sizeMB,
+            isSharded: !!detailedMetrics.storage?.isSharded,
+            shardCount: detailedMetrics.storage?.shardCount || 1,
+            keysWritten: detailedMetrics.storage?.keys || 0 // If available
+        },
+
+        // Health & Diagnostics
+        anomalies: anomalies, // Logs "Consistent 0s", "N/As" etc.
+        
+        // Metadata
+        _schemaVersion: '2.0'
     };
 
+    // Attach Error Details if present
     if (error) {
-        entry.error = {
+        runEntry.error = {
             message: error.message || 'Unknown Error',
-            // Capture specific sharding/firestore stages if available
             stage: error.stage || 'UNKNOWN',
-            code: error.code || null,
-            stack: error.stack || null
+            stack: error.stack ? error.stack.substring(0, 1000) : null, // Truncate stack
+            code: error.code || null
         };
     }
 
-    // Fire and forget (await but catch to ensure logging doesn't crash the worker)
+    // 4. Prepare Aggregation Update (Atomic Increments)
+    const statsUpdate = {
+        lastRunAt: now,
+        lastRunStatus: status,
+        totalRuns: FieldValue.increment(1)
+    };
+
+    if (status === 'SUCCESS') {
+        statsUpdate.successCount = FieldValue.increment(1);
+    } else {
+        statsUpdate.failureCount = FieldValue.increment(1);
+        // Increment specific error type counter
+        if (error) {
+            const safeKey = sanitizeErrorKey(error.message);
+            statsUpdate[`errorCounts.${safeKey}`] = FieldValue.increment(1);
+        }
+    }
+
+    // 5. Execute as Batch
     try {
-        await docRef.set(entry);
+        const batch = db.batch();
+        
+        // Set the specific run log
+        batch.set(runDocRef, runEntry);
+        
+        // Merge updates into the parent computation document
+        // We use { merge: true } implicitly with set or explicit update. 
+        // Using set({ merge: true }) ensures doc creation if it doesn't exist.
+        batch.set(computationDocRef, statsUpdate, { merge: true });
+
+        await batch.commit();
+
     } catch (e) {
-        console.error(`[RunRecorder] Failed to save history for ${computation}:`, e.message);
+        // Fallback logging if Firestore fails (prevents infinite loop crashing)
+        console.error(`[RunRecorder] ❌ CRITICAL: Failed to write audit log for ${computation}`, e);
     }
 }
 

package/functions/computation-system/topology/ManifestLoader.js

@@ -5,35 +5,39 @@
 const { build } = require('../context/ManifestBuilder');
 const { StructuredLogger, PROCESS_TYPES, generateProcessId } = require('../logger/logger');
 
-// Cache the manifest in global scope (warm start optimization)
-let cachedManifest = null;
+// [FIX] Cache using a Map to handle different productLine combinations
+const manifestCache = new Map();
 
 function getManifest(productLines = [], calculationsDir, dependencies = {}) {
-    if (cachedManifest) {
-        return cachedManifest;
+    // Generate a unique key for this specific request configuration
+    const cacheKey = JSON.stringify(productLines ? productLines.slice().sort() : ['ALL']);
+
+    if (manifestCache.has(cacheKey)) {
+        return manifestCache.get(cacheKey);
     }
 
     const logger = dependencies.logger || new StructuredLogger();
     const pid = generateProcessId(PROCESS_TYPES.MANIFEST, 'build', new Date().toISOString().slice(0,10));
     
-    logger.log('INFO', 'Starting Manifest Build...', { processId: pid });
+    logger.log('INFO', 'Starting Manifest Build...', { processId: pid, scope: cacheKey });
     
     const startTime = Date.now();
     try {
-        cachedManifest = build(productLines, calculationsDir);
+        const manifest = build(productLines, calculationsDir);
         
         // Log Topology Stats
         const passCounts = {};
-        cachedManifest.forEach(c => { passCounts[c.pass] = (passCounts[c.pass] || 0) + 1; });
+        manifest.forEach(c => { passCounts[c.pass] = (passCounts[c.pass] || 0) + 1; });
         
         logger.log('INFO', 'Manifest Build Success', { 
             processId: pid, 
             durationMs: Date.now() - startTime,
-            totalCalculations: cachedManifest.length,
+            totalCalculations: manifest.length,
             topology: passCounts
         });
 
-        return cachedManifest;
+        manifestCache.set(cacheKey, manifest);
+        return manifest;
     } catch (e) {
         logger.log('FATAL', 'Manifest Build Failed', { processId: pid, error: e.message });
         throw e;

package/functions/computation-system/utils/data_loader.js

@@ -174,21 +174,32 @@ async function getPriceShardRefs(config, deps) {
  * @param {object} deps 
  * @returns {Promise<Object>} The lookup map { "instrumentId": "shardDocId" }
  */
+/**
+ * Ensures the Price Shard Index exists. If not, builds it by scanning all shards.
+ * [FIX] Added TTL check to ensure new instruments are discovered.
+ */
 async function ensurePriceShardIndex(config, deps) {
     const { db, logger } = deps;
     const metadataCol = config.metadataCollection || 'system_metadata';
-    const indexDocRef = db.collection(metadataCol).doc('price_shard_index'); // TODO. TEST THIS SHARD INDEX SYSTEM, CURRENTLY UNUSED IN COMPUTATIONS BUT IS EXTREMELY EFFICIENT AND GREAT FOR COST REDUCTION
+    const indexDocRef = db.collection(metadataCol).doc('price_shard_index');
     
     // 1. Try to fetch existing index
     const snap = await indexDocRef.get();
     if (snap.exists) {
         const data = snap.data();
-        // Simple expiry check (optional): Rebuild if older than 24h
-        // For now, we trust it exists.
-        return data.index || {};
-    }
+        
+        // [FIX] Check TTL (24 hours)
+        const lastUpdated = data.lastUpdated ? new Date(data.lastUpdated).getTime() : 0;
+        const now = Date.now();
+        const oneDayMs = 24 * 60 * 60 * 1000;
 
-    logger.log('INFO', '[ShardIndex] Index not found. Building new Price Shard Index (Scanning all shards)...');
+        if ((now - lastUpdated) < oneDayMs) {
+            return data.index || {};
+        }
+        logger.log('INFO', '[ShardIndex] Index is stale (>24h). Rebuilding...');
+    } else {
+        logger.log('INFO', '[ShardIndex] Index not found. Building new Price Shard Index...');
+    }
     
     // 2. Build Index
     const collection = config.priceCollection || 'asset_prices';
@@ -199,9 +210,8 @@ async function ensurePriceShardIndex(config, deps) {
     
     snapshot.forEach(doc => {
         shardCount++;
-        const data = doc.data(); // This loads the shard into memory, intensive but necessary once
+        const data = doc.data(); 
         if (data.history) {
-            // Keys of history are Instrument IDs
             Object.keys(data.history).forEach(instId => {
                 index[instId] = doc.id;
             });

package/functions/core/utils/intelligent_proxy_manager.js

@@ -3,7 +3,7 @@
  * It selects an available (unlocked) proxy for each request and locks it upon failure.
  * * This module is designed to be reusable and receives all dependencies
  * (firestore, logger) and configuration via its constructor.
- * --- MODIFIED: Now includes exponential backoff and retries specifically for rate-limit errors. ---
+ * --- MODIFIED: Fixed Hostname Collision Bug in _loadConfig ---
  */
 const { FieldValue } = require('@google-cloud/firestore');
 const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
@@ -44,17 +44,44 @@ class IntelligentProxyManager {
     async _loadConfig() {
         if (Date.now() - this.configLastLoaded < this.CONFIG_CACHE_DURATION_MS) { return;  }
         if (this.proxyUrls.length === 0) { return; }
+        
         this.logger.log('INFO', "[ProxyManager] Refreshing proxy configuration and lock status...");
-        try { const tempProxyStatus = {};
-            for (const url of this.proxyUrls) { const owner = new URL(url).hostname;  tempProxyStatus[owner] = { owner, url, status: 'unlocked' };  }
-            if (this.proxyLockingEnabled) { const doc = await this.firestore.doc(this.PERFORMANCE_DOC_PATH).get();
-                if (doc.exists) { const data = doc.data(); if (data.locks) { for (const owner in data.locks) { if (tempProxyStatus[owner] && data.locks[owner].locked === true) { tempProxyStatus[owner].status = 'locked'; } } } }
-            } else { this.logger.log('TRACE', '[ProxyManager] Proxy locking is disabled, skipping lock status check.'); }
+        
+        try { 
+            const tempProxyStatus = {};
+            for (const url of this.proxyUrls) { 
+                // [FIX] Use the full URL as the unique ID, sanitized for Firestore usage.
+                // Replaces all non-alphanumeric characters with underscores.
+                // Old logic: new URL(url).hostname -> caused collision because all are script.google.com
+                const owner = url.replace(/[^a-zA-Z0-9]/g, '_');
+                
+                tempProxyStatus[owner] = { owner, url, status: 'unlocked' };  
+            }
+
+            if (this.proxyLockingEnabled) { 
+                const doc = await this.firestore.doc(this.PERFORMANCE_DOC_PATH).get();
+                if (doc.exists) { 
+                    const data = doc.data(); 
+                    if (data.locks) { 
+                        for (const owner in data.locks) { 
+                            // If the sanitized URL key exists in locks and is locked, update status
+                            if (tempProxyStatus[owner] && data.locks[owner].locked === true) { 
+                                tempProxyStatus[owner].status = 'locked'; 
+                            } 
+                        } 
+                    } 
+                }
+            } else { 
+                this.logger.log('TRACE', '[ProxyManager] Proxy locking is disabled, skipping lock status check.'); 
+            }
+            
             this.proxies = tempProxyStatus;
             this.configLastLoaded = Date.now();
             this.logger.log('SUCCESS', `[ProxyManager] Refreshed ${Object.keys(this.proxies).length} proxy statuses.`);
+            
         } catch (error) {
-            this.logger.log('ERROR', '[ProxyManager] Failed to load proxy config from Firestore.', { errorMessage: error.message, path: this.PERFORMANCE_DOC_PATH }); }
+            this.logger.log('ERROR', '[ProxyManager] Failed to load proxy config from Firestore.', { errorMessage: error.message, path: this.PERFORMANCE_DOC_PATH }); 
+        }
     }
 
     /**
@@ -65,6 +92,7 @@ class IntelligentProxyManager {
         await this._loadConfig();
         const availableProxies = this.proxyLockingEnabled ? Object.values(this.proxies).filter(p => p.status === 'unlocked') : Object.values(this.proxies);
         if (availableProxies.length === 0) { const errorMsg = this.proxyLockingEnabled  ? "All proxies are locked. No proxy available." : "No proxies are loaded. Cannot make request.";  this.logger.log('ERROR', `[ProxyManager] ${errorMsg}`); throw new Error(errorMsg); }
+        // Random selection to distribute load (consider Round Robin in future for 20k scale)
         const selected = availableProxies[Math.floor(Math.random() * availableProxies.length)];
         return { owner: selected.owner, url: selected.url };
     }
@@ -77,12 +105,17 @@ class IntelligentProxyManager {
         if (!this.proxyLockingEnabled) { this.logger.log('TRACE', `[ProxyManager] Locking skipped for ${owner} (locking is disabled).`); return; }
         if (this.proxies[owner]) { this.proxies[owner].status = 'locked'; }
         this.logger.log('WARN', `[ProxyManager] Locking proxy: ${owner}`);
-        try { const docRef = this.firestore.doc(this.PERFORMANCE_DOC_PATH); await docRef.set({ locks: { [owner]: { locked: true, lastLocked: FieldValue.serverTimestamp() } } }, { merge: true });
-        } catch (error) { this.logger.log('ERROR', `[ProxyManager] Failed to write lock for ${owner} to Firestore.`, { errorMessage: error.message }); }
+        try { 
+            const docRef = this.firestore.doc(this.PERFORMANCE_DOC_PATH); 
+            // Use the sanitized owner key
+            await docRef.set({ locks: { [owner]: { locked: true, lastLocked: FieldValue.serverTimestamp() } } }, { merge: true });
+        } catch (error) { 
+            this.logger.log('ERROR', `[ProxyManager] Failed to write lock for ${owner} to Firestore.`, { errorMessage: error.message }); 
+        }
     }
 
     /**
-     * --- CORRECTED LOGIC: Makes a fetch request by trying different proxies ---
+     * Makes a fetch request by trying different proxies.
      * @param {string} targetUrl - The URL to fetch.
      * @param {object} options - Fetch options (e.g., headers).
      * @returns {Promise<object>} A mock Response object.
@@ -105,7 +138,7 @@ class IntelligentProxyManager {
 
             // 2. Make a SINGLE attempt with this selected proxy.
             const response = await this._fetchViaAppsScript(proxy.url, targetUrl, options);
-            lastResponse = response; // Save this response in case it's the last one
+            lastResponse = response; 
 
             // 3. Case 1: Success! Return immediately.
             if (response.ok) {
@@ -124,7 +157,7 @@ class IntelligentProxyManager {
                 // LOCK THE FAILED PROXY so _selectProxy() won't pick it again.
                 await this.lockProxy(proxy.owner);
 
-                // Back off slightly before trying the *next* proxy to avoid a thundering herd.
+                // Back off slightly before trying the *next* proxy.
                 await sleep(this.INITIAL_BACKOFF_MS * attempt);
 
                 continue; // Go to the next loop iteration to select a *new* proxy.
@@ -136,22 +169,18 @@ class IntelligentProxyManager {
             return response;
         }
 
-        // 6. If loop finishes, all (this.MAX_RETRIES) proxy attempts failed.
+        // 6. If loop finishes, all proxy attempts failed.
         this.logger.log('ERROR', `[ProxyManager] Request failed after ${this.MAX_RETRIES} proxy attempts.`, { url: targetUrl, lastStatus: lastResponse?.status });
-        return lastResponse; // Return the last failed response
+        return lastResponse; 
     }
 
-
-    // Inside backend_npm_pkgs/bulltrackers-module/functions/core/utils/intelligent_proxy_manager.js
-
     /**
      * Internal function to call the Google AppScript proxy.
-     * --- MODIFIED: Now checks Content-Type for HTML to robustly detect rate limits ---
      * @private
      */
     async _fetchViaAppsScript(proxyUrl, targetUrl, options) {
         const payload = { url: targetUrl, ...options };
-        let response; // Declare response here to access in catch block
+        let response; 
 
         try {
             response = await fetch(proxyUrl, { 
@@ -160,7 +189,6 @@ class IntelligentProxyManager {
                 body: JSON.stringify(payload) 
             });
 
-            // --- THIS IS THE DOCTYPE CHECK --- 
             // Check the response headers from the proxy itself.
             const contentType = response.headers.get('content-type') || '';
             if (contentType.includes('text/html')) {
@@ -169,20 +197,19 @@ class IntelligentProxyManager {
                 this.logger.log('WARN', `[ProxyManager] Proxy returned HTML error page (rate limit).`, {  
                     status: response.status,  
                     proxy: proxyUrl,  
-                    errorSnippet: errorText.substring(0, 150) // Log a snippet
+                    errorSnippet: errorText.substring(0, 150) 
                 });
                 
                 return {  
                     ok: false,  
-                    status: response.status, // Will be 500, 503, etc.
+                    status: response.status, 
                     isUrlFetchError: true, 
-                    isRateLimitError: true, // <--- This is the key change
+                    isRateLimitError: true,
                     error: { message: `Proxy returned HTML error page (likely rate limit).` }, 
                     headers: response.headers, 
                     text: () => Promise.resolve(errorText) 
                 };
             }
-            // --- END DOCTYPE CHECK ---
 
             // If it's not HTML, but still not OK (e.g., 400 Bad Request),
             // it's a non-rate-limit proxy error.
@@ -194,7 +221,6 @@ class IntelligentProxyManager {
                      error: errorText  
                  });
                  
-                 // We can still check 429 here, just in case Google sends one.
                  const isRateLimit = response.status === 429;
                  
                  return {  
@@ -211,13 +237,9 @@ class IntelligentProxyManager {
             // If we are here, Content-Type was application/json and status was OK.
             const proxyResponse = await response.json();
             
-            // Now we check for errors *inside* the JSON 
-            // (e.g., the Apps Script caught an error and reported it).
+            // Check for errors *inside* the JSON (caught by Apps Script)
             if (proxyResponse.error) {
                 const errorMsg = proxyResponse.error.message || '';
-                
-                // Fallback check for "invoked too many times" *inside* the JSON error,
-                // just in case. The HTML check is now our primary defense.
                 const isRateLimit = errorMsg.toLowerCase().includes('service invoked too many times');
                 
                 if (isRateLimit) {
@@ -225,7 +247,6 @@ class IntelligentProxyManager {
                      return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, isRateLimitError: true, headers: new Headers() };
                 }
                 
-                // Other non-rate-limit error caught by the script
                 return { ok: false, status: 500, error: proxyResponse.error, isUrlFetchError: true, isRateLimitError: false, headers: new Headers(), text: () => Promise.resolve(errorMsg) };
             }
 
@@ -246,7 +267,7 @@ class IntelligentProxyManager {
                 ok: false, 
                 status: 0,  
                 isUrlFetchError: true, 
-                isRateLimitError: false, // Not a rate limit, a network failure
+                isRateLimitError: false, 
                 error: { message: `Network error: ${networkError.message}` }, 
                 headers: new Headers() 
             };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.260",
+  "version": "1.0.262",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [