bulltrackers-module 1.0.291 → 1.0.293

package/functions/computation-system/context/ManifestBuilder.js

@@ -89,12 +89,12 @@ function getDependencySet(endpoints, adjacencyList) {
  * Returns a string description of the first cycle found.
  */
 function detectCircularDependencies(manifestMap) {
-    let index = 0;
-    const stack = [];
-    const indices = new Map();
+    let index      = 0;
+    const stack    = [];
+    const indices  = new Map();
     const lowLinks = new Map();
-    const onStack = new Set();
-    const cycles = [];
+    const onStack  = new Set();
+    const cycles   = [];
 
     function strongconnect(v) {
         indices.set(v, index);

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -2,6 +2,7 @@
  * FILENAME: computation-system/helpers/computation_dispatcher.js
  * PURPOSE: "Smart Dispatcher" - Analyzes state, initializes Run Counters, and dispatches tasks.
  * UPDATED: Implements Callback Pattern. Initializes 'computation_runs' doc for worker coordination.
+ * UPDATED: Implements Forensic Crash Analysis & Intelligent Resource Routing.
  */
 
 const { getExpectedDateStrings, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
@@ -13,12 +14,49 @@ const { generateCodeHash }                      = require('../topology/HashManag
 const pLimit                                    = require('p-limit');
 const crypto                                    = require('crypto');
 
-const TOPIC_NAME = 'computation-tasks';
-const STATUS_IMPOSSIBLE = 'IMPOSSIBLE';
+const STATUS_IMPOSSIBLE   = 'IMPOSSIBLE';
+
+// Threshold to trigger high-mem routing (e.g., 1.5 GB for a 2GB worker)
+const OOM_THRESHOLD_MB = 1500; 
+
+/**
+ * [NEW] Forensics: Checks if the calculation crashed previously due to Memory.
+ * Reads the 'telemetry.lastMemory' from the audit ledger.
+ */
+async function checkCrashForensics(db, date, pass, computationName) {
+    try {
+        const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${computationName}`;
+        const doc = await db.doc(ledgerPath).get();
+
+        if (!doc.exists) return 'standard';
+
+        const data = doc.data();
+        
+        // Check if we have telemetry from a previous run
+        if (data.telemetry && data.telemetry.lastMemory) {
+            const lastRSS = data.telemetry.lastMemory.rssMB || 0;
+            
+            if (lastRSS > OOM_THRESHOLD_MB) {
+                console.log(`[Dispatcher] 🕵️‍♀️ Forensics: ${computationName} likely OOM'd at ${lastRSS}MB. Routing to HIGH-MEM.`);
+                return 'high-mem';
+            }
+        }
+        
+        // Also check if it's explicitly marked FAILED with 'Memory' in error
+        if (data.status === 'FAILED' && data.error && /memory/i.test(data.error)) {
+            return 'high-mem';
+        }
+
+    } catch (e) {
+        console.warn(`[Dispatcher] Forensics check failed for ${computationName}: ${e.message}`);
+    }
+    
+    return 'standard';
+}
 
 /**
  * Dispatches computation tasks for a specific pass.
- * @param {Object} config - System config
+ * @param {Object} config - System config (Injected with topics)
  * @param {Object} dependencies - { db, logger, ... }
  * @param {Array} computationManifest - List of calculations
  * @param {Object} reqBody - (Optional) HTTP Body containing 'callbackUrl' and 'date'
@@ -28,7 +66,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const pubsubUtils = new PubSubUtils(dependencies);
     const passToRun   = String(config.COMPUTATION_PASS_TO_RUN);
     
-    // [NEW] Extract Date and Callback from request body (pushed by Workflow)
+    // Extract Date and Callback from request body (pushed by Workflow)
     const dateStr     = reqBody.date || config.date; 
     const callbackUrl = reqBody.callbackUrl || null;
 
@@ -44,18 +82,14 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
     if (!calcsInThisPass.length) { return logger.log('WARN', `[Dispatcher] No calcs for Pass ${passToRun}. Exiting.`); }
 
-    const calcNames = calcsInThisPass.map(c => c.name);
     logger.log('INFO', `🚀 [Dispatcher] Smart-Dispatching PASS ${passToRun} for ${dateStr}`);
     
-    // -- DATE ANALYSIS LOGIC (Unchanged) --
-    const passEarliestDate = Object.values(DEFINITIVE_EARLIEST_DATES).reduce((a, b) => a < b ? a : b);
-    const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
-    
-    // We only analyze the specific requested date to keep dispatch fast for the workflow
+    // -- DATE ANALYSIS LOGIC --
     const allExpectedDates = [dateStr]; 
-
     const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
     const tasksToDispatch = [];
+    
+    // Concurrency limit for analysis & forensics
     const limit = pLimit(20);
 
     const analysisPromises = allExpectedDates.map(d => limit(async () => {
@@ -87,18 +121,16 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
             const report = analyzeDateExecution(d, calcsInThisPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
 
+            // Handle Status Updates (Impossible / Blocked)
             const statusUpdates = {};
-            
             report.impossible.forEach(item => {
                 if (dailyStatus[item.name]?.hash !== STATUS_IMPOSSIBLE) {
                     statusUpdates[item.name] = { hash: STATUS_IMPOSSIBLE, category: 'unknown', reason: item.reason };
                 }
             });
-
             report.blocked.forEach(item => {
                  statusUpdates[item.name] = { hash: false, category: 'unknown', reason: item.reason };
             });
-
             report.failedDependency.forEach(item => {
                  const missingStr = item.missing ? item.missing.join(', ') : 'unknown';
                  statusUpdates[item.name] = { hash: false, category: 'unknown', reason: `Dependency Missing: ${missingStr}` };
@@ -109,21 +141,29 @@ async function dispatchComputationPass(config, dependencies, computationManifest
             }
 
             const validToRun = [...report.runnable, ...report.reRuns];
-            validToRun.forEach(item => {
+            
+            // [NEW] Parallel Forensics Check
+            await Promise.all(validToRun.map(item => limit(async () => {
+                const compName = normalizeName(item.name);
+                
+                // 1. Determine Resource Requirements
+                const requiredResource = await checkCrashForensics(db, d, passToRun, compName);
+                
                 const uniqueDispatchId = crypto.randomUUID(); 
                 tasksToDispatch.push({
                     action: 'RUN_COMPUTATION_DATE',
                     dispatchId: uniqueDispatchId,
                     date: d,
                     pass: passToRun,
-                    computation: normalizeName(item.name),
+                    computation: compName,
                     hash: item.hash || item.newHash, 
                     previousCategory: item.previousCategory || null,
                     triggerReason: item.reason || "Unknown",
                     dependencyResultHashes: item.dependencyResultHashes || {},
-                    timestamp: Date.now()
+                    timestamp: Date.now(),
+                    resources: requiredResource // 'standard' or 'high-mem'
                 });
-            });
+            })));
 
         } catch (e) {
             logger.log('ERROR', `[Dispatcher] Failed analysis for ${d}: ${e.message}`);
@@ -132,10 +172,9 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
     await Promise.all(analysisPromises);
 
-    // -- NEW: CALLBACK & COUNTER INITIALIZATION --
+    // -- CALLBACK & COUNTER INITIALIZATION --
 
     if (tasksToDispatch.length > 0) {
-        logger.log('INFO', `[Dispatcher] 📝 Preparing ${tasksToDispatch.length} tasks for execution...`);
         
         // 1. Initialize Shared State Document (The Counter)
         const runId = crypto.randomUUID();
@@ -147,14 +186,14 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                 date: dateStr,
                 pass: passToRun,
                 totalTasks: tasksToDispatch.length,
-                remainingTasks: tasksToDispatch.length, // <--- The Countdown
-                callbackUrl: callbackUrl,               // <--- The Workflow Hook
+                remainingTasks: tasksToDispatch.length, 
+                callbackUrl: callbackUrl,               
                 status: 'IN_PROGRESS'
             });
-            logger.log('INFO', `[Dispatcher] 🏁 Run State Initialized: ${runId}`);
+            logger.log('INFO', `[Dispatcher] 🏁 Run State Initialized: ${runId}. Tasks: ${tasksToDispatch.length}`);
         }
 
-        // 2. Attach Run Metadata to every task
+        // 2. Attach Run Metadata
         tasksToDispatch.forEach(task => {
             task.runId = runId;
             task.metaStatePath = callbackUrl ? metaStatePath : null; 
@@ -180,12 +219,13 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                     t.set(ledgerRef, {
                         status: 'PENDING',
                         dispatchId: task.dispatchId,
-                        runId: task.runId, // Track the batch ID
+                        runId: task.runId,
                         computation: task.computation,
                         expectedHash: task.hash || 'unknown',
                         createdAt: new Date(),
                         dispatcherHash: currentManifestHash,
                         triggerReason: task.triggerReason,
+                        resources: task.resources, // Log intended resource type
                         retries: 0
                     }, { merge: true });
                     
@@ -201,22 +241,36 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
         await Promise.all(txnPromises);
 
-        // 4. Publish to Pub/Sub
+        // 4. Publish to Pub/Sub (Segregated by Resources)
         if (finalDispatched.length > 0) {
-            logger.log('INFO', `[Dispatcher] ✅ Publishing ${finalDispatched.length} tasks to Pub/Sub...`);
             
-            await pubsubUtils.batchPublishTasks(dependencies, {
-                topicName: TOPIC_NAME,
-                tasks: finalDispatched,
-                taskType: `computation-pass-${passToRun}`,
-                maxPubsubBatchSize: 100
-            });
+            const standardTasks = finalDispatched.filter(t => t.resources !== 'high-mem');
+            const highMemTasks  = finalDispatched.filter(t => t.resources === 'high-mem');
+
+            // Publish Standard
+            if (standardTasks.length > 0) {
+                logger.log('INFO', `[Dispatcher] ✅ Publishing ${standardTasks.length} Standard tasks...`);
+                await pubsubUtils.batchPublishTasks(dependencies, {
+                    topicName: config.computationTopicStandard || 'computation-tasks',
+                    tasks: standardTasks,
+                    taskType: `computation-pass-${passToRun}-std`,
+                    maxPubsubBatchSize: 100
+                });
+            }
+
+            // Publish High-Mem
+            if (highMemTasks.length > 0) {
+                logger.log('INFO', `[Dispatcher] 🏋️‍♀️ Publishing ${highMemTasks.length} tasks to HIGH-MEM infrastructure.`);
+                await pubsubUtils.batchPublishTasks(dependencies, {
+                    topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
+                    tasks: highMemTasks,
+                    taskType: `computation-pass-${passToRun}-highmem`,
+                    maxPubsubBatchSize: 100
+                });
+            }
             
-            // Return count so workflow knows to wait
             return { dispatched: finalDispatched.length, runId };
         } else {
-            // Edge Case: Analysis said "Run", but Ledger said "Already Done"
-            // We must update the state doc to 0 or delete it, OR return 0 so workflow doesn't wait.
             logger.log('INFO', `[Dispatcher] All tasks were already COMPLETED.`);
             return { dispatched: 0 };
         }
@@ -227,4 +281,4 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     }
 }
 
-module.exports = { dispatchComputationPass };
+module.exports = { dispatchComputationPass };

package/functions/computation-system/helpers/computation_worker.js

@@ -2,6 +2,7 @@
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes tasks, executes logic, and signals Workflow upon Batch Completion.
  * UPDATED: Implements IAM Auth for Workflow Callbacks.
+ * UPDATED: Implements Memory Heartbeat (Flight Recorder) for OOM detection.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
@@ -9,7 +10,7 @@ const { getManifest }         = require('../topology/ManifestLoader');
 const { StructuredLogger }    = require('../logger/logger'); 
 const { recordRunAttempt }    = require('../persistence/RunRecorder'); 
 const https                   = require('https'); 
-const { GoogleAuth }          = require('google-auth-library'); // [NEW] Required for Auth
+const { GoogleAuth }          = require('google-auth-library'); 
 
 let calculationPackage;
 try { calculationPackage = require('aiden-shared-calculations-unified');
@@ -19,51 +20,68 @@ const calculations = calculationPackage.calculations;
 const MAX_RETRIES = 3;
 
 /**
- * [NEW] Helper: Fires the webhook back to Google Cloud Workflows.
- * UPDATED: Now generates an IAM Bearer Token to authenticate the request.
+ * [NEW] Helper: Starts a background heartbeat to track memory usage.
+ * This acts as a "Black Box Recorder". If the worker crashes (OOM),
+ * the last written value will remain in Firestore for the Dispatcher to analyze.
+ */
+function startMemoryHeartbeat(db, ledgerPath, intervalMs = 2000) {
+    const getMemStats = () => {
+        const mem = process.memoryUsage();
+        return {
+            rssMB: Math.round(mem.rss / 1024 / 1024),       // Resident Set Size (OOM Killer Metric)
+            heapUsedMB: Math.round(mem.heapUsed / 1024 / 1024),
+            timestamp: new Date()
+        };
+    };
+
+    const timer = setInterval(async () => {
+        try {
+            const stats = getMemStats();
+            // Use update() to minimize payload size and avoid overwriting status
+            await db.doc(ledgerPath).update({
+                'telemetry.lastMemory': stats,
+                'telemetry.lastHeartbeat': new Date()
+            }).catch(() => {}); // Ignore write errors to prevent crashing the worker
+        } catch (e) {
+            // Silently fail on telemetry errors
+        }
+    }, intervalMs);
+
+    // Unref so this timer doesn't prevent the process from exiting naturally
+    timer.unref();
+
+    return timer;
+}
+
+/**
+ * Helper: Fires the webhook back to Google Cloud Workflows.
  */
 async function triggerWorkflowCallback(url, status, logger) {
     if (!url) return;
     logger.log('INFO', `[Worker] 🔔 BATCH COMPLETE! Triggering Workflow Callback: ${status}`);
     
     try {
-        // 1. Get OAuth2 Access Token (Required for Workflows Callbacks)
-        const auth = new GoogleAuth({
-            scopes: ['https://www.googleapis.com/auth/cloud-platform']
-        });
+        const auth = new GoogleAuth({ scopes: ['https://www.googleapis.com/auth/cloud-platform'] });
         const client = await auth.getClient();
         const accessToken = await client.getAccessToken();
         const token = accessToken.token;
 
-        // 2. Send Authenticated Request
         return new Promise((resolve, reject) => {
-            const body = JSON.stringify({ 
-                status: status, 
-                timestamp: new Date().toISOString() 
-            });
+            const body = JSON.stringify({ status: status, timestamp: new Date().toISOString() });
             
             const req = https.request(url, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
                     'Content-Length': Buffer.byteLength(body),
-                    'Authorization': `Bearer ${token}` // <--- CRITICAL FIX
+                    'Authorization': `Bearer ${token}`
                 }
             }, (res) => {
-                if (res.statusCode >= 200 && res.statusCode < 300) {
-                    resolve();
-                } else {
-                    logger.log('WARN', `Callback responded with ${res.statusCode}`);
-                    // We resolve anyway to avoid crashing the worker logic
-                    resolve(); 
-                }
+                if (res.statusCode >= 200 && res.statusCode < 300) { resolve(); } 
+                else { logger.log('WARN', `Callback responded with ${res.statusCode}`); resolve(); }
             });
 
-            req.on('error', (e) => {
-                logger.log('ERROR', `Failed to trigger callback: ${e.message}`);
-                resolve(); 
-            });
-            
+            req.on('error', (e) => { logger.log('ERROR', `Failed to trigger callback: ${e.message}`); resolve(); });
             req.write(body);
             req.end();
         });
@@ -73,37 +91,21 @@ async function triggerWorkflowCallback(url, status, logger) {
 }
 
 /**
- * [NEW] Helper: Decrements 'remainingTasks' in Firestore.
- * Returns the callbackUrl IF this was the last task.
+ * Helper: Decrements 'remainingTasks' in Firestore.
  */
 async function decrementAndCheck(db, metaStatePath, logger) {
     if (!metaStatePath) return null;
-    
     try {
         const result = await db.runTransaction(async (t) => {
             const ref = db.doc(metaStatePath);
             const doc = await t.get(ref);
-            
-            if (!doc.exists) return null; // State might have expired or been deleted
+            if (!doc.exists) return null;
             const data = doc.data();
-            
             const newRemaining = (data.remainingTasks || 0) - 1;
-            
-            t.update(ref, { 
-                remainingTasks: newRemaining,
-                lastUpdated: new Date()
-            });
-            
-            // Return needed data only if we hit 0 (or lower, for safety)
-            return { 
-                remaining: newRemaining, 
-                callbackUrl: data.callbackUrl 
-            };
+            t.update(ref, { remainingTasks: newRemaining, lastUpdated: new Date() });
+            return { remaining: newRemaining, callbackUrl: data.callbackUrl };
         });
-
-        if (result && result.remaining <= 0) {
-            return result.callbackUrl;
-        }
+        if (result && result.remaining <= 0) return result.callbackUrl;
     } catch (e) {
         logger.log('ERROR', `[Worker] Failed to decrement batch counter: ${e.message}`);
     }
@@ -125,12 +127,12 @@ async function handleComputationTask(message, config, dependencies) {
 
     if (!data || data.action !== 'RUN_COMPUTATION_DATE') { return; }
     
-    // Extract fields including new metaStatePath
     const { date, pass, computation, previousCategory, triggerReason, dispatchId, dependencyResultHashes, metaStatePath } = data;
     
     if (!date || !pass || !computation) { logger.log('ERROR', `[Worker] Invalid payload.`, data); return; }
 
     const retryCount = message.deliveryAttempt || 1;
+    const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${computation}`;
 
     // --- POISON MESSAGE HANDLING (DLQ) ---
     if (retryCount > MAX_RETRIES) {
@@ -144,36 +146,38 @@ async function handleComputationTask(message, config, dependencies) {
                     failureReason: 'MAX_RETRIES_EXCEEDED'
                 });
 
-                await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).doc(computation).set({
+                await db.doc(ledgerPath).set({
                     status: 'FAILED',
                     error: 'Max Retries Exceeded (Poison Message)',
                     failedAt: new Date()
                 }, { merge: true });
 
                 const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
-                if (callbackUrl) {
-                    await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger);
-                }
-
+                if (callbackUrl) { await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger); }
                 return;
             } catch (dlqErr) { logger.log('FATAL', `[Worker] Failed to write to DLQ`, dlqErr); }
     }
 
     logger.log('INFO', `[Worker] 📥 Received Task: ${computation} (${date}) [Attempt ${retryCount}/${MAX_RETRIES}]`);
 
-    // Update Status to IN_PROGRESS
+    // 1. Update Status to IN_PROGRESS & Initialize Telemetry
     try {
-        await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).doc(computation).set({
+        await db.doc(ledgerPath).set({
             status: 'IN_PROGRESS',
             workerId: process.env.K_REVISION || 'unknown',
             startedAt: new Date(),
-            dispatchId: dispatchId
+            dispatchId: dispatchId,
+            telemetry: { startTime: new Date(), lastMemory: null } // Init for heartbeat
         }, { merge: true });
     } catch (leaseErr) {}
 
+    // 2. START HEARTBEAT (The Flight Recorder)
+    const heartbeatTimer = startMemoryHeartbeat(db, ledgerPath, 2000);
+
     let computationManifest;
     try { computationManifest = getManifest(config.activeProductLines || [], calculations, runDependencies);
     } catch (manifestError) { 
+        clearInterval(heartbeatTimer); // Stop if we fail early
         logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
         return; 
     }
@@ -186,6 +190,9 @@ async function handleComputationTask(message, config, dependencies) {
         );
         const duration = Date.now() - startTime;
 
+        // STOP HEARTBEAT ON SUCCESS
+        clearInterval(heartbeatTimer);
+
         const failureReport  = result?.updates?.failureReport  || [];
         const successUpdates = result?.updates?.successUpdates || {};
 
@@ -194,26 +201,23 @@ async function handleComputationTask(message, config, dependencies) {
             throw new Error(failReason.error.message || 'Computation Logic Failed');
         } 
         else {
-            if (Object.keys(successUpdates).length > 0) {
-                logger.log('INFO', `[Worker] ✅ Stored: ${computation}`);
-            } else {
-                logger.log('WARN', `[Worker] ⚠️ Empty Result: ${computation}`);
-            }
+            if (Object.keys(successUpdates).length > 0) { logger.log('INFO', `[Worker] ✅ Stored: ${computation}`); } 
+            else { logger.log('WARN', `[Worker] ⚠️ Empty Result: ${computation}`); }
 
-            await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).doc(computation).update({
+            await db.doc(ledgerPath).update({
                 status: 'COMPLETED',
                 completedAt: new Date()
             }).catch(() => {});
 
             await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, { durationMs: duration }, triggerReason);
 
-            // Decrement & Callback
             const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
-            if (callbackUrl) {
-                await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger);
-            }
+            if (callbackUrl) { await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger); }
         }
     } catch (err) { 
+        // STOP HEARTBEAT ON ERROR
+        clearInterval(heartbeatTimer);
+
         // --- ERROR HANDLING ---
         const isDeterministicError = err.stage === 'SHARDING_LIMIT_EXCEEDED' || 
                                      err.stage === 'QUALITY_CIRCUIT_BREAKER' ||
@@ -231,7 +235,7 @@ async function handleComputationTask(message, config, dependencies) {
                      failureReason: 'PERMANENT_DETERMINISTIC_ERROR'
                  });
                  
-                 await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).doc(computation).set({
+                 await db.doc(ledgerPath).set({
                     status: 'FAILED',
                     error: err.message || 'Permanent Deterministic Error',
                     failedAt: new Date()
@@ -240,23 +244,17 @@ async function handleComputationTask(message, config, dependencies) {
                  await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'PERMANENT_FAIL' }, { durationMs: 0 }, triggerReason);
                  
                  const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
-                 if (callbackUrl) {
-                     await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger);
-                 }
-
+                 if (callbackUrl) { await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger); }
                  return; 
              } catch (dlqErr) { logger.log('FATAL', `[Worker] Failed to write to DLQ`, dlqErr); }
         }
 
-        if (retryCount >= MAX_RETRIES) {
-             throw err; 
-        }
+        if (retryCount >= MAX_RETRIES) { throw err; }
 
         logger.log('ERROR', `[Worker] ❌ Crash: ${computation}: ${err.message}`); 
         await recordRunAttempt(db, { date, computation, pass }, 'CRASH', {  message: err.message, stack: err.stack, stage: 'SYSTEM_CRASH' }, { durationMs: 0 }, triggerReason);
-        
         throw err; 
     }
 }
 
-module.exports = { handleComputationTask };
+module.exports = { handleComputationTask };

package/functions/computation-system/tools/BuildReporter.js

@@ -6,41 +6,29 @@ const SimRunner = require('../simulation/SimRunner');
 const pLimit = require('p-limit');
 const path = require('path');
 const crypto = require('crypto');
-const fs = require('fs'); // [NEW] Required for file reading
+const fs = require('fs'); 
 const packageJson = require(path.join(__dirname, '..', '..', '..', 'package.json'));
 const packageVersion = packageJson.version;
-const { generateCodeHash } = require('../utils/utils'); // Reuse your standard logic
+const { generateCodeHash } = require('../utils/utils'); 
 
-// Persistent Registry for SimHashes
 const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
 
-// --- [NEW] RECURSIVE SYSTEM HASHING ---
-
-// 1. define the Root of the system (one level up from 'tools')
+// --- RECURSIVE SYSTEM HASHING ---
 const SYSTEM_ROOT = path.resolve(__dirname, '..');
-
-// 2. Define what to ignore to prevent noise or infinite loops
 const IGNORED_DIRS = new Set(['node_modules', '.git', '.idea', 'coverage', 'logs', 'tests']);
 const IGNORED_FILES = new Set(['package-lock.json', '.DS_Store', '.env']);
 
-/**
- * Recursively walks a directory and returns a list of file paths.
- */
 function walkSync(dir, fileList = []) {
     const files = fs.readdirSync(dir);
-    
     files.forEach(file => {
         if (IGNORED_FILES.has(file)) return;
-        
         const filePath = path.join(dir, file);
         const stat = fs.statSync(filePath);
-        
         if (stat.isDirectory()) {
             if (!IGNORED_DIRS.has(file)) {
                 walkSync(filePath, fileList);
             }
         } else {
-            // Only hash code files (add .yaml if you want workflows included)
             if (file.endsWith('.js') || file.endsWith('.json') || file.endsWith('.yaml')) {
                 fileList.push(filePath);
             }
@@ -49,69 +37,41 @@ function walkSync(dir, fileList = []) {
     return fileList;
 }
 
-/**
- * Generates a single hash representing the entire infrastructure code state.
- */
 function getInfrastructureHash() {
     try {
         const allFiles = walkSync(SYSTEM_ROOT);
-        allFiles.sort(); // Crucial for determinism
-
+        allFiles.sort(); 
         const bigHash = crypto.createHash('sha256');
-
         for (const filePath of allFiles) {
             const content = fs.readFileSync(filePath, 'utf8');
             const relativePath = path.relative(SYSTEM_ROOT, filePath);
-
-            // DECISION: How to clean?
             let cleanContent = content;
-
-            // 1. If it's JS, use your system standard for code hashing
             if (filePath.endsWith('.js')) {
-                // This strips comments and whitespace consistently with ManifestBuilder
-                // Note: generateCodeHash returns a hash, we can just use that hash
                 cleanContent = generateCodeHash(content); 
-            } 
-            // 2. If it's JSON/YAML, just strip basic whitespace to ignore indent changes
-            else {
+            } else {
                 cleanContent = content.replace(/\s+/g, '');
             }
-
-            // Feed the PATH and the CLEAN CONTENT into the master hash
             bigHash.update(`${relativePath}:${cleanContent}|`);
         }
-
         return bigHash.digest('hex');
     } catch (e) {
         console.warn(`[BuildReporter] ⚠️ Failed to generate infra hash: ${e.message}`);
-        return 'infra_hash_error'; // Fail safe so we run the report rather than crash
+        return 'infra_hash_error'; 
     }
 }
 
-/**
- * UPDATED: System Fingerprint = Manifest Hash + Infrastructure Hash
- */
 function getSystemFingerprint(manifest) {
-    // 1. Business Logic Hash (The Calculations)
     const sortedManifestHashes = manifest.map(c => c.hash).sort().join('|');
-    
-    // 2. Infrastructure Hash (The System Code)
     const infraHash = getInfrastructureHash();
-
-    // 3. Combine
     return crypto.createHash('sha256')
         .update(sortedManifestHashes + infraHash)
         .digest('hex');
 }
 
-/**
- * Helper: Determines if a calculation should be excluded from the report.
- */
 function isDateBeforeAvailability(dateStr, calcManifest) {
     const targetDate = new Date(dateStr + 'T00:00:00Z');
     const deps = calcManifest.rootDataDependencies || [];
     if (deps.length === 0) return false;
-
     for (const dep of deps) {
         let startDate = null;
         if (dep === 'portfolio')      startDate = DEFINITIVE_EARLIEST_DATES.portfolio;
@@ -119,19 +79,14 @@ function isDateBeforeAvailability(dateStr, calcManifest) {
         else if (dep === 'social')    startDate = DEFINITIVE_EARLIEST_DATES.social;
         else if (dep === 'insights')  startDate = DEFINITIVE_EARLIEST_DATES.insights;
         else if (dep === 'price')     startDate = DEFINITIVE_EARLIEST_DATES.price;
-
         if (startDate && targetDate < startDate) { return true; }
     }
     return false;
 }
 
-/**
- * Helper: Calculates the transitive closure of dependents (Blast Radius).
- */
 function calculateBlastRadius(targetCalcName, reverseGraph) {
     const impactSet = new Set();
     const queue = [targetCalcName];
-    
     while(queue.length > 0) {
         const current = queue.shift();
         const dependents = reverseGraph.get(current) || [];
@@ -142,7 +97,6 @@ function calculateBlastRadius(targetCalcName, reverseGraph) {
             }
         });
     }
-
     return {
         directDependents: (reverseGraph.get(targetCalcName) || []).length,
         totalCascadingDependents: impactSet.size,
@@ -150,14 +104,11 @@ function calculateBlastRadius(targetCalcName, reverseGraph) {
     };
 }
 
-/**
- * [NEW] Helper: Runs SimHash check with Caching and Registry Persistence.
- */
 async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, logger, simHashCache, db) {
     const trueReRuns = [];
     const stableUpdates = [];
     
-    // Limit concurrency for simulations
+    // Concurrency for simulations
     const limit = pLimit(10);
 
     const checks = candidates.map(item => limit(async () => {
@@ -170,16 +121,10 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
                 return;
             }
 
-            // 1. Check Cache first (Avoid re-simulating the same code for 100 different dates)
             let newSimHash = simHashCache.get(manifest.hash);
-
-            // 2. If Miss, Run Simulation & Persist to Registry
             if (!newSimHash) {
                 newSimHash = await SimRunner.run(manifest, manifestMap);
                 simHashCache.set(manifest.hash, newSimHash);
-                
-                // Write to Registry so Production Workers can find it without running SimRunner
-                // Fire-and-forget write to reduce latency
                 db.collection(SIMHASH_REGISTRY_COLLECTION).doc(manifest.hash).set({
                     simHash: newSimHash,
                     createdAt: new Date(),
@@ -187,17 +132,14 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
                 }).catch(err => logger.log('WARN', `Failed to write SimHash registry for ${manifest.name}: ${err.message}`));
             }
 
-            // 3. Compare
             if (newSimHash === stored.simHash) {
-                // BEHAVIORAL MATCH: Code changed, but output is identical.
                 stableUpdates.push({
                     ...item,
                     reason: "Code Updated (Logic Stable)",
-                    simHash: newSimHash, // New SimHash (same as old)
-                    newHash: manifest.hash // New Code Hash
+                    simHash: newSimHash, 
+                    newHash: manifest.hash 
                 });
             } else {
-                // BEHAVIORAL MISMATCH: Logic changed.
                 trueReRuns.push({
                     ...item,
                     reason: item.reason + ` [SimHash Mismatch]`,
@@ -215,9 +157,6 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
     return { trueReRuns, stableUpdates };
 }
 
-/**
- * AUTO-RUN ENTRY POINT
- */
 async function ensureBuildReport(config, dependencies, manifest) {
     const { db, logger } = dependencies;
     const now = new Date();
@@ -234,40 +173,28 @@ async function ensureBuildReport(config, dependencies, manifest) {
 
         if (!shouldRun) { logger.log('INFO', `[BuildReporter] 🔒 Report for v${packageVersion} locked. Skipping.`); return; }
 
-        // [NEW] 1. Calculate Current System Fingerprint
         const currentSystemHash = getSystemFingerprint(manifest);
-
-        // [NEW] 2. Fetch Last Build's Fingerprint
         const latestBuildDoc = await db.collection('computation_build_records').doc('latest').get();
         
         if (latestBuildDoc.exists) {
             const latestData = latestBuildDoc.data();
-            
-            // [OPTIMIZATION] If signatures match, we can clone the report or just skip
             if (latestData.systemFingerprint === currentSystemHash) {
                 logger.log('INFO', `[BuildReporter] ⚡ System Fingerprint (${currentSystemHash.substring(0,8)}) matches latest build. Skipping Report.`);
-                
-                // Create a "Skipped" record for the new version so we know it deployed
                 await db.collection('computation_build_records').doc(buildId).set({
                     buildId,
                     packageVersion,
                     systemFingerprint: currentSystemHash,
                     status: 'SKIPPED_IDENTICAL',
-                    referenceBuild: latestData.buildId, // Pointer to the build that actually did the work
+                    referenceBuild: latestData.buildId,
                     generatedAt: new Date().toISOString()
                 });
-                
-                // Release lock and exit
                 lockRef.update({ status: 'SKIPPED', completedAt: new Date() }).catch(() => {});
                 return;
             }
         }
 
         logger.log('INFO', `[BuildReporter] 🚀 Change Detected. Running Pre-flight Report for v${packageVersion}...`);
-        
-        // Pass the fingerprint to generateBuildReport so it can save it
         await generateBuildReport(config, dependencies, manifest, 90, buildId, currentSystemHash);
-        
         lockRef.update({ status: 'COMPLETED', completedAt: new Date() }).catch(() => {});
 
     } catch (e) {
@@ -275,14 +202,9 @@ async function ensureBuildReport(config, dependencies, manifest) {
     }
 }
 
-/**
- * Generates the report, writes to Firestore, AND FIXES STABLE UPDATES.
- */
 async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null, systemFingerprint = null) {
     const { db, logger } = dependencies;
     const buildId = customBuildId || `manual_${Date.now()}`;
-    
-    // Calculate fingerprint if not provided (for manual runs)
     const finalFingerprint = systemFingerprint || getSystemFingerprint(manifest);
 
     logger.log('INFO', `[BuildReporter] Generating Build Report: ${buildId} (Scope: ${daysBack} days, Fingerprint: ${finalFingerprint.substring(0,8)})...`);
@@ -293,8 +215,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     
     const datesToCheck = getExpectedDateStrings(startDate, today);
     const manifestMap  = new Map(manifest.map(c => [normalizeName(c.name), c]));
-    
-    // [OPTIMIZATION] Cache SimHashes across dates so we only calculate once per code version
     const simHashCache = new Map();
 
     const reverseGraph = new Map();
@@ -312,16 +232,16 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const reportHeader = { 
         buildId, 
         packageVersion, 
-        systemFingerprint: finalFingerprint, // Saved to Firestore
+        systemFingerprint: finalFingerprint, 
         generatedAt: new Date().toISOString(), 
         summary: {}, 
         _sharded: true 
     };
 
     let totalRun = 0, totalReRun = 0, totalStable = 0;
-    const detailWrites = []; 
 
-    const limit = pLimit(20);
+    // [FIX] Reduced concurrency from 20 to 5 to avoid Firestore DEADLINE_EXCEEDED
+    const limit = pLimit(5);
 
     const processingPromises = datesToCheck.map(dateStr => limit(async () => {
         try {
@@ -374,29 +294,22 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             // 1. RUN
             analysis.runnable.forEach(item => pushIfValid(dateSummary.run, item, "New Calculation"));
 
-            // 2. RE-RUN & STABLE Analysis (SimHash Integration)
+            // 2. RE-RUN & STABLE Analysis
             if (analysis.reRuns.length > 0) {
-                // Pass simHashCache and db for registry writes
                 const { trueReRuns, stableUpdates } = await verifyBehavioralStability(analysis.reRuns, manifestMap, dailyStatus, logger, simHashCache, db);
                 
                 trueReRuns.forEach(item => pushIfValid(dateSummary.rerun, item, "Logic Changed"));
                 stableUpdates.forEach(item => pushIfValid(dateSummary.stable, item, "Cosmetic Change"));
 
-                // [CRITICAL FIX] "Fix the Blast Radius"
-                // If updates are STABLE, we update the status NOW. 
-                // This implies: Code Hash changes, but Sim Hash stays same.
-                // The Dispatcher will see the new Code Hash in status matches the Manifest, so it won't dispatch.
                 if (stableUpdates.length > 0) {
                     const updatesPayload = {};
                     for (const stable of stableUpdates) {
                         const m = manifestMap.get(stable.name);
-                        // We preserve the *existing* resultHash because the logic is proven stable.
-                        // We update the 'hash' to the NEW code hash.
                         if (m && dailyStatus[stable.name]) {
                             updatesPayload[stable.name] = {
-                                hash: m.hash, // New Code Hash
-                                simHash: stable.simHash, // Same Sim Hash
-                                resultHash: dailyStatus[stable.name].resultHash, // Same Result Hash
+                                hash: m.hash, 
+                                simHash: stable.simHash, 
+                                resultHash: dailyStatus[stable.name].resultHash, 
                                 dependencyResultHashes: dailyStatus[stable.name].dependencyResultHashes || {},
                                 category: m.category,
                                 composition: m.composition,
@@ -404,7 +317,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
                             };
                         }
                     }
-                    // Perform the "Fix"
                     if (Object.keys(updatesPayload).length > 0) {
                         await updateComputationStatus(dateStr, updatesPayload, config, dependencies);
                         logger.log('INFO', `[BuildReporter] 🩹 Fixed ${Object.keys(updatesPayload).length} stable items for ${dateStr}. They will NOT re-run.`);
@@ -413,22 +325,19 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             }
 
             // 3. BLOCKED / IMPOSSIBLE / UPTODATE
-            analysis.blocked.forEach(item => pushIfValid(dateSummary.blocked, item));
-            analysis.failedDependency.forEach(item => pushIfValid(dateSummary.blocked, item, "Dependency Missing"));
-            analysis.impossible.forEach(item => pushIfValid(dateSummary.impossible, item));
-            analysis.skipped.forEach(item => pushIfValid(dateSummary.uptodate, item, "Up To Date"));
+            analysis.blocked.forEach          (item => pushIfValid(dateSummary.blocked,    item));
+            analysis.failedDependency.forEach (item => pushIfValid(dateSummary.blocked,    item, "Dependency Missing"));
+            analysis.impossible.forEach       (item => pushIfValid(dateSummary.impossible, item));
+            analysis.skipped.forEach          (item => pushIfValid(dateSummary.uptodate,   item, "Up To Date"));
 
             // Meta stats
-            const includedCount = dateSummary.run.length + dateSummary.rerun.length + dateSummary.stable.length +
+            const includedCount = dateSummary.run.length     + dateSummary.rerun.length      + dateSummary.stable.length +
                                   dateSummary.blocked.length + dateSummary.impossible.length + dateSummary.uptodate.length;
             dateSummary.meta.totalIncluded = includedCount;
             dateSummary.meta.match = (includedCount === expectedCount);
 
-            // REMOVE THIS: Inefficient memory usage causes timeouts
-            // detailWrites.push({ ref: db.collection(...), data: dateSummary });
-
-            // ADD THIS (Write immediately):
-           await db.collection('computation_build_records')
+            // Write Immediately
+            await db.collection('computation_build_records')
             .doc(buildId)
             .collection('details')
             .doc(dateStr)
@@ -450,10 +359,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
 
     await db.collection('computation_build_records').doc(buildId).set(reportHeader);
     
-    // Parallel write details ---> Now redundant.
-    //const writeLimit = pLimit(15);
-    //await Promise.all(detailWrites.map(w => writeLimit(() => w.ref.set(w.data))));
-
     await db.collection('computation_build_records').doc('latest').set({ ...reportHeader, note: "Latest build report pointer." });
     
     logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, Stable (Fixed): ${totalStable}, New: ${totalRun}.`);

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,7 +1,6 @@
 # Cloud Workflows Definition for BullTrackers Computation Pipeline
 # Orchestrates 5 sequential passes using Event-Driven Callbacks (Zero Polling).
-# FIXED: Replaced invalid 'sys' callback functions with 'events' library functions.
-# FIXED: Proper extraction of 'callback_details.url' for the dispatcher.
+# FIXED: Restored 'passes' and 'max_retries' variables in init step.
 
 main:
   params: [input]
@@ -10,8 +9,14 @@ main:
         assign:
           - project: ${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}
           - location: "europe-west1"
-          # If 'date' is provided in input, use it. Otherwise default to today (YYYY-MM-DD).
-          - date_to_run: ${default(map.get(input, "date"), text.substring(time.format(sys.now()), 0, 10))}
+          
+          # T-1 Date Logic (Process Yesterday)
+          - now: ${sys.now()}
+          - yesterday_timestamp: ${now - 86400} 
+          - yesterday_str: ${text.substring(time.format(yesterday_timestamp), 0, 10)}
+          - date_to_run: ${default(map.get(input, "date"), yesterday_str)}
+
+          # Configuration Variables (Restored)
           - passes: ["1", "2", "3", "4", "5"]
           - max_retries: 3
 
@@ -42,7 +47,6 @@ main:
                             - attempt_count: ${attempt_count + 1}
                       
                       # 1. GENERATE CALLBACK ENDPOINT
-                      # We use the 'events' library. This returns an object containing the URL.
                       - create_callback:
                           call: events.create_callback_endpoint
                           args:
@@ -60,7 +64,6 @@ main:
                             severity: "INFO"
 
                       # 2. TRIGGER DISPATCHER
-                      # We pass the extracted 'callback_url' string to the dispatcher.
                       - trigger_dispatcher:
                           call: http.post
                           args:
@@ -89,12 +92,11 @@ main:
                                     next: pass_retry_loop
 
                       # 4. WAIT FOR WORKER SIGNAL
-                      # We must pass the original 'callback_details' object here, not the URL string.
                       - wait_for_completion:
                           call: events.await_callback
                           args:
                             callback: ${callback_details}
-                            timeout: 86400 # Wait up to 24 hours
+                            timeout: 10800 # UPDATED: Reduced from 86400 (24h) to 10800 (3h) to detect crashes faster
                           result: callback_request
 
                       # 5. PROCESS SIGNAL

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.291",
+  "version": "1.0.293",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [