bulltrackers-module 1.0.292 → 1.0.294

package/functions/computation-system/context/ManifestBuilder.js

@@ -89,12 +89,12 @@ function getDependencySet(endpoints, adjacencyList) {
  * Returns a string description of the first cycle found.
  */
 function detectCircularDependencies(manifestMap) {
-    let index = 0;
-    const stack = [];
-    const indices = new Map();
+    let index      = 0;
+    const stack    = [];
+    const indices  = new Map();
     const lowLinks = new Map();
-    const onStack = new Set();
-    const cycles = [];
+    const onStack  = new Set();
+    const cycles   = [];
 
     function strongconnect(v) {
         indices.set(v, index);

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,7 +1,8 @@
 /**
  * @fileoverview Executor for "Standard" (per-user) calculations.
  * UPDATED: Implements Batch Flushing to prevent OOM on large datasets.
- * UPDATED: Removes manual global.gc() calls.
+ * UPDATED: Implements "Circuit Breaker" to fail fast on high error rates.
+ * UPDATED: Implements "Adaptive Flushing" based on V8 Heap usage.
  * UPDATED: Manages incremental sharding states.
  * UPDATED: Implements 'isInitialWrite' flag for robust cleanup.
  */
@@ -12,6 +13,7 @@ const { ContextFactory }                                               = require
 const { commitResults }                                                = require('../persistence/ResultCommitter');
 const mathLayer                                                        = require('../layers/index');
 const { performance }                                                  = require('perf_hooks');
+const v8                                                               = require('v8'); // [NEW] For Memory introspection
 
 class StandardExecutor {
     static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
@@ -59,6 +61,9 @@ class StandardExecutor {
         const aggregatedSuccess = {};
         const aggregatedFailures = [];
 
+        // [NEW] Global Error Tracking for Circuit Breaker
+        const errorStats = { count: 0, total: 0 };
+
         Object.keys(state).forEach(name => {
             executionStats[name] = { 
                 processedUsers: 0, 
@@ -89,7 +94,7 @@ class StandardExecutor {
 
         let yP_chunk = {}, tH_chunk = {};
         
-        const BATCH_SIZE = 5000;
+        const MIN_BATCH_SIZE = 1000; // Minimum to process before checking stats
         let usersSinceLastFlush = 0;
 
         try {
@@ -103,6 +108,8 @@ class StandardExecutor {
                 const chunkSize = Object.keys(tP_chunk).length;
 
                 const startProcessing = performance.now();
+                
+                // [UPDATED] Collect execution results (success/failure counts)
                 const promises = streamingCalcs.map(calc =>  
                     StandardExecutor.executePerUser(
                         calc, calc.manifest, dateStr, tP_chunk, yP_chunk, tH_chunk, 
@@ -110,15 +117,37 @@ class StandardExecutor {
                         executionStats[normalizeName(calc.manifest.name)]
                     ) 
                 );
-                await Promise.all(promises);
+                
+                const batchResults = await Promise.all(promises);
                 const procDuration = performance.now() - startProcessing;
                 
                 Object.keys(executionStats).forEach(name => executionStats[name].timings.processing += procDuration);
 
+                // [NEW] Update Error Stats
+                batchResults.forEach(r => {
+                    errorStats.total += (r.success + r.failures);
+                    errorStats.count += r.failures;
+                });
+
+                // [NEW] Circuit Breaker: Fail fast if error rate > 10% after processing 100+ items
+                // We check total > 100 to avoid failing on the very first user if they happen to be bad.
+                if (errorStats.total > 100 && (errorStats.count / errorStats.total) > 0.10) {
+                    const failRate = (errorStats.count / errorStats.total * 100).toFixed(1);
+                    throw new Error(`[Circuit Breaker] High failure rate detected (${failRate}%). Aborting batch to prevent silent data loss.`);
+                }
+
                 usersSinceLastFlush += chunkSize;
 
-                if (usersSinceLastFlush >= BATCH_SIZE) {
-                    logger.log('INFO', `[${passName}] 🛁 Flushing buffer after ${usersSinceLastFlush} users...`);
+                // [NEW] Adaptive Flushing (Memory Pressure Check)
+                const heapStats = v8.getHeapStatistics();
+                const heapUsedRatio = heapStats.used_heap_size / heapStats.heap_size_limit;
+                const MEMORY_THRESHOLD = 0.70; // 70% of available RAM
+                const COUNT_THRESHOLD = 5000;
+
+                if (usersSinceLastFlush >= COUNT_THRESHOLD || heapUsedRatio > MEMORY_THRESHOLD) {
+                    const reason = heapUsedRatio > MEMORY_THRESHOLD ? `MEMORY_PRESSURE (${(heapUsedRatio*100).toFixed(0)}%)` : 'BATCH_LIMIT';
+                    
+                    logger.log('INFO', `[${passName}] 🛁 Flushing buffer after ${usersSinceLastFlush} users. Reason: ${reason}`);
                     
                     // [UPDATED] Pass isInitialWrite: true only on the first flush
                     const flushResult = await StandardExecutor.flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, 'INTERMEDIATE', true, !hasFlushed);
@@ -171,6 +200,7 @@ class StandardExecutor {
                 _executionStats: executionStats[name]
             };
 
+            // Clear the memory immediately after preparing the commit
             inst.results = {};
         }
 
@@ -226,6 +256,10 @@ class StandardExecutor {
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
         const SCHEMAS  = mathLayer.SCHEMAS;
 
+        // [NEW] Track local batch success/failure
+        let chunkSuccess = 0;
+        let chunkFailures = 0;
+
         for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
             const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
             const todayHistory       = historyData ? historyData[userId] : null;
@@ -249,10 +283,16 @@ class StandardExecutor {
             try { 
                 await calcInstance.process(context); 
                 if (stats) stats.processedUsers++;
+                chunkSuccess++;
             } 
-            catch (e) { logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); }
+            catch (e) { 
+                logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); 
+                chunkFailures++;
+            }
         }
+
+        return { success: chunkSuccess, failures: chunkFailures };
     }
 }
 
-module.exports = { StandardExecutor };
+module.exports = { StandardExecutor };

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -2,9 +2,11 @@
  * FILENAME: computation-system/helpers/computation_dispatcher.js
  * PURPOSE: "Smart Dispatcher" - Analyzes state, initializes Run Counters, and dispatches tasks.
  * UPDATED: Implements Callback Pattern. Initializes 'computation_runs' doc for worker coordination.
+ * UPDATED: Implements Forensic Crash Analysis & Intelligent Resource Routing.
+ * FIXED: Implemented "Catch-Up" logic to scan full history (Start -> Target Date) instead of just Target Date.
  */
 
-const { getExpectedDateStrings, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
+const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
 const { groupByPass, analyzeDateExecution }     = require('../WorkflowOrchestrator.js');
 const { PubSubUtils }                           = require('../../core/utils/pubsub_utils');
 const { fetchComputationStatus, updateComputationStatus } = require('../persistence/StatusRepository');
@@ -13,12 +15,49 @@ const { generateCodeHash }                      = require('../topology/HashManag
 const pLimit                                    = require('p-limit');
 const crypto                                    = require('crypto');
 
-const TOPIC_NAME = 'computation-tasks';
-const STATUS_IMPOSSIBLE = 'IMPOSSIBLE';
+const STATUS_IMPOSSIBLE   = 'IMPOSSIBLE';
+
+// Threshold to trigger high-mem routing (e.g., 1.5 GB for a 2GB worker)
+const OOM_THRESHOLD_MB = 1500; 
+
+/**
+ * [NEW] Forensics: Checks if the calculation crashed previously due to Memory.
+ * Reads the 'telemetry.lastMemory' from the audit ledger.
+ */
+async function checkCrashForensics(db, date, pass, computationName) {
+    try {
+        const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${computationName}`;
+        const doc = await db.doc(ledgerPath).get();
+
+        if (!doc.exists) return 'standard';
+
+        const data = doc.data();
+        
+        // Check if we have telemetry from a previous run
+        if (data.telemetry && data.telemetry.lastMemory) {
+            const lastRSS = data.telemetry.lastMemory.rssMB || 0;
+            
+            if (lastRSS > OOM_THRESHOLD_MB) {
+                console.log(`[Dispatcher] 🕵️‍♀️ Forensics: ${computationName} likely OOM'd at ${lastRSS}MB. Routing to HIGH-MEM.`);
+                return 'high-mem';
+            }
+        }
+        
+        // Also check if it's explicitly marked FAILED with 'Memory' in error
+        if (data.status === 'FAILED' && data.error && /memory/i.test(data.error)) {
+            return 'high-mem';
+        }
+
+    } catch (e) {
+        console.warn(`[Dispatcher] Forensics check failed for ${computationName}: ${e.message}`);
+    }
+    
+    return 'standard';
+}
 
 /**
  * Dispatches computation tasks for a specific pass.
- * @param {Object} config - System config
+ * @param {Object} config - System config (Injected with topics)
  * @param {Object} dependencies - { db, logger, ... }
  * @param {Array} computationManifest - List of calculations
  * @param {Object} reqBody - (Optional) HTTP Body containing 'callbackUrl' and 'date'
@@ -28,7 +67,8 @@ async function dispatchComputationPass(config, dependencies, computationManifest
     const pubsubUtils = new PubSubUtils(dependencies);
     const passToRun   = String(config.COMPUTATION_PASS_TO_RUN);
     
-    // [NEW] Extract Date and Callback from request body (pushed by Workflow)
+    // Extract Date and Callback from request body (pushed by Workflow)
+    // NOTE: 'dateStr' acts as the "Target Date" (Ceiling), usually T-1.
     const dateStr     = reqBody.date || config.date; 
     const callbackUrl = reqBody.callbackUrl || null;
 
@@ -44,18 +84,30 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
     if (!calcsInThisPass.length) { return logger.log('WARN', `[Dispatcher] No calcs for Pass ${passToRun}. Exiting.`); }
 
-    const calcNames = calcsInThisPass.map(c => c.name);
-    logger.log('INFO', `🚀 [Dispatcher] Smart-Dispatching PASS ${passToRun} for ${dateStr}`);
+    logger.log('INFO', `🚀 [Dispatcher] Smart-Dispatching PASS ${passToRun} (Target: ${dateStr})`);
     
-    // -- DATE ANALYSIS LOGIC (Unchanged) --
-    const passEarliestDate = Object.values(DEFINITIVE_EARLIEST_DATES).reduce((a, b) => a < b ? a : b);
-    const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
+    // -- DATE ANALYSIS LOGIC (FIXED: RANGE SCAN) --
     
-    // We only analyze the specific requested date to keep dispatch fast for the workflow
-    const allExpectedDates = [dateStr]; 
+    // 1. Determine the absolute start of data history
+    const earliestDates = await getEarliestDataDates(config, dependencies);
+    const startDate     = earliestDates.absoluteEarliest;
+    const endDate       = new Date(dateStr + 'T00:00:00Z');
+
+    // 2. Generate the full range of dates to check
+    let allExpectedDates = getExpectedDateStrings(startDate, endDate);
+
+    // Safety fallback: if range is invalid or empty, default to target date only
+    if (!allExpectedDates || allExpectedDates.length === 0) {
+        logger.log('WARN', `[Dispatcher] Date range calculation returned empty (Start: ${startDate.toISOString()} -> End: ${endDate.toISOString()}). Defaulting to single target date.`);
+        allExpectedDates = [dateStr];
+    } else {
+        logger.log('INFO', `[Dispatcher] 📅 Analysis Range: ${allExpectedDates.length} days (${allExpectedDates[0]} to ${allExpectedDates[allExpectedDates.length-1]})`);
+    }
 
     const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
     const tasksToDispatch = [];
+    
+    // Concurrency limit for analysis & forensics (Parallelize the historical scan)
     const limit = pLimit(20);
 
     const analysisPromises = allExpectedDates.map(d => limit(async () => {
@@ -71,6 +123,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                 prevDate.setUTCDate(prevDate.getUTCDate() - 1);
                 prevDateStr = prevDate.toISOString().slice(0, 10);
                 
+                // Only fetch previous status if it's within valid range
                 if (prevDate >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
                     fetchPromises.push(fetchComputationStatus(prevDateStr, config, dependencies));
                 }
@@ -87,18 +140,16 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
             const report = analyzeDateExecution(d, calcsInThisPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
 
+            // Handle Status Updates (Impossible / Blocked)
             const statusUpdates = {};
-            
             report.impossible.forEach(item => {
                 if (dailyStatus[item.name]?.hash !== STATUS_IMPOSSIBLE) {
                     statusUpdates[item.name] = { hash: STATUS_IMPOSSIBLE, category: 'unknown', reason: item.reason };
                 }
             });
-
             report.blocked.forEach(item => {
                  statusUpdates[item.name] = { hash: false, category: 'unknown', reason: item.reason };
             });
-
             report.failedDependency.forEach(item => {
                  const missingStr = item.missing ? item.missing.join(', ') : 'unknown';
                  statusUpdates[item.name] = { hash: false, category: 'unknown', reason: `Dependency Missing: ${missingStr}` };
@@ -109,21 +160,29 @@ async function dispatchComputationPass(config, dependencies, computationManifest
             }
 
             const validToRun = [...report.runnable, ...report.reRuns];
-            validToRun.forEach(item => {
+            
+            // [NEW] Parallel Forensics Check
+            await Promise.all(validToRun.map(item => limit(async () => {
+                const compName = normalizeName(item.name);
+                
+                // 1. Determine Resource Requirements
+                const requiredResource = await checkCrashForensics(db, d, passToRun, compName);
+                
                 const uniqueDispatchId = crypto.randomUUID(); 
                 tasksToDispatch.push({
                     action: 'RUN_COMPUTATION_DATE',
                     dispatchId: uniqueDispatchId,
                     date: d,
                     pass: passToRun,
-                    computation: normalizeName(item.name),
+                    computation: compName,
                     hash: item.hash || item.newHash, 
                     previousCategory: item.previousCategory || null,
                     triggerReason: item.reason || "Unknown",
                     dependencyResultHashes: item.dependencyResultHashes || {},
-                    timestamp: Date.now()
+                    timestamp: Date.now(),
+                    resources: requiredResource // 'standard' or 'high-mem'
                 });
-            });
+            })));
 
         } catch (e) {
             logger.log('ERROR', `[Dispatcher] Failed analysis for ${d}: ${e.message}`);
@@ -132,10 +191,9 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
     await Promise.all(analysisPromises);
 
-    // -- NEW: CALLBACK & COUNTER INITIALIZATION --
+    // -- CALLBACK & COUNTER INITIALIZATION --
 
     if (tasksToDispatch.length > 0) {
-        logger.log('INFO', `[Dispatcher] 📝 Preparing ${tasksToDispatch.length} tasks for execution...`);
         
         // 1. Initialize Shared State Document (The Counter)
         const runId = crypto.randomUUID();
@@ -144,17 +202,17 @@ async function dispatchComputationPass(config, dependencies, computationManifest
         if (callbackUrl) {
             await db.doc(metaStatePath).set({
                 createdAt: new Date(),
-                date: dateStr,
+                date: dateStr, // Acts as the "Job Label" (target date)
                 pass: passToRun,
                 totalTasks: tasksToDispatch.length,
-                remainingTasks: tasksToDispatch.length, // <--- The Countdown
-                callbackUrl: callbackUrl,               // <--- The Workflow Hook
+                remainingTasks: tasksToDispatch.length, 
+                callbackUrl: callbackUrl,               
                 status: 'IN_PROGRESS'
             });
-            logger.log('INFO', `[Dispatcher] 🏁 Run State Initialized: ${runId}`);
+            logger.log('INFO', `[Dispatcher] 🏁 Run State Initialized: ${runId}. Tasks: ${tasksToDispatch.length}`);
         }
 
-        // 2. Attach Run Metadata to every task
+        // 2. Attach Run Metadata
         tasksToDispatch.forEach(task => {
             task.runId = runId;
             task.metaStatePath = callbackUrl ? metaStatePath : null; 
@@ -180,12 +238,13 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                     t.set(ledgerRef, {
                         status: 'PENDING',
                         dispatchId: task.dispatchId,
-                        runId: task.runId, // Track the batch ID
+                        runId: task.runId,
                         computation: task.computation,
                         expectedHash: task.hash || 'unknown',
                         createdAt: new Date(),
                         dispatcherHash: currentManifestHash,
                         triggerReason: task.triggerReason,
+                        resources: task.resources, // Log intended resource type
                         retries: 0
                     }, { merge: true });
                     
@@ -201,22 +260,36 @@ async function dispatchComputationPass(config, dependencies, computationManifest
 
         await Promise.all(txnPromises);
 
-        // 4. Publish to Pub/Sub
+        // 4. Publish to Pub/Sub (Segregated by Resources)
         if (finalDispatched.length > 0) {
-            logger.log('INFO', `[Dispatcher] ✅ Publishing ${finalDispatched.length} tasks to Pub/Sub...`);
             
-            await pubsubUtils.batchPublishTasks(dependencies, {
-                topicName: TOPIC_NAME,
-                tasks: finalDispatched,
-                taskType: `computation-pass-${passToRun}`,
-                maxPubsubBatchSize: 100
-            });
+            const standardTasks = finalDispatched.filter(t => t.resources !== 'high-mem');
+            const highMemTasks  = finalDispatched.filter(t => t.resources === 'high-mem');
+
+            // Publish Standard
+            if (standardTasks.length > 0) {
+                logger.log('INFO', `[Dispatcher] ✅ Publishing ${standardTasks.length} Standard tasks...`);
+                await pubsubUtils.batchPublishTasks(dependencies, {
+                    topicName: config.computationTopicStandard || 'computation-tasks',
+                    tasks: standardTasks,
+                    taskType: `computation-pass-${passToRun}-std`,
+                    maxPubsubBatchSize: 100
+                });
+            }
+
+            // Publish High-Mem
+            if (highMemTasks.length > 0) {
+                logger.log('INFO', `[Dispatcher] 🏋️‍♀️ Publishing ${highMemTasks.length} tasks to HIGH-MEM infrastructure.`);
+                await pubsubUtils.batchPublishTasks(dependencies, {
+                    topicName: config.computationTopicHighMem || 'computation-tasks-highmem',
+                    tasks: highMemTasks,
+                    taskType: `computation-pass-${passToRun}-highmem`,
+                    maxPubsubBatchSize: 100
+                });
+            }
             
-            // Return count so workflow knows to wait
             return { dispatched: finalDispatched.length, runId };
         } else {
-            // Edge Case: Analysis said "Run", but Ledger said "Already Done"
-            // We must update the state doc to 0 or delete it, OR return 0 so workflow doesn't wait.
             logger.log('INFO', `[Dispatcher] All tasks were already COMPLETED.`);
             return { dispatched: 0 };
         }

package/functions/computation-system/helpers/computation_worker.js

@@ -2,6 +2,7 @@
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes tasks, executes logic, and signals Workflow upon Batch Completion.
  * UPDATED: Implements IAM Auth for Workflow Callbacks.
+ * UPDATED: Implements Memory Heartbeat (Flight Recorder) for OOM detection.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
@@ -9,7 +10,7 @@ const { getManifest }         = require('../topology/ManifestLoader');
 const { StructuredLogger }    = require('../logger/logger'); 
 const { recordRunAttempt }    = require('../persistence/RunRecorder'); 
 const https                   = require('https'); 
-const { GoogleAuth }          = require('google-auth-library'); // [NEW] Required for Auth
+const { GoogleAuth }          = require('google-auth-library'); 
 
 let calculationPackage;
 try { calculationPackage = require('aiden-shared-calculations-unified');
@@ -19,51 +20,68 @@ const calculations = calculationPackage.calculations;
 const MAX_RETRIES = 3;
 
 /**
- * [NEW] Helper: Fires the webhook back to Google Cloud Workflows.
- * UPDATED: Now generates an IAM Bearer Token to authenticate the request.
+ * [NEW] Helper: Starts a background heartbeat to track memory usage.
+ * This acts as a "Black Box Recorder". If the worker crashes (OOM),
+ * the last written value will remain in Firestore for the Dispatcher to analyze.
+ */
+function startMemoryHeartbeat(db, ledgerPath, intervalMs = 2000) {
+    const getMemStats = () => {
+        const mem = process.memoryUsage();
+        return {
+            rssMB: Math.round(mem.rss / 1024 / 1024),       // Resident Set Size (OOM Killer Metric)
+            heapUsedMB: Math.round(mem.heapUsed / 1024 / 1024),
+            timestamp: new Date()
+        };
+    };
+
+    const timer = setInterval(async () => {
+        try {
+            const stats = getMemStats();
+            // Use update() to minimize payload size and avoid overwriting status
+            await db.doc(ledgerPath).update({
+                'telemetry.lastMemory': stats,
+                'telemetry.lastHeartbeat': new Date()
+            }).catch(() => {}); // Ignore write errors to prevent crashing the worker
+        } catch (e) {
+            // Silently fail on telemetry errors
+        }
+    }, intervalMs);
+
+    // Unref so this timer doesn't prevent the process from exiting naturally
+    timer.unref();
+
+    return timer;
+}
+
+/**
+ * Helper: Fires the webhook back to Google Cloud Workflows.
  */
 async function triggerWorkflowCallback(url, status, logger) {
     if (!url) return;
     logger.log('INFO', `[Worker] 🔔 BATCH COMPLETE! Triggering Workflow Callback: ${status}`);
     
     try {
-        // 1. Get OAuth2 Access Token (Required for Workflows Callbacks)
-        const auth = new GoogleAuth({
-            scopes: ['https://www.googleapis.com/auth/cloud-platform']
-        });
+        const auth = new GoogleAuth({ scopes: ['https://www.googleapis.com/auth/cloud-platform'] });
         const client = await auth.getClient();
         const accessToken = await client.getAccessToken();
         const token = accessToken.token;
 
-        // 2. Send Authenticated Request
         return new Promise((resolve, reject) => {
-            const body = JSON.stringify({ 
-                status: status, 
-                timestamp: new Date().toISOString() 
-            });
+            const body = JSON.stringify({ status: status, timestamp: new Date().toISOString() });
             
             const req = https.request(url, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
                     'Content-Length': Buffer.byteLength(body),
-                    'Authorization': `Bearer ${token}` // <--- CRITICAL FIX
+                    'Authorization': `Bearer ${token}`
                 }
             }, (res) => {
-                if (res.statusCode >= 200 && res.statusCode < 300) {
-                    resolve();
-                } else {
-                    logger.log('WARN', `Callback responded with ${res.statusCode}`);
-                    // We resolve anyway to avoid crashing the worker logic
-                    resolve(); 
-                }
+                if (res.statusCode >= 200 && res.statusCode < 300) { resolve(); } 
+                else { logger.log('WARN', `Callback responded with ${res.statusCode}`); resolve(); }
             });
 
-            req.on('error', (e) => {
-                logger.log('ERROR', `Failed to trigger callback: ${e.message}`);
-                resolve(); 
-            });
-            
+            req.on('error', (e) => { logger.log('ERROR', `Failed to trigger callback: ${e.message}`); resolve(); });
             req.write(body);
             req.end();
         });
@@ -73,37 +91,21 @@ async function triggerWorkflowCallback(url, status, logger) {
 }
 
 /**
- * [NEW] Helper: Decrements 'remainingTasks' in Firestore.
- * Returns the callbackUrl IF this was the last task.
+ * Helper: Decrements 'remainingTasks' in Firestore.
  */
 async function decrementAndCheck(db, metaStatePath, logger) {
     if (!metaStatePath) return null;
-    
     try {
         const result = await db.runTransaction(async (t) => {
             const ref = db.doc(metaStatePath);
             const doc = await t.get(ref);
-            
-            if (!doc.exists) return null; // State might have expired or been deleted
+            if (!doc.exists) return null;
             const data = doc.data();
-            
             const newRemaining = (data.remainingTasks || 0) - 1;
-            
-            t.update(ref, { 
-                remainingTasks: newRemaining,
-                lastUpdated: new Date()
-            });
-            
-            // Return needed data only if we hit 0 (or lower, for safety)
-            return { 
-                remaining: newRemaining, 
-                callbackUrl: data.callbackUrl 
-            };
+            t.update(ref, { remainingTasks: newRemaining, lastUpdated: new Date() });
+            return { remaining: newRemaining, callbackUrl: data.callbackUrl };
         });
-
-        if (result && result.remaining <= 0) {
-            return result.callbackUrl;
-        }
+        if (result && result.remaining <= 0) return result.callbackUrl;
     } catch (e) {
         logger.log('ERROR', `[Worker] Failed to decrement batch counter: ${e.message}`);
     }
@@ -125,12 +127,12 @@ async function handleComputationTask(message, config, dependencies) {
 
     if (!data || data.action !== 'RUN_COMPUTATION_DATE') { return; }
     
-    // Extract fields including new metaStatePath
     const { date, pass, computation, previousCategory, triggerReason, dispatchId, dependencyResultHashes, metaStatePath } = data;
     
     if (!date || !pass || !computation) { logger.log('ERROR', `[Worker] Invalid payload.`, data); return; }
 
     const retryCount = message.deliveryAttempt || 1;
+    const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks/${computation}`;
 
     // --- POISON MESSAGE HANDLING (DLQ) ---
     if (retryCount > MAX_RETRIES) {
@@ -144,36 +146,38 @@ async function handleComputationTask(message, config, dependencies) {
                     failureReason: 'MAX_RETRIES_EXCEEDED'
                 });
 
-                await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).doc(computation).set({
+                await db.doc(ledgerPath).set({
                     status: 'FAILED',
                     error: 'Max Retries Exceeded (Poison Message)',
                     failedAt: new Date()
                 }, { merge: true });
 
                 const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
-                if (callbackUrl) {
-                    await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger);
-                }
-
+                if (callbackUrl) { await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger); }
                 return;
             } catch (dlqErr) { logger.log('FATAL', `[Worker] Failed to write to DLQ`, dlqErr); }
     }
 
     logger.log('INFO', `[Worker] 📥 Received Task: ${computation} (${date}) [Attempt ${retryCount}/${MAX_RETRIES}]`);
 
-    // Update Status to IN_PROGRESS
+    // 1. Update Status to IN_PROGRESS & Initialize Telemetry
     try {
-        await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).doc(computation).set({
+        await db.doc(ledgerPath).set({
             status: 'IN_PROGRESS',
             workerId: process.env.K_REVISION || 'unknown',
             startedAt: new Date(),
-            dispatchId: dispatchId
+            dispatchId: dispatchId,
+            telemetry: { startTime: new Date(), lastMemory: null } // Init for heartbeat
         }, { merge: true });
     } catch (leaseErr) {}
 
+    // 2. START HEARTBEAT (The Flight Recorder)
+    const heartbeatTimer = startMemoryHeartbeat(db, ledgerPath, 2000);
+
     let computationManifest;
     try { computationManifest = getManifest(config.activeProductLines || [], calculations, runDependencies);
     } catch (manifestError) { 
+        clearInterval(heartbeatTimer); // Stop if we fail early
         logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
         return; 
     }
@@ -186,6 +190,9 @@ async function handleComputationTask(message, config, dependencies) {
         );
         const duration = Date.now() - startTime;
 
+        // STOP HEARTBEAT ON SUCCESS
+        clearInterval(heartbeatTimer);
+
         const failureReport  = result?.updates?.failureReport  || [];
         const successUpdates = result?.updates?.successUpdates || {};
 
@@ -194,26 +201,23 @@ async function handleComputationTask(message, config, dependencies) {
             throw new Error(failReason.error.message || 'Computation Logic Failed');
         } 
         else {
-            if (Object.keys(successUpdates).length > 0) {
-                logger.log('INFO', `[Worker] ✅ Stored: ${computation}`);
-            } else {
-                logger.log('WARN', `[Worker] ⚠️ Empty Result: ${computation}`);
-            }
+            if (Object.keys(successUpdates).length > 0) { logger.log('INFO', `[Worker] ✅ Stored: ${computation}`); } 
+            else { logger.log('WARN', `[Worker] ⚠️ Empty Result: ${computation}`); }
 
-            await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).doc(computation).update({
+            await db.doc(ledgerPath).update({
                 status: 'COMPLETED',
                 completedAt: new Date()
             }).catch(() => {});
 
             await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, { durationMs: duration }, triggerReason);
 
-            // Decrement & Callback
             const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
-            if (callbackUrl) {
-                await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger);
-            }
+            if (callbackUrl) { await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger); }
         }
     } catch (err) { 
+        // STOP HEARTBEAT ON ERROR
+        clearInterval(heartbeatTimer);
+
         // --- ERROR HANDLING ---
         const isDeterministicError = err.stage === 'SHARDING_LIMIT_EXCEEDED' || 
                                      err.stage === 'QUALITY_CIRCUIT_BREAKER' ||
@@ -231,7 +235,7 @@ async function handleComputationTask(message, config, dependencies) {
                      failureReason: 'PERMANENT_DETERMINISTIC_ERROR'
                  });
                  
-                 await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).doc(computation).set({
+                 await db.doc(ledgerPath).set({
                     status: 'FAILED',
                     error: err.message || 'Permanent Deterministic Error',
                     failedAt: new Date()
@@ -240,23 +244,17 @@ async function handleComputationTask(message, config, dependencies) {
                  await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'PERMANENT_FAIL' }, { durationMs: 0 }, triggerReason);
                  
                  const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
-                 if (callbackUrl) {
-                     await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger);
-                 }
-
+                 if (callbackUrl) { await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger); }
                  return; 
              } catch (dlqErr) { logger.log('FATAL', `[Worker] Failed to write to DLQ`, dlqErr); }
         }
 
-        if (retryCount >= MAX_RETRIES) {
-             throw err; 
-        }
+        if (retryCount >= MAX_RETRIES) { throw err; }
 
         logger.log('ERROR', `[Worker] ❌ Crash: ${computation}: ${err.message}`); 
         await recordRunAttempt(db, { date, computation, pass }, 'CRASH', {  message: err.message, stack: err.stack, stage: 'SYSTEM_CRASH' }, { durationMs: 0 }, triggerReason);
-        
         throw err; 
     }
 }
 
-module.exports = { handleComputationTask };
+module.exports = { handleComputationTask };

package/functions/computation-system/paper.md

@@ -0,0 +1,93 @@
+# The BullTrackers Computation System: An Advanced DAG-Based Architecture for High-Fidelity Financial Simulation
+
+## Abstract
+
+This paper details the design, implementation, and theoretical underpinnings of the BullTrackers Computation System, a proprietary high-performance execution engine designed for complex financial modeling and user behavior analysis. The system leverages a Directed Acyclic Graph (DAG) architecture to orchestrate interdependent calculations, employing Kahn’s Algorithm for topological sorting and Tarjan’s Algorithm for cycle detection. Key innovations include "Content-Based Dependency Short-Circuiting" for massive optimization, a "System Epoch" and "Infrastructure Hash" based auditing system for absolute reproducibility, and a batch-flushing execution model designed to mitigate Out-Of-Memory (OOM) errors during high-volume processing. We further explore the application of this system in running advanced psychometric and risk-geometry models ("Smart Money" scoring) and how the architecture supports self-healing workflows through granular state management.
+
+## 1. Introduction
+
+In modern financial analytics, derived data often depends on a complex web of varying input frequencies—real-time price ticks, daily portfolio snapshots, and historical trade logs. Traditional linear batch processing protocols fail to capture the nuances of these interdependencies, often leading to race conditions or redundant computations.
+
+The BullTrackers Computation System was devised to solve this by treating the entire domain logic as a **Directed Acyclic Graph (DAG)**. Every calculation is a node, and every data requirement is an edge. By resolving the topography of this graph dynamically at runtime, the system ensures that:
+1.  Data is always available before it is consumed (referential integrity).
+2.  Only necessary computations are executed (efficiency).
+3.  Changes in code or infrastructure propagate deterministically through the graph (auditability).
+
+## 2. Theoretical Foundations
+
+The core utility of the system is its ability to turn a collection of loosely coupled JavaScript classes into a strictly ordered execution plan.
+
+### 2.1 Directed Acyclic Graphs (DAGs)
+We model the computation space as a DAG where $G = (V, E)$.
+*   **Vertices ($V$)**: Individual Calculation Units (e.g., `NetProfit`, [SmartMoneyScore](file:///C:/Users/aiden/Desktop/code_projects/Bulltrackers2025/Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/computation-system/layers/profiling.js#24-236)).
+*   **Edges ($E$)**: Data dependencies, where an edge $(u, v)$ implies $v$ requires the output of $u$.
+
+### 2.2 Topological Sorting (Kahn’s Algorithm)
+To execute the graph, we must linearize it such that for every dependency $u \rightarrow v$, $u$ precedes $v$ in the execution order. We implement **Kahn’s Algorithm** within [ManifestBuilder.js](file:///C:/Users/aiden/Desktop/code_projects/Bulltrackers2025/Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/computation-system/context/ManifestBuilder.js) to achieve this:
+1.  Calculate the **in-degree** (number of incoming edges) for all nodes.
+2.  Initialize a queue with all nodes having an in-degree of 0 (independent nodes).
+3.  While the queue is not empty:
+    *   Dequeue node $N$ and add it to the `SortedManifest`.
+    *   For each neighbor $M$ dependent on $N$, decrement $M$'s in-degree.
+    *   If $M$'s in-degree becomes 0, enqueue $M$.
+4.  This generates a series of "Passes" or "Waves" of execution, allowing parallel processing of independent nodes within the same pass.
+
+### 2.3 Cycle Detection (Tarjan’s Algorithm)
+A critical failure mode in DAGs is the introduction of a cycle (e.g., A needs B, B needs A), effectively turning the DAG into a DCG (Directed Cyclic Graph), which is unresolvable.
+If Kahn’s algorithm fails to visit all nodes (indicating a cycle exists), the system falls back to **Tarjan’s Strongly Connected Components (SCC) Algorithm**. This uses depth-first search to identify the exact cycle chain (e.g., `Calc A -> Calc B -> Calc C -> Calc A`), reporting the "First Cycle Found" to the developer for immediate remediation.
+
+## 3. System Architecture & "Source of Truth"
+
+The architecture is centered around the **Manifest**, a dynamic, immutable registry of all capabilities within the system.
+
+### 3.1 The Dynamic Manifest
+Unlike static build tools, the Manifest is built at runtime by [ManifestLoader.js](file:///C:/Users/aiden/Desktop/code_projects/Bulltrackers2025/Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/computation-system/topology/ManifestLoader.js) and [ManifestBuilder.js](file:///C:/Users/aiden/Desktop/code_projects/Bulltrackers2025/Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/computation-system/context/ManifestBuilder.js). It employs an **Auto-Discovery** mechanism that scans directories for calculation classes.
+*   **Static Metadata**: Each class exposes `getMetadata()` and `getDependencies()`.
+*   **Product Line Filtering**: The builder can slice the graph, generating a subgraph relevant only to specific product lines (e.g., "Crypto", "Stocks"), reducing overhead.
+
+### 3.2 Granular Hashing & The Audit Chain
+To ensure that "if the code hasn't changed, the result shouldn't change," the system implements a multi-layered hashing strategy ([HashManager.js](file:///C:/Users/aiden/Desktop/code_projects/Bulltrackers2025/Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/computation-system/topology/HashManager.js)):
+1.  **Code Hash**: The raw string content of the calculation class.
+2.  **Layer Hash**: Hashes of shared utility layers (`mathematics`, `profiling`) used by the class.
+3.  **Dependency Hash**: A composite hash of all upstream dependencies.
+4.  **Infrastructure Hash**: A hash representing the underlying system environment.
+5.  **System Epoch**: A manual versioning flag to force global re-computation.
+
+This results in a `Composite Hash`. If this hash matches the `storedHash` in the database, execution can be skipped entirely.
+
+## 4. Execution Engine: Flow, Resilience & Optimization
+
+The `WorkflowOrchestrator` acts as the runtime kernel, utilizing [StandardExecutor](file:///C:/Users/aiden/Desktop/code_projects/Bulltrackers2025/Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/computation-system/executors/StandardExecutor.js#16-257) and [MetaExecutor](file:///C:/Users/aiden/Desktop/code_projects/Bulltrackers2025/Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/computation-system/executors/MetaExecutor.js#12-83) for the heavy lifting.
+
+### 4.1 Content-Based Dependency Short-Circuiting
+A major optimization (O(n) gain) is the **Content-Based Short-Circuiting** logic found in [WorkflowOrchestrator.js](file:///C:/Users/aiden/Desktop/code_projects/Bulltrackers2025/Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/computation-system/WorkflowOrchestrator.js):
+Even if an upstream dependency *re-runs* (e.g., its timestamp changed), its *output* might be identical to the previous run.
+1.  The system tracks `ResultHash` (hash of the actual output data).
+2.  When checking dependencies for Node B (which depends on A), if A has re-run but its `ResultHash` is unchanged from what B used last time, B **does not need to re-run**.
+3.  This effectively stops "change propagation" dead in its tracks if the data change is semantically null.
+
+### 4.2 Batch Flushing & OOM Prevention
+Financial datasets (processing 100k+ users with daily portfolios) often exceed Node.js heap limits. The [StandardExecutor](file:///C:/Users/aiden/Desktop/code_projects/Bulltrackers2025/Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/computation-system/executors/StandardExecutor.js#16-257) implements a **Streaming & Flushing** architecture:
+*   **Streams** inputs (Portfolio/History) using generators (`yield`), preventing loading all users into memory.
+*   **Buffers** results in a `state` object.
+*   **Flushes** to the database (Firestore/Storage) every $N$ users (e.g., 5000), clearing the internal buffer helps avoid Out-Of-Memory crashes.
+*   **Incremental Sharding**: It manages shard indices dynamically to split massive result sets into retrievable chunks.
+
+### 4.3 Handling "Impossible" States
+If a dependency fails or is missing critical data, the Orchestrator marks dependent nodes as `IMPOSSIBLE` rather than failing them. This allows the rest of the graph (independent branches) to continue execution, maximizing system throughput even in a partially degraded state.
+
+## 5. Advanced Application: Psychometrics & Risk Geometry
+
+The capabilities of this computation engine are best demonstrated by the [profiling.js](file:///C:/Users/aiden/Desktop/code_projects/Bulltrackers2025/Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/computation-system/layers/profiling.js) layer it powers. Because the DAG ensures all historical and portfolio data is perfectly aligned, we can run sophisticated O(n^2) or O(n log n) algorithms on user data reliably.
+
+### 5.1 "Smart Money" & Cognitive Profiling
+The system executes a [UserClassifier](file:///C:/Users/aiden/Desktop/code_projects/Bulltrackers2025/Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/computation-system/layers/profiling.js#382-399) that computes:
+*   **Risk Geometry**: Using the **Monotone Chain** algorithm to compute the Convex Hull of a user's risk/reward performance (Efficient Frontier analysis).
+*   **Psychometrics**: Detecting "Revenge Trading" (increasing risk after losses) and "Disposition Skew" (holding losers too long).
+*   **Attribution**: Separating "Luck" (market beta) from "Skill" (Alpha) by comparing performance against sector benchmarks.
+
+These complex models depend on the *guarantee* provided by the DAG that all necessary history and price data is pre-computed and available in the [Context](file:///C:/Users/aiden/Desktop/code_projects/Bulltrackers2025/Backend/Entrypoints/BullTrackers/Backend/Core/bulltrackers-module/functions/computation-system/simulation/Fabricator.js#20-69).
+
+## 6. Conclusion
+
+The BullTrackers Computation System represents a shift from "Action-Based" to "State-Based" architecture. By encoding the domain logic into a Directed Acyclic Graph, we achieve a system that is self-healing, massively scalable via short-circuiting and batching, and capable of supporting deep analytical models. It provides the robustness required for high-stakes financial simulation, ensuring that every decimal point is traceable, reproducible, and verifiable.

package/functions/computation-system/persistence/RunRecorder.js

@@ -43,14 +43,14 @@ async function recordRunAttempt(db, context, status, error = null, detailedMetri
     const timings = rawExecStats.timings || {};
 
     const runEntry = {
-        runId: runId,
+        runId:           runId,
         computationName: computation,
-        pass: String(pass),
-        workerId: workerId,
-        targetDate: targetDate,
-        triggerTime: now.toISOString(),
-        durationMs: detailedMetrics.durationMs || 0,
-        status: status,
+        pass:            String(pass),
+        workerId:        workerId,
+        targetDate:      targetDate,
+        triggerTime:     now.toISOString(),
+        durationMs:      detailedMetrics.durationMs || 0,
+        status:          status,
         
         // [NEW] Trigger Context
         trigger: {

package/functions/computation-system/tools/BuildReporter.js

@@ -325,13 +325,13 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             }
 
             // 3. BLOCKED / IMPOSSIBLE / UPTODATE
-            analysis.blocked.forEach(item => pushIfValid(dateSummary.blocked, item));
-            analysis.failedDependency.forEach(item => pushIfValid(dateSummary.blocked, item, "Dependency Missing"));
-            analysis.impossible.forEach(item => pushIfValid(dateSummary.impossible, item));
-            analysis.skipped.forEach(item => pushIfValid(dateSummary.uptodate, item, "Up To Date"));
+            analysis.blocked.forEach          (item => pushIfValid(dateSummary.blocked,    item));
+            analysis.failedDependency.forEach (item => pushIfValid(dateSummary.blocked,    item, "Dependency Missing"));
+            analysis.impossible.forEach       (item => pushIfValid(dateSummary.impossible, item));
+            analysis.skipped.forEach          (item => pushIfValid(dateSummary.uptodate,   item, "Up To Date"));
 
             // Meta stats
-            const includedCount = dateSummary.run.length + dateSummary.rerun.length + dateSummary.stable.length +
+            const includedCount = dateSummary.run.length     + dateSummary.rerun.length      + dateSummary.stable.length +
                                   dateSummary.blocked.length + dateSummary.impossible.length + dateSummary.uptodate.length;
             dateSummary.meta.totalIncluded = includedCount;
             dateSummary.meta.match = (includedCount === expectedCount);

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,7 +1,6 @@
 # Cloud Workflows Definition for BullTrackers Computation Pipeline
 # Orchestrates 5 sequential passes using Event-Driven Callbacks (Zero Polling).
-# FIXED: Replaced invalid 'sys' callback functions with 'events' library functions.
-# FIXED: Proper extraction of 'callback_details.url' for the dispatcher.
+# FIXED: Restored 'passes' and 'max_retries' variables in init step.
 
 main:
   params: [input]
@@ -10,8 +9,14 @@ main:
         assign:
           - project: ${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}
           - location: "europe-west1"
-          # If 'date' is provided in input, use it. Otherwise default to today (YYYY-MM-DD).
-          - date_to_run: ${default(map.get(input, "date"), text.substring(time.format(sys.now()), 0, 10))}
+          
+          # T-1 Date Logic (Process Yesterday)
+          - now: ${sys.now()}
+          - yesterday_timestamp: ${now - 86400} 
+          - yesterday_str: ${text.substring(time.format(yesterday_timestamp), 0, 10)}
+          - date_to_run: ${default(map.get(input, "date"), yesterday_str)}
+
+          # Configuration Variables (Restored)
           - passes: ["1", "2", "3", "4", "5"]
           - max_retries: 3
 
@@ -42,7 +47,6 @@ main:
                             - attempt_count: ${attempt_count + 1}
                       
                       # 1. GENERATE CALLBACK ENDPOINT
-                      # We use the 'events' library. This returns an object containing the URL.
                       - create_callback:
                           call: events.create_callback_endpoint
                           args:
@@ -60,7 +64,6 @@ main:
                             severity: "INFO"
 
                       # 2. TRIGGER DISPATCHER
-                      # We pass the extracted 'callback_url' string to the dispatcher.
                       - trigger_dispatcher:
                           call: http.post
                           args:
@@ -89,12 +92,11 @@ main:
                                     next: pass_retry_loop
 
                       # 4. WAIT FOR WORKER SIGNAL
-                      # We must pass the original 'callback_details' object here, not the URL string.
                       - wait_for_completion:
                           call: events.await_callback
                           args:
                             callback: ${callback_details}
-                            timeout: 86400 # Wait up to 24 hours
+                            timeout: 10800 # UPDATED: Reduced from 86400 (24h) to 10800 (3h) to detect crashes faster
                           result: callback_request
 
                       # 5. PROCESS SIGNAL

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.292",
+  "version": "1.0.294",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [

package/functions/computation-system/onboarding.md

@@ -1,210 +0,0 @@
-# BullTrackers Computation System: Architecture & Operational Manual
-
-This document provides a comprehensive overview of the BullTrackers Computation System, a distributed, deterministic, and self-optimizing data pipeline. Unlike traditional task schedulers, this system operates on "Build System" principles, treating data calculations as compiled artifacts with strict versioning and dependency guarantees.
-
----
-
-## 1. System Philosophy & Core Concepts
-
-### The "Build System" Paradigm
-We treat the computation pipeline like a large-scale software build system (e.g., Bazel or Make). Every data point is an "artifact" produced by a specific version of code (Code Hash) acting on specific versions of dependencies (Dependency Hashes).
-*   **Determinism**: If the input data and code haven't changed, the output *must* be identical. We verify this to skip unnecessary work.
-*   **Merkle Tree Structure**: The state of the system is a DAG (Directed Acyclic Graph) of hashes. A change in a root node propagates potential invalidation down the tree, but invalidation stops as soon as a node produces the same output as before (Short-Circuiting).
-
-### Source-of-Truth Architecture
-The **Root Data Index** is the absolute source of truth. No computation can start until the underlying raw data (prices, signals) is indexed and verified "Available" for the target date. This prevents partial runs and "garbage-in-garbage-out".
-
-### The Three-Layer Hash Model
-To optimize execution, we track three distinct hashes for every calculation:
-1.  **Code Hash (Static)**: A SHA-256 hash of the cleaned source code (comments and whitespace stripped). This tells us if the logic *might* have changed.
-2.  **SimHash (Behavioral)**: Generated by running the code against a deterministic "Fabricated" context. This tells us if the logic *actually* changed behavior (e.g., a refactor that changes variable names but not logic will have a different Code Hash but the same SimHash).
-3.  **ResultHash (Output)**: A hash of the actual production output from a run. This tells us if the data changed. Used for downstream short-circuiting.
-
----
-
-## 2. Core Components Overview
-
-### Root Data Indexer
-A scheduled crawler that verifies the availability of raw external data (e.g., asset prices, global signals) for a given date. It produces an "Availability Manifest" that the Dispatcher consults before scheduling anything.
-
-### Manifest Builder
-*   **Role**: Topology Discovery.
-*   **Mechanism**: It scans the `calculations/` directory, loads every module, and builds the global Dependency Graph (DAG) in memory.
-*   **Output**: A topological sort of all calculations assigned to "Passes" (Pass 0, Pass 1, etc.).
-
-### The Dispatcher (`WorkflowOrchestrator.js`)
-The "Brain" of the system. It runs largely stateless, analyzing the `StatusRepository` against the `Manifest`.
-*   **Responsibility**: For a given Grid (Date x Calculation), it determines if the state is `RUNNABLE`, `BLOCKED`, `SKIPPED`, or `IMPOSSIBLE`.
-*   **Key Logic**: It implements the "Short-Circuiting" and "Historical Continuity" checks.
-
-### The Build Optimizer
-A pre-flight tool that attempts to avoiding running tasks by proving they are identical to previous versions.
-*   **Mechanism**: If a calculation's Code Hash changes, the Optimizer runs a **Simulation** (using `SimRunner`) to generate a SimHash. If the SimHash matches the registry, the system acts as if the code never changed, skipping the production re-run.
-
-### The Worker (`StandardExecutor` / `MetaExecutor`)
-The execution unit. It is unaware of the broader topology.
-*   **Input**: A target Calculation and Date.
-*   **Action**: Fetches inputs, runs `process()`, validates results, and writes to Firestore.
-*   **Output**: The computed data + the **ResultHash**.
-
----
-
-## 3. The Daily Lifecycle (Chronological Process)
-
-### Phase 1: Indexing
-The system waits for the `SystemEpoch` to advance. The Root Data Indexer checks for "Canary Blocks" (indicators that external data providers have finished for the day). Once confirmed, the date is marked `OPEN`.
-
-### Phase 2: Pre-Flight Optimization
-Before dispatching workers:
-1.  The system identifies all calculations with new **Code Hashes**.
-2.  It runs `SimRunner` for these calculations to generate fresh **SimHashes**.
-3.  If `SimHash(New) == SimHash(Old)`, the system updates the Status Ledger to enable the new Code Hash without flagging it as "Changed".
-
-### Phase 3: Dispatch Analysis
-The Dispatcher iterates through the Topological Passes (0 -> N). For each calculation, it queries `calculateExecutionStatus`:
-*   Are dependencies done?
-*   Did dependencies change their output (`ResultHash`)?
-*   Is historical context available?
-
-### Phase 4: Execution Waves
-Workers are triggered via Pub/Sub or direct method invocation.
-*   **Pass 1**: Primitive conversions (e.g., Price Extractor).
-*   **Pass 2**: Technical Indicators that depend on Pass 1.
-*   **Pass 3**: Aggregations and Complex Metrics.
-
-### Phase 5: Reconciliation
-After all queues drain, the system performs a final sweep. Any tasks marked `FAILED` are retried (up to a limit). Impossible tasks are finalized as `IMPOSSIBLE`.
-
----
-
-## 4. Deep Dive: Hashing & Dependency Logic
-
-### Intrinsic Code Hashing
-Located in `topology/HashManager.js`.
-We generate a unique fingerprint for every calculation file:
-```javascript
-clean = codeString.replace(comments).replace(whitespace);
-hash = sha256(clean);
-```
-This ensures that changes to comments or formatting do *not* trigger re-runs.
-
-### Behavioral Hashing (SimHash)
-Located in `simulation/SimRunner.js`.
-When code changes, we can't be 100% sure it's safe just by looking at the source.
-1.  **The Fabricator**: Generates a deterministic mock `Context` (prices, previous results) based on the input schema.
-2.  **Simulation Run**: The calculation `process()` method is executed against this mock data.
-3.  **The Registry**: The hash of the *output* of this simulation is stored.
-If a refactor results in the exact same Mock Output, the system considers the change "Cosmetic".
-
-### Dependency Short-Circuiting
-Implemented in `WorkflowOrchestrator.js` (`analyzeDateExecution`).
-Even if an upstream calculation re-runs, downstream dependents might not need to.
-*   **Logic**:
-    *   Calc A (Upstream) re-runs. Old Output Hash: `HashX`. New Output Hash: `HashX`.
-    *   Calc B (Downstream) sees that Calc A "changed" (new timestamp), BUT the content hash `HashX` is identical to what Calc B used last time.
-    *   **Result**: Calc B is `SKIPPED`.
-
----
-
-## 5. Decision Logic & Edge Case Scenarios
-
-### Scenario A: Standard Code Change (Logic)
-*   **Trigger**: You change the formula for `RSI`. Code Hash changes. SimHash changes.
-*   **Dispatcher**: Sees `storedHash !== currentHash`.
-*   **Result**: Marks as `RUNNABLE`. Worker runs.
-
-### Scenario B: Cosmetic Code Change (Refactor)
-*   **Trigger**: You rename a variable in `RSI`. Code Hash changes. SimHash remains identical.
-*   **Optimizer**: Updates the centralized Status Ledger: "Version `Desc_v2` is equivalent to `Desc_v1`".
-*   **Dispatcher**: Sees the new hash in the ledger as "Verified".
-*   **Result**: Task is `SKIPPED`.
-
-### Scenario C: Upstream Invalidation (The Cascade)
-*   **Condition**: `PriceExtractor` fixes a bug. `ResultHash` changes from `HashA` to `HashB`.
-*   **Downstream**: `RSI` checks detailed dependency report.
-*   **Check**: `LastRunDeps['PriceExtractor'] (HashA) !== CurrentDeps['PriceExtractor'] (HashB)`.
-*   **Result**: `RSI` is forced to re-run.
-
-### Scenario D: Upstream Stability (The Firewall)
-*   **Condition**: `PriceExtractor` runs an optimization. Output is exact same data. `ResultHash` remains `HashA`.
-*   **Downstream**: `RSI` checks dependency report.
-*   **Check**: `LastRunDeps['PriceExtractor'] (HashA) === CurrentDeps['PriceExtractor'] (HashA)`.
-*   **Result**: `RSI` is `SKIPPED`. This firewall prevents massive re-calculation storms for non-functional upstream changes.
-
-### Scenario E: The "Impossible" State
-*   **Condition**: Core market data is missing for `1990-01-01`.
-*   **Root Indexer**: Marks date as providing `[]` (empty) for critical inputs.
-*   **Dispatcher**: Marks `PriceExtractor` as `IMPOSSIBLE: NO_DATA`.
-*   **Propagation**: Any calculation depending on `PriceExtractor` sees the `IMPOSSIBLE` status and marks *itself* as `IMPOSSIBLE: UPSTREAM`.
-*   **Benefit**: The system doesn't waste cycles retrying calculations that can never succeed.
-
-### Scenario F: Category Migration
-*   **Condition**: You change `getMetadata()` for a calculation, moving it from `signals` to `risk`.
-*   **Dispatcher**: Detects `storedCategory !== newCategory`.
-*   **Worker**:
-    1.  Runs `process()` and writes to the *new* path (`risk/CalculateX`).
-    2.  Detects the `previousCategory` flag.
-    3.  Deletes the data at the *old* path (`signals/CalculateX`) to prevent orphan data.
-
----
-
-## 6. Data Management & Storage
-
-### Input Streaming
-To handle large datasets without OOM (Out Of Memory) errors:
-*   `StandardExecutor` does not load all users/tickers at once.
-*   It utilizes wait-and-stream logic (e.g., batches of 50 ids) to process the `Context`.
-
-### Transparent Auto-Sharding
-Firestore has a 1MB document limit.
-*   **Write Path**: If a calculation result > 900KB, it is split into `DocID`, `DocID_shard1`, `DocID_shard2`.
-*   **Read Path**: The `DependencyFetcher` automatically detects sharding pointers and re-assembles (hydrates) the full object before passing it to `process()`.
-
-### Compression Strategy
-*   Payloads are inspected before write.
-*   If efficient (high entropy text/JSON), Zlib compression is applied.
-*   Metadata is tagged `encoding: 'zlib'` so readers know to inflate.
-
----
-
-## 7. Quality Assurance & Self-Healing
-
-### The Heuristic Validator
-Before saving *any* result, the Executor runs heuristics:
-*   **NaN Check**: Are there `NaN` or `Infinity` values in key fields?
-*   **Flatline Check**: Is the data variance 0.00 across a large timespan?
-*   **Null Density**: Is >50% of the dataset null?
-*   **Circuit Breaker**: If heuristics fail, the task throws an error. It is better to fail and alert than to persist corrupted data that pollutes the cache.
-
-### Zombie Task Recovery
-*   **Lease Mechanism**: When a task starts, it sets a `startedAt` timestamp.
-*   **Detection**: The Dispatcher checks for tasks marked `RUNNING` where `startedAt` > 15 minutes ago.
-*   **Resolution**: These are assumed crashed (OOM/Timeout). They are reset to `PENDING` (or `FAILED` if retry count exceeded).
-
-### Dead Letter Queue (DLQ)
-Tasks that deterministically fail (crash every time) after N retries are moved to a special DLQ status. This prevents the system from getting stuck in an infinite retry loop.
-
----
-
-## 8. Developer Workflows
-
-### How to Add a New Calculation
-1.  Create `calculations/category/MyNewCalc.js`.
-2.  Implement `getMetadata()` to define dependencies.
-3.  Implement `process(context)`.
-4.  Run `npm run build-manifest` to register it in the topology.
-
-### How to Force a Global Re-Run
-*   Change the `SYSTEM_EPOCH` constant in `system_epoch.js`.
-*   This changes the "Global Salt" for all hashes, processing every calculation as "New".
-
-### How to Backfill History
-*   **Standard Dispatcher**: Good for recent history (last 30 days).
-*   **BatchPriceExecutor**: Specialized for massive historical backfills (e.g., 20 years of price data). It bypasses some topology checks for raw speed.
-
-### Local Debugging
-Run the orchestrator in "Dry Run" mode:
-```bash
-node scripts/run_orchestrator.js --date=2024-01-01 --dry-run
-```
-This prints the `Analysis Report` (Runnable/Blocked lists) without actually triggering workers.