bulltrackers-module 1.0.256 → 1.0.258

package/functions/computation-system/WorkflowOrchestrator.js

@@ -12,43 +12,35 @@ const { generateProcessId, PROCESS_TYPES }                 = require('./logger/l
 
 const STATUS_IMPOSSIBLE = 'IMPOSSIBLE';
 
-function groupByPass(manifest) { 
-    return manifest.reduce((acc, calc) => { 
-        (acc[calc.pass] = acc[calc.pass] || []).push(calc); 
-        return acc; 
-    }, {}); 
-}
+function groupByPass(manifest) {  return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc);  return acc;  }, {}); }
 
 /**
  * Analyzes whether calculations should run, be skipped, or are blocked.
  */
 function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
-    const report = { runnable: [], blocked: [], impossible: [], failedDependency: [], reRuns: [], skipped: [] };
+    const report           = { runnable: [], blocked: [], impossible: [], failedDependency: [], reRuns: [], skipped: [] };
     const simulationStatus = { ...dailyStatus };
-    const isTargetToday = (dateStr === new Date().toISOString().slice(0, 10));
+    const isTargetToday    = (dateStr === new Date().toISOString().slice(0, 10));
 
     const isDepSatisfied = (depName, currentStatusMap, manifestMap) => {
-        const norm = normalizeName(depName);
-        const stored = currentStatusMap[norm]; 
+        const norm        = normalizeName(depName);
+        const stored      = currentStatusMap[norm]; 
         const depManifest = manifestMap.get(norm);
-        if (!stored) return false;
+        if (!stored)                           return false;
         if (stored.hash === STATUS_IMPOSSIBLE) return false;
-        if (!depManifest) return false;
-        if (stored.hash !== depManifest.hash) return false;
+        if (!depManifest)                      return false;
+        if (stored.hash !== depManifest.hash)  return false;
         return true;
     };
 
     for (const calc of calcsInPass) {
-        const cName = normalizeName(calc.name);
-        const stored = simulationStatus[cName];
-        const storedHash = stored ? stored.hash : null;
+        const cName          = normalizeName(calc.name);
+        const stored         = simulationStatus[cName];
+        const storedHash     = stored ? stored.hash : null;
         const storedCategory = stored ? stored.category : null;
-        const currentHash = calc.hash;
+        const currentHash    = calc.hash;
 
-        const markImpossible = (reason) => {
-            report.impossible.push({ name: cName, reason });
-            simulationStatus[cName] = { hash: STATUS_IMPOSSIBLE, category: calc.category };
-        };
+        const markImpossible = (reason) => { report.impossible.push({ name: cName, reason }); simulationStatus[cName] = { hash: STATUS_IMPOSSIBLE, category: calc.category }; };
 
         const markRunnable = (isReRun = false, reRunDetails = null) => {
             if (isReRun) report.reRuns.push(reRunDetails);
@@ -58,33 +50,33 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
 
         let migrationOldCategory = null;
         if (storedCategory && storedCategory !== calc.category) { migrationOldCategory = storedCategory; }
-
-        if (storedHash === STATUS_IMPOSSIBLE) {
-            report.skipped.push({ name: cName, reason: 'Permanently Impossible' });
-            continue;
-        }
-
-        // [FIX] Use the shared, strict checking logic from AvailabilityChecker
-        // This ensures 'speculator' calculations check 'speculatorPortfolio', etc.
+        if (storedHash === STATUS_IMPOSSIBLE) { report.skipped.push({ name: cName, reason: 'Permanently Impossible' }); continue; }
         const rootCheck = checkRootDependencies(calc, rootDataStatus);
         
+        // Check Root Data Availability 
+            // LOGIC : Root data is essential for any calculation
+            // Therefore if a computation has a dependency on rootdata that does not exist for the dates the computation requires, then the computation is impossible to run.
+            // However, to handle edge cases where we might test trigger the computation system early, we do not mark impossible if the computation requires data for today, it might arrive later, we just block and skip.
+
         if (!rootCheck.canRun) {
             const missingStr = rootCheck.missing.join(', ');
             if (!isTargetToday) {
-                // Historical missing data is usually permanent/impossible
                 markImpossible(`Missing Root Data: ${missingStr} (Historical)`);
             } else {
-                // Today's missing data might just be late (Blocked)
                 report.blocked.push({ name: cName, reason: `Missing Root Data: ${missingStr} (Waiting)` });
             }
             continue;
         }
 
+        // Check Calculation Dependencies
+            // LOGIC : If a calc B depends on calc A, and calc A is impossible, then calc B is always impossible
+            // This has a cascading effect, if calc C depends on calc B and calc B depends on calc A and calc A is impossible, then calc B and calc C are also impossible.
+
         let dependencyIsImpossible = false;
         const missingDeps = [];
         if (calc.dependencies) {
             for (const dep of calc.dependencies) {
-                const normDep = normalizeName(dep);
+                const normDep   = normalizeName(dep);
                 const depStored = simulationStatus[normDep];
                 if (depStored && depStored.hash === STATUS_IMPOSSIBLE) { dependencyIsImpossible = true; break; }
                 if (!isDepSatisfied(dep, simulationStatus, manifestMap)) { missingDeps.push(dep); }
@@ -94,6 +86,13 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         if (dependencyIsImpossible) { markImpossible('Dependency is Impossible'); continue; }
         if (missingDeps.length > 0) { report.failedDependency.push({ name: cName, missing: missingDeps }); continue; }
 
+        // Historical Continuity Check
+            // LOGIC : For computations that require historical data, we process them chronologically
+            // This is to handle the edge case where calc B runs for Tuesday data, but requires Mondays results from calc B. 
+            // If we triggered a hash mismatch through updating the code of calc B, it would overwrite the results for Tuesday and Monday but without this, 
+            // it would never be guaranteed that Monday runs before Tuesday, and so Tuesday would run with the old Monday hash data, or no data.
+            // This fixes this edge case by ensuring that historical computations only run if the previous day's computation has run with the latest hash, if not, it blocks and waits.
+
         if (calc.isHistorical && prevDailyStatus) {
             const yesterday = new Date(dateStr + 'T00:00:00Z');
             yesterday.setUTCDate(yesterday.getUTCDate() - 1);
@@ -105,10 +104,13 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
                 }
             }
         }
-
+        // Final Hash Comparison
+            // LOGIC : If the stored hash matches the current hash, we don't need to run the computation again, unless the category stored does not match the current computation category
+            // This is to handle the edge case where a developer changes the category of a computation, the stored results need to be moved into the new location so we trigger a re-run to move the data and also delete the old category stored data.
+        
         if (!storedHash) { markRunnable(); } 
         else if (storedHash !== currentHash) { markRunnable(true, { name: cName, oldHash: storedHash, newHash: currentHash, previousCategory: migrationOldCategory }); } 
-        else if (migrationOldCategory) { markRunnable(true, { name: cName, reason: 'Category Migration', previousCategory: migrationOldCategory, newCategory: calc.category }); } 
+        else if (migrationOldCategory) {       markRunnable(true, { name: cName, reason: 'Category Migration', previousCategory: migrationOldCategory, newCategory: calc.category }); } 
         else { report.skipped.push({ name: cName }); simulationStatus[cName] = { hash: currentHash, category: calc.category }; }
     }
     return report;
@@ -117,8 +119,9 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
 /**
  * DIRECT EXECUTION PIPELINE (For Workers)
  * Skips analysis. Assumes the calculation is valid and runnable.
+ * [UPDATED] Accepted previousCategory argument to handle migrations.
  */
-async function executeDispatchTask(dateStr, pass, targetComputation, config, dependencies, computationManifest) {
+async function executeDispatchTask(dateStr, pass, targetComputation, config, dependencies, computationManifest, previousCategory = null) {
     const { logger } = dependencies;
     const pid = generateProcessId(PROCESS_TYPES.EXECUTOR, targetComputation, dateStr);
     
@@ -128,8 +131,13 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
     
     if (!calcManifest) { throw new Error(`Calculation '${targetComputation}' not found in manifest.`); }
 
+    // [UPDATED] Attach migration context if present
+    if (previousCategory) {
+        calcManifest.previousCategory = previousCategory;
+        logger.log('INFO', `[Executor] Migration detected for ${calcManifest.name}. Old data will be cleaned from: ${previousCategory}`);
+    }
+
     // 2. Fetch Root Data Availability
-    // Note: this returns { status: {...}, portfolioRefs: null, historyRefs: null, ... }
     const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
     
     if (!rootData) {
@@ -154,11 +162,8 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
     let resultUpdates = {};
 
     try {
-        if (calcManifest.type === 'standard') {
-             // StandardExecutor handles the null refs in rootData by fetching on demand
-             resultUpdates = await StandardExecutor.run(new Date(dateStr + 'T00:00:00Z'), [calcManifest], `Pass ${pass}`, config, dependencies, rootData, existingResults, previousResults);
-        } else if (calcManifest.type === 'meta') {
-            resultUpdates = await MetaExecutor.run(new Date(dateStr + 'T00:00:00Z'), [calcManifest], `Pass ${pass}`, config, dependencies, existingResults, previousResults, rootData);
+        if (calcManifest.type === 'standard')    { resultUpdates = await StandardExecutor.run(new Date(dateStr + 'T00:00:00Z'), [calcManifest], `Pass ${pass}`, config, dependencies, rootData, existingResults, previousResults);
+        } else if (calcManifest.type === 'meta') { resultUpdates = await MetaExecutor.run    (new Date(dateStr + 'T00:00:00Z'), [calcManifest], `Pass ${pass}`, config, dependencies, existingResults, previousResults, rootData);
         }
         logger.log('INFO', `[Executor] Success: ${calcManifest.name} for ${dateStr}`);
         return { date: dateStr, updates: resultUpdates };
@@ -168,6 +173,5 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
     }
 }
 
-async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) { /* Legacy support stub */ }
 
-module.exports = { runDateComputation, executeDispatchTask, groupByPass, analyzeDateExecution };
+module.exports = { executeDispatchTask, groupByPass, analyzeDateExecution };

package/functions/computation-system/config/validation_overrides.js

@@ -0,0 +1,6 @@
+module.exports = {
+    // Only add calculations here if the HeuristicValidator is too aggressive
+    // EXAMPLES :
+    // "bankruptcy-detector": { maxZeroPct: 100 }, // It's rare, so 100% 0s is fine
+    // "sparse-signal-generator": { maxNullPct: 99 }
+};

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -104,6 +104,7 @@ async function dispatchComputationPass(config, dependencies, computationManifest
                     pass: passToRun,
                     computation: normalizeName(item.name),
                     hash: item.hash || item.newHash, // [NEW] Ensure Hash is passed for Ledger
+                    previousCategory: item.previousCategory || null, // [UPDATED] Pass migration context
                     timestamp: Date.now()
                 });
             });

package/functions/computation-system/helpers/computation_worker.js

@@ -1,12 +1,13 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes computation tasks from Pub/Sub and executes them.
- * UPDATED: Simplified "Dumb Worker" - Trusts Dispatcher validation.
+ * UPDATED: Integrated Run Ledger for per-run/per-date success/failure tracking.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
 const { getManifest }         = require('../topology/ManifestLoader');
 const { StructuredLogger }    = require('../logger/logger'); 
+const { recordRunAttempt }    = require('../persistence/RunRecorder'); // [NEW IMPORT]
 
 // 1. IMPORT CALCULATIONS
 let calculationPackage;
@@ -21,7 +22,6 @@ const calculations = calculationPackage.calculations;
 
 /**
  * Handles a single Pub/Sub message.
- * Assumes the message contains a VALID, RUNNABLE task from the Smart Dispatcher.
  */
 async function handleComputationTask(message, config, dependencies) {
     
@@ -33,7 +33,7 @@ async function handleComputationTask(message, config, dependencies) {
     });
 
     const runDependencies = { ...dependencies, logger: systemLogger };
-    const { logger } = runDependencies;
+    const { logger, db } = runDependencies;
 
     // 3. PARSE PAYLOAD
     let data;
@@ -54,7 +54,8 @@ async function handleComputationTask(message, config, dependencies) {
 
     if (!data || data.action !== 'RUN_COMPUTATION_DATE') { return; }
     
-    const { date, pass, computation } = data;
+    // [UPDATED] Destructure previousCategory from payload
+    const { date, pass, computation, previousCategory } = data;
     
     if (!date || !pass || !computation) { 
         logger.log('ERROR', `[Worker] Invalid payload: Missing date, pass, or computation.`, data); 
@@ -67,29 +68,67 @@ async function handleComputationTask(message, config, dependencies) {
         computationManifest = getManifest(config.activeProductLines || [], calculations, runDependencies);
     } catch (manifestError) {
         logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
+        // Record Fatal Manifest Error
+        await recordRunAttempt(db, { date, computation, pass }, 'CRASH', { 
+            message: manifestError.message, 
+            stage: 'MANIFEST_LOAD' 
+        });
         return; 
     }
 
-    // 5. EXECUTE (TRUSTED MODE)
-    // We do not check DB status or analyze feasibility. We assume Dispatcher did its job.
+    // 5. EXECUTE (With Run Ledger)
     try {
         logger.log('INFO', `[Worker] 📥 Received: ${computation} for ${date}`);
         
+        const startTime = Date.now();
+        // [UPDATED] Pass previousCategory to executor
         const result = await executeDispatchTask(
             date, 
             pass, 
             computation, 
             config, 
             runDependencies, 
-            computationManifest
+            computationManifest,
+            previousCategory
         );
-        
-        if (result && result.updates) {
-             logger.log('INFO', `[Worker] ✅ Stored: ${computation} for ${date}`);
+        const duration = Date.now() - startTime;
+
+        // CHECK FOR INTERNAL FAILURES (Trapped by ResultCommitter)
+        const failureReport = result?.updates?.failureReport || [];
+        const successUpdates = result?.updates?.successUpdates || {};
+
+        if (failureReport.length > 0) {
+            // Task ran, but logic or storage failed (e.g., Sharding Limit)
+            const failReason = failureReport[0]; // Assuming 1 calc per task
+            
+            logger.log('ERROR', `[Worker] ❌ Failed logic/storage for ${computation}`, failReason.error);
+            
+            await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', failReason.error, { durationMs: duration });
+            
+            // Throw error to ensure Pub/Sub retry (if transient) or Visibility (if permanent)
+            throw new Error(failReason.error.message || 'Computation Logic Failed');
+        } 
+        else if (Object.keys(successUpdates).length > 0) {
+            // Success
+            logger.log('INFO', `[Worker] ✅ Stored: ${computation} for ${date}`);
+            await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, { durationMs: duration });
+        } 
+        else {
+            // No updates, but no error (e.g. Empty Result) - Log as Success/Skipped
+            logger.log('WARN', `[Worker] ⚠️ No results produced for ${computation} (Empty?)`);
+            await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', { message: 'Empty Result' }, { durationMs: duration });
         }
 
     } catch (err) { 
-        logger.log('ERROR', `[Worker] ❌ Failed: ${computation} for ${date}: ${err.message}`); 
+        // Catch System Crashes (OOM, Timeout, Unhandled Exception)
+        logger.log('ERROR', `[Worker] ❌ Crash: ${computation} for ${date}: ${err.message}`); 
+        
+        await recordRunAttempt(db, { date, computation, pass }, 'CRASH', { 
+            message: err.message, 
+            stack: err.stack, 
+            stage: 'SYSTEM_CRASH' 
+        });
+        
         throw err; // Trigger Pub/Sub retry
     }
 }

package/functions/computation-system/persistence/ResultCommitter.js

@@ -1,45 +1,45 @@
 /**
  * @fileoverview Handles saving computation results with observability and Smart Cleanup.
- * UPDATED: Implements Audit Ledger completion logic ("Closing the Ledger").
+ * UPDATED: Returns detailed failure reports for the Run Ledger.
  */
 const { commitBatchInChunks }     = require('./FirestoreUtils');
 const { updateComputationStatus } = require('./StatusRepository');
 const { batchStoreSchemas }       = require('../utils/schema_capture');
 const { generateProcessId, PROCESS_TYPES } = require('../logger/logger');
-// Note: normalizeName is typically needed for doc IDs, but keys in stateObj are usually already normalized.
-// If not, ensure it is imported. Based on StandardExecutor, keys are normalized.
+
+const { HeuristicValidator } = require('./ResultsValidator'); // Validator
+const validationOverrides = require('../config/validation_overrides'); // Override file
 
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
     const successUpdates = {};
+    const failureReport  = []; // [NEW] Track failures per calculation
     const schemas        = [];
-    const cleanupTasks   = []; // Tasks to delete old data
+    const cleanupTasks   = []; 
     const { logger, db } = deps;
     const pid            = generateProcessId(PROCESS_TYPES.STORAGE, passName, dStr);
     
-    // [NEW] Extract numeric pass ID from string (e.g., "Pass 1" -> "1")
     const passNum = passName.replace(/[^0-9]/g, '');
 
     for (const name in stateObj) {
         const calc = stateObj[name];
         try {
             const result = await calc.getResult();
-            
-            // Validate Result: Check for Null, Empty Object, or Zero
-            const isEmpty = !result || 
-                            (typeof result === 'object' && Object.keys(result).length === 0) ||
-                            (typeof result === 'number' && result === 0);
-
-            if (isEmpty) {
-                // Mark status as FALSE (Failed/Empty) so it re-runs or is flagged
-                if (calc.manifest.hash) {
-                    successUpdates[name] = { 
-                        hash: false, 
-                        category: calc.manifest.category 
-                    };
-                }
-                // Do not store empty results
-                continue; 
+
+            const overrides = validationOverrides[calc.manifest.name] || {};
+            const healthCheck = HeuristicValidator.analyze(calc.manifest.name, result, overrides);
+
+            if (!healthCheck.valid) {
+            // We throw a specific error stage so we know to BLOCK it, not retry it.
+                throw { 
+                    message: healthCheck.reason, 
+                    stage: 'QUALITY_CIRCUIT_BREAKER' 
+                };
             }
+            
+            // Validate Result
+            const isEmpty = !result ||  (typeof result === 'object' && Object.keys(result).length === 0) || (typeof result === 'number' && result === 0);
+
+            if (isEmpty) { if (calc.manifest.hash) { successUpdates[name] = {  hash: false,  category: calc.manifest.category  }; } continue; }
 
             const mainDocRef = db.collection(config.resultsCollection)
                 .doc(dStr)
@@ -48,9 +48,16 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 .collection(config.computationsSubcollection)
                 .doc(name);
 
-            const updates = await prepareAutoShardedWrites(result, mainDocRef, logger);
+            // [CRITICAL UPDATE] Catch errors specifically during Sharding/Prep
+            let updates;
+            try {
+                updates = await prepareAutoShardedWrites(result, mainDocRef, logger);
+            } catch (prepError) {
+                // If this fails, it's likely a memory or logic issue before DB commit
+                throw { message: prepError.message, stack: prepError.stack, stage: 'PREPARE_SHARDS' };
+            }
 
-            // --- [NEW] ADD AUDIT LEDGER COMPLETION TO BATCH ---
+            // Audit Ledger
             if (passNum && calc.manifest) {
                 const ledgerRef = db.collection(`computation_audit_ledger/${dStr}/passes/${passNum}/tasks`).doc(name);
                 updates.push({
@@ -64,7 +71,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                     options: { merge: true }
                 });
             }
-            // --------------------------------------------------
 
             // Capture Schema
             if (calc.manifest.class.getSchema) {
@@ -81,52 +87,59 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 const totalSize = updates.reduce((acc, u) => acc + (u.data ? JSON.stringify(u.data).length : 0), 0);
                 const isSharded = updates.some(u => u.data._sharded === true);
                 
-                await commitBatchInChunks(config, deps, updates, `${name} Results`);
-                
-                // Structured Storage Log
-                if (logger && logger.logStorage) {
-                    logger.logStorage(pid, name, dStr, mainDocRef.path, totalSize, isSharded);
+                try {
+                    await commitBatchInChunks(config, deps, updates, `${name} Results`);
+                } catch (commitErr) {
+                    // Check for Firestore specific limits
+                    let stage = 'COMMIT_BATCH';
+                    let msg = commitErr.message;
+                    if (msg.includes('Transaction too big') || msg.includes('payload is too large')) { stage = 'SHARDING_LIMIT_EXCEEDED'; msg = `Firestore Limit Exceeded: ${msg}`; }
+                    throw { message: msg, stack: commitErr.stack, stage };
                 }
                 
-                // Update success tracking (Include Category)
-                if (calc.manifest.hash) {
-                    successUpdates[name] = { 
-                        hash: calc.manifest.hash, 
-                        category: calc.manifest.category 
-                    };
-                }
+                // Log Storage
+                if (logger && logger.logStorage) { logger.logStorage(pid, name, dStr, mainDocRef.path, totalSize, isSharded); }
+                
+                // Mark Success
+                if (calc.manifest.hash) { successUpdates[name] = {  hash: calc.manifest.hash, category: calc.manifest.category  }; }
 
-                // CHECK FOR MIGRATION CLEANUP
+                // Cleanup Migration
                 if (calc.manifest.previousCategory && calc.manifest.previousCategory !== calc.manifest.category) {
-                    logger.log('INFO', `[Migration] Scheduled cleanup for ${name} from '${calc.manifest.previousCategory}'`);
                     cleanupTasks.push(deleteOldCalculationData(dStr, calc.manifest.previousCategory, name, config, deps));
                 }
             }
         } catch (e) {
-            if (logger && logger.log) {
-                 logger.log('ERROR', `Commit failed for ${name}`, { processId: pid, error: e.message });
-            }
+            // [NEW] Intelligent Failure Reporting
+            const stage = e.stage || 'EXECUTION';
+            const msg   = e.message || 'Unknown error';
+            
+            if (logger && logger.log) { logger.log('ERROR', `Commit failed for ${name} [${stage}]`, { processId: pid, error: msg }); }
+            
+            failureReport.push({
+                name,
+                error: { message: msg, stack: e.stack, stage }
+            });
         }
     }
 
     if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(() => {});
 
-    // Execute Cleanup Tasks (orphaned data from category changes)
-    if (cleanupTasks.length > 0) {
-        await Promise.allSettled(cleanupTasks);
-    }
+    if (cleanupTasks.length > 0) { await Promise.allSettled(cleanupTasks); }
 
-    if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
-        await updateComputationStatus(dStr, successUpdates, config, deps);
-    }
-    return successUpdates;
+    if (!skipStatusWrite && Object.keys(successUpdates).length > 0) { await updateComputationStatus(dStr, successUpdates, config, deps); }
+    
+    // [UPDATE] Return both success and failures so the Worker can log them
+    return { successUpdates, failureReport };
 }
 
 /**
  * Deletes result documents from a previous category location.
- * Must handle standard docs AND sharded docs (subcollections).
+ *  This function exists to handle the deleteion of old data,
+ *  The general use case is that if a developer changes a calculations' category, 
+ *  We want to clean up and delete the old path, the change of a category would trigger a re-run of the calculation to naturally move the data to the new location
  */
 async function deleteOldCalculationData(dateStr, oldCategory, calcName, config, deps) {
+
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils || { withRetry: (fn) => fn() };
     
@@ -138,20 +151,13 @@ async function deleteOldCalculationData(dateStr, oldCategory, calcName, config,
             .collection(config.computationsSubcollection)
             .doc(calcName);
 
-        // 1. Check for Shards Subcollection
         const shardsCol = oldDocRef.collection('_shards');
         const shardsSnap = await withRetry(() => shardsCol.listDocuments(), 'ListOldShards');
         
         const batch = db.batch();
         let ops = 0;
 
-        // Delete shards
-        for (const shardDoc of shardsSnap) {
-            batch.delete(shardDoc);
-            ops++;
-        }
-
-        // Delete main doc
+        for (const shardDoc of shardsSnap) { batch.delete(shardDoc); ops++; }
         batch.delete(oldDocRef);
         ops++;
 
@@ -188,7 +194,6 @@ async function prepareAutoShardedWrites(result, docRef, logger) {
     let currentChunkSize         = 0;
     let shardIndex               = 0;
 
-    // [UPDATE] Add _lastUpdated to non-sharded writes
     if ((totalSize + docPathSize) < CHUNK_LIMIT) {  
         const data = { 
             ...result, 

package/functions/computation-system/persistence/ResultsValidator.js

@@ -0,0 +1,136 @@
+/**
+ * @fileoverview HeuristicValidator.js
+ * "Grey Box" validation that infers health using statistical analysis and structural sanity checks.
+ * UPDATED: Added NaN detection, Flatline (Variance) checks, and Vector/Array depth checks.
+ */
+
+class HeuristicValidator {
+    /**
+     * @param {string} calcName - Name for logging
+     * @param {Object} data - The result data to inspect
+     * @param {Object} [overrides] - Optional central config overrides
+     */
+    static analyze(calcName, data, overrides = {}) {
+        // 1. Structure Check
+        if (!data || typeof data !== 'object') return { valid: true }; // Let scalar types pass
+        
+        const keys = Object.keys(data);
+        const totalItems = keys.length;
+        
+        // Skip tiny datasets (statistically insignificant)
+        if (totalItems < 5) return { valid: true };
+
+        // 2. Sampling Configuration
+        const sampleSize = Math.min(totalItems, 100);
+        const step = Math.floor(totalItems / sampleSize);
+        
+        let zeroCount = 0;
+        let nullCount = 0;
+        let nanCount = 0; // NEW: Track NaNs
+        let emptyVectorCount = 0; // NEW: Track empty arrays in complex objects
+        let analyzedCount = 0;
+        
+        // For Variance/Flatline Check
+        const numericValues = []; 
+
+        for (let i = 0; i < totalItems; i += step) {
+            const key = keys[i];
+            const val = data[key];
+            if (!val) { // Catch null/undefined immediately
+                nullCount++;
+                analyzedCount++;
+                continue;
+            }
+            analyzedCount++;
+
+            // --- TYPE A: Object / Complex Result ---
+            // Example: { "profile": [...], "current_price": 100 } or { "signal": "Buy", "score": 0.5 }
+            if (typeof val === 'object') {
+                const subValues = Object.values(val);
+                
+                // Dead Object Check: All props are null/0/undefined
+                const isDeadObject = subValues.every(v => v === 0 || v === null || v === undefined);
+                if (isDeadObject) nullCount++;
+
+                // NaN Check in Properties
+                const hasNan = subValues.some(v => typeof v === 'number' && (isNaN(v) || !isFinite(v)));
+                if (hasNan) nanCount++;
+
+                // Vector/Profile Empty Check (Specific to your System)
+                // If result contains 'profile', 'history', 'sparkline', or 'buckets' arrays
+                const arrayProps = ['profile', 'history', 'sparkline', 'buckets', 'prices'];
+                for (const prop of arrayProps) {
+                    if (Array.isArray(val[prop]) && val[prop].length === 0) {
+                        emptyVectorCount++;
+                    }
+                }
+
+                // Extract primary numeric score for Flatline check (heuristically guessing the 'main' metric)
+                const numericProp = subValues.find(v => typeof v === 'number' && v !== 0);
+                if (numericProp !== undefined) numericValues.push(numericProp);
+            } 
+            // --- TYPE B: Scalar / Primitive Result ---
+            else if (typeof val === 'number') {
+                if (val === 0) zeroCount++;
+                if (isNaN(val) || !isFinite(val)) nanCount++;
+                else numericValues.push(val);
+            }
+        }
+
+        // 3. Thresholds
+        const thresholds = {
+            maxZeroPct: overrides.maxZeroPct ?? 99, 
+            maxNullPct: overrides.maxNullPct ?? 90,
+            maxNanPct:  overrides.maxNanPct  ?? 0,   // Strict: NaNs are usually bad bugs
+            maxFlatlinePct: 95 // If >95% of data is identical, it's suspicious
+        };
+
+        // 4. Calculate Stats
+        const zeroPct = (zeroCount / analyzedCount) * 100;
+        const nullPct = (nullCount / analyzedCount) * 100;
+        const nanPct  = (nanCount  / analyzedCount) * 100;
+        
+        // 5. Variance / Flatline Analysis
+        // If we found numeric values, check if they are all the same
+        let isFlatline = false;
+        if (numericValues.length > 5) {
+            const first = numericValues[0];
+            const identicalCount = numericValues.filter(v => Math.abs(v - first) < 0.000001).length;
+            const flatlinePct = (identicalCount / numericValues.length) * 100;
+            
+            // Only flag flatline if the value isn't 0 (0 is handled by maxZeroPct)
+            if (flatlinePct > thresholds.maxFlatlinePct && Math.abs(first) > 0.0001) {
+                isFlatline = true;
+            }
+        }
+
+        // 6. Evaluations
+        if (nanPct > thresholds.maxNanPct) {
+            return { valid: false, reason: `Mathematical Error: ${nanPct.toFixed(1)}% of sampled results contain NaN or Infinity.` };
+        }
+
+        if (zeroPct > thresholds.maxZeroPct) {
+            return { valid: false, reason: `Data Integrity: ${zeroPct.toFixed(1)}% of sampled results are 0. (Suspected Logic Failure)` };
+        }
+
+        if (nullPct > thresholds.maxNullPct) {
+            return { valid: false, reason: `Data Integrity: ${nullPct.toFixed(1)}% of sampled results are Empty/Null.` };
+        }
+
+        if (isFlatline) {
+            return { valid: false, reason: `Anomaly: Detected Result Flatline. >${thresholds.maxFlatlinePct}% of outputs are identical (non-zero).` };
+        }
+        
+        // Special check for Distribution/Profile calculations
+        if (calcName.includes('profile') || calcName.includes('distribution')) {
+            const vectorEmptyPct = (emptyVectorCount / analyzedCount) * 100;
+            if (vectorEmptyPct > 90) {
+                return { valid: false, reason: `Data Integrity: ${vectorEmptyPct.toFixed(1)}% of distribution profiles are empty.` };
+            }
+        }
+
+        return { valid: true };
+    }
+}
+
+module.exports = { HeuristicValidator };

package/functions/computation-system/persistence/RunRecorder.js

@@ -0,0 +1,54 @@
+/**
+ * @fileoverview Utility for recording computation run attempts (The Run Ledger).
+ * Tracks success, failure, and error contexts for every execution pass.
+ */
+const { generateProcessId } = require('../logger/logger');
+
+/**
+ * Records a run attempt to the computation_run_history collection.
+ * * @param {Firestore} db - Firestore instance
+ * @param {Object} context - { date, computation, pass }
+ * @param {string} status - 'SUCCESS', 'FAILURE', or 'CRASH'
+ * @param {Object|null} error - Error object or null
+ * @param {Object} metrics - { durationMs, ... }
+ */
+async function recordRunAttempt(db, context, status, error = null, metrics = {}) {
+    if (!db || !context) return;
+
+    const { date, computation, pass } = context;
+    // Generate a unique ID for this specific run attempt
+    const runId = `${Date.now()}_${generateProcessId('run', computation, date)}`;
+    
+    const docRef = db.collection('computation_run_history')
+        .doc(date)
+        .collection('runs')
+        .doc(runId);
+
+    const entry = {
+        computationName: computation,
+        date: date,
+        pass: String(pass),
+        timestamp: new Date().toISOString(),
+        status: status,
+        metrics: metrics
+    };
+
+    if (error) {
+        entry.error = {
+            message: error.message || 'Unknown Error',
+            // Capture specific sharding/firestore stages if available
+            stage: error.stage || 'UNKNOWN',
+            code: error.code || null,
+            stack: error.stack || null
+        };
+    }
+
+    // Fire and forget (await but catch to ensure logging doesn't crash the worker)
+    try {
+        await docRef.set(entry);
+    } catch (e) {
+        console.error(`[RunRecorder] Failed to save history for ${computation}:`, e.message);
+    }
+}
+
+module.exports = { recordRunAttempt };

package/index.js

@@ -27,7 +27,7 @@ const { handleUpdate }                                    = require('./functions
 
 // Computation System
 const { build: buildManifest }                            = require('./functions/computation-system/context/ManifestBuilder');
-const { runDateComputation: runComputationPass }          = require('./functions/computation-system/WorkflowOrchestrator');
+// const { runDateComputation: runComputationPass }          = require('./functions/computation-system/WorkflowOrchestrator'); Depreciated
 const { dispatchComputationPass }                         = require('./functions/computation-system/helpers/computation_dispatcher');
 const { handleComputationTask }                           = require('./functions/computation-system/helpers/computation_worker');
 // [NEW] Import Report Tools

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.256",
+  "version": "1.0.258",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [