bulltrackers-module 1.0.229 → 1.0.231

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,6 +1,6 @@
 /**
  * @fileoverview Main Orchestrator. Coordinates the topological execution.
- * UPDATED: Includes defensive checks for Logger compatibility.
+ * UPDATED: Removed legacy boolean 'true' logic.
  */
 const { normalizeName }                                    = require('./utils/utils');
 const { checkRootDataAvailability }                        = require('./data/AvailabilityChecker');
@@ -10,6 +10,8 @@ const { StandardExecutor }                                 = require('./executor
 const { MetaExecutor }                                     = require('./executors/MetaExecutor');
 const { generateProcessId, PROCESS_TYPES }                 = require('./logger/logger');
 
+const STATUS_IMPOSSIBLE = 'IMPOSSIBLE';
+
 function groupByPass(manifest) { 
     return manifest.reduce((acc, calc) => { 
         (acc[calc.pass] = acc[calc.pass] || []).push(calc); 
@@ -17,29 +19,24 @@ function groupByPass(manifest) {
     }, {}); 
 }
 
-/**
- * Performs strict analysis of what can run based on availability and hash states.
- * AIRTIGHT LOGIC:
- * 1. Missing Root Data -> Blocked (Writes 'false' to DB)
- * 2. Missing/Stale Dependency -> FailedDependency (Writes 'false' to DB)
- * 3. Hash Mismatch -> ReRun (Cascade or Code Change)
- * 4. No Result -> Run
- * 5. Result Exists & Hash Match -> Skip
- */
 function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap) {
     const report = {
         runnable: [],
-        blocked: [],          // Missing Root Data
-        failedDependency: [], // Missing Dependency OR Stale Dependency
-        reRuns: [],           // Hash Mismatch
-        skipped: []           // Already done & valid
+        blocked: [],          
+        impossible: [],       
+        failedDependency: [], 
+        reRuns: [],           
+        skipped: []           
     };
 
+    const isTargetToday = (dateStr === new Date().toISOString().slice(0, 10));
+
     const isDepSatisfied = (depName, dailyStatus, manifestMap) => {
         const norm = normalizeName(depName);
         const storedDepHash = dailyStatus[norm];
         const depManifest = manifestMap.get(norm);
 
+        if (storedDepHash === STATUS_IMPOSSIBLE) return false; 
         if (!storedDepHash) return false; 
         if (!depManifest) return false;
         if (storedDepHash !== depManifest.hash) return false;
@@ -52,7 +49,13 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         const storedHash = dailyStatus[cName];
         const currentHash = calc.hash;
 
-        // 1. Root Data Check
+        // 1. Check Impossible
+        if (storedHash === STATUS_IMPOSSIBLE) {
+            report.skipped.push({ name: cName, reason: 'Permanently Impossible' });
+            continue;
+        }
+
+        // 2. Root Data Check
         const missingRoots = [];
         if (calc.rootDataDependencies) {
             for (const dep of calc.rootDataDependencies) {
@@ -65,33 +68,50 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         }
 
         if (missingRoots.length > 0) {
-            report.blocked.push({ name: cName, reason: `Missing Root Data: ${missingRoots.join(', ')}` });
+            if (!isTargetToday) {
+                report.impossible.push({ name: cName, reason: `Missing Root Data: ${missingRoots.join(', ')} (Historical)` });
+            } else {
+                report.blocked.push({ name: cName, reason: `Missing Root Data: ${missingRoots.join(', ')} (Waiting)` });
+            }
             continue;
         }
 
-        // 2. Dependency Check
+        // 3. Dependency Check
+        let dependencyIsImpossible = false;
         const missingDeps = [];
+        
         if (calc.dependencies) {
             for (const dep of calc.dependencies) {
+                const normDep = normalizeName(dep);
+                
+                if (dailyStatus[normDep] === STATUS_IMPOSSIBLE) {
+                    dependencyIsImpossible = true;
+                    break; 
+                }
+
                 if (!isDepSatisfied(dep, dailyStatus, manifestMap)) {
                     missingDeps.push(dep);
                 }
             }
         }
 
+        if (dependencyIsImpossible) {
+            report.impossible.push({ name: cName, reason: 'Dependency is Impossible' });
+            continue;
+        }
+
         if (missingDeps.length > 0) {
             report.failedDependency.push({ name: cName, missing: missingDeps });
             continue;
         }
 
-        // 3. Hash / State Check
-        if (!storedHash) {
+        // 4. Hash / State Check (Legacy 'true' logic removed)
+        if (!storedHash || storedHash === false) { 
             report.runnable.push(calc);
         } else if (storedHash !== currentHash) {
             report.reRuns.push({ name: cName, oldHash: storedHash, newHash: currentHash });
-        } else if (storedHash === true) {
-            report.reRuns.push({ name: cName, reason: 'Legacy Upgrade' });
         } else {
+            // Stored Hash === Current Hash
             report.skipped.push({ name: cName });
         }
     }
@@ -123,23 +143,23 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
     const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
     const analysisReport = analyzeDateExecution(dateStr, calcsInThisPass, rootStatus, dailyStatus, manifestMap);
     
-    // 4. LOG ANALYSIS (SAFEGUARDED)
+    // 4. LOG ANALYSIS
     if (logger && typeof logger.logDateAnalysis === 'function') {
         logger.logDateAnalysis(dateStr, analysisReport);
     } else {
-        // Fallback if injected logger is outdated
-        const logMsg = `[Analysis] Date: ${dateStr} | Runnable: ${analysisReport.runnable.length} | Blocked: ${analysisReport.blocked.length} | ReRuns: ${analysisReport.reRuns.length}`;
+        const logMsg = `[Analysis] Date: ${dateStr} | Runnable: ${analysisReport.runnable.length} | Blocked: ${analysisReport.blocked.length} | Impossible: ${analysisReport.impossible.length}`;
         if (logger && logger.info) logger.info(logMsg);
         else console.log(logMsg);
     }
 
-    // 5. MARK FAILURES (Write 'false' to DB for blocked items)
-    const failureUpdates = {};
-    analysisReport.blocked.forEach(item => failureUpdates[item.name] = false);
-    analysisReport.failedDependency.forEach(item => failureUpdates[item.name] = false);
+    // 5. UPDATE STATUS FOR NON-RUNNABLE ITEMS
+    const statusUpdates = {};
+    analysisReport.blocked.forEach(item => statusUpdates[item.name] = false);
+    analysisReport.failedDependency.forEach(item => statusUpdates[item.name] = false);
+    analysisReport.impossible.forEach(item => statusUpdates[item.name] = STATUS_IMPOSSIBLE);
     
-    if (Object.keys(failureUpdates).length > 0) {
-        await updateComputationStatus(dateStr, failureUpdates, config, dependencies);
+    if (Object.keys(statusUpdates).length > 0) {
+        await updateComputationStatus(dateStr, statusUpdates, config, dependencies);
     }
 
     // 6. EXECUTE RUNNABLES
@@ -151,7 +171,12 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
     const finalRunList = calcsInThisPass.filter(c => calcsToRunNames.has(normalizeName(c.name)));
 
     if (!finalRunList.length) {
-        return { date: dateStr, updates: {}, skipped: analysisReport.skipped.length };
+        return { 
+            date: dateStr, 
+            updates: {}, 
+            skipped: analysisReport.skipped.length, 
+            impossible: analysisReport.impossible.length 
+        };
     }
 
     if (logger && logger.log) {

package/functions/computation-system/logger/logger.js

@@ -1,26 +1,14 @@
 /**
  * @fileoverview Structured Logging System for Computation Engine
- * Provides comprehensive tracking with process IDs, context, and filtering capabilities.
- * UPDATED: Fixed Date Analysis keys to match WorkflowOrchestrator (failedDependency vs resolving).
+ * UPDATED: Added 'impossible' category handling to Date Analysis.
  */
 
 const crypto = require('crypto');
 
-/**
- * Log Levels (Ordered by Severity)
- */
 const LOG_LEVELS = {
-    TRACE: 0,
-    DEBUG: 1,
-    INFO: 2,
-    WARN: 3,
-    ERROR: 4,
-    FATAL: 5
+    TRACE: 0, DEBUG: 1, INFO: 2, WARN: 3, ERROR: 4, FATAL: 5
 };
 
-/**
- * Process Types for Tracking
- */
 const PROCESS_TYPES = {
     MANIFEST: 'manifest',
     ORCHESTRATOR: 'orchestrator',
@@ -31,17 +19,11 @@ const PROCESS_TYPES = {
     DISPATCH: 'dispatch'
 };
 
-/**
- * Generates a deterministic process ID.
- */
 function generateProcessId(type, identifier, date = '') {
     const input = `${type}|${identifier}|${date}`;
     return crypto.createHash('sha256').update(input).digest('hex').substring(0, 16);
 }
 
-/**
- * Formats a log entry into structured JSON
- */
 function formatLogEntry(entry) {
     return JSON.stringify({
         timestamp: entry.timestamp,
@@ -68,7 +50,6 @@ class StructuredLogger {
             includeStackTrace: config.includeStackTrace !== false,
             ...config
         };
-        
         this.activeProcesses = new Map();
     }
 
@@ -78,7 +59,6 @@ class StructuredLogger {
             : crypto.randomBytes(8).toString('hex');
 
         const processLogger = new ProcessLogger(this, processType, processId, computationName, date);
-        
         this.activeProcesses.set(processId, {
             type: processType,
             computationName,
@@ -86,7 +66,6 @@ class StructuredLogger {
             startTime: Date.now(),
             logger: processLogger
         });
-        
         return processLogger;
     }
 
@@ -101,17 +80,17 @@ class StructuredLogger {
     }
 
     /**
-     * The Main Date Analysis Logger (FIXED)
-     * Handles 'failedDependency', 'skipped', and 'blocked' correctly.
+     * The Main Date Analysis Logger (UPDATED)
+     * Handles 'impossible' category for permanent failures.
      */
     logDateAnalysis(dateStr, analysisReport) {
-        // Destructure with SAFETY DEFAULTS to prevent crashes
         const { 
             runnable = [], 
             blocked = [], 
+            impossible = [], // New Category
             reRuns = [], 
-            failedDependency = [], // Matches WorkflowOrchestrator
-            skipped = []           // Matches WorkflowOrchestrator
+            failedDependency = [], 
+            skipped = []           
         } = analysisReport;
         
         // 1. Structured Output
@@ -125,6 +104,7 @@ class StructuredLogger {
                 stats: {
                     runnable: runnable.length,
                     blocked: blocked.length,
+                    impossible: impossible.length,
                     reRuns: reRuns.length,
                     failedDependency: failedDependency.length,
                     skipped: skipped.length
@@ -135,7 +115,10 @@ class StructuredLogger {
 
         // 2. Human Readable Output
         if (this.config.enableConsole) {
-            const symbols = { info: 'ℹ️', warn: '⚠️', check: '✅', block: '⛔', link: '🔗', cycle: '🔄', skip: '⏭️' };
+            const symbols = { 
+                info: 'ℹ️', warn: '⚠️', check: '✅', block: '⛔', 
+                cycle: '🔄', skip: '⏭️', dead: '💀' 
+            };
             
             console.log(`\n🔍 === DATE ANALYSIS REPORT: ${dateStr} ===`);
             
@@ -146,29 +129,34 @@ class StructuredLogger {
                 });
             }
 
-            // Updated to handle failedDependency instead of resolving
             if (failedDependency.length) {
-                console.log(`\n${symbols.block} [FAILED DEPENDENCIES] (Missing upstream data)`);
+                console.log(`\n${symbols.block} [FAILED DEPENDENCIES] (Upstream failure)`);
                 failedDependency.forEach(item => {
-                    // item.missing comes from Orchestrator (not item.missingDeps)
                     console.log(`   • ${item.name}: Missing ${item.missing ? item.missing.join(', ') : 'dependencies'}`);
                 });
             }
 
+            // NEW: Impossible items
+            if (impossible.length) {
+                console.log(`\n${symbols.dead} [IMPOSSIBLE] (Permanent Failure - Will not retry)`);
+                impossible.forEach(item => {
+                    console.log(`   • ${item.name}: ${item.reason}`);
+                });
+            }
+
             if (runnable.length) {
                 console.log(`\n${symbols.check} [READY TO RUN]`);
                 runnable.forEach(item => console.log(`   • ${item.name}`));
             }
 
             if (blocked.length) {
-                console.log(`\n${symbols.warn} [BLOCKED] (Missing Root Data)`);
+                console.log(`\n${symbols.warn} [BLOCKED] (Waiting for data - Retriable)`);
                 blocked.forEach(item => {
                     console.log(`   • ${item.name}: ${item.reason}`);
                 });
             }
 
             if (skipped.length) {
-                 // Optional: Only show count if list is huge
                 if (skipped.length > 10) {
                      console.log(`\n${symbols.skip} [SKIPPED] (${skipped.length} calculations up-to-date)`);
                 } else {
@@ -264,7 +252,6 @@ class StructuredLogger {
     }
 }
 
-// ... ProcessLogger and Templates (unchanged) ...
 class ProcessLogger {
     constructor(parent, processType, processId, computationName, date) {
         this.parent = parent;

package/functions/computation-system/persistence/ResultCommitter.js

@@ -45,12 +45,19 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 await commitBatchInChunks(config, deps, updates, `${name} Results`);
                 
                 // Structured Storage Log
-                logger.logStorage(pid, name, dStr, mainDocRef.path, totalSize, isSharded);
+                if (logger && logger.logStorage) {
+                    logger.logStorage(pid, name, dStr, mainDocRef.path, totalSize, isSharded);
+                }
                 
-                successUpdates[name] = calc.manifest.hash || true;
+                // Update success tracking
+                if (calc.manifest.hash) {
+                    successUpdates[name] = calc.manifest.hash;
+                }
             }
         } catch (e) {
-            logger.log('ERROR', `Commit failed for ${name}`, { processId: pid, error: e.message });
+            if (logger && logger.log) {
+                 logger.log('ERROR', `Commit failed for ${name}`, { processId: pid, error: e.message });
+            }
         }
     }
 
@@ -62,10 +69,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
     return successUpdates;
 }
 
-// ... rest of file (calculateFirestoreBytes, prepareAutoShardedWrites) remains same ...
-// Just ensure prepareAutoShardedWrites uses the provided logger if it logs internal warnings.
 function calculateFirestoreBytes(value) {
-    // ... same as before
     if (value === null) return 1;
     if (value === undefined) return 0;
     if (typeof value === 'boolean') return 1;
@@ -79,8 +83,6 @@ function calculateFirestoreBytes(value) {
 }
 
 async function prepareAutoShardedWrites(result, docRef, logger) {
-    // ... same logic, just ensure existing logs inside here use the logger properly if needed
-    // Copied from previous logic, essentially checks size > 900KB and splits
     const SAFETY_THRESHOLD_BYTES = 1000 * 1024; 
     const OVERHEAD_ALLOWANCE     = 20 * 1024;
     const CHUNK_LIMIT            = SAFETY_THRESHOLD_BYTES - OVERHEAD_ALLOWANCE;
@@ -94,8 +96,6 @@ async function prepareAutoShardedWrites(result, docRef, logger) {
 
     if ((totalSize + docPathSize) < CHUNK_LIMIT) {  const data = { ...result, _completed: true, _sharded: false };  return [{ ref: docRef, data, options: { merge: true } }];  }
 
-    // Note: We don't log "Sharding..." here anymore because we log the structured event in commitResults
-    
     for (const [key, value] of Object.entries(result)) { 
         if (key.startsWith('_')) continue;
         const keySize   = Buffer.byteLength(key, 'utf8') + 1;         

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.229",
+  "version": "1.0.231",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [