bulltrackers-module 1.0.224 → 1.0.225

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,18 +1,15 @@
 /**
  * @fileoverview Main Orchestrator. Coordinates the topological execution.
- * UPDATED: Includes comprehensive Date Analysis logic.
+ * UPDATED: Strict Dependency & Hash Cascade Logic with Explicit Failure Marking.
  */
-const { normalizeName, getExpectedDateStrings }            = require('./utils/utils');
+const { normalizeName }                                    = require('./utils/utils');
 const { checkRootDataAvailability }                        = require('./data/AvailabilityChecker');
 const { fetchExistingResults }                             = require('./data/DependencyFetcher');
 const { fetchComputationStatus, updateComputationStatus }  = require('./persistence/StatusRepository');
-const { runBatchPriceComputation }                         = require('./executors/PriceBatchExecutor');
 const { StandardExecutor }                                 = require('./executors/StandardExecutor');
 const { MetaExecutor }                                     = require('./executors/MetaExecutor');
 const { generateProcessId, PROCESS_TYPES }                 = require('./logger/logger');
 
-const PARALLEL_BATCH_SIZE = 7;
-
 function groupByPass(manifest) { 
     return manifest.reduce((acc, calc) => { 
         (acc[calc.pass] = acc[calc.pass] || []).push(calc); 
@@ -21,22 +18,42 @@ function groupByPass(manifest) {
 }
 
 /**
- * Performs the logical analysis requested by the user.
- * Determines exactly what can run, what is blocked, and why.
+ * Performs strict analysis of what can run based on availability and hash states.
+ * AIRTIGHT LOGIC:
+ * 1. Missing Root Data -> Blocked (Writes 'false' to DB)
+ * 2. Missing/Stale Dependency -> FailedDependency (Writes 'false' to DB)
+ * 3. Hash Mismatch -> ReRun (Cascade or Code Change)
+ * 4. No Result -> Run
+ * 5. Result Exists & Hash Match -> Skip
  */
 function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap) {
     const report = {
         runnable: [],
-        blocked: [],   // Missing Root Data
-        resolving: [], // Missing Dependency (but might be fixed by run)
-        reRuns: []     // Hash Mismatch
+        blocked: [],          // Missing Root Data
+        failedDependency: [], // Missing Dependency OR Stale Dependency
+        reRuns: [],           // Hash Mismatch (Triggered by self code change OR upstream cascade)
+        skipped: []           // Already done & valid
     };
 
-    // Helper: Is a dependency satisfied?
-    const isDepSatisfied = (depName, dailyStatus) => {
+    // Helper: Is a dependency satisfied AND valid (matching hash)?
+    const isDepSatisfied = (depName, dailyStatus, manifestMap) => {
         const norm = normalizeName(depName);
-        if (dailyStatus[norm]) return true; // It exists
-        return false;
+        const storedDepHash = dailyStatus[norm];
+        const depManifest = manifestMap.get(norm);
+
+        // 1. Must exist in DB. If missing, we cannot run.
+        if (!storedDepHash) return false; 
+        
+        // 2. Must exist in Manifest (Sanity check)
+        if (!depManifest) return false;
+
+        // 3. STRICT: The dependency's stored hash must match its current manifest hash.
+        // If 'A' changed code, 'A' has a new hash. If we are running 'B' (Pass 2),
+        // we expect 'A' (Pass 1) to have already run and updated the DB with the NEW hash.
+        // If DB still has OLD hash, 'A' failed or didn't run. 'B' is unsafe to run.
+        if (storedDepHash !== depManifest.hash) return false;
+
+        return true;
     };
 
     for (const calc of calcsInPass) {
@@ -44,7 +61,7 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         const storedHash = dailyStatus[cName];
         const currentHash = calc.hash;
 
-        // 1. Root Data Check
+        // 1. Root Data Check (FATAL)
         const missingRoots = [];
         if (calc.rootDataDependencies) {
             for (const dep of calc.rootDataDependencies) {
@@ -58,54 +75,41 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
 
         if (missingRoots.length > 0) {
             report.blocked.push({ name: cName, reason: `Missing Root Data: ${missingRoots.join(', ')}` });
-            continue;
+            continue; // Cannot proceed
         }
 
-        // 2. Hash / Version Check
-        let isReRun = false;
-        if (storedHash && storedHash !== currentHash) {
-            report.reRuns.push({ name: cName, oldHash: storedHash, newHash: currentHash });
-            isReRun = true;
-        } else if (storedHash === true) {
-             // Legacy upgrade
-            report.reRuns.push({ name: cName, reason: 'Legacy Upgrade' });
-            isReRun = true;
-        } else if (storedHash && storedHash === currentHash) {
-            // Already done, and hashes match.
-            // Check if we need to run implies we ignore this? 
-            // Usually we skip if done. But for "Analysis" log, we treat it as "Skipped/Done".
-            // If the user wants to FORCE run, that's different. 
-            // Assuming standard flow: if done & hash match, we don't run.
-            // report.blocked.push({ name: cName, reason: 'Already up to date' });
-            // continue;
-        }
-
-        // 3. Dependency Check
+        // 2. Dependency Check (FATAL)
+        // Since we are in a topological pass system, all dependencies SHOULD be satisfied 
+        // by previous passes. If not, it is a fatal error for this calculation.
         const missingDeps = [];
         if (calc.dependencies) {
             for (const dep of calc.dependencies) {
-                // If it's a historical dependency (yesterday's data), we assume availability or check elsewhere
-                // If it's a current pass dependency:
-                if (!isDepSatisfied(dep, dailyStatus)) {
+                if (!isDepSatisfied(dep, dailyStatus, manifestMap)) {
                     missingDeps.push(dep);
                 }
             }
         }
 
         if (missingDeps.length > 0) {
-            // It's missing a dependency. Is it fatal?
-            // Since we are inside a Pass, typically dependencies are from PREVIOUS passes.
-            // If it's from a previous pass and missing, it's blocked.
-            // But for the sake of the report:
-            report.resolving.push({ name: cName, missingDeps });
-            // In strict execution, this might be "runnable" if we assume the dep runs first in this batch,
-            // but usually dependencies are strictly lower passes.
-            // We'll mark it as resolving/blocked.
+            report.failedDependency.push({ name: cName, missing: missingDeps });
+            continue; // Cannot proceed
+        }
+
+        // 3. Hash / State Check
+        if (!storedHash) {
+            // Case A: No result exists (or status is 'false' from previous failure) -> RUN
+            // Note: If storedHash is boolean false, !storedHash is true, so we retry.
+            report.runnable.push(calc);
+        } else if (storedHash !== currentHash) {
+            // Case B: Result exists, but hash mismatch -> RE-RUN
+            // This covers code changes in THIS calc, AND cascading changes from dependencies
+            report.reRuns.push({ name: cName, oldHash: storedHash, newHash: currentHash });
+        } else if (storedHash === true) {
+            // Case C: Legacy boolean status -> RE-RUN (Upgrade to hash)
+            report.reRuns.push({ name: cName, reason: 'Legacy Upgrade' });
         } else {
-            // Dependencies met.
-            if (!dailyStatus[cName] || isReRun) {
-                report.runnable.push(calc);
-            }
+            // Case D: Result exists, Hash Matches -> SKIP
+            report.skipped.push({ name: cName });
         }
     }
 
@@ -124,7 +128,7 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
     // 2. Check Data Availability (One shot)
     const earliestDates = {
         portfolio: new Date('2025-09-25T00:00:00Z'),
-        history:   new Date('2025-11-05T00:00:00Z'), // Configurable in prod
+        history:   new Date('2025-11-05T00:00:00Z'), 
         social:    new Date('2025-10-30T00:00:00Z'),
         insights:  new Date('2025-08-26T00:00:00Z'),
         price:     new Date('2025-08-01T00:00:00Z')
@@ -133,39 +137,48 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
     const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
     const rootStatus = rootData ? rootData.status : { hasPortfolio: false, hasPrices: false, hasInsights: false, hasSocial: false, hasHistory: false };
 
-    // 3. PERFORM DATE ANALYSIS (Log the report)
+    // 3. ANALYZE EXECUTION
     const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
     const analysisReport = analyzeDateExecution(dateStr, calcsInThisPass, rootStatus, dailyStatus, manifestMap);
     
-    // LOG THE REPORT
+    // 4. LOG ANALYSIS
     logger.logDateAnalysis(dateStr, analysisReport);
 
-    // 4. Filter Run List based on Analysis
-    // We combine 'runnable' and 'reRuns'. 'resolving' are skipped (unless we support intra-pass resolution).
-    // Note: analyzeDateExecution returns plain objects or calc objects. We need the calc objects.
+    // 5. MARK FAILURES (Explicitly write 'false' to DB for blocked items)
+    // This prevents UI/downstream consumers from waiting indefinitely.
+    const failureUpdates = {};
+    analysisReport.blocked.forEach(item => failureUpdates[item.name] = false);
+    analysisReport.failedDependency.forEach(item => failureUpdates[item.name] = false);
+    
+    if (Object.keys(failureUpdates).length > 0) {
+        await updateComputationStatus(dateStr, failureUpdates, config, dependencies);
+    }
+
+    // 6. EXECUTE RUNNABLES
     const calcsToRunNames = new Set([
         ...analysisReport.runnable.map(c => c.name),
-        ...analysisReport.reRuns.map(c => c.name) // Re-runs are valid to run
+        ...analysisReport.reRuns.map(c => c.name)
     ]);
 
     const finalRunList = calcsInThisPass.filter(c => calcsToRunNames.has(normalizeName(c.name)));
 
     if (!finalRunList.length) {
-        return null;
+        // Nothing to run (everything either skipped, blocked, or failed)
+        return { date: dateStr, updates: {}, skipped: analysisReport.skipped.length };
     }
 
     logger.log('INFO', `[Orchestrator] Executing ${finalRunList.length} calculations for ${dateStr}`, { processId: orchestratorPid });
 
-    // 5. Execution
     const standardToRun = finalRunList.filter(c => c.type === 'standard');
     const metaToRun     = finalRunList.filter(c => c.type === 'meta');
     
     const dateUpdates = {};
 
     try {
-        const calcsRunning    = [...standardToRun, ...metaToRun];
+        const calcsRunning = [...standardToRun, ...metaToRun];
         
         // Fetch dependencies (Previous pass results)
+        // includeSelf=false: We are re-running, so we ignore our own old results.
         const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
         
         // Fetch Yesterday's results (if Historical)
@@ -183,15 +196,12 @@ async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, d
         }
 
     } catch (err) {
+        // If execution CRASHES (code bug, timeout), we log and throw.
+        // Pub/Sub will retry this message. This is correct for transient errors.
         logger.log('ERROR', `[Orchestrator] Failed execution for ${dateStr}`, { processId: orchestratorPid, error: err.message });
         throw err; 
     }
 
-    // 6. Status Update happens inside Executors, but we can log final success here
-    if (Object.keys(dateUpdates).length > 0) { 
-        // await updateComputationStatus... (Handled by Executors currently)
-    }
-
     return { date: dateStr, updates: dateUpdates };
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.224",
+  "version": "1.0.225",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [