bulltrackers-module 1.0.308 → 1.0.309

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,7 +1,6 @@
 /**
  * FILENAME: computation-system/WorkflowOrchestrator.js
- * UPDATED: Added missing groupByPass export.
- * Includes Content-Based Short-Circuiting for both Upstream and Historical dependencies.
+ * UPDATED: Implements Data-Drift Detection (Content-Addressable Execution).
  */
 
 const { normalizeName, DEFINITIVE_EARLIEST_DATES }         = require('./utils/utils');
@@ -14,7 +13,7 @@ const { MetaExecutor }                                     = require('./executor
 const STATUS_IMPOSSIBLE_PREFIX = 'IMPOSSIBLE';
 
 /**
- * [FIX] Groups manifest entries by their pass number.
+ * Groups manifest entries by their pass number.
  * Required by the Dispatcher to identify current work-sets.
  */
 function groupByPass(manifest) {
@@ -27,28 +26,39 @@ function groupByPass(manifest) {
 }
 
 /**
- * [NEW] Core Short-Circuit Logic.
- * Checks if a dependency (either a different node or "yesterday's self") is satisfied.
+ * Core Short-Circuit Logic.
+ * Checks if a dependency is satisfied AND checks for Data Drift.
+ * Returns { ready: boolean, dataChanged: boolean, reason: string }
  */
 function isDependencyReady(depName, isHistoricalSelf, currentStatusMap, prevStatusMap, manifestMap, storedStatus) {
     const norm = normalizeName(depName);
     const targetStatus = isHistoricalSelf ? (prevStatusMap ? prevStatusMap[norm] : null) : currentStatusMap[norm];
     const depManifest = manifestMap.get(norm);
     
+    // 1. Availability Check
     if (!targetStatus) return { ready: false, reason: 'Missing' };
     if (String(targetStatus.hash).startsWith(STATUS_IMPOSSIBLE_PREFIX)) return { ready: false, reason: 'Impossible Upstream' };
     
-    // 1. Code Hash Match (Strict)
-    if (targetStatus.hash === depManifest.hash) return { ready: true };
-
-    // 2. Content Hash Match (Short-Circuit)
-    // If our code didn't change, check if the output of the dependency is what we expect.
-    const lastSeenResultHash = storedStatus?.dependencyResultHashes?.[depName];
-    if (lastSeenResultHash && targetStatus.resultHash === lastSeenResultHash) {
-        return { ready: true, shortCircuited: true };
+    // 2. Code Hash Check (The dependency must be running the correct version)
+    // If the dependency's hash doesn't match its manifest, it means the dependency itself needs to run/update first.
+    if (depManifest && targetStatus.hash !== depManifest.hash) {
+        return { ready: false, reason: 'Dependency Version Mismatch' };
     }
 
-    return { ready: false, reason: 'Hash Mismatch' };
+    // 3. Data Integrity Check (The Short-Circuit Logic)
+    // We check if the result hash of the dependency matches what we remember using last time.
+    if (storedStatus && storedStatus.dependencyResultHashes) {
+        const lastSeenResultHash = storedStatus.dependencyResultHashes[depName];
+        
+        // If we recorded a dependency hash last time, and it differs from the current live status,
+        // then the dependency has produced NEW data. We are NOT ready to skip.
+        // We return 'ready: true' (it exists) but we flag 'dataChanged: true' to force execution.
+        if (lastSeenResultHash && targetStatus.resultHash !== lastSeenResultHash) {
+             return { ready: true, dataChanged: true, reason: 'Dependency Data Update' };
+        }
+    }
+
+    return { ready: true, dataChanged: false };
 }
 
 function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
@@ -74,17 +84,22 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
 
         // 2. Dependency & Temporal Check
         const missingDeps = [];
-        let isBlockedByHistory = false;
+        let hasDataDrift = false; // Tracks if any dependency produced new results
+        let isBlocked = false;
 
         // A. Standard Upstream Dependencies
         if (calc.dependencies) {
             for (const dep of calc.dependencies) {
                 const check = isDependencyReady(dep, false, simulationStatus, null, manifestMap, stored);
-                if (!check.ready) missingDeps.push(dep);
+                if (!check.ready) {
+                     missingDeps.push(dep);
+                } else if (check.dataChanged) {
+                    hasDataDrift = true;
+                }
             }
         }
 
-        // B. [UPGRADED] Temporal Dependency (Yesterday's Self)
+        // B. Temporal Dependency (Yesterday's Self)
         if (calc.isHistorical) {
             const yesterday = new Date(dateStr + 'T00:00:00Z');
             yesterday.setUTCDate(yesterday.getUTCDate() - 1);
@@ -92,7 +107,8 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             // Only block if yesterday is a valid data date. 
             if (yesterday >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
                 const check = isDependencyReady(calc.name, true, null, prevDailyStatus, manifestMap, stored);
-                if (!check.ready) isBlockedByHistory = true;
+                if (!check.ready) isBlocked = true;
+                else if (check.dataChanged) hasDataDrift = true; // Historical drift implies we need to re-run
             }
         }
 
@@ -107,12 +123,14 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             continue;
         }
 
-        if (isBlockedByHistory) {
+        if (isBlocked) {
             report.blocked.push({ name: cName, reason: 'Waiting for Yesterday' });
             continue;
         }
 
-        // 3. Runnable / Skip Logic
+        // 3. Execution Decision
+        
+        // Collect current dependency result hashes to be saved if we run
         const currentDependencyResultHashes = {};
         if (calc.dependencies) {
             calc.dependencies.forEach(d => {
@@ -120,16 +138,26 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
                 if (resHash) currentDependencyResultHashes[d] = resHash;
             });
         }
-
+        
         const taskPayload = { name: cName, dependencyResultHashes: currentDependencyResultHashes };
 
         if (!stored?.hash) {
+            // Case A: New Calculation (Never run)
             report.runnable.push({ ...taskPayload, reason: "New Calculation" });
             simulationStatus[cName] = { hash: currentHash, resultHash: 'SIMULATED' };
-        } else if (stored.hash !== currentHash) {
+        } 
+        else if (stored.hash !== currentHash) {
+            // Case B: Code Hash Mismatch (Logic Changed)
             report.reRuns.push({ ...taskPayload, oldHash: stored.hash, newHash: currentHash, reason: "Hash Mismatch" });
             simulationStatus[cName] = { hash: currentHash, resultHash: 'SIMULATED' };
-        } else {
+        } 
+        else if (hasDataDrift) {
+            // Case C: Code Matches, BUT Input Data Changed (The Holy Grail Optimization)
+            report.runnable.push({ ...taskPayload, reason: "Input Data Changed" });
+            simulationStatus[cName] = { hash: currentHash, resultHash: 'SIMULATED' };
+        } 
+        else {
+            // Case D: Code Matches AND Data Matches -> Short Circuit
             report.skipped.push({ name: cName, reason: "Up To Date" });
         }
     }
@@ -142,6 +170,9 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
     const calcManifest = manifestMap.get(normalizeName(targetComputation));
     
     if (!calcManifest) throw new Error(`Calc '${targetComputation}' not found.`);
+    
+    // [CRITICAL] Inject the fresh dependency result hashes so they are saved to DB on commit.
+    // This enables the "lastSeenResultHash" check in future runs.
     calcManifest.dependencyResultHashes = dependencyResultHashes;
 
     const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);

package/functions/computation-system/context/ManifestBuilder.js

@@ -1,10 +1,14 @@
+/**
+ * {
+ * type: uploaded file
+ * fileName: computation-system/context/ManifestBuilder.js
+ * }
+ */
 /**
  * @fileoverview Dynamic Manifest Builder - Handles Topological Sort and Auto-Discovery.
- * UPDATED: Generates Granular Hash Composition for Audit Trails.
- * UPGRADE: Implements Tarjan's Algorithm for Precise Cycle Detection.
- * FIXED: Now incorporates System Infrastructure Hash into Calculation Hashes.
+ * UPDATED: Removed Automatic Infra Hashing. Now relies strictly on SYSTEM_EPOCH.
  */
-const { generateCodeHash, getInfrastructureHash, LEGACY_MAPPING } = require('../topology/HashManager.js');
+const { generateCodeHash, LEGACY_MAPPING } = require('../topology/HashManager.js');
 const { normalizeName } = require('../utils/utils');
 
 const SYSTEM_EPOCH = require('../system_epoch');
@@ -86,7 +90,6 @@ function getDependencySet(endpoints, adjacencyList) {
 
 /**
  * Helper: Detects cycles using Tarjan's SCC Algorithm.
- * Returns a string description of the first cycle found.
  */
 function detectCircularDependencies(manifestMap) {
     let index      = 0;
@@ -106,8 +109,7 @@ function detectCircularDependencies(manifestMap) {
         const entry = manifestMap.get(v);
         if (entry && entry.dependencies) {
             for (const w of entry.dependencies) {
-                if (!manifestMap.has(w)) continue; // Skip external/missing deps (handled elsewhere)
-                
+                if (!manifestMap.has(w)) continue; 
                 if (!indices.has(w)) {
                     strongconnect(w);
                     lowLinks.set(v, Math.min(lowLinks.get(v), lowLinks.get(w)));
@@ -147,10 +149,7 @@ function detectCircularDependencies(manifestMap) {
 
 function buildManifest(productLinesToRun = [], calculations) {
     log.divider('Building Dynamic Manifest');
-    
-    // [CRITICAL FIX] Calculate Infrastructure Hash once per build
-    const INFRA_HASH = getInfrastructureHash();
-    log.info(`[ManifestBuilder] System Infrastructure Hash: ${INFRA_HASH.substring(0, 8)}`);
+    log.info(`[ManifestBuilder] Global System Epoch: ${SYSTEM_EPOCH}`);
 
     const requestedLog = (!productLinesToRun || productLinesToRun.length === 0) 
         ? "ALL (Wildcard/Empty)" 
@@ -175,9 +174,8 @@ function buildManifest(productLinesToRun = [], calculations) {
         const codeStr      = Class.toString();
         const selfCodeHash = generateCodeHash(codeStr);
         
-        // [CRITICAL FIX] Include INFRA_HASH in the composite signature
-        // This ensures that if the system platform changes, ALL calculations are considered "changed"
-        let compositeHashString = selfCodeHash + `|EPOCH:${SYSTEM_EPOCH}|INFRA:${INFRA_HASH}`; 
+        // [UPDATED] Composite Hash now depends ONLY on Code + Epoch + Layers
+        let compositeHashString = selfCodeHash + `|EPOCH:${SYSTEM_EPOCH}`; 
         
         const usedDeps = [];
         const usedLayerHashes = {}; 
@@ -227,7 +225,6 @@ function buildManifest(productLinesToRun = [], calculations) {
             composition: {
                 epoch: SYSTEM_EPOCH,
                 code: selfCodeHash,
-                infra: INFRA_HASH, // Stored in composition for audit
                 layers: layerComposition,
                 deps: {} 
             },
@@ -279,7 +276,6 @@ function buildManifest(productLinesToRun = [], calculations) {
     
     const activeList = Array.from(activePackages).sort().join(', ');
     log.info(`[ManifestBuilder] ✅ FINAL ACTIVE PRODUCT LINES: [${activeList}]`);
-    log.info(`[ManifestBuilder] Total Active Calculations: ${requiredCalcs.size}`);
 
     const filteredManifestMap = new Map();
     const filteredInDegree = new Map();

package/functions/computation-system/tools/BuildReporter.js

@@ -1,70 +1,31 @@
+/**
+ * {
+ * type: uploaded file
+ * fileName: computation-system/tools/BuildReporter.js
+ * }
+ */
 const { analyzeDateExecution } = require('../WorkflowOrchestrator');
 const { fetchComputationStatus, updateComputationStatus } = require('../persistence/StatusRepository');
-const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils');
+const { normalizeName, getExpectedDateStrings, DEFINITIVE_EARLIEST_DATES, generateCodeHash } = require('../utils/utils');
 const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
 const SimRunner = require('../simulation/SimRunner');
+const SYSTEM_EPOCH = require('../system_epoch'); // Relies on manual epoch
 const pLimit = require('p-limit');
 const path = require('path');
 const crypto = require('crypto');
 const fs = require('fs'); 
 const packageJson = require(path.join(__dirname, '..', '..', '..', 'package.json'));
 const packageVersion = packageJson.version;
-const { generateCodeHash } = require('../utils/utils'); 
 
 const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
 
-// --- RECURSIVE SYSTEM HASHING ---
-const SYSTEM_ROOT = path.resolve(__dirname, '..');
-const IGNORED_DIRS = new Set(['node_modules', '.git', '.idea', 'coverage', 'logs', 'tests']);
-const IGNORED_FILES = new Set(['package-lock.json', '.DS_Store', '.env']);
-
-function walkSync(dir, fileList = []) {
-    const files = fs.readdirSync(dir);
-    files.forEach(file => {
-        if (IGNORED_FILES.has(file)) return;
-        const filePath = path.join(dir, file);
-        const stat = fs.statSync(filePath);
-        if (stat.isDirectory()) {
-            if (!IGNORED_DIRS.has(file)) {
-                walkSync(filePath, fileList);
-            }
-        } else {
-            if (file.endsWith('.js') || file.endsWith('.json') || file.endsWith('.yaml')) {
-                fileList.push(filePath);
-            }
-        }
-    });
-    return fileList;
-}
-
-function getInfrastructureHash() {
-    try {
-        const allFiles = walkSync(SYSTEM_ROOT);
-        allFiles.sort(); 
-        const bigHash = crypto.createHash('sha256');
-        for (const filePath of allFiles) {
-            const content = fs.readFileSync(filePath, 'utf8');
-            const relativePath = path.relative(SYSTEM_ROOT, filePath);
-            let cleanContent = content;
-            if (filePath.endsWith('.js')) {
-                cleanContent = generateCodeHash(content); 
-            } else {
-                cleanContent = content.replace(/\s+/g, '');
-            }
-            bigHash.update(`${relativePath}:${cleanContent}|`);
-        }
-        return bigHash.digest('hex');
-    } catch (e) {
-        console.warn(`[BuildReporter] ⚠️ Failed to generate infra hash: ${e.message}`);
-        return 'infra_hash_error'; 
-    }
-}
-
+/**
+ * Replaces expensive file walking with System Epoch + Manifest Hash.
+ */
 function getSystemFingerprint(manifest) {
     const sortedManifestHashes = manifest.map(c => c.hash).sort().join('|');
-    const infraHash = getInfrastructureHash();
     return crypto.createHash('sha256')
-        .update(sortedManifestHashes + infraHash)
+        .update(sortedManifestHashes + SYSTEM_EPOCH)
         .digest('hex');
 }
 
@@ -104,11 +65,14 @@ function calculateBlastRadius(targetCalcName, reverseGraph) {
     };
 }
 
+/**
+ * [OPTIMIZED] Logic Stability Check
+ * We trust SimHash here. If the code logic (behavior) is stable, we mark it stable.
+ * We do NOT check dependency drift here. The WorkflowOrchestrator handles that at runtime.
+ */
 async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, logger, simHashCache, db) {
     const trueReRuns = [];
     const stableUpdates = [];
-    
-    // Concurrency for simulations
     const limit = pLimit(10);
 
     const checks = candidates.map(item => limit(async () => {
@@ -116,11 +80,13 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
             const manifest = manifestMap.get(item.name);
             const stored = dailyStatus[item.name];
 
+            // 1. If no history exists, it must run.
             if (!stored || !stored.simHash || !manifest) {
                 trueReRuns.push(item);
                 return;
             }
 
+            // 2. Fetch or Compute SimHash
             let newSimHash = simHashCache.get(manifest.hash);
             if (!newSimHash) {
                 newSimHash = await SimRunner.run(manifest, manifestMap);
@@ -132,7 +98,11 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
                 }).catch(err => logger.log('WARN', `Failed to write SimHash registry for ${manifest.name}: ${err.message}`));
             }
 
+            // 3. Behavioral Comparison
             if (newSimHash === stored.simHash) {
+                // STABLE: The logic is identical for mocked data.
+                // We mark this as stable, allowing the Orchestrator to skip it 
+                // UNLESS the actual production input data has changed.
                 stableUpdates.push({
                     ...item,
                     reason: "Code Updated (Logic Stable)",
@@ -140,6 +110,7 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
                     newHash: manifest.hash 
                 });
             } else {
+                // UNSTABLE: The logic actually produces different results.
                 trueReRuns.push({
                     ...item,
                     reason: item.reason + ` [SimHash Mismatch]`,
@@ -240,7 +211,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
 
     let totalRun = 0, totalReRun = 0, totalStable = 0;
 
-    // [FIX] Reduced concurrency from 20 to 5 to avoid Firestore DEADLINE_EXCEEDED
     const limit = pLimit(5);
 
     const processingPromises = datesToCheck.map(dateStr => limit(async () => {
@@ -305,12 +275,15 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
                     const updatesPayload = {};
                     for (const stable of stableUpdates) {
                         const m = manifestMap.get(stable.name);
-                        if (m && dailyStatus[stable.name]) {
+                        // We must grab the EXISTING data from the DB to preserve the OLD result hash.
+                        // This allows the Orchestrator to compare Old Result vs New Dependency Result.
+                        const stored = dailyStatus[stable.name];
+                        if (m && stored) {
                             updatesPayload[stable.name] = {
-                                hash: m.hash, 
-                                simHash: stable.simHash, 
-                                resultHash: dailyStatus[stable.name].resultHash, 
-                                dependencyResultHashes: dailyStatus[stable.name].dependencyResultHashes || {},
+                                hash: m.hash, // The NEW Code Hash
+                                simHash: stable.simHash, // The NEW SimHash (Same as old)
+                                resultHash: stored.resultHash, // The OLD Result Hash (Preserved)
+                                dependencyResultHashes: stored.dependencyResultHashes || {}, // OLD Deps (Preserved)
                                 category: m.category,
                                 composition: m.composition,
                                 lastUpdated: new Date()
@@ -324,19 +297,16 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
                 }
             }
 
-            // 3. BLOCKED / IMPOSSIBLE / UPTODATE
             analysis.blocked.forEach          (item => pushIfValid(dateSummary.blocked,    item));
             analysis.failedDependency.forEach (item => pushIfValid(dateSummary.blocked,    item, "Dependency Missing"));
             analysis.impossible.forEach       (item => pushIfValid(dateSummary.impossible, item));
             analysis.skipped.forEach          (item => pushIfValid(dateSummary.uptodate,   item, "Up To Date"));
 
-            // Meta stats
             const includedCount = dateSummary.run.length     + dateSummary.rerun.length      + dateSummary.stable.length +
                                   dateSummary.blocked.length + dateSummary.impossible.length + dateSummary.uptodate.length;
             dateSummary.meta.totalIncluded = includedCount;
             dateSummary.meta.match = (includedCount === expectedCount);
 
-            // Write Immediately
             await db.collection('computation_build_records')
             .doc(buildId)
             .collection('details')

package/functions/computation-system/topology/HashManager.js

@@ -1,10 +1,14 @@
 /**
- * @fileoverview Manages code versioning, hashing, and legacy mappings.
- * UPDATED: Includes Centralized Infrastructure Hashing to track system-level changes.
+ * {
+ * type: uploaded file
+ * fileName: computation-system/topology/HashManager.js
+ * }
+ */
+/**
+ * @fileoverview Manages code versioning and legacy mappings.
+ * UPDATED: Removed global infrastructure scanning. Now relies on Manual Epochs.
  */
 const crypto = require('crypto');
-const fs = require('fs');
-const path = require('path');
 
 // Legacy Keys Mapping (Ensures backward compatibility)
 const LEGACY_MAPPING = {
@@ -39,66 +43,13 @@ function generateCodeHash(codeString) {
     return crypto.createHash('sha256').update(clean).digest('hex');
 }
 
-// --- INFRASTRUCTURE HASHING (The "System Fingerprint") ---
-
-const SYSTEM_ROOT = path.resolve(__dirname, '..');
-const IGNORED_DIRS = new Set(['node_modules', '.git', '.idea', 'coverage', 'logs', 'tests', 'docs']);
-const IGNORED_FILES = new Set(['package-lock.json', '.DS_Store', '.env', 'README.md']);
-
-function walkSync(dir, fileList = []) {
-    const files = fs.readdirSync(dir);
-    files.forEach(file => {
-        if (IGNORED_FILES.has(file)) return;
-        const filePath = path.join(dir, file);
-        const stat = fs.statSync(filePath);
-        if (stat.isDirectory()) {
-            if (!IGNORED_DIRS.has(file)) {
-                walkSync(filePath, fileList);
-            }
-        } else {
-            // Hash JS, JSON, and YAML (Workflows)
-            if (file.endsWith('.js') || file.endsWith('.json') || file.endsWith('.yaml')) {
-                fileList.push(filePath);
-            }
-        }
-    });
-    return fileList;
-}
-
 /**
- * Generates a hash representing the state of the entire computation-system codebase.
- * This ensures that changes to infrastructure (like ContextFactory or Executors)
- * trigger a re-run of calculations even if the calculation logic itself didn't change.
+ * DEPRECATED: Previously walked the file system.
+ * Now returns a static string because we rely on SYSTEM_EPOCH for global versioning.
+ * Kept for backward compatibility with older consumers.
  */
 function getInfrastructureHash() {
-    try {
-        const allFiles = walkSync(SYSTEM_ROOT);
-        allFiles.sort(); // Crucial for determinism
-
-        const bigHash = crypto.createHash('sha256');
-
-        for (const filePath of allFiles) {
-            const content = fs.readFileSync(filePath, 'utf8');
-            const relativePath = path.relative(SYSTEM_ROOT, filePath);
-            
-            let cleanContent = content;
-
-            // Reuse the standard code hash logic for JS files to be consistent
-            if (filePath.endsWith('.js')) {
-                cleanContent = generateCodeHash(content);
-            } else {
-                // For JSON/YAML, just strip whitespace
-                cleanContent = content.replace(/\s+/g, '');
-            }
-
-            bigHash.update(`${relativePath}:${cleanContent}|`);
-        }
-
-        return bigHash.digest('hex');
-    } catch (e) {
-        console.warn(`[HashManager] ⚠️ Failed to generate infra hash: ${e.message}`);
-        return 'infra_error'; 
-    }
+    return 'MANUAL_EPOCH_MODE';
 }
 
 module.exports = { LEGACY_MAPPING, generateCodeHash, getInfrastructureHash };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.308",
+  "version": "1.0.309",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [