bulltrackers-module 1.0.263 → 1.0.265

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,6 +1,6 @@
 /**
  * @fileoverview Main Orchestrator. Coordinates the topological execution.
- * UPDATED: Uses centralized AvailabilityChecker for strict UserType validation.
+ * UPDATED: Implements Smart Audit logic to detect WHY a hash mismatch occurred.
  */
 const { normalizeName, DEFINITIVE_EARLIEST_DATES }         = require('./utils/utils');
 const { checkRootDataAvailability, checkRootDependencies } = require('./data/AvailabilityChecker');
@@ -10,13 +10,13 @@ const { StandardExecutor }                                 = require('./executor
 const { MetaExecutor }                                     = require('./executors/MetaExecutor');
 const { generateProcessId, PROCESS_TYPES }                 = require('./logger/logger');
 
-// [FIX] Split IMPOSSIBLE into semantic categories
 const STATUS_IMPOSSIBLE_PREFIX = 'IMPOSSIBLE';
 
 function groupByPass(manifest) {  return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc);  return acc;  }, {}); }
 
 /**
  * Analyzes whether calculations should run, be skipped, or are blocked.
+ * Now performs Deep Hash Analysis to explain Re-Runs.
  */
 function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
     const report           = { runnable: [], blocked: [], impossible: [], failedDependency: [], reRuns: [], skipped: [] };
@@ -28,7 +28,6 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         const stored      = currentStatusMap[norm]; 
         const depManifest = manifestMap.get(norm);
         if (!stored)                           return false;
-        // [FIX] Check for any IMPOSSIBLE variant
         if (typeof stored.hash === 'string' && stored.hash.startsWith(STATUS_IMPOSSIBLE_PREFIX)) return false;
         if (!depManifest)                      return false;
         if (stored.hash !== depManifest.hash)  return false;
@@ -42,7 +41,6 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         const storedCategory = stored ? stored.category : null;
         const currentHash    = calc.hash;
 
-        // [FIX] Granular impossible marking
         const markImpossible = (reason, type = 'GENERIC') => { 
             report.impossible.push({ name: cName, reason }); 
             const statusHash = `${STATUS_IMPOSSIBLE_PREFIX}:${type}`;
@@ -58,7 +56,6 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         let migrationOldCategory = null;
         if (storedCategory && storedCategory !== calc.category) { migrationOldCategory = storedCategory; }
         
-        // [FIX] Check for any IMPOSSIBLE variant in storage
         if (typeof storedHash === 'string' && storedHash.startsWith(STATUS_IMPOSSIBLE_PREFIX)) { 
             report.skipped.push({ name: cName, reason: `Permanently Impossible (${storedHash})` }); 
             continue; 
@@ -69,7 +66,6 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         if (!rootCheck.canRun) {
             const missingStr = rootCheck.missing.join(', ');
             if (!isTargetToday) {
-                // [FIX] Mark specifically as NO_DATA
                 markImpossible(`Missing Root Data: ${missingStr} (Historical)`, 'NO_DATA');
             } else {
                 report.blocked.push({ name: cName, reason: `Missing Root Data: ${missingStr} (Waiting)` });
@@ -83,7 +79,6 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             for (const dep of calc.dependencies) {
                 const normDep   = normalizeName(dep);
                 const depStored = simulationStatus[normDep];
-                // [FIX] Check for any IMPOSSIBLE variant in dependencies
                 if (depStored && typeof depStored.hash === 'string' && depStored.hash.startsWith(STATUS_IMPOSSIBLE_PREFIX)) { 
                     dependencyIsImpossible = true; 
                     break; 
@@ -93,7 +88,6 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
         }
 
         if (dependencyIsImpossible) { 
-            // [FIX] Mark specifically as UPSTREAM failure
             markImpossible('Dependency is Impossible', 'UPSTREAM'); 
             continue; 
         }
@@ -111,44 +105,88 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
             }
         }
         
-        if (!storedHash) { markRunnable(); } 
-        else if (storedHash !== currentHash) { markRunnable(true, { name: cName, oldHash: storedHash, newHash: currentHash, previousCategory: migrationOldCategory }); } 
-        else if (migrationOldCategory) {       markRunnable(true, { name: cName, reason: 'Category Migration', previousCategory: migrationOldCategory, newCategory: calc.category }); } 
-        else { report.skipped.push({ name: cName }); simulationStatus[cName] = { hash: currentHash, category: calc.category }; }
+        // --- HASH CHECK LOGIC ---
+        if (!storedHash) { 
+            markRunnable(false); // New Calculation
+        } 
+        else if (storedHash !== currentHash) {
+            // Smart Logic: Why did it change?
+            let changeReason = "Hash Mismatch (Unknown)";
+            const oldComp = stored.composition;
+            const newComp = calc.composition;
+
+            if (oldComp && newComp) {
+                // 1. Check Code
+                if (oldComp.code !== newComp.code) {
+                    changeReason = "Code Changed";
+                }
+                // 2. Check Layers
+                else if (JSON.stringify(oldComp.layers) !== JSON.stringify(newComp.layers)) {
+                    // Find specific layer
+                    const changedLayers = [];
+                    for(const lKey in newComp.layers) {
+                        if (newComp.layers[lKey] !== oldComp.layers[lKey]) changedLayers.push(lKey);
+                    }
+                    changeReason = `Layer Update: [${changedLayers.join(', ')}]`;
+                }
+                // 3. Check Dependencies
+                else if (JSON.stringify(oldComp.deps) !== JSON.stringify(newComp.deps)) {
+                    // Find specific dep
+                    const changedDeps = [];
+                    for(const dKey in newComp.deps) {
+                        if (newComp.deps[dKey] !== oldComp.deps[dKey]) changedDeps.push(dKey);
+                    }
+                    changeReason = `Upstream Change: [${changedDeps.join(', ')}]`;
+                }
+                else {
+                    changeReason = "Logic/Epoch Change";
+                }
+            } else {
+                changeReason = "Hash Mismatch (No prior composition)";
+            }
+
+            markRunnable(true, { 
+                name: cName, 
+                oldHash: storedHash, 
+                newHash: currentHash, 
+                previousCategory: migrationOldCategory,
+                reason: changeReason // <--- Passed to Reporter
+            }); 
+        } 
+        else if (migrationOldCategory) {       
+            markRunnable(true, { name: cName, reason: 'Category Migration', previousCategory: migrationOldCategory, newCategory: calc.category }); 
+        } 
+        else { 
+            report.skipped.push({ name: cName }); 
+            simulationStatus[cName] = { hash: currentHash, category: calc.category, composition: calc.composition }; 
+        }
     }
     return report;
 }
 
 /**
  * DIRECT EXECUTION PIPELINE (For Workers)
- * Skips analysis. Assumes the calculation is valid and runnable.
- * [UPDATED] Accepted previousCategory argument to handle migrations.
  */
 async function executeDispatchTask(dateStr, pass, targetComputation, config, dependencies, computationManifest, previousCategory = null) {
     const { logger } = dependencies;
     const pid = generateProcessId(PROCESS_TYPES.EXECUTOR, targetComputation, dateStr);
     
-    // 1. Get Calculation Manifest
     const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
     const calcManifest = manifestMap.get(normalizeName(targetComputation));
     
     if (!calcManifest) { throw new Error(`Calculation '${targetComputation}' not found in manifest.`); }
 
-    // [UPDATED] Attach migration context if present
     if (previousCategory) {
         calcManifest.previousCategory = previousCategory;
         logger.log('INFO', `[Executor] Migration detected for ${calcManifest.name}. Old data will be cleaned from: ${previousCategory}`);
     }
 
-    // 2. Fetch Root Data Availability
     const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
-    
     if (!rootData) {
-        logger.log('ERROR', `[Executor] FATAL: Root data check failed for ${targetComputation} on ${dateStr}. Index might be missing.`);
+        logger.log('ERROR', `[Executor] FATAL: Root data check failed for ${targetComputation} on ${dateStr}.`);
         return;
     }
 
-    // 3. Fetch Dependencies
     const calcsToRun = [calcManifest];
     const existingResults = await fetchExistingResults(dateStr, calcsToRun, computationManifest, config, dependencies, false);
     
@@ -160,7 +198,6 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
         previousResults = await fetchExistingResults(prevDateStr, calcsToRun, computationManifest, config, dependencies, true);
     }
 
-    // 4. Execute
     logger.log('INFO', `[Executor] Running ${calcManifest.name} for ${dateStr}`, { processId: pid });
     let resultUpdates = {};
 
@@ -176,5 +213,4 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
     }
 }
 
-
 module.exports = { executeDispatchTask, groupByPass, analyzeDateExecution };

package/functions/computation-system/context/ManifestBuilder.js

@@ -1,5 +1,6 @@
 /**
  * @fileoverview Dynamic Manifest Builder - Handles Topological Sort and Auto-Discovery.
+ * UPDATED: Generates Granular Hash Composition for Audit Trails.
  */
 const { generateCodeHash, LEGACY_MAPPING } = require('../topology/HashManager.js');
 const { normalizeName }                    = require('../utils/utils');
@@ -106,9 +107,13 @@ function buildManifest(productLinesToRun = [], calculations) {
         const metadata     = Class.getMetadata();
         const dependencies = Class.getDependencies().map(normalizeName);
         const codeStr      = Class.toString();
+        const selfCodeHash = generateCodeHash(codeStr);
+        
+        let compositeHashString = selfCodeHash + `|EPOCH:${SYSTEM_EPOCH}`; 
         
-        let compositeHashString = generateCodeHash(codeStr) + `|EPOCH:${SYSTEM_EPOCH}`; // Here we build the hash
         const usedDeps = [];
+        // Track layer hashes for composition analysis
+        const usedLayerHashes = {}; 
 
         for (const [layerName, exportsMap] of Object.entries(LAYER_TRIGGERS)) {
             const layerHashes = LAYER_HASHES[layerName];
@@ -118,19 +123,30 @@ function buildManifest(productLinesToRun = [], calculations) {
                     if (exportHash) {
                         compositeHashString += exportHash;
                         usedDeps.push(`${layerName}.${exportName}`);
+                        
+                        // Group hashes by layer for the composition report
+                        if (!usedLayerHashes[layerName]) usedLayerHashes[layerName] = '';
+                        usedLayerHashes[layerName] += exportHash;
                     }
                 }
             }
         }
         
+        // Simplify layer hashes to one hash per layer for the report
+        const layerComposition = {};
+        for(const [lName, lStr] of Object.entries(usedLayerHashes)) {
+            layerComposition[lName] = generateCodeHash(lStr);
+        }
+
         // Safe Mode Fallback
         let isSafeMode = false;
         if (usedDeps.length === 0) {
             isSafeMode = true;
             Object.values(LAYER_HASHES).forEach(layerObj => { Object.values(layerObj).forEach(h => compositeHashString += h); }); 
+            layerComposition['ALL_SAFE_MODE'] = 'ALL';
         }
 
-        const baseHash = generateCodeHash(compositeHashString);
+        const intrinsicHash = generateCodeHash(compositeHashString);
 
         const manifestEntry = {
             name: normalizedName,
@@ -143,7 +159,16 @@ function buildManifest(productLinesToRun = [], calculations) {
             userType: metadata.userType,
             dependencies: dependencies,
             pass: 0,
-            hash: baseHash,
+            hash: intrinsicHash, // Will be updated with deps
+            
+            // [NEW] Composition Object for Audit
+            composition: {
+                epoch: SYSTEM_EPOCH,
+                code: selfCodeHash,
+                layers: layerComposition,
+                deps: {} // Will be populated after topo sort
+            },
+            
             debugUsedLayers: isSafeMode ? ['ALL (Safe Mode)'] : usedDeps
         };
         
@@ -174,8 +199,6 @@ function buildManifest(productLinesToRun = [], calculations) {
     }
 
     const productLineEndpoints = [];
-    
-    // [UPDATE] Check if we should run ALL product lines (if empty or wildcard)
     const runAll = !productLinesToRun || productLinesToRun.length === 0 || productLinesToRun.includes('*');
 
     for (const [name, entry] of manifestMap.entries()) {  
@@ -187,7 +210,6 @@ function buildManifest(productLinesToRun = [], calculations) {
     const requiredCalcs = getDependencySet(productLineEndpoints, adjacency);
     log.info(`Filtered down to ${requiredCalcs.size} active calculations.`);
 
-    // [LOG VERIFICATION] Final Proof of Active Lines
     const activePackages = new Set();
     requiredCalcs.forEach(name => {
         const entry = manifestMap.get(name);
@@ -240,11 +262,17 @@ function buildManifest(productLinesToRun = [], calculations) {
 
     // --- Cascading Hash (Phase 2) ---
     for (const entry of sortedManifest) {
-        let dependencySignature = entry.hash;
+        let dependencySignature = entry.hash; // Start with intrinsic
+        
         if (entry.dependencies && entry.dependencies.length > 0) {
             const depHashes = entry.dependencies.map(depName => { 
-                const depEntry = filteredManifestMap.get(depName); 
-                return depEntry ? depEntry.hash : ''; 
+                const depEntry = filteredManifestMap.get(depName);
+                if (depEntry) {
+                    // Populate Composition
+                    entry.composition.deps[depName] = depEntry.hash;
+                    return depEntry.hash;
+                }
+                return ''; 
             }).join('|'); 
             dependencySignature += `|DEPS:${depHashes}`;
         }

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,6 +1,7 @@
 /**
  * @fileoverview Executor for "Standard" (per-user) calculations.
- * UPDATED: Handles lazy loading of data references (accepts null refs from Indexer).
+ * UPDATED: Handles lazy loading of data references.
+ * UPDATED: Supports Multi-Date Fan-Out Aggregation (Time Machine Mode).
  */
 const { normalizeName }                                                = require('../utils/utils');
 const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs } = require('../utils/data_loader');
@@ -39,8 +40,46 @@ class StandardExecutor {
         // 3. Stream & Process
         await StandardExecutor.streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
         
-        // 4. Commit
-        return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
+        // 4. Pre-Commit Transformation for Fan-Out
+        // If a calc produced multi-date output per user, we must transpose it:
+        // FROM: UserA -> { "2024-01-01": data, "2024-01-02": data }
+        // TO:   "2024-01-01" -> { UserA: data }, "2024-01-02" -> { UserA: data }
+        
+        const transformedState = {};
+        for (const [name, inst] of Object.entries(state)) {
+            const result = await inst.getResult(); // { userId: { date: data } } or { userId: data }
+            const firstUser = Object.keys(result)[0];
+            
+            // Check if the inner value is a Date Map
+            // Only checks the first user as heuristic; implies uniform return type
+            if (firstUser && result[firstUser] && typeof result[firstUser] === 'object') {
+                const innerKeys = Object.keys(result[firstUser]);
+                // Check if keys look like YYYY-MM-DD
+                const isDateMap = innerKeys.length > 0 && innerKeys.every(k => /^\d{4}-\d{2}-\d{2}$/.test(k));
+                
+                if (isDateMap) {
+                    const transposed = {};
+                    for (const [userId, dateMap] of Object.entries(result)) {
+                        for (const [dateKey, dailyData] of Object.entries(dateMap)) {
+                            if (!transposed[dateKey]) transposed[dateKey] = {};
+                            transposed[dateKey][userId] = dailyData;
+                        }
+                    }
+                    
+                    // Mock a "getResult" for the committer that returns the Transposed Map
+                    transformedState[name] = {
+                        manifest: inst.manifest,
+                        getResult: async () => transposed
+                    };
+                    continue;
+                }
+            }
+            // Normal behavior
+            transformedState[name] = inst;
+        }
+
+        // 5. Commit
+        return await commitResults(transformedState, dStr, passName, config, deps, skipStatusWrite);
     }
 
     static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
@@ -58,20 +97,16 @@ class StandardExecutor {
         const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
         const prevDateStr = prevDate.toISOString().slice(0, 10);
         
-        // [FIX] pass null if portfolioRefs is null; streamPortfolioData handles the fetch
         const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
         
         const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-        // yesterdayPortfolioRefs are manually fetched in run(), so they are usually populated
         const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
         
         const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-        // [FIX] Removed '&& historyRefs' check. We pass null to streamHistoryData if refs are missing, allowing it to fetch them.
         const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
 
         let yP_chunk = {}, tH_chunk = {};
         
-        // [FIX] Ensure manual iterators are closed if loop fails
         try {
             for await (const tP_chunk of tP_iter) {
                 if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};