bulltrackers-module 1.0.223 → 1.0.225

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,15 +1,14 @@
 /**
- * @fileoverview Main Orchestrator. Coordinates the topological execution of calculations.
+ * @fileoverview Main Orchestrator. Coordinates the topological execution.
+ * UPDATED: Strict Dependency & Hash Cascade Logic with Explicit Failure Marking.
  */
-const { normalizeName, getExpectedDateStrings }            = require('./utils/utils');
-const { checkRootDataAvailability, getViableCalculations } = require('./data/AvailabilityChecker');
+const { normalizeName }                                    = require('./utils/utils');
+const { checkRootDataAvailability }                        = require('./data/AvailabilityChecker');
 const { fetchExistingResults }                             = require('./data/DependencyFetcher');
 const { fetchComputationStatus, updateComputationStatus }  = require('./persistence/StatusRepository');
-const { runBatchPriceComputation }                         = require('./executors/PriceBatchExecutor');
 const { StandardExecutor }                                 = require('./executors/StandardExecutor');
 const { MetaExecutor }                                     = require('./executors/MetaExecutor');
-
-const PARALLEL_BATCH_SIZE = 7;
+const { generateProcessId, PROCESS_TYPES }                 = require('./logger/logger');
 
 function groupByPass(manifest) { 
     return manifest.reduce((acc, calc) => { 
@@ -18,135 +17,192 @@ function groupByPass(manifest) {
     }, {}); 
 }
 
-async function runComputationPass(config, dependencies, computationManifest) {
-    const { logger } = dependencies;
-    const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
-    if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
-
-    logger.log('INFO', `🚀 Starting PASS ${passToRun}...`);
+/**
+ * Performs strict analysis of what can run based on availability and hash states.
+ * AIRTIGHT LOGIC:
+ * 1. Missing Root Data -> Blocked (Writes 'false' to DB)
+ * 2. Missing/Stale Dependency -> FailedDependency (Writes 'false' to DB)
+ * 3. Hash Mismatch -> ReRun (Cascade or Code Change)
+ * 4. No Result -> Run
+ * 5. Result Exists & Hash Match -> Skip
+ */
+function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap) {
+    const report = {
+        runnable: [],
+        blocked: [],          // Missing Root Data
+        failedDependency: [], // Missing Dependency OR Stale Dependency
+        reRuns: [],           // Hash Mismatch (Triggered by self code change OR upstream cascade)
+        skipped: []           // Already done & valid
+    };
 
-    const earliestDates = {
-        portfolio: new Date('2025-09-25T00:00:00Z'),
-        history:   new Date('2025-11-05T00:00:00Z'),
-        social:    new Date('2025-10-30T00:00:00Z'),
-        insights:  new Date('2025-08-26T00:00:00Z'),
-        price:     new Date('2025-08-01T00:00:00Z')
+    // Helper: Is a dependency satisfied AND valid (matching hash)?
+    const isDepSatisfied = (depName, dailyStatus, manifestMap) => {
+        const norm = normalizeName(depName);
+        const storedDepHash = dailyStatus[norm];
+        const depManifest = manifestMap.get(norm);
+
+        // 1. Must exist in DB. If missing, we cannot run.
+        if (!storedDepHash) return false; 
+        
+        // 2. Must exist in Manifest (Sanity check)
+        if (!depManifest) return false;
+
+        // 3. STRICT: The dependency's stored hash must match its current manifest hash.
+        // If 'A' changed code, 'A' has a new hash. If we are running 'B' (Pass 2),
+        // we expect 'A' (Pass 1) to have already run and updated the DB with the NEW hash.
+        // If DB still has OLD hash, 'A' failed or didn't run. 'B' is unsafe to run.
+        if (storedDepHash !== depManifest.hash) return false;
+
+        return true;
     };
-    earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a, b) => a < b ? a : b);
-    const passes                   = groupByPass(computationManifest);
-    const calcsInThisPass          = passes[passToRun] || [];
 
-    if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
+    for (const calc of calcsInPass) {
+        const cName = normalizeName(calc.name);
+        const storedHash = dailyStatus[cName];
+        const currentHash = calc.hash;
 
-    const passEarliestDate = earliestDates.absoluteEarliest;
-    const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
-    const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
-    
-    // Separate specialized batch calcs
-    const priceBatchCalcs           = calcsInThisPass.filter(c => c.type === 'meta' && c.rootDataDependencies?.includes('price'));
-    const standardAndOtherMetaCalcs = calcsInThisPass.filter(c => !priceBatchCalcs.includes(c));
-
-    if (priceBatchCalcs.length > 0) {
-        try { 
-            await runBatchPriceComputation(config, dependencies, allExpectedDates, priceBatchCalcs); 
-        } catch (e) { 
-            logger.log('ERROR', 'Batch Price failed', e); 
+        // 1. Root Data Check (FATAL)
+        const missingRoots = [];
+        if (calc.rootDataDependencies) {
+            for (const dep of calc.rootDataDependencies) {
+                if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missingRoots.push('portfolio');
+                if (dep === 'insights'  && !rootDataStatus.hasInsights)  missingRoots.push('insights');
+                if (dep === 'social'    && !rootDataStatus.hasSocial)    missingRoots.push('social');
+                if (dep === 'history'   && !rootDataStatus.hasHistory)   missingRoots.push('history');
+                if (dep === 'price'     && !rootDataStatus.hasPrices)    missingRoots.push('price');
+            }
         }
-    }
 
-    if (standardAndOtherMetaCalcs.length === 0) return;
+        if (missingRoots.length > 0) {
+            report.blocked.push({ name: cName, reason: `Missing Root Data: ${missingRoots.join(', ')}` });
+            continue; // Cannot proceed
+        }
 
-    for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) {
-        const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
-        await Promise.all(batch.map(dateStr => 
-            runDateComputation(dateStr, passToRun, standardAndOtherMetaCalcs, config, dependencies, computationManifest)
-        ));
+        // 2. Dependency Check (FATAL)
+        // Since we are in a topological pass system, all dependencies SHOULD be satisfied 
+        // by previous passes. If not, it is a fatal error for this calculation.
+        const missingDeps = [];
+        if (calc.dependencies) {
+            for (const dep of calc.dependencies) {
+                if (!isDepSatisfied(dep, dailyStatus, manifestMap)) {
+                    missingDeps.push(dep);
+                }
+            }
+        }
+
+        if (missingDeps.length > 0) {
+            report.failedDependency.push({ name: cName, missing: missingDeps });
+            continue; // Cannot proceed
+        }
+
+        // 3. Hash / State Check
+        if (!storedHash) {
+            // Case A: No result exists (or status is 'false' from previous failure) -> RUN
+            // Note: If storedHash is boolean false, !storedHash is true, so we retry.
+            report.runnable.push(calc);
+        } else if (storedHash !== currentHash) {
+            // Case B: Result exists, but hash mismatch -> RE-RUN
+            // This covers code changes in THIS calc, AND cascading changes from dependencies
+            report.reRuns.push({ name: cName, oldHash: storedHash, newHash: currentHash });
+        } else if (storedHash === true) {
+            // Case C: Legacy boolean status -> RE-RUN (Upgrade to hash)
+            report.reRuns.push({ name: cName, reason: 'Legacy Upgrade' });
+        } else {
+            // Case D: Result exists, Hash Matches -> SKIP
+            report.skipped.push({ name: cName });
+        }
     }
+
+    return report;
 }
 
 async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) {
     const { logger } = dependencies;
+    const orchestratorPid = generateProcessId(PROCESS_TYPES.ORCHESTRATOR, passToRun, dateStr);
+
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
     
-    // 1. Version Check
-    const dailyStatus    = await fetchComputationStatus(dateStr, config, dependencies);
-    const calcsToAttempt = [];
+    // 1. Fetch State
+    const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
     
-    for (const calc of calcsInThisPass) {
-        const cName = normalizeName(calc.name);
-        const storedStatus = dailyStatus[cName];
-        const currentHash = calc.hash;
-
-        if (!storedStatus) { 
-            calcsToAttempt.push(calc); continue; 
-        }
-        if (typeof storedStatus === 'string' && currentHash && storedStatus !== currentHash) { 
-            logger.log('INFO', `[Versioning] ${cName}: Code Changed.`); 
-            calcsToAttempt.push(calc); continue; 
-        }
-        if (storedStatus === true && currentHash) { 
-            logger.log('INFO', `[Versioning] ${cName}: Upgrading legacy status.`); 
-            calcsToAttempt.push(calc); continue; 
-        }
-    }
-
-    if (!calcsToAttempt.length) return null;
-
-    // 2. Data Availability Check
+    // 2. Check Data Availability (One shot)
     const earliestDates = {
         portfolio: new Date('2025-09-25T00:00:00Z'),
-        history:   new Date('2025-11-05T00:00:00Z'),
+        history:   new Date('2025-11-05T00:00:00Z'), 
         social:    new Date('2025-10-30T00:00:00Z'),
         insights:  new Date('2025-08-26T00:00:00Z'),
         price:     new Date('2025-08-01T00:00:00Z')
     };
-
+    
     const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
-    if (!rootData) { logger.log('INFO', `[DateRunner] Root data missing for ${dateStr}. Skipping.`); return null; }
+    const rootStatus = rootData ? rootData.status : { hasPortfolio: false, hasPrices: false, hasInsights: false, hasSocial: false, hasHistory: false };
+
+    // 3. ANALYZE EXECUTION
+    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
+    const analysisReport = analyzeDateExecution(dateStr, calcsInThisPass, rootStatus, dailyStatus, manifestMap);
+    
+    // 4. LOG ANALYSIS
+    logger.logDateAnalysis(dateStr, analysisReport);
+
+    // 5. MARK FAILURES (Explicitly write 'false' to DB for blocked items)
+    // This prevents UI/downstream consumers from waiting indefinitely.
+    const failureUpdates = {};
+    analysisReport.blocked.forEach(item => failureUpdates[item.name] = false);
+    analysisReport.failedDependency.forEach(item => failureUpdates[item.name] = false);
     
-    // 3. Viability Check (Smart Execution Map)
-    // Filter candidates: Remove any calculation that misses Root Data OR Matches Stale Dependencies
-    // PASSED: computationManifest (needed for hash lookup)
-    const runnableCalcs = getViableCalculations(calcsToAttempt, computationManifest, rootData.status, dailyStatus);
-
-    if (!runnableCalcs.length) {
-        // logger.log('INFO', `[DateRunner] ${dateStr}: Candidates pruned due to missing deps/data/stale hashes.`);
-        return null;
+    if (Object.keys(failureUpdates).length > 0) {
+        await updateComputationStatus(dateStr, failureUpdates, config, dependencies);
+    }
+
+    // 6. EXECUTE RUNNABLES
+    const calcsToRunNames = new Set([
+        ...analysisReport.runnable.map(c => c.name),
+        ...analysisReport.reRuns.map(c => c.name)
+    ]);
+
+    const finalRunList = calcsInThisPass.filter(c => calcsToRunNames.has(normalizeName(c.name)));
+
+    if (!finalRunList.length) {
+        // Nothing to run (everything either skipped, blocked, or failed)
+        return { date: dateStr, updates: {}, skipped: analysisReport.skipped.length };
     }
 
-    const standardToRun = runnableCalcs.filter(c => c.type === 'standard');
-    const metaToRun = runnableCalcs.filter(c => c.type === 'meta');
-    logger.log('INFO', `[DateRunner] Running ${dateStr}: ${standardToRun.length} std, ${metaToRun.length} meta`);
+    logger.log('INFO', `[Orchestrator] Executing ${finalRunList.length} calculations for ${dateStr}`, { processId: orchestratorPid });
+
+    const standardToRun = finalRunList.filter(c => c.type === 'standard');
+    const metaToRun     = finalRunList.filter(c => c.type === 'meta');
     
     const dateUpdates = {};
 
     try {
-        const calcsRunning    = [...standardToRun, ...metaToRun];
+        const calcsRunning = [...standardToRun, ...metaToRun];
+        
+        // Fetch dependencies (Previous pass results)
+        // includeSelf=false: We are re-running, so we ignore our own old results.
         const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
+        
+        // Fetch Yesterday's results (if Historical)
         const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
         const prevDateStr     = prevDate.toISOString().slice(0, 10);
         const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
 
         if (standardToRun.length) { 
-            const updates = await StandardExecutor.run(dateToProcess, standardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, false); 
+            const updates = await StandardExecutor.run(dateToProcess, standardToRun, `Pass ${passToRun}`, config, dependencies, rootData, existingResults, previousResults, false); 
             Object.assign(dateUpdates, updates); 
         }
         if (metaToRun.length) { 
-            const updates = await MetaExecutor.run(dateToProcess, metaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, false); 
+            const updates = await MetaExecutor.run(dateToProcess, metaToRun, `Pass ${passToRun}`, config, dependencies, existingResults, previousResults, rootData, false); 
             Object.assign(dateUpdates, updates); 
         }
 
     } catch (err) {
-        logger.log('ERROR', `[DateRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
-        [...standardToRun, ...metaToRun].forEach(c => dateUpdates[normalizeName(c.name)] = false);
+        // If execution CRASHES (code bug, timeout), we log and throw.
+        // Pub/Sub will retry this message. This is correct for transient errors.
+        logger.log('ERROR', `[Orchestrator] Failed execution for ${dateStr}`, { processId: orchestratorPid, error: err.message });
         throw err; 
     }
 
-    if (Object.keys(dateUpdates).length > 0) { 
-        await updateComputationStatus(dateStr, dateUpdates, config, dependencies); 
-    }
-
     return { date: dateStr, updates: dateUpdates };
 }
 
-module.exports = { runComputationPass, runDateComputation, groupByPass };
+module.exports = { runDateComputation, groupByPass };

package/functions/computation-system/context/ManifestBuilder.js

@@ -50,18 +50,17 @@ function buildDynamicTriggers() {
 }
 
 const LAYER_HASHES = {};
-for (const [name, exports] of Object.entries(LAYER_GROUPS)) {
-    LAYER_HASHES[name] = generateLayerHashes(exports, name);
-}
+for (const [name, exports] of Object.entries(LAYER_GROUPS)) { LAYER_HASHES[name] = generateLayerHashes(exports, name); }
+
 const LAYER_TRIGGERS = buildDynamicTriggers();
 
 const log = {
-    info: (msg) => console.log('ℹ︎ ' + msg),
-    step: (msg) => console.log('› ' + msg),
-    warn: (msg) => console.warn('⚠︎ ' + msg),
+    info:    (msg) => console.log('ℹ︎ ' + msg),
+    step:    (msg) => console.log('› ' + msg),
+    warn:    (msg) => console.warn('⚠︎ ' + msg),
     success: (msg) => console.log('✔︎ ' + msg),
-    error: (msg) => console.error('✖ ' + msg),
-    fatal: (msg) => { console.error('✖ FATAL ✖ ' + msg); console.error('✖ FATAL ✖ Manifest build FAILED.'); },
+    error:   (msg) => console.error('✖ ' + msg),
+    fatal:   (msg) => { console.error('✖ FATAL ✖ ' + msg); console.error('✖ FATAL ✖ Manifest build FAILED.'); },
     divider: (label) => { const line = ''.padEnd(60, '─'); console.log(`\n${line}\n${label}\n${line}\n`); },
 };
 

package/functions/computation-system/controllers/computation_controller.js

@@ -22,7 +22,6 @@ const LEGACY_MAPPING = {
     priceExtractor: 'priceExtractor',
     InsightsExtractor: 'insights',
     UserClassifier: 'classifier',
-    // Added based on your profiling.js file:
     Psychometrics: 'psychometrics',
     CognitiveBiases: 'bias',
     SkillAttribution: 'skill',
@@ -47,7 +46,6 @@ class ContextBuilder {
         for (const [key, value] of Object.entries(mathLayer)) { mathContext[key] = value; const legacyKey = LEGACY_MAPPING[key]; if (legacyKey) { mathContext[legacyKey] = value; } }
         return mathContext;
     }
-    // ... (rest of class remains identical)
     static buildPerUserContext(options) {
         const { todayPortfolio, yesterdayPortfolio, todayHistory, yesterdayHistory, userId, userType, dateStr, metadata, mappings, insights, socialData, computedDependencies, previousComputedDependencies, config, deps } = options;
         return {
@@ -80,7 +78,6 @@ class ContextBuilder {
 }
 
 class ComputationExecutor {
-    // ... (remains identical to previous version)
     constructor(config, dependencies, dataLoader) {
         this.config = config;
         this.deps   = dependencies;
@@ -146,5 +143,4 @@ class ComputationController {
     }
 }
 
-// EXPORT LEGACY_MAPPING SO BUILDER CAN USE IT
 module.exports = { ComputationController, LEGACY_MAPPING };

package/functions/computation-system/data/AvailabilityChecker.js

@@ -52,29 +52,18 @@ function getViableCalculations(candidates, fullManifest, rootDataStatus, dailySt
                 const depManifest = manifestMap.get(normDep);
                 
                 // If dependency is missing from manifest, we can't verify it (shouldn't happen)
-                if (!depManifest) {
-                    dependenciesMet = false;
-                    break;
-                }
+                if (!depManifest) { dependenciesMet = false; break; }
 
                 // CHECK: Does the dependency exist in DB?
-                if (!storedHash) {
-                    dependenciesMet = false;
-                    break;
-                }
+                if (!storedHash) { dependenciesMet = false; break; }
 
                 // CHECK: Does the stored hash match the current code hash?
                 // This prevents running on stale data if a dependency failed to update.
-                if (storedHash !== depManifest.hash) {
-                    dependenciesMet = false;
-                    break;
-                }
+                if (storedHash !== depManifest.hash) { dependenciesMet = false; break; }
             }
         }
 
-        if (dependenciesMet) {
-            viable.push(calc);
-        }
+        if (dependenciesMet) { viable.push(calc); }
     }
 
     return viable;

package/functions/computation-system/helpers/computation_worker.js

@@ -1,25 +1,57 @@
 /**
- * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_worker.js
+ * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes computation tasks from Pub/Sub and executes them.
- * REFACTORED: Now imports logic from the new WorkflowOrchestrator.
+ * UPDATED: Correctly points to layers/index.js and wraps it for the ManifestBuilder.
  */
 
 const { runDateComputation, groupByPass } = require('../WorkflowOrchestrator.js');
+const { getManifest }                     = require('../topology/ManifestLoader');
+
+// 1. IMPORT CALCULATIONS
+// User confirmation: Calculations are barrel-loaded into layers/index.js
+const rawLayers = require('../layers/index');
+
+// 2. PREPARE FOR MANIFEST
+// The ManifestBuilder expects a structure like: { packageName: { CalculationClass, ... } }
+// Since layers/index.js returns a flat object, we wrap it in a 'core' group here.
+const calculations = { 
+    core: rawLayers 
+};
 
 /**
  * Handles a single Pub/Sub message for a computation task.
  * Supports both Gen 1 (Message) and Gen 2 (CloudEvent) formats.
+ * * @param {object} message - The Pub/Sub message payload.
+ * @param {object} config - System configuration (must include activeProductLines).
+ * @param {object} dependencies - System dependencies (logger, db, etc.).
  */
-async function handleComputationTask(message, config, dependencies, computationManifest) {
+async function handleComputationTask(message, config, dependencies) {
     const { logger } = dependencies;
 
+    // 3. LAZY LOAD MANIFEST
+    // This ensures we only build the manifest when we actually receive a task.
+    // We pass our wrapped 'calculations' object here.
+    let computationManifest;
+    try {
+        computationManifest = getManifest(
+            config.activeProductLines || [], 
+            calculations, 
+            dependencies
+        );
+    } catch (manifestError) {
+        logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
+        return; 
+    }
+
+    // 4. PARSE PUB/SUB MESSAGE
     let data;
     try {
-        // 1. Handle Cloud Functions Gen 2 (CloudEvent) -> Gen 1 -> Direct JSON -> Message
         if (message.data && message.data.message && message.data.message.data) { 
-            const buffer = Buffer.from(message.data.message.data, 'base64'); data = JSON.parse(buffer.toString()); 
+            const buffer = Buffer.from(message.data.message.data, 'base64'); 
+            data = JSON.parse(buffer.toString()); 
         } else if (message.data && typeof message.data === 'string') { 
-            const buffer = Buffer.from(message.data, 'base64'); data = JSON.parse(buffer.toString()); 
+            const buffer = Buffer.from(message.data, 'base64'); 
+            data = JSON.parse(buffer.toString()); 
         } else if (message.json) { 
             data = message.json; 
         } else { 
@@ -30,8 +62,8 @@ async function handleComputationTask(message, config, dependencies, computationM
         return; 
     }
 
+    // 5. EXECUTE TASK
     try {
-        // Validate Action
         if (!data || data.action !== 'RUN_COMPUTATION_DATE') { 
             if (data) logger.log('WARN', `[Worker] Unknown or missing action: ${data?.action}. Ignoring.`); 
             return; 
@@ -53,17 +85,24 @@ async function handleComputationTask(message, config, dependencies, computationM
             return; 
         }
         
-        const result = await runDateComputation(date, pass, calcsInThisPass, config, dependencies, computationManifest);
+        const result = await runDateComputation(
+            date, 
+            pass, 
+            calcsInThisPass, 
+            config, 
+            dependencies, 
+            computationManifest
+        );
         
-        if (result) { 
-            logger.log('INFO', `[Worker] Successfully processed ${date} (Pass ${pass}). Updates: ${Object.keys(result.updates || {}).length}`);
+        if (result && result.updates && Object.keys(result.updates).length > 0) { 
+            logger.log('INFO', `[Worker] Successfully processed ${date} (Pass ${pass}). Updates: ${Object.keys(result.updates).length}`);
         } else { 
-            logger.log('INFO', `[Worker] Processed ${date} (Pass ${pass}) - Skipped (Dependencies missing or already done).`); 
+            logger.log('INFO', `[Worker] Processed ${date} (Pass ${pass}) - No updates.`); 
         }
         
     } catch (err) { 
         logger.log('ERROR', `[Worker] Fatal error processing task: ${err.message}`, { stack: err.stack }); 
-        throw err;  
+        throw err;
     }
 }
 

package/functions/computation-system/logger/logger.js

@@ -0,0 +1,454 @@
+/**
+ * @fileoverview Structured Logging System for Computation Engine
+ * Provides comprehensive tracking with process IDs, context, and filtering capabilities.
+ * UPDATED: Includes Pre-flight Date Analysis and Storage Observability.
+ */
+
+const crypto = require('crypto');
+
+/**
+ * Log Levels (Ordered by Severity)
+ */
+const LOG_LEVELS = {
+    TRACE: 0,
+    DEBUG: 1,
+    INFO: 2,
+    WARN: 3,
+    ERROR: 4,
+    FATAL: 5
+};
+
+/**
+ * Process Types for Tracking
+ */
+const PROCESS_TYPES = {
+    MANIFEST: 'manifest',
+    ORCHESTRATOR: 'orchestrator',
+    EXECUTOR: 'executor',
+    STORAGE: 'storage',
+    WORKER: 'worker',
+    ANALYSIS: 'analysis',
+    // Legacy / Specific aliases
+    SCHEMA_GENERATION: 'schema_generation',
+    COMPUTATION_EXECUTION: 'computation_execution',
+    DEPENDENCY_FETCH: 'dependency_fetch',
+    DATA_AVAILABILITY: 'data_availability',
+    DISPATCH: 'dispatch'
+};
+
+/**
+ * Generates a deterministic process ID from inputs.
+ * Ensures that logs for the same computation/date always have the same Trace ID.
+ * @param {string} type - Process type (e.g., 'orchestrator')
+ * @param {string} identifier - Unique key (e.g., pass name, calculation name)
+ * @param {string} date - Date string YYYY-MM-DD (optional)
+ * @returns {string} 16-character hexadecimal process ID
+ */
+function generateProcessId(type, identifier, date = '') {
+    const input = `${type}|${identifier}|${date}`;
+    return crypto.createHash('sha256').update(input).digest('hex').substring(0, 16);
+}
+
+/**
+ * Formats a log entry into structured JSON
+ */
+function formatLogEntry(entry) {
+    return JSON.stringify({
+        timestamp: entry.timestamp,
+        level: entry.level,
+        processType: entry.processType,
+        processId: entry.processId,
+        computationName: entry.computationName,
+        date: entry.date,
+        message: entry.message,
+        context: entry.context,
+        metadata: entry.metadata,
+        // Specific fields for specialized logs
+        stats: entry.stats,
+        storage: entry.storage,
+        details: entry.details
+    });
+}
+
+/**
+ * Main Structured Logger Class
+ */
+class StructuredLogger {
+    constructor(config = {}) {
+        this.config = {
+            minLevel: config.minLevel || LOG_LEVELS.INFO,
+            enableConsole: config.enableConsole !== false,
+            enableStructured: config.enableStructured !== false,
+            includeStackTrace: config.includeStackTrace !== false,
+            ...config
+        };
+        
+        this.activeProcesses = new Map();
+    }
+
+    /**
+     * Starts a tracked process and returns a ProcessLogger
+     * @param {string} processType - Type from PROCESS_TYPES
+     * @param {string} computationName - Optional computation name
+     * @param {string} date - Optional date string
+     * @returns {ProcessLogger}
+     */
+    startProcess(processType, computationName = null, date = null) {
+        // Use deterministic ID if components are present, else random/time-based fallback
+        const processId = (computationName || date) 
+            ? generateProcessId(processType, computationName || 'general', date)
+            : crypto.randomBytes(8).toString('hex');
+
+        const processLogger = new ProcessLogger(this, processType, processId, computationName, date);
+        
+        this.activeProcesses.set(processId, {
+            type: processType,
+            computationName,
+            date,
+            startTime: Date.now(),
+            logger: processLogger
+        });
+        
+        return processLogger;
+    }
+
+    /**
+     * Ends a tracked process
+     */
+    endProcess(processId) {
+        const process = this.activeProcesses.get(processId);
+        if (process) {
+            const duration = Date.now() - process.startTime;
+            this.activeProcesses.delete(processId);
+            return duration;
+        }
+        return null;
+    }
+
+    /**
+     * The Main Date Analysis Logger (NEW)
+     * Aggregates the status of all calculations for a specific date into one readable report.
+     */
+    logDateAnalysis(dateStr, analysisReport) {
+        const { runnable, blocked, reRuns, resolving } = analysisReport;
+        
+        // 1. Structured Output (Machine Readable)
+        if (this.config.enableStructured) {
+            console.log(JSON.stringify({
+                timestamp: new Date().toISOString(),
+                level: 'INFO',
+                processType: PROCESS_TYPES.ANALYSIS,
+                date: dateStr,
+                message: `Date Analysis for ${dateStr}`,
+                stats: {
+                    runnable: runnable.length,
+                    blocked: blocked.length,
+                    reRuns: reRuns.length,
+                    resolving: resolving.length
+                },
+                details: analysisReport
+            }));
+        }
+
+        // 2. Human Readable Output (Console)
+        if (this.config.enableConsole) {
+            const symbols = { info: 'ℹ️', warn: '⚠️', check: '✅', block: '⛔', link: '🔗', cycle: '🔄' };
+            
+            console.log(`\n🔍 === DATE ANALYSIS REPORT: ${dateStr} ===`);
+            
+            if (reRuns.length) {
+                console.log(`\n${symbols.cycle} [HASH MISMATCH / RE-RUNS]`);
+                reRuns.forEach(item => {
+                    console.log(`   • ${item.name}: Hash changed (Old: ${item.oldHash?.substring(0,6)}... New: ${item.newHash?.substring(0,6)}...). Cascade: ${item.cascade?.length || 0} dependents.`);
+                });
+            }
+
+            if (resolving.length) {
+                console.log(`\n${symbols.link} [DEPENDENCY RESOLUTION] (Will run after deps)`);
+                resolving.forEach(item => {
+                    console.log(`   • ${item.name}: Waiting for [${item.missingDeps.join(', ')}]`);
+                });
+            }
+
+            if (runnable.length) {
+                console.log(`\n${symbols.check} [READY TO RUN]`);
+                runnable.forEach(item => console.log(`   • ${item.name}`));
+            }
+
+            if (blocked.length) {
+                console.log(`\n${symbols.block} [BLOCKED / SKIPPED]`);
+                blocked.forEach(item => {
+                    console.log(`   • ${item.name}: ${item.reason}`);
+                });
+            }
+            console.log(`\n=============================================\n`);
+        }
+    }
+
+    /**
+     * Storage Observability Logger (NEW)
+     */
+    logStorage(processId, calcName, date, path, sizeBytes, isSharded) {
+        // Standard Log Call with extended metadata
+        this.log(LOG_LEVELS.INFO, `Results stored for ${calcName}`, {
+            storage: { 
+                path, 
+                sizeBytes, 
+                isSharded, 
+                sizeMB: (sizeBytes / 1024 / 1024).toFixed(2) 
+            }
+        }, PROCESS_TYPES.STORAGE, processId, calcName, date);
+    }
+
+    /**
+     * Core logging method
+     */
+    log(level, message, context = {}, processType = null, processId = null, computationName = null, date = null) {
+        const numericLevel = typeof level === 'string' ? LOG_LEVELS[level] : level;
+        
+        if (numericLevel < this.config.minLevel) return;
+
+        // Support passing "meta" object in place of context for newer calls
+        // Logic: If context contains 'processId' or 'storage', it's likely a meta object
+        let finalContext = context;
+        let finalMetadata = {};
+        let finalStats = undefined;
+        let finalStorage = undefined;
+
+        if (context && (context.processId || context.storage || context.stats)) {
+            if (context.processId) processId = context.processId;
+            if (context.processType) processType = context.processType;
+            if (context.computationName) computationName = context.computationName;
+            if (context.date) date = context.date;
+            if (context.stats) finalStats = context.stats;
+            if (context.storage) finalStorage = context.storage;
+            // Clean up context to be just the data remaining
+            finalContext = { ...context };
+            delete finalContext.processId; delete finalContext.processType;
+            delete finalContext.computationName; delete finalContext.date;
+            delete finalContext.stats; delete finalContext.storage;
+        }
+
+        const entry = {
+            timestamp: new Date().toISOString(),
+            level: Object.keys(LOG_LEVELS).find(k => LOG_LEVELS[k] === numericLevel) || 'INFO',
+            processType,
+            processId,
+            computationName,
+            date,
+            message,
+            context: typeof finalContext === 'string' ? { error: finalContext } : finalContext,
+            metadata: finalMetadata,
+            stats: finalStats,
+            storage: finalStorage
+        };
+
+        // Add stack trace for errors
+        if (numericLevel >= LOG_LEVELS.ERROR && this.config.includeStackTrace && finalContext.stack) {
+            entry.metadata.stackTrace = finalContext.stack;
+        }
+
+        // Console output (pretty-printed for development)
+        if (this.config.enableConsole) {
+            this._consoleLog(entry);
+        }
+
+        // Structured output (for log aggregation systems)
+        if (this.config.enableStructured) {
+            console.log(formatLogEntry(entry));
+        }
+    }
+
+    /**
+     * Pretty console output for development
+     */
+    _consoleLog(entry) {
+        const symbols = { TRACE: '🔍', DEBUG: '🐛', INFO: 'ℹ️', WARN: '⚠️', ERROR: '❌', FATAL: '💀' };
+        const colors = {
+            TRACE: '\x1b[90m', DEBUG: '\x1b[36m', INFO: '\x1b[32m',
+            WARN: '\x1b[33m', ERROR: '\x1b[31m', FATAL: '\x1b[35m'
+        };
+        const reset = '\x1b[0m';
+        const color = colors[entry.level] || '';
+        const symbol = symbols[entry.level] || 'ℹ️';
+
+        let output = `${color}${symbol} [${entry.level}]${reset}`;
+        
+        if (entry.processType) output += ` [${entry.processType}]`;
+        if (entry.processId) output += ` [${entry.processId.substring(0, 8)}]`;
+        if (entry.computationName) output += ` [${entry.computationName}]`;
+        if (entry.date) output += ` [${entry.date}]`;
+        
+        output += ` ${entry.message}`;
+
+        console.log(output);
+
+        // Print context if present and not empty
+        if (entry.context && Object.keys(entry.context).length > 0) {
+            console.log(`  ${color}Context:${reset}`, entry.context);
+        }
+        if (entry.storage) {
+             console.log(`  ${color}Storage:${reset}`, entry.storage);
+        }
+    }
+
+    // Convenience methods
+    trace(message, context = {}) { this.log(LOG_LEVELS.TRACE, message, context); }
+    debug(message, context = {}) { this.log(LOG_LEVELS.DEBUG, message, context); }
+    info(message, context = {}) { this.log(LOG_LEVELS.INFO, message, context); }
+    warn(message, context = {}) { this.log(LOG_LEVELS.WARN, message, context); }
+    error(message, context = {}) { this.log(LOG_LEVELS.ERROR, message, context); }
+    fatal(message, context = {}) { this.log(LOG_LEVELS.FATAL, message, context); }
+}
+
+/**
+ * Process-scoped Logger
+ * Automatically includes process context in all log calls
+ */
+class ProcessLogger {
+    constructor(parent, processType, processId, computationName, date) {
+        this.parent = parent;
+        this.processType = processType;
+        this.processId = processId;
+        this.computationName = computationName;
+        this.date = date;
+        this.startTime = Date.now();
+        this.metrics = {
+            operations: 0,
+            errors: 0,
+            warnings: 0
+        };
+    }
+
+    log(level, message, context = {}) {
+        const numericLevel = typeof level === 'string' ? LOG_LEVELS[level] : level;
+        
+        this.metrics.operations++;
+        if (numericLevel === LOG_LEVELS.ERROR || numericLevel === LOG_LEVELS.FATAL) {
+            this.metrics.errors++;
+        }
+        if (numericLevel === LOG_LEVELS.WARN) {
+            this.metrics.warnings++;
+        }
+
+        this.parent.log(
+            level,
+            message,
+            context,
+            this.processType,
+            this.processId,
+            this.computationName,
+            this.date
+        );
+    }
+
+    /**
+     * Complete the process and log summary
+     */
+    complete(success = true, finalMessage = null) {
+        const duration = Date.now() - this.startTime;
+        const level = success ? LOG_LEVELS.INFO : LOG_LEVELS.ERROR;
+        
+        const summaryMessage = finalMessage || (success 
+            ? `Process completed successfully`
+            : `Process completed with errors`);
+
+        this.log(level, summaryMessage, {
+            stats: {
+                duration: `${duration}ms`,
+                durationMs: duration,
+                operations: this.metrics.operations,
+                errors: this.metrics.errors,
+                warnings: this.metrics.warnings,
+                success
+            }
+        });
+
+        this.parent.endProcess(this.processId);
+        return duration;
+    }
+
+    // Convenience methods
+    trace(message, context = {}) { this.log(LOG_LEVELS.TRACE, message, context); }
+    debug(message, context = {}) { this.log(LOG_LEVELS.DEBUG, message, context); }
+    info(message, context = {}) { this.log(LOG_LEVELS.INFO, message, context); }
+    warn(message, context = {}) { this.log(LOG_LEVELS.WARN, message, context); }
+    error(message, context = {}) { this.log(LOG_LEVELS.ERROR, message, context); }
+    fatal(message, context = {}) { this.log(LOG_LEVELS.FATAL, message, context); }
+}
+
+/**
+ * Log Message Templates
+ */
+const LOG_TEMPLATES = {
+    // Schema Generation
+    SCHEMA_SUCCESS: (computationName) => 
+        `Schema generation successful for ${computationName}`,
+    SCHEMA_FAILURE: (computationName, reason) => 
+        `Schema generation failed for ${computationName}: ${reason}`,
+
+    // Computation Execution
+    COMPUTATION_START: (computationName, date) => 
+        `Starting computation ${computationName} for ${date}`,
+    COMPUTATION_SUCCESS: (computationName, date) => 
+        `Computation successful for ${computationName} on ${date}`,
+    COMPUTATION_FAILURE: (computationName, date, reason) => 
+        `Computation failed for ${computationName} on ${date}: ${reason}`,
+
+    // Storage
+    STORAGE_SUCCESS: (computationName, date, path, size) => 
+        `Results stored for ${computationName} on ${date} at ${path} (${size} bytes)`,
+    STORAGE_FAILURE: (computationName, date, path, reason) => 
+        `Failed to store results for ${computationName} on ${date} at ${path}: ${reason}`,
+
+    // Hash Validation
+    HASH_MISMATCH: (computationName, storedHash, currentHash) => 
+        `Hash mismatch for ${computationName}: stored=${storedHash}, current=${currentHash}`,
+    HASH_MATCH: (computationName) => 
+        `Hash match for ${computationName}, no code changes detected`,
+    HASH_CASCADE: (computationName, affectedComputations) => 
+        `Code change in ${computationName} will cascade to: ${affectedComputations.join(', ')}`,
+
+    // Manifest
+    MANIFEST_SUCCESS: (computationCount) => 
+        `Manifest built successfully with ${computationCount} computations`,
+    MANIFEST_TREE: (tree) => 
+        `Dependency tree:\n${tree}`,
+
+    // Date Analysis
+    DATE_ANALYSIS: (date, runnable, notRunnable) => 
+        `Date ${date}: ${runnable.length} runnable, ${notRunnable.length} blocked`,
+    DATE_MISSING_ROOTDATA: (date, computationName, missingData) => 
+        `${computationName} on ${date}: Cannot run due to missing root data: ${missingData.join(', ')}`,
+    DATE_MISSING_DEPENDENCY: (date, computationName, missingDep) => 
+        `${computationName} on ${date}: Will resolve after ${missingDep} completes`,
+    DATE_HASH_RERUN: (date, computationName, affectedDeps) => 
+        `${computationName} on ${date}: Hash mismatch, re-running (affects: ${affectedDeps.join(', ')})`,
+
+    // Availability
+    DATA_AVAILABLE: (date, types) => 
+        `Data available for ${date}: ${types.join(', ')}`,
+    DATA_MISSING: (date, types) => 
+        `Data missing for ${date}: ${types.join(', ')}`,
+
+    // Dispatch/Worker
+    DISPATCH_START: (pass, dateCount) => 
+        `Dispatching Pass ${pass} for ${dateCount} dates`,
+    DISPATCH_COMPLETE: (pass, dispatched) => 
+        `Dispatch complete: ${dispatched} tasks for Pass ${pass}`,
+    WORKER_TASK_START: (date, pass) => 
+        `Worker starting task: Date=${date}, Pass=${pass}`,
+    WORKER_TASK_COMPLETE: (date, pass, updateCount) => 
+        `Worker completed task: Date=${date}, Pass=${pass}, Updates=${updateCount}`,
+    WORKER_TASK_SKIP: (date, pass, reason) => 
+        `Worker skipped task: Date=${date}, Pass=${pass}, Reason=${reason}`
+};
+
+module.exports = {
+    StructuredLogger,
+    ProcessLogger,
+    LOG_LEVELS,
+    PROCESS_TYPES,
+    LOG_TEMPLATES,
+    generateProcessId
+};

package/functions/computation-system/persistence/ResultCommitter.js

@@ -1,22 +1,22 @@
 /**
- * @fileoverview Handles saving computation results with transparent auto-sharding.
+ * @fileoverview Handles saving computation results with observability.
  */
 const { commitBatchInChunks }     = require('./FirestoreUtils');
 const { updateComputationStatus } = require('./StatusRepository');
 const { batchStoreSchemas }       = require('../utils/schema_capture');
+const { generateProcessId, PROCESS_TYPES } = require('../logger/logger');
 
 async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
     const successUpdates = {};
     const schemas        = [];
+    const { logger }     = deps;
+    const pid            = generateProcessId(PROCESS_TYPES.STORAGE, passName, dStr);
 
     for (const name in stateObj) {
         const calc = stateObj[name];
         try {
             const result = await calc.getResult();
-            if (!result) { 
-                deps.logger.log('INFO', `${name} for ${dStr}: Skipped (Empty Result)`); 
-                continue; 
-            }
+            if (!result) continue;
 
             const mainDocRef = deps.db.collection(config.resultsCollection)
                 .doc(dStr)
@@ -25,8 +25,9 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 .collection(config.computationsSubcollection)
                 .doc(name);
 
-            const updates = await prepareAutoShardedWrites(result, mainDocRef, deps.logger);
+            const updates = await prepareAutoShardedWrites(result, mainDocRef, logger);
 
+            // Capture Schema
             if (calc.manifest.class.getSchema) {
                 const { class: _cls, ...safeMetadata } = calc.manifest;
                 schemas.push({ 
@@ -38,15 +39,18 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             }
 
             if (updates.length > 0) {
+                const totalSize = updates.reduce((acc, u) => acc + (u.data ? JSON.stringify(u.data).length : 0), 0);
+                const isSharded = updates.some(u => u.data._sharded === true);
+                
                 await commitBatchInChunks(config, deps, updates, `${name} Results`);
+                
+                // Structured Storage Log
+                logger.logStorage(pid, name, dStr, mainDocRef.path, totalSize, isSharded);
+                
                 successUpdates[name] = calc.manifest.hash || true;
-                const isSharded      = updates.some(u => u.data._sharded === true);
-                deps.logger.log('INFO', `${name} for ${dStr}: \u2714 Success (Written ${isSharded ? 'Sharded' : 'Standard'})`);
-            } else {
-                deps.logger.log('INFO', `${name} for ${dStr}: - Empty Data`);
             }
         } catch (e) {
-            deps.logger.log('ERROR', `${name} for ${dStr}: \u2716 FAILED Commit: ${e.message}`);
+            logger.log('ERROR', `Commit failed for ${name}`, { processId: pid, error: e.message });
         }
     }
 
@@ -54,12 +58,14 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 
     if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
         await updateComputationStatus(dStr, successUpdates, config, deps);
-        deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} successful computations.`);
     }
     return successUpdates;
 }
 
+// ... rest of file (calculateFirestoreBytes, prepareAutoShardedWrites) remains same ...
+// Just ensure prepareAutoShardedWrites uses the provided logger if it logs internal warnings.
 function calculateFirestoreBytes(value) {
+    // ... same as before
     if (value === null) return 1;
     if (value === undefined) return 0;
     if (typeof value === 'boolean') return 1;
@@ -73,7 +79,9 @@ function calculateFirestoreBytes(value) {
 }
 
 async function prepareAutoShardedWrites(result, docRef, logger) {
-    const SAFETY_THRESHOLD_BYTES = 1000 * 1024; // 1MB Limit
+    // ... same logic, just ensure existing logs inside here use the logger properly if needed
+    // Copied from previous logic, essentially checks size > 900KB and splits
+    const SAFETY_THRESHOLD_BYTES = 1000 * 1024; 
     const OVERHEAD_ALLOWANCE     = 20 * 1024;
     const CHUNK_LIMIT            = SAFETY_THRESHOLD_BYTES - OVERHEAD_ALLOWANCE;
     const totalSize              = calculateFirestoreBytes(result);
@@ -84,19 +92,16 @@ async function prepareAutoShardedWrites(result, docRef, logger) {
     let currentChunkSize         = 0;
     let shardIndex               = 0;
 
-    // If under limit, write directly
     if ((totalSize + docPathSize) < CHUNK_LIMIT) {  const data = { ...result, _completed: true, _sharded: false };  return [{ ref: docRef, data, options: { merge: true } }];  }
 
-    logger.log('INFO', `[AutoShard] Result size ~${Math.round(totalSize/1024)}KB exceeds limit. Sharding...`);
-
-    // iF over limit, shard the document
+    // Note: We don't log "Sharding..." here anymore because we log the structured event in commitResults
+    
     for (const [key, value] of Object.entries(result)) { 
         if (key.startsWith('_')) continue;
         const keySize   = Buffer.byteLength(key, 'utf8') + 1;         
         const valueSize = calculateFirestoreBytes(value);
         const itemSize  = keySize + valueSize;
 
-        // If adding this item exceeds the chunk limit, commit current chunk
         if (currentChunkSize + itemSize > CHUNK_LIMIT) { 
             writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: currentChunk, options: { merge: false } });
             shardIndex++;
@@ -107,10 +112,8 @@ async function prepareAutoShardedWrites(result, docRef, logger) {
         currentChunkSize += itemSize;
     }
 
-    // Write the final chunk
     if (Object.keys(currentChunk).length > 0) {  writes.push({ ref: shardCollection.doc(`shard_${shardIndex}`), data: currentChunk, options: { merge: false } });  }
     
-    // Finally, write the pointer document
     const pointerData = { _completed: true, _sharded: true, _shardCount: shardIndex + 1, _lastUpdated: new Date().toISOString() };
     writes.push({ ref: docRef, data: pointerData, options: { merge: false } });
     return writes;

package/functions/computation-system/topology/ManifestLoader.js

@@ -0,0 +1,43 @@
+/**
+ * @fileoverview Singleton Loader for the Manifest.
+ * Prevents expensive manifest rebuilding on every function invocation if not needed.
+ */
+const { build } = require('../context/ManifestBuilder');
+const { StructuredLogger, PROCESS_TYPES, generateProcessId } = require('../logger/logger');
+
+// Cache the manifest in global scope (warm start optimization)
+let cachedManifest = null;
+
+function getManifest(productLines = [], calculationsDir, dependencies = {}) {
+    if (cachedManifest) {
+        return cachedManifest;
+    }
+
+    const logger = dependencies.logger || new StructuredLogger();
+    const pid = generateProcessId(PROCESS_TYPES.MANIFEST, 'build', new Date().toISOString().slice(0,10));
+    
+    logger.log('INFO', 'Starting Manifest Build...', { processId: pid });
+    
+    const startTime = Date.now();
+    try {
+        cachedManifest = build(productLines, calculationsDir);
+        
+        // Log Topology Stats
+        const passCounts = {};
+        cachedManifest.forEach(c => { passCounts[c.pass] = (passCounts[c.pass] || 0) + 1; });
+        
+        logger.log('INFO', 'Manifest Build Success', { 
+            processId: pid, 
+            durationMs: Date.now() - startTime,
+            totalCalculations: cachedManifest.length,
+            topology: passCounts
+        });
+
+        return cachedManifest;
+    } catch (e) {
+        logger.log('FATAL', 'Manifest Build Failed', { processId: pid, error: e.message });
+        throw e;
+    }
+}
+
+module.exports = { getManifest };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.223",
+  "version": "1.0.225",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [