bulltrackers-module 1.0.174 → 1.0.176

package/functions/computation-system/controllers/computation_controller.js

@@ -171,7 +171,7 @@ class ComputationExecutor {
 
             const context = ContextBuilder.buildPerUserContext({
                 todayPortfolio, yesterdayPortfolio,
-                todayHistory, // Passed to context
+                todayHistory, 
                 userId, userType: actualUserType, dateStr, metadata, mappings, insights,
                 computedDependencies: computedDeps, 
                 previousComputedDependencies: prevDeps,

package/functions/computation-system/helpers/computation_manifest_builder.js

@@ -17,10 +17,8 @@ process.env.TMPDIR = '/tmp';
 process.env.TMP    = '/tmp';
 process.env.TEMP   = '/tmp';
 const os           = require('os');
-os.tmpdir = () => '/tmp';
-try { const temp = require('temp'); const path = require('path'); const fs = require('fs'); const tmp = '/tmp';
-if (!fs.existsSync(tmp)) fs.mkdirSync(tmp); temp.dir = tmp; temp.path = () => path.join(tmp, 'temp-' + Math.random().toString(36).slice(2)); } catch {}
-const Viz = require('graphviz'); 
+
+ const path = require('path');
 
 /* --------------------------------------------------
  * Pretty Console Helpers
@@ -42,7 +40,7 @@ const log = {
 const normalizeName = (name) => { if (typeof name !== 'string') return name; return name.trim().replace(/,$/, '').replace(/_/g, '-').toLowerCase(); };
 
 /**
- * Finds the closest string matches for a typo.
+ * Finds the closest string matches for a typo. // https://medium.com/@ethannam/understanding-the-levenshtein-distance-equation-for-beginners-c4285a5604f0
  */
 function suggestClosest(name, candidates, n = 3) {
     const levenshtein = (a = '', b = '') => {
@@ -125,7 +123,7 @@ function buildManifest(productLinesToRun = [], calculations) {
         const metadata     = Class.getMetadata();
         const dependencies = Class.getDependencies().map(normalizeName);
         // --- RULE 4: Check for isHistorical mismatch ---
-        if (metadata.isHistorical === true && !Class.toString().includes('yesterday')) { // UPDATED FOR MATH LAYER, THIS IS A LITTLE BRITTLE, BUT FOR NOW FINE...
+        if (metadata.isHistorical === true && !Class.toString().includes('yesterday')) { // UPDATED FOR MATH LAYER, TODO THIS IS A LITTLE BRITTLE, BUT FOR NOW FINE...
             log.warn(`Calculation "${normalizedName}" is marked 'isHistorical: true' but does not seem to reference 'yesterday' data.`); 
         }
         const manifestEntry = {
@@ -144,8 +142,6 @@ function buildManifest(productLinesToRun = [], calculations) {
         inDegree.set(normalizedName, dependencies.length);
         dependencies.forEach(dep => { if (!reverseAdjacency.has(dep)) reverseAdjacency.set(dep, []); reverseAdjacency.get(dep).push(normalizedName); }); }
     
-    // --- UPDATED ---
-    // This 'calculations' object is now the one passed in as an argument.
     if (!calculations || typeof calculations !== 'object') {
         log.fatal('Calculations object was not provided or is invalid.');
         throw new Error('Manifest build failed: Invalid calculations object.');
@@ -185,14 +181,18 @@ function buildManifest(productLinesToRun = [], calculations) {
     /* ---------------- 3. Filter for Product Lines ---------------- */
     log.divider('Filtering by Product Line');
     // 1. Find all "endpoint" calculations (the final signals) in the target product lines.
+
     const productLineEndpoints = [];
     for (const [name, entry] of manifestMap.entries()) { if (productLinesToRun.includes(entry.category)) { productLineEndpoints.push(name); } }
+
     // 2. Add 'core' calculations as they are always included.
     for (const [name, entry] of manifestMap.entries()) { if (entry.category === 'core') { productLineEndpoints.push(name); } }
+
     // 3. Trace all dependencies upwards from these endpoints.
     const requiredCalcs = getDependencySet(productLineEndpoints, adjacency);
     log.info(`Identified ${productLineEndpoints.length} endpoint/core calculations.`);
     log.info(`Traced dependencies: ${requiredCalcs.size} total calculations are required.`);
+
     // 4. Create the final, filtered maps for sorting.
     const filteredManifestMap      = new Map();
     const filteredInDegree         = new Map();
@@ -233,37 +233,7 @@ function buildManifest(productLinesToRun = [], calculations) {
     return sortedManifest;
 }
 
-/**
- * Generates an SVG dependency graph for a given manifest.
- * @param {object[]} manifest - The sorted manifest array.
- * @param {string} filename - The output filename (e.g., "full-dependency-tree.svg").
- */
-async function generateSvgGraph(manifest, filename = 'dependency-tree.svg') {
-    log.divider(`Generating SVG Graph: ${filename}`);
-    const viz = new Viz({ worker: false });
-    const categories = [...new Set(manifest.map(e => e.category))];
-    const colors = ['#e6194b', '#3cb44b', '#ffe119', '#4363d8', '#f58231', '#911eb4', '#46f0f0', '#f032e6', '#bcf60c', '#fabebe', '#008080', '#e6beff', '#9a6324', '#fffac8', '#800000', '#aaffc3', '#808000', '#ffd8b1', '#000075', '#808080'];
-    const colorMap = new Map(categories.map((cat, i) => [cat, colors[i % colors.length]]));
-    colorMap.set('core', '#a9a9a9');
-    let dot = `digraph Manifest {\n  rankdir=LR;\n  node [shape=box, style=filled, fontname="Helvetica"];\n  layout=dot;\n  overlap=false;\n  splines=true;\n`;
-    for (const category of categories) {
-        if (category === 'core') continue;
-        dot += `  subgraph "cluster_${category}" {\n    label="${category.toUpperCase()} Product Line";\n    style=filled;\n    color="#f0f0f0";\n`;
-        for (const entry of manifest.filter(e => e.category === category)) { dot += `    "${entry.name}" [label="${entry.name}\\n(Pass ${entry.pass})", fillcolor="${colorMap.get(category)}"];\n`; }
-        dot += `  }\n`; }
-    dot += `  subgraph "cluster_core" {\n    label="CORE";\n    style=filled;\n    color="#e0e0e0";\n`;
-    for (const entry of manifest.filter(e => e.category === 'core')) { dot += `    "${entry.name}" [label="${entry.name}\\n(Pass ${entry.pass})", fillcolor="${colorMap.get('core')}"];\n`; }
-    dot += `  }\n`;
-    for (const entry of manifest) { for (const dep of entry.dependencies || []) { dot += `  "${dep}" -> "${entry.name}";\n`; } }
-    dot += '}\n';
-    try {
-        const svg = await viz.renderString(dot, { format: 'svg' });
-        const out = path.join('/tmp', filename);
-        fs.writeFileSync(out, svg);
-        log.success(`Dependency tree generated at ${out}`);
 
-    } catch (e) { log.error(`SVG generation failed: ${e.message}`); }
-}
 
 
 /**
@@ -282,4 +252,4 @@ function build(productLinesToRun, calculations) {
 }
 
 
-module.exports = { build, generateSvgGraph }; 
+module.exports = { build}; 

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,9 +1,17 @@
 /**
- * FIXED: computation_pass_runner.js
- * V3: Fetches Previous Day's Results to enable State Persistence.
+ * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_pass_runner.js
  */
 
-const { groupByPass, checkRootDataAvailability, fetchExistingResults, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
+const { 
+    groupByPass, 
+    checkRootDataAvailability, 
+    fetchExistingResults, 
+    fetchGlobalComputationStatus,  // <--- New Import
+    updateGlobalComputationStatus, // <--- New Import
+    runStandardComputationPass, 
+    runMetaComputationPass,
+    checkRootDependencies
+} = require('./orchestration_helpers.js');
 const { getExpectedDateStrings } = require('../utils/utils.js');
 const PARALLEL_BATCH_SIZE = 7;
 
@@ -13,93 +21,121 @@ async function runComputationPass(config, dependencies, computationManifest) {
     if (!passToRun) 
         return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
     
-    logger.log('INFO', `🚀 Starting PASS ${passToRun}...`);
+    logger.log('INFO', `🚀 Starting PASS ${passToRun} with Global Status Check...`);
 
-    const earliestDates            = { portfolio: new Date('2025-09-25T00:00:00Z'), history: new Date('2025-11-05T00:00:00Z'), social: new Date('2025-10-30T00:00:00Z'), insights: new Date('2025-08-26T00:00:00Z') };
+    // Hardcoded earliest dates
+    const earliestDates = { portfolio: new Date('2025-09-25T00:00:00Z'), history: new Date('2025-11-05T00:00:00Z'), social: new Date('2025-10-30T00:00:00Z'), insights: new Date('2025-08-26T00:00:00Z') };
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
 
-    const passes          = groupByPass(computationManifest);
+    const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
     if (!calcsInThisPass.length) 
         return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
     
-    const calcEarliestDates = new Map();
+    const passEarliestDate = earliestDates.absoluteEarliest; 
+    const endDateUTC = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
+    const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
-    for (const calc of calcsInThisPass) { 
-        const deps = calc.rootDataDependencies || []; 
+    const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
+    const metaCalcs     = calcsInThisPass.filter(c => c.type === 'meta');
 
-        if (!deps.length) 
-            { calcEarliestDates.set(calc.name, earliestDates.absoluteEarliest); continue; }
+    // 1. Fetch Global Status ONCE (Memory Cache)
+    // Returns { "2023-10-27": { calcA: true, calcB: false }, ... }
+    const globalStatusData = await fetchGlobalComputationStatus(config, dependencies);
 
-        const latestDep = new Date(Math.max(...deps.map(d => earliestDates[d]?.getTime() || 0)));
-        const calcDate = calc.isHistorical 
-            ? new Date(latestDep.getTime() + 86400000) 
-            : latestDep; 
+    // Helper: Check status using in-memory data
+    const shouldRun = (calc, dateStr) => {
+        const dailyStatus = globalStatusData[dateStr] || {};
         
-            calcEarliestDates.set(calc.name, calcDate);
-    }
+        // 1. If explicitly TRUE, ignore.
+        if (dailyStatus[calc.name] === true) return false;
+
+        // 2. Check dependencies (using same in-memory status)
+        if (calc.dependencies && calc.dependencies.length > 0) {
+            const depsMet = calc.dependencies.every(depName => dailyStatus[depName] === true);
+            if (!depsMet) return false; 
+        }
+        return true;
+    };
 
-    const passEarliestDate = new Date(Math.min(...Array.from(calcEarliestDates.values()).map(d => d.getTime())));
-    const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
-    const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
-    const standardCalcs    = calcsInThisPass.filter(c => c.type === 'standard');
-    const metaCalcs        = calcsInThisPass.filter(c => c.type === 'meta');
+    // Process a single date and RETURN updates (do not write)
+    const processDate = async (dateStr) => {
+        const dateToProcess = new Date(dateStr + 'T00:00:00Z');
 
-    const checkDeps = (calc, rootData, existingResults, dateToProcess) => { 
-        if (existingResults[calc.name]) 
-            return false;
+        // Filter using in-memory status
+        const standardToRun = standardCalcs.filter(c => shouldRun(c, dateStr));
+        const metaToRun     = metaCalcs.filter(c => shouldRun(c, dateStr));
+        
+        if (!standardToRun.length && !metaToRun.length) return null; // No work
 
-        const earliest = calcEarliestDates.get(calc.name);
+        const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
+        if (!rootData) return null;
 
-        if (earliest && dateToProcess < earliest) 
-            return false;
+        const finalStandardToRun = standardToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
+        const finalMetaToRun     = metaToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
 
-        const missingRoot = (calc.rootDataDependencies || []).filter(dep => !rootData.status[`has${dep[0].toUpperCase() + dep.slice(1)}`]);
+        if (!finalStandardToRun.length && !finalMetaToRun.length) return null;
 
-        if (missingRoot.length) 
-            return false;
+        logger.log('INFO', `[PassRunner] Running ${dateStr}: ${finalStandardToRun.length} std, ${finalMetaToRun.length} meta`);
 
-        if (calc.type === 'meta') 
-            { const missingComputed = (calc.dependencies || []).filter(d => !existingResults[d]);
-                 if (missingComputed.length)
-                     return false; } 
-        return true; 
-    };
+        const dateUpdates = {}; // { calcName: true/false }
 
-    const processDate = async (dateStr) => {
-        const dateToProcess = new Date(dateStr + 'T00:00:00Z');
         try {
-            const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
-            if (!rootData) return logger.log('WARN', `[PassRunner] Skipping ${dateStr}: No root data.`);
-            
-            // 1. Fetch TODAY'S results (Check what is already done)
-            const existingResults = await fetchExistingResults(dateStr, calcsInThisPass, computationManifest, config, dependencies, false);
+            const calcsRunning = [...finalStandardToRun, ...finalMetaToRun];
+            const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
             
-            // 2. Fetch YESTERDAY'S results (For State Persistence) We calculate T-1 date string
-            const prevDate    = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+            const prevDate = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
             const prevDateStr = prevDate.toISOString().slice(0, 10);
+            const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
 
-            // We pass 'true' to includeSelf, because we need the calc's OWN history (e.g. mimetic-latency needs previous mimetic-latency)
-            const previousResults = await fetchExistingResults(prevDateStr, calcsInThisPass, computationManifest, config, dependencies, true);
-
-            const standardToRun = standardCalcs.filter(c => checkDeps(c, rootData, existingResults, dateToProcess));
-            const metaToRun     = metaCalcs.filter(c => checkDeps(c, rootData, existingResults, dateToProcess));
-            
-            if (!standardToRun.length && !metaToRun.length) return logger.log('INFO', `[PassRunner] All calcs complete for ${dateStr}. Skipping.`);
+            if (finalStandardToRun.length) {
+                const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, true); // skipStatusWrite=true
+                Object.assign(dateUpdates, updates);
+            }
             
-            logger.log('INFO', `[PassRunner] Running ${dateStr}: ${standardToRun.length} standard, ${metaToRun.length} meta`);
+            if (finalMetaToRun.length) {
+                const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, true); // skipStatusWrite=true
+                Object.assign(dateUpdates, updates);
+            }
             
-            // Pass 'previousResults' to the runners
-            if (standardToRun.length) await runStandardComputationPass(dateToProcess, standardToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData, existingResults, previousResults);
-            if (metaToRun.length)     await runMetaComputationPass(dateToProcess, metaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData);
-            
-        } catch (err) { logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack }); }
+        } catch (err) { 
+            logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
+            // Mark failures
+            [...finalStandardToRun, ...finalMetaToRun].forEach(c => dateUpdates[c.name] = false);
+        }
+
+        // Return the updates for this date
+        return { date: dateStr, updates: dateUpdates };
     };
 
+    // Batch process dates
     for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) { 
         const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
-        await Promise.all(batch.map(processDate)); 
+        
+        // Run batch in parallel
+        const results = await Promise.all(batch.map(processDate));
+        
+        // Aggregate updates from the batch
+        const batchUpdates = {};
+        let hasUpdates = false;
+        
+        results.forEach(res => {
+            if (res && res.updates && Object.keys(res.updates).length > 0) {
+                batchUpdates[res.date] = res.updates;
+                hasUpdates = true;
+                
+                // Also update our local in-memory copy so subsequent logic in this run sees it (though passes usually rely on prev days)
+                if (!globalStatusData[res.date]) globalStatusData[res.date] = {};
+                Object.assign(globalStatusData[res.date], res.updates);
+            }
+        });
+
+        // Write status ONCE per batch
+        if (hasUpdates) {
+            await updateGlobalComputationStatus(batchUpdates, config, dependencies);
+            logger.log('INFO', `[PassRunner] Batched status update for ${Object.keys(batchUpdates).length} dates.`);
+        }
     }
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }