bulltrackers-module 1.0.175 → 1.0.177

package/functions/computation-system/controllers/computation_controller.js

@@ -147,37 +147,25 @@ class ComputationExecutor {
      */
     async executePerUser(calcInstance, metadata, dateStr, portfolioData, yesterdayPortfolioData, historyData, computedDeps, prevDeps) {
         const { logger } = this.deps;
-        
-        // Fix for the 'all' userType discrepancy:
-        const targetUserType = metadata.userType; // 'all', 'normal', or 'speculator'
-
+        const targetUserType = metadata.userType; 
         const mappings = await this.loader.loadMappings();
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
-
         for (const [userId, todayPortfolio] of Object.entries(portfolioData)) {
-            // 1. Get Yesterday's Portfolio (if available)
             const yesterdayPortfolio = yesterdayPortfolioData ? yesterdayPortfolioData[userId] : null;
-            
-            // 2. Get Today's Trading History (if available)
             const todayHistory = historyData ? historyData[userId] : null;
-
             const actualUserType = todayPortfolio.PublicPositions ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
-            
-            // Filtering Logic
             if (targetUserType !== 'all') {
                  const mappedTarget = (targetUserType === 'speculator') ? SCHEMAS.USER_TYPES.SPECULATOR : SCHEMAS.USER_TYPES.NORMAL;
                  if (mappedTarget !== actualUserType) continue;
             }
-
             const context = ContextBuilder.buildPerUserContext({
                 todayPortfolio, yesterdayPortfolio,
-                todayHistory, // Passed to context
+                todayHistory, 
                 userId, userType: actualUserType, dateStr, metadata, mappings, insights,
                 computedDependencies: computedDeps, 
                 previousComputedDependencies: prevDeps,
                 config: this.config, deps: this.deps
             });
-
             try { await calcInstance.process(context); } 
             catch (e) { logger.log('WARN', `Calc ${metadata.name} failed for user ${userId}: ${e.message}`); }
         }
@@ -187,14 +175,12 @@ class ComputationExecutor {
         const mappings = await this.loader.loadMappings();
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await this.loader.loadInsights(dateStr) } : null;
         const social = metadata.rootDataDependencies?.includes('social') ? { today: await this.loader.loadSocial(dateStr) } : null;
-
         const context = ContextBuilder.buildMetaContext({
             dateStr, metadata, mappings, insights, socialData: social,
             computedDependencies: computedDeps,
             previousComputedDependencies: prevDeps,
             config: this.config, deps: this.deps
         });
-
         return await calcInstance.process(context);
     }
 }

package/functions/computation-system/helpers/computation_manifest_builder.js

@@ -11,17 +11,6 @@
  * It has removed all logic for deprecated folder structures (e.g., /historical/).
  */
 
-const fs                 = require('fs');
-//Hacky solution to force tmp writes TODO : This Tmp write is really dodgy, not ideal but works, consider less hacky solutions to writing to filesystem
-process.env.TMPDIR = '/tmp';
-process.env.TMP    = '/tmp';
-process.env.TEMP   = '/tmp';
-const os           = require('os');
-os.tmpdir = () => '/tmp';
-try { const temp = require('temp'); const path = require('path'); const fs = require('fs'); const tmp = '/tmp';
-if (!fs.existsSync(tmp)) fs.mkdirSync(tmp); temp.dir = tmp; temp.path = () => path.join(tmp, 'temp-' + Math.random().toString(36).slice(2)); } catch {}
-const Viz = require('graphviz'); 
-
 /* --------------------------------------------------
  * Pretty Console Helpers
  * -------------------------------------------------- */
@@ -42,7 +31,7 @@ const log = {
 const normalizeName = (name) => { if (typeof name !== 'string') return name; return name.trim().replace(/,$/, '').replace(/_/g, '-').toLowerCase(); };
 
 /**
- * Finds the closest string matches for a typo.
+ * Finds the closest string matches for a typo. // https://medium.com/@ethannam/understanding-the-levenshtein-distance-equation-for-beginners-c4285a5604f0
  */
 function suggestClosest(name, candidates, n = 3) {
     const levenshtein = (a = '', b = '') => {
@@ -56,7 +45,7 @@ function suggestClosest(name, candidates, n = 3) {
         return dp[m][n];
     };
     const scores = candidates.map(c => [c, levenshtein(name, c)]);
-    scores.sort((a, b) => a[1] - b[1]);
+    scores.sort((a, b)              => a[1] - b[1]);
     return scores.slice(0, n).map(s => s[0]);
 }
 
@@ -125,7 +114,7 @@ function buildManifest(productLinesToRun = [], calculations) {
         const metadata     = Class.getMetadata();
         const dependencies = Class.getDependencies().map(normalizeName);
         // --- RULE 4: Check for isHistorical mismatch ---
-        if (metadata.isHistorical === true && !Class.toString().includes('yesterday')) { // UPDATED FOR MATH LAYER, THIS IS A LITTLE BRITTLE, BUT FOR NOW FINE...
+        if (metadata.isHistorical === true && !Class.toString().includes('yesterday')) { // UPDATED FOR MATH LAYER, TODO THIS IS A LITTLE BRITTLE, BUT FOR NOW FINE...
             log.warn(`Calculation "${normalizedName}" is marked 'isHistorical: true' but does not seem to reference 'yesterday' data.`); 
         }
         const manifestEntry = {
@@ -144,8 +133,6 @@ function buildManifest(productLinesToRun = [], calculations) {
         inDegree.set(normalizedName, dependencies.length);
         dependencies.forEach(dep => { if (!reverseAdjacency.has(dep)) reverseAdjacency.set(dep, []); reverseAdjacency.get(dep).push(normalizedName); }); }
     
-    // --- UPDATED ---
-    // This 'calculations' object is now the one passed in as an argument.
     if (!calculations || typeof calculations !== 'object') {
         log.fatal('Calculations object was not provided or is invalid.');
         throw new Error('Manifest build failed: Invalid calculations object.');
@@ -185,14 +172,18 @@ function buildManifest(productLinesToRun = [], calculations) {
     /* ---------------- 3. Filter for Product Lines ---------------- */
     log.divider('Filtering by Product Line');
     // 1. Find all "endpoint" calculations (the final signals) in the target product lines.
+
     const productLineEndpoints = [];
     for (const [name, entry] of manifestMap.entries()) { if (productLinesToRun.includes(entry.category)) { productLineEndpoints.push(name); } }
+
     // 2. Add 'core' calculations as they are always included.
     for (const [name, entry] of manifestMap.entries()) { if (entry.category === 'core') { productLineEndpoints.push(name); } }
+
     // 3. Trace all dependencies upwards from these endpoints.
     const requiredCalcs = getDependencySet(productLineEndpoints, adjacency);
     log.info(`Identified ${productLineEndpoints.length} endpoint/core calculations.`);
     log.info(`Traced dependencies: ${requiredCalcs.size} total calculations are required.`);
+
     // 4. Create the final, filtered maps for sorting.
     const filteredManifestMap      = new Map();
     const filteredInDegree         = new Map();
@@ -206,17 +197,20 @@ function buildManifest(productLinesToRun = [], calculations) {
     const sortedManifest = [];
     const queue          = [];
     let maxPass          = 0;
+
     for (const [name, degree] of filteredInDegree) { if (degree === 0) { queue.push(name); filteredManifestMap.get(name).pass = 1; maxPass = 1; } }
     queue.sort();
     while (queue.length) {
         const currentName = queue.shift();
         const currentEntry = filteredManifestMap.get(currentName);
         sortedManifest.push(currentEntry);
+        
         for (const neighborName of (filteredReverseAdjacency.get(currentName) || [])) { const newDegree = filteredInDegree.get(neighborName) - 1; filteredInDegree.set(neighborName, newDegree);
             const neighborEntry = filteredManifestMap.get(neighborName);
             if (neighborEntry.pass <= currentEntry.pass) { neighborEntry.pass = currentEntry.pass + 1; if (neighborEntry.pass > maxPass) maxPass = neighborEntry.pass; }
             if (newDegree === 0) { queue.push(neighborName); } }
         queue.sort(); }
+
     if (sortedManifest.length !== filteredManifestMap.size) {
         log.divider('Circular Dependency Detected');
         const cycles = findCycles(filteredManifestMap, adjacency);
@@ -233,37 +227,7 @@ function buildManifest(productLinesToRun = [], calculations) {
     return sortedManifest;
 }
 
-/**
- * Generates an SVG dependency graph for a given manifest.
- * @param {object[]} manifest - The sorted manifest array.
- * @param {string} filename - The output filename (e.g., "full-dependency-tree.svg").
- */
-async function generateSvgGraph(manifest, filename = 'dependency-tree.svg') {
-    log.divider(`Generating SVG Graph: ${filename}`);
-    const viz = new Viz({ worker: false });
-    const categories = [...new Set(manifest.map(e => e.category))];
-    const colors = ['#e6194b', '#3cb44b', '#ffe119', '#4363d8', '#f58231', '#911eb4', '#46f0f0', '#f032e6', '#bcf60c', '#fabebe', '#008080', '#e6beff', '#9a6324', '#fffac8', '#800000', '#aaffc3', '#808000', '#ffd8b1', '#000075', '#808080'];
-    const colorMap = new Map(categories.map((cat, i) => [cat, colors[i % colors.length]]));
-    colorMap.set('core', '#a9a9a9');
-    let dot = `digraph Manifest {\n  rankdir=LR;\n  node [shape=box, style=filled, fontname="Helvetica"];\n  layout=dot;\n  overlap=false;\n  splines=true;\n`;
-    for (const category of categories) {
-        if (category === 'core') continue;
-        dot += `  subgraph "cluster_${category}" {\n    label="${category.toUpperCase()} Product Line";\n    style=filled;\n    color="#f0f0f0";\n`;
-        for (const entry of manifest.filter(e => e.category === category)) { dot += `    "${entry.name}" [label="${entry.name}\\n(Pass ${entry.pass})", fillcolor="${colorMap.get(category)}"];\n`; }
-        dot += `  }\n`; }
-    dot += `  subgraph "cluster_core" {\n    label="CORE";\n    style=filled;\n    color="#e0e0e0";\n`;
-    for (const entry of manifest.filter(e => e.category === 'core')) { dot += `    "${entry.name}" [label="${entry.name}\\n(Pass ${entry.pass})", fillcolor="${colorMap.get('core')}"];\n`; }
-    dot += `  }\n`;
-    for (const entry of manifest) { for (const dep of entry.dependencies || []) { dot += `  "${dep}" -> "${entry.name}";\n`; } }
-    dot += '}\n';
-    try {
-        const svg = await viz.renderString(dot, { format: 'svg' });
-        const out = path.join('/tmp', filename);
-        fs.writeFileSync(out, svg);
-        log.success(`Dependency tree generated at ${out}`);
 
-    } catch (e) { log.error(`SVG generation failed: ${e.message}`); }
-}
 
 
 /**
@@ -282,4 +246,4 @@ function build(productLinesToRun, calculations) {
 }
 
 
-module.exports = { build, generateSvgGraph }; 
+module.exports = { build}; 

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,17 +1,13 @@
 /**
- * FIXED: computation_pass_runner.js
- * V3.4: Optimized Orchestration using Status Document (Single Source of Truth).
- * - Reads one 'computation_status' doc per day to decide what to run.
- * - Reduces Firestore reads significantly.
- * - explicitly marks failures as false in the status doc.
+ * FILENAME: bulltrackers-module/functions/computation-system/helpers/computation_pass_runner.js
  */
 
 const { 
     groupByPass, 
     checkRootDataAvailability, 
     fetchExistingResults, 
-    fetchComputationStatus, 
-    updateComputationStatus,
+    fetchGlobalComputationStatus,  // <--- New Import
+    updateGlobalComputationStatus, // <--- New Import
     runStandardComputationPass, 
     runMetaComputationPass,
     checkRootDependencies
@@ -25,113 +21,117 @@ async function runComputationPass(config, dependencies, computationManifest) {
     if (!passToRun) 
         return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
     
-    logger.log('INFO', `🚀 Starting PASS ${passToRun} with Optimized Status Check...`);
+    logger.log('INFO', `🚀 Starting PASS ${passToRun} with Global Status Check...`);
 
-    // Hardcoded earliest dates for global availability checks
-    const earliestDates            = { portfolio: new Date('2025-09-25T00:00:00Z'), history: new Date('2025-11-05T00:00:00Z'), social: new Date('2025-10-30T00:00:00Z'), insights: new Date('2025-08-26T00:00:00Z') };
+    // Hardcoded earliest dates
+    const earliestDates = { portfolio: new Date('2025-09-25T00:00:00Z'), history: new Date('2025-11-05T00:00:00Z'), social: new Date('2025-10-30T00:00:00Z'), insights: new Date('2025-08-26T00:00:00Z') };
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
 
-    const passes          = groupByPass(computationManifest);
+    const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
 
     if (!calcsInThisPass.length) 
         return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
     
-    // Determine date range to process
-    // We no longer need complex per-calc date derivation since the status doc handles "done-ness".
-    // We just iterate from absolute earliest to yesterday.
     const passEarliestDate = earliestDates.absoluteEarliest; 
-    const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
+    const endDateUTC = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
 
-    const standardCalcs    = calcsInThisPass.filter(c => c.type === 'standard');
-    const metaCalcs        = calcsInThisPass.filter(c => c.type === 'meta');
+    const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
+    const metaCalcs     = calcsInThisPass.filter(c => c.type === 'meta');
 
-    // Helper: Decide if a calculation should run based on the status document
-    const shouldRun = (calc, statusData) => {
-        // 1. If explicitly TRUE, it ran fine. Ignore.
-        if (statusData[calc.name] === true) return false;
+    // 1. Fetch Global Status ONCE (Memory Cache)
+    // Returns { "2023-10-27": { calcA: true, calcB: false }, ... }
+    const globalStatusData = await fetchGlobalComputationStatus(config, dependencies);
 
-        // 2. If missing or FALSE, it needs to run.
-        //    But first, check if its dependencies are ready (if any).
-        //    Note: Dependencies from previous passes must be TRUE.
+    // Helper: Check status using in-memory data
+    const shouldRun = (calc, dateStr) => {
+        const dailyStatus = globalStatusData[dateStr] || {};
+        
+        // 1. If explicitly TRUE, ignore.
+        if (dailyStatus[calc.name] === true) return false;
+
+        // 2. Check dependencies (using same in-memory status)
         if (calc.dependencies && calc.dependencies.length > 0) {
-            const depsMet = calc.dependencies.every(depName => statusData[depName] === true);
-            if (!depsMet) return false; // Dependency not ready yet
+            const depsMet = calc.dependencies.every(depName => dailyStatus[depName] === true);
+            if (!depsMet) return false; 
         }
-
         return true;
     };
 
+    // Process a single date and RETURN updates (do not write)
     const processDate = async (dateStr) => {
-        // 1. Fetch the Single Status Document (Cheap Read)
-        const statusData = await fetchComputationStatus(dateStr, config, dependencies);
         const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-
-        // 2. Filter calculations based on Status Doc
-        const standardToRun = standardCalcs.filter(c => shouldRun(c, statusData));
-        const metaToRun     = metaCalcs.filter(c => shouldRun(c, statusData));
+        const standardToRun = standardCalcs.filter(c => shouldRun(c, dateStr));
+        const metaToRun     = metaCalcs.filter(c => shouldRun(c, dateStr));
         
-        // Optimization: If nothing needs to run, stop here. 
-        // No checking root data, no loading results.
-        if (!standardToRun.length && !metaToRun.length) {
-             return; // logger.log('INFO', `[PassRunner] ${dateStr} complete or waiting for deps.`);
-        }
+        if (!standardToRun.length && !metaToRun.length) return null; // No work
 
-        // 3. Check Root Data Availability (Only done if we have work to do)
-        // We filter standardToRun further based on root data requirements
-        // (e.g. if calc needs history but history isn't ready for this date)
         const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
-        
-        if (!rootData) {
-            // If root data is completely missing for the day, we can't run anything.
-            // We do NOT mark as false, because it's not a failure, just data unavailability.
-            // We just skip.
-            return;
-        }
+        if (!rootData) return null;
 
-        // Further filter based on specific root data needs (e.g. checks "hasHistory")
         const finalStandardToRun = standardToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
         const finalMetaToRun     = metaToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
 
-        if (!finalStandardToRun.length && !finalMetaToRun.length) return;
+        if (!finalStandardToRun.length && !finalMetaToRun.length) return null;
 
-        logger.log('INFO', `[PassRunner] Running ${dateStr}: ${finalStandardToRun.length} standard, ${finalMetaToRun.length} meta`);
+        logger.log('INFO', `[PassRunner] Running ${dateStr}: ${finalStandardToRun.length} std, ${finalMetaToRun.length} meta`);
+
+        const dateUpdates = {}; // { calcName: true/false }
 
         try {
-            // 4. Fetch Data for Execution (Dependencies)
-            // We only fetch data required by the calcs that are actually running.
             const calcsRunning = [...finalStandardToRun, ...finalMetaToRun];
             const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
-            
-            // Fetch Previous Day's Results (for State Persistence)
-            const prevDate    = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
-            const prevDateStr = prevDate.toISOString().slice(0, 10);
+            const prevDate        = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+            const prevDateStr     = prevDate.toISOString().slice(0, 10);
             const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
-
-            // 5. Execute
-            if (finalStandardToRun.length) 
-                await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData, existingResults, previousResults);
-            
-            if (finalMetaToRun.length)     
-                await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData);
-            
+            if (finalStandardToRun.length) {
+                const updates = await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Std)`, config, dependencies, rootData, existingResults, previousResults, true); // skipStatusWrite=true
+                Object.assign(dateUpdates, updates);
+            }
+            if (finalMetaToRun.length) {
+                const updates = await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData, true); // skipStatusWrite=true
+                Object.assign(dateUpdates, updates);
+            }
         } catch (err) { 
             logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
-            
-            // 6. Explicitly Mark Failures as FALSE
-            const failedUpdates = {};
-            [...finalStandardToRun, ...finalMetaToRun].forEach(c => failedUpdates[c.name] = false);
-            await updateComputationStatus(dateStr, failedUpdates, config, dependencies);
+            // Mark failures
+            [...finalStandardToRun, ...finalMetaToRun].forEach(c => dateUpdates[c.name] = false);
         }
+
+        // Return the updates for this date
+        return { date: dateStr, updates: dateUpdates };
     };
 
     // Batch process dates
     for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) { 
         const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
-        await Promise.all(batch.map(processDate)); 
+        
+        // Run batch in parallel
+        const results = await Promise.all(batch.map(processDate));
+        
+        // Aggregate updates from the batch
+        const batchUpdates = {};
+        let hasUpdates = false;
+        
+        results.forEach(res => {
+            if (res && res.updates && Object.keys(res.updates).length > 0) {
+                batchUpdates[res.date] = res.updates;
+                hasUpdates = true;
+                
+                // Also update our local in-memory copy so subsequent logic in this run sees it (though passes usually rely on prev days)
+                if (!globalStatusData[res.date]) globalStatusData[res.date] = {};
+                Object.assign(globalStatusData[res.date], res.updates);
+            }
+        });
+
+        // Write status ONCE per batch
+        if (hasUpdates) {
+            await updateGlobalComputationStatus(batchUpdates, config, dependencies);
+            logger.log('INFO', `[PassRunner] Batched status update for ${Object.keys(batchUpdates).length} dates.`);
+        }
     }
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 
-module.exports = { runComputationPass };
+module.exports = { runComputationPass };