bulltrackers-module 1.0.159 → 1.0.161

package/functions/computation-system/helpers/computation_manifest_builder.js

@@ -21,12 +21,12 @@ const { Module, render } = require('viz.js/full.render.js');
  * Pretty Console Helpers
  * -------------------------------------------------- */
 const log = {
-    info: (msg) => console.log('ℹ︎ ' + msg),
-    step: (msg) => console.log('› ' + msg),
-    warn: (msg) => console.warn('⚠︎ ' + msg),
-    success: (msg) => console.log('✔︎ ' + msg),
-    error: (msg) => console.error('✖ ' + msg),
-    fatal: (msg) => { console.error('✖ FATAL ✖ ' + msg); console.error('✖ FATAL ✖ Manifest build FAILED.'); },
+    info:    (msg)   =>  console.log('ℹ︎ ' + msg),
+    step:    (msg)   =>  console.log('› ' + msg),
+    warn:    (msg)   =>  console.warn('⚠︎ ' + msg),
+    success: (msg)   =>  console.log('✔︎ ' + msg),
+    error:   (msg)   =>  console.error('✖ ' + msg),
+    fatal:   (msg)   => { console.error('✖ FATAL ✖ ' + msg); console.error('✖ FATAL ✖ Manifest build FAILED.'); },
     divider: (label) => { const line = ''.padEnd(60, '─'); console.log(`\n${line}\n${label}\n${line}\n`); },
 };
 
@@ -237,7 +237,11 @@ async function generateSvgGraph(manifest, filename = 'dependency-tree.svg') {
     dot += '}\n';
     try {
         const svg = await viz.renderString(dot, { format: 'svg' });
-        const out = path.join(__dirname, '..', '..', '..', '..', 'config', filename);
+        
+        // --- MODIFIED PATH ---
+        // Writes to the root of the 'bulltrackers-module' package directory, 
+        // which is a safer, more predictable location.
+        const out = path.join(__dirname, '..', '..', '..', filename); 
         fs.writeFileSync(out, svg);
         log.success(`Dependency tree generated at ${out}`);
     } catch (e) { log.error(`SVG generation failed: ${e.message}`); }
@@ -248,17 +252,43 @@ async function generateSvgGraph(manifest, filename = 'dependency-tree.svg') {
  * Main entry point for building and exporting the manifest.
  * @param {string[]} productLinesToRun - Array of product line categories (folder names) to build for.
  */
-async function build(productLinesToRun) {
+function build(productLinesToRun) {
     try {
+        // This function MUST NOT fail or generate SVGs.
+        // It has one job: build and return the manifest array.
         const manifest = buildManifest(productLinesToRun);
-        await generateSvgGraph(manifest, `dependency-tree-filtered.svg`);
-        const allProductLines = Object.keys(calculations).filter(c => c !== 'legacy');
-        const fullManifest = buildManifest(allProductLines);
-        await generateSvgGraph(fullManifest, 'dependency-tree-full.svg');
-        return manifest;
-    } catch (error) { log.error(error.message); return null; }
+        return manifest; // Returns the array
+    } catch (error) { 
+        log.error(error.message); 
+        // If buildManifest fails, it will throw.
+        // We return null so the caller can handle the failure.
+        return null; 
+    }
 }
 
 module.exports = { build };
 
-if (require.main === module) { (async () => { log.info('Running manifest builder in local debug mode...'); const productLines = Object.keys(calculations).filter(c => c !== 'legacy');await build(productLines); })(); }
+// --- MODIFIED ---
+// This block is ONLY executed when you run `node computation_manifest_builder.js`
+// This is now the ONLY place SVG generation happens.
+if (require.main === module) { 
+    (async () => { // This part remains async for the SVG generation
+        log.info('Running manifest builder in local debug mode...');
+        const allProductLines = Object.keys(calculations).filter(c => c !== 'legacy');
+        
+        try {
+            // 1. Build the full manifest
+            const fullManifest = buildManifest(allProductLines); // Sync call
+            
+            if (fullManifest) {
+                // 2. Generate the full SVG graph
+                await generateSvgGraph(fullManifest, 'dependency-tree-full.svg');
+                log.info('Local debug build and graph generation complete.');
+            } else {
+                log.fatal('Local debug build FAILED.');
+            }
+        } catch (error) {
+            log.error(error.message);
+        }
+    })(); 
+}

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -99,7 +99,13 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
     const fetched = {};
     if (docRefs.length) {
         (await db.getAll(...docRefs)).forEach((doc, i) => {
-            fetched[depNames[i]] = doc.exists ? doc.data() : null;
+            // --- FIX [PROBLEM 8]: Add completion marker check ---
+            const data = doc.exists ? doc.data() : null;
+            if (data && data._completed === true) {
+                fetched[depNames[i]] = data;
+            } else {
+                fetched[depNames[i]] = null; // Treat as not existing if incomplete
+            }
         });
     }
     
@@ -139,19 +145,18 @@ function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingRe
         return earliestRunDate;
     };
     const filterCalc = (calc) => {
-        if (existingResults[calc.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists.`); skipped.add(calc.name); return false;}
+        // --- FIX [PROBLEM 8]: The check for existingResults is now correct ---
+        if (existingResults[calc.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists (and is complete).`); skipped.add(calc.name); return false;}
+        
         const earliestRunDate = getTrueEarliestRunDate(calc);
         if (dateToProcess < earliestRunDate) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`); skipped.add(calc.name); return false; }
         const { canRun, missing: missingRoot } = checkRootDependencies(calc, rootDataStatus);
         if (!canRun) {logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);skipped.add(calc.name); return false;}
         
-        // --- FIX: This check is now robust ---
-        // 'existingResults' now contains all dependencies, so this check
-        // will correctly find 'pnl_distribution_per_stock' and *not* skip.
         if (calc.type === 'meta') { 
             const missingDeps = (calc.dependencies || [])
                 .map(normalizeName)
-                .filter(d => !existingResults[d]); 
+                .filter(d => !existingResults[d]); // This check is now robust
             if (missingDeps.length > 0) { 
                 logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`); 
                 skipped.add(calc.name); 
@@ -177,15 +182,20 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
     const needsYesterdayInsights  = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
     const needsYesterdaySocial    = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
     const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
+    
+    // --- FIX: Add T-1 COMPUTED dependency loading ---
+    const needsYesterdayDependencies = calcs.some(c => c.isHistorical && c.dependencies && c.dependencies.length > 0);
+    
     const prev = new Date(date);
     prev.setUTCDate(prev.getUTCDate() - 1);
     const prevStr = prev.toISOString().slice(0, 10);
+    
     if(needsYesterdayInsights) {
-        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
+        tasks.push((async()=>{ 
         logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
         updated.yesterdayInsights=await loadDailyInsights(config,deps,prevStr); })());}
     if(needsYesterdaySocial) {
-        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
+        tasks.push((async()=>{ 
         logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
         updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); })());}
     
@@ -195,6 +205,17 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
             updated.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr); 
         })());
     }
+
+    // --- FIX: Load T-1 COMPUTED dependencies ---
+    if(needsYesterdayDependencies) {
+        tasks.push((async()=>{ 
+            logger.log('INFO', `[PassRunner] Loading YESTERDAY computed dependencies for ${prevStr}`);
+            // This is a simplified fetch, assuming all calcs in this pass share the same T-1 deps
+            // A more robust solution would aggregate all unique T-1 deps.
+            updated.yesterdayDependencyData = await fetchExistingResults(prevStr, calcs, calcs.map(c => c.manifest), config, deps);
+        })());
+    }
+
     await Promise.all(tasks); 
     return updated; 
 }
@@ -204,9 +225,12 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
  */
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs) { 
     const { logger, calculationUtils } = deps;
-    const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights } = rootData;
+    const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, yesterdayDependencyData } = rootData;
     const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
-    const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
+    
+    // --- FIX: Add yesterday's computed data to context ---
+    const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config, yesterdaysDependencyData: yesterdayDependencyData};
+    
     let firstUser=true; 
     for(const name in state){  const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
     const cat=calc.manifest.category;
@@ -249,6 +273,11 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     firstUser=false; 
     if (pY) { delete yesterdayPortfolios[uid]; }
     if (hT) { delete todayHistoryData[uid]; } }  }
+    
+    // --- FIX [PROBLEM 7]: Clear stale data to prevent memory leak ---
+    yesterdayPortfolios = {};
+    todayHistoryData = {};
+
     logger.log('INFO', `[${passName}] Finished streaming data for ${dateStr}.`);
 }
 
@@ -279,6 +308,8 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     } else { standardResult[key] = result[key]; }}
     if (Object.keys(standardResult).length > 0) {
     const docRef = deps.db.collection(config.resultsCollection).doc(dStr) .collection(config.resultsSubcollection).doc(calc.manifest.category) .collection(config.computationsSubcollection).doc(name);
+    // --- FIX [PROBLEM 8]: Add completion marker ---
+    standardResult._completed = true;
     standardWrites.push({ ref: docRef, data: standardResult });}
     const calcClass = calc.manifest.class;
     let staticSchema = null;
@@ -297,7 +328,10 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     if (docPath.includes('/')) { docRef = deps.db.doc(docPath); } else {
     const collection = (docPath.startsWith('user_profile_history'))  ? config.shardedUserProfileCollection : config.shardedProfitabilityCollection;
     docRef = deps.db.collection(collection).doc(docPath); }
-    if (docData && typeof docData === 'object' && !Array.isArray(docData)) {shardedDocWrites.push({ ref: docRef, data: docData });
+    if (docData && typeof docData === 'object' && !Array.isArray(docData)) {
+    // --- FIX [PROBLEM 8]: Add completion marker to sharded writes ---
+    docData._completed = true;
+    shardedDocWrites.push({ ref: docRef, data: docData });
     } else { logger.log('ERROR', `[${passName}] Invalid sharded document data for ${docPath}. Not an object.`, { data: docData }); }
     if (shardedDocWrites.length > 0) { await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${docPath} ${dStr}`); } }
     const logMetadata = {};
@@ -336,6 +370,8 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
             }
             if (Object.keys(standardResult).length > 0) {
                 const docRef = deps.db.collection(config.resultsCollection).doc(dStr) .collection(config.resultsSubcollection).doc(mCalc.category) .collection(config.computationsSubcollection).doc(name);
+                // --- FIX [PROBLEM 8]: Add completion marker ---
+                standardResult._completed = true;
                 standardWrites.push({ ref: docRef, data: standardResult }); 
             }
             const calcClass = mCalc.class;
@@ -353,7 +389,13 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
     for (const collectionName in shardedWrites) {
     const docs = shardedWrites[collectionName];
     const shardedDocWrites = [];
-    for (const docId in docs) { const docRef = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(collectionName).doc(docId); shardedDocWrites.push({ ref: docRef, data: docs[docId] }); }
+    for (const docId in docs) { 
+        const docRef = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(collectionName).doc(docId); 
+        // --- FIX [PROBLEM 8]: Add completion marker to sharded writes ---
+        const docData = docs[docId];
+        docData._completed = true;
+        shardedDocWrites.push({ ref: docRef, data: docData }); 
+    }
     if (shardedDocWrites.length > 0) { await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`); } }    
     const logMetadata = {};
     if (failedCalcs.length > 0) { logMetadata.failedComputations = failedCalcs; }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.159",
+  "version": "1.0.161",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [