bulltrackers-module 1.0.159 → 1.0.160

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -99,7 +99,13 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
     const fetched = {};
     if (docRefs.length) {
         (await db.getAll(...docRefs)).forEach((doc, i) => {
-            fetched[depNames[i]] = doc.exists ? doc.data() : null;
+            // --- FIX [PROBLEM 8]: Add completion marker check ---
+            const data = doc.exists ? doc.data() : null;
+            if (data && data._completed === true) {
+                fetched[depNames[i]] = data;
+            } else {
+                fetched[depNames[i]] = null; // Treat as not existing if incomplete
+            }
         });
     }
     
@@ -139,19 +145,18 @@ function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingRe
         return earliestRunDate;
     };
     const filterCalc = (calc) => {
-        if (existingResults[calc.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists.`); skipped.add(calc.name); return false;}
+        // --- FIX [PROBLEM 8]: The check for existingResults is now correct ---
+        if (existingResults[calc.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists (and is complete).`); skipped.add(calc.name); return false;}
+        
         const earliestRunDate = getTrueEarliestRunDate(calc);
         if (dateToProcess < earliestRunDate) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`); skipped.add(calc.name); return false; }
         const { canRun, missing: missingRoot } = checkRootDependencies(calc, rootDataStatus);
         if (!canRun) {logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);skipped.add(calc.name); return false;}
         
-        // --- FIX: This check is now robust ---
-        // 'existingResults' now contains all dependencies, so this check
-        // will correctly find 'pnl_distribution_per_stock' and *not* skip.
         if (calc.type === 'meta') { 
             const missingDeps = (calc.dependencies || [])
                 .map(normalizeName)
-                .filter(d => !existingResults[d]); 
+                .filter(d => !existingResults[d]); // This check is now robust
             if (missingDeps.length > 0) { 
                 logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`); 
                 skipped.add(calc.name); 
@@ -177,15 +182,20 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
     const needsYesterdayInsights  = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
     const needsYesterdaySocial    = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
     const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
+    
+    // --- FIX: Add T-1 COMPUTED dependency loading ---
+    const needsYesterdayDependencies = calcs.some(c => c.isHistorical && c.dependencies && c.dependencies.length > 0);
+    
     const prev = new Date(date);
     prev.setUTCDate(prev.getUTCDate() - 1);
     const prevStr = prev.toISOString().slice(0, 10);
+    
     if(needsYesterdayInsights) {
-        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
+        tasks.push((async()=>{ 
         logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
         updated.yesterdayInsights=await loadDailyInsights(config,deps,prevStr); })());}
     if(needsYesterdaySocial) {
-        tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
+        tasks.push((async()=>{ 
         logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
         updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); })());}
     
@@ -195,6 +205,17 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
             updated.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr); 
         })());
     }
+
+    // --- FIX: Load T-1 COMPUTED dependencies ---
+    if(needsYesterdayDependencies) {
+        tasks.push((async()=>{ 
+            logger.log('INFO', `[PassRunner] Loading YESTERDAY computed dependencies for ${prevStr}`);
+            // This is a simplified fetch, assuming all calcs in this pass share the same T-1 deps
+            // A more robust solution would aggregate all unique T-1 deps.
+            updated.yesterdayDependencyData = await fetchExistingResults(prevStr, calcs, calcs.map(c => c.manifest), config, deps);
+        })());
+    }
+
     await Promise.all(tasks); 
     return updated; 
 }
@@ -204,9 +225,12 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
  */
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs) { 
     const { logger, calculationUtils } = deps;
-    const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights } = rootData;
+    const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, yesterdayDependencyData } = rootData;
     const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
-    const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
+    
+    // --- FIX: Add yesterday's computed data to context ---
+    const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config, yesterdaysDependencyData: yesterdayDependencyData};
+    
     let firstUser=true; 
     for(const name in state){  const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
     const cat=calc.manifest.category;
@@ -249,6 +273,11 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     firstUser=false; 
     if (pY) { delete yesterdayPortfolios[uid]; }
     if (hT) { delete todayHistoryData[uid]; } }  }
+    
+    // --- FIX [PROBLEM 7]: Clear stale data to prevent memory leak ---
+    yesterdayPortfolios = {};
+    todayHistoryData = {};
+
     logger.log('INFO', `[${passName}] Finished streaming data for ${dateStr}.`);
 }
 
@@ -279,6 +308,8 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     } else { standardResult[key] = result[key]; }}
     if (Object.keys(standardResult).length > 0) {
     const docRef = deps.db.collection(config.resultsCollection).doc(dStr) .collection(config.resultsSubcollection).doc(calc.manifest.category) .collection(config.computationsSubcollection).doc(name);
+    // --- FIX [PROBLEM 8]: Add completion marker ---
+    standardResult._completed = true;
     standardWrites.push({ ref: docRef, data: standardResult });}
     const calcClass = calc.manifest.class;
     let staticSchema = null;
@@ -297,7 +328,10 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     if (docPath.includes('/')) { docRef = deps.db.doc(docPath); } else {
     const collection = (docPath.startsWith('user_profile_history'))  ? config.shardedUserProfileCollection : config.shardedProfitabilityCollection;
     docRef = deps.db.collection(collection).doc(docPath); }
-    if (docData && typeof docData === 'object' && !Array.isArray(docData)) {shardedDocWrites.push({ ref: docRef, data: docData });
+    if (docData && typeof docData === 'object' && !Array.isArray(docData)) {
+    // --- FIX [PROBLEM 8]: Add completion marker to sharded writes ---
+    docData._completed = true;
+    shardedDocWrites.push({ ref: docRef, data: docData });
     } else { logger.log('ERROR', `[${passName}] Invalid sharded document data for ${docPath}. Not an object.`, { data: docData }); }
     if (shardedDocWrites.length > 0) { await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${docPath} ${dStr}`); } }
     const logMetadata = {};
@@ -336,6 +370,8 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
             }
             if (Object.keys(standardResult).length > 0) {
                 const docRef = deps.db.collection(config.resultsCollection).doc(dStr) .collection(config.resultsSubcollection).doc(mCalc.category) .collection(config.computationsSubcollection).doc(name);
+                // --- FIX [PROBLEM 8]: Add completion marker ---
+                standardResult._completed = true;
                 standardWrites.push({ ref: docRef, data: standardResult }); 
             }
             const calcClass = mCalc.class;
@@ -353,7 +389,13 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
     for (const collectionName in shardedWrites) {
     const docs = shardedWrites[collectionName];
     const shardedDocWrites = [];
-    for (const docId in docs) { const docRef = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(collectionName).doc(docId); shardedDocWrites.push({ ref: docRef, data: docs[docId] }); }
+    for (const docId in docs) { 
+        const docRef = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(collectionName).doc(docId); 
+        // --- FIX [PROBLEM 8]: Add completion marker to sharded writes ---
+        const docData = docs[docId];
+        docData._completed = true;
+        shardedDocWrites.push({ ref: docRef, data: docData }); 
+    }
     if (shardedDocWrites.length > 0) { await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`); } }    
     const logMetadata = {};
     if (failedCalcs.length > 0) { logMetadata.failedComputations = failedCalcs; }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.159",
+  "version": "1.0.160",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [