bulltrackers-module 1.0.143 → 1.0.145

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -12,8 +12,7 @@ function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[ca
  */
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
-    if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) {
-        return { canRun: true, missing };}
+    if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) { return { canRun: true, missing };}
     for (const dep of calcManifest.rootDataDependencies) {
         if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
         else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
@@ -34,8 +33,6 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
     try {
         const tasks = [];
-        // This logic is correct. It *avoids* calling get...Refs
-        // if the dateToProcess is before the earliest known data.
         if (dateToProcess >= earliestDates.portfolio)
             {tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(res => {portfolioRefs = res;hasPortfolio = !!(res?.length);}));}
         if (dateToProcess >= earliestDates.insights) {
@@ -44,18 +41,11 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
             tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {socialData = res;hasSocial = !!res;}));}
         if (dateToProcess >= earliestDates.history) {
             tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(res => {historyRefs = res;hasHistory = !!(res?.length);}));}
-
         await Promise.all(tasks);
-
-        // --- NEW: Log what was *actually* found ---
         logger.log('INFO', `[PassRunner] Data availability for ${dateStr}: P:${hasPortfolio}, I:${hasInsights}, S:${hasSocial}, H:${hasHistory}`);
 
-        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) {
-            logger.log('WARN', `[PassRunner] No root data at all for ${dateStr}.`);
-            // We return null to skip the entire day
-            return null;
-        }
-        return {portfolioRefs, insightsData,socialData,historyRefs,status: { hasPortfolio, hasInsights, hasSocial, hasHistory }};
+        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) { logger.log('WARN', `[PassRunner] No root data at all for ${dateStr}.`); return null; }
+        return { portfolioRefs, todayInsights: insightsData, todaySocialPostInsights: socialData, historyRefs, status: { hasPortfolio, hasInsights, hasSocial, hasHistory } };
     } catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
 }
 
@@ -91,76 +81,32 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
 function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates, logger) {
     const skipped = new Set();
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-
-    /**
-     * --- CORRECTED LOGIC ---
-     * Helper to get the true earliest date a calc can run.
-     */
     const getTrueEarliestRunDate = (calc) => {
-        let earliestRunDate = new Date('1970-01-01T00:00:00Z'); // Start at zero
+        let earliestRunDate = new Date('1970-01-01T00:00:00Z'); 
         const dependencies = calc.rootDataDependencies || [];
-
-        // 1. Find the LATEST "today" dependency
-        // This is the date where all *today* data is first available
         for (const dep of dependencies) {
             if (dep === 'portfolio' && earliestDates.portfolio > earliestRunDate) earliestRunDate = earliestDates.portfolio;
             if (dep === 'history' && earliestDates.history > earliestRunDate) earliestRunDate = earliestDates.history;
             if (dep === 'social' && earliestDates.social > earliestRunDate) earliestRunDate = earliestDates.social;
             if (dep === 'insights' && earliestDates.insights > earliestRunDate) earliestRunDate = earliestDates.insights;
         }
-
-        // 2. If the calc is historical, shift the *final* date by +1
-        // This universally applies the "+1" rule if *any* yesterday data is needed,
-        // (as long as we found a dependency in step 1).
-        if (calc.isHistorical && earliestRunDate.getTime() > 0) {
-            earliestRunDate.setUTCDate(earliestRunDate.getUTCDate() + 1);
-        }
-        
+        if (calc.isHistorical && earliestRunDate.getTime() > 0) { earliestRunDate.setUTCDate(earliestRunDate.getUTCDate() + 1); }
         return earliestRunDate;
     };
-
     const filterCalc = (calc) => {
-        // 1. Skip if result already exists
-        if (existingResults[calc.name]) {
-            logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists.`);
-            skipped.add(calc.name);
-            return false;
-        }
+        if (existingResults[calc.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Result already exists.`); skipped.add(calc.name); return false;}
 
-        // 2. Check *true* earliest run date
         const earliestRunDate = getTrueEarliestRunDate(calc);
-        if (dateToProcess < earliestRunDate) {
-            logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`);
-            skipped.add(calc.name);
-            return false;
-        }
-
-        // 3. Check if *today's* root data was *actually* found
-        // This handles gaps *after* the earliest date
+        if (dateToProcess < earliestRunDate) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Date is before true earliest run date (${earliestRunDate.toISOString().slice(0, 10)}).`); skipped.add(calc.name); return false; }
+        
         const { canRun, missing: missingRoot } = checkRootDependencies(calc, rootDataStatus);
-        if (!canRun) {
-            logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);
-            skipped.add(calc.name);
-            return false;
-        }
-
-        // 4. (For Meta Calcs) Check computed dependencies
-        if (calc.type === 'meta') {
-            const missingDeps = (calc.dependencies || []).map(normalizeName).filter(d => !existingResults[d]);
-            if (missingDeps.length > 0) {
-                logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);
-                skipped.add(calc.name);
-                return false;
-            }
-        }
+        if (!canRun) {logger.log('INFO', `[Pass ${passToRun}] Skipping ${calc.name} for ${dateStr}. Data missing for this date: [${missingRoot.join(', ')}]`);skipped.add(calc.name); return false;}
         
-        // If it passed all checks, run it.
+        if (calc.type === 'meta') { const missingDeps = (calc.dependencies || []).map(normalizeName).filter(d => !existingResults[d]); if (missingDeps.length > 0) { logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${calc.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`); skipped.add(calc.name); return false;} }
         return true;
     };
-
     const standardCalcsToRun = standardCalcs.filter(filterCalc);
     const metaCalcsToRun = metaCalcs.filter(filterCalc);
-
     return { standardCalcsToRun, metaCalcsToRun };
 }
 
@@ -169,50 +115,35 @@ function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingRe
 function initializeCalculators(calcs, logger) { const state = {}; for (const c of calcs) { const name=normalizeName(c.name), Cl=c.class; if(typeof Cl==='function') try { const inst=new Cl(); inst.manifest=c; state[name]=inst; } catch(e){logger.warn(`Init failed ${name}`,{errorMessage:e.message}); state[name]=null;} else {logger.warn(`Class missing ${name}`); state[name]=null;} } return state; }
 
 /** * Stage 7: Load historical data required for calculations 
- * --- THIS FUNCTION IS NOW FIXED ---
  */
 async function loadHistoricalData(date, calcs, config, deps, rootData) { 
-    const { logger } = deps; // <--- THIS WAS THE MISSING LINE
-    const updated = {...rootData}, dStr=date.toISOString().slice(0,10); const tasks = [];
-    
+    const { logger }              = deps;
+    const updated                 = {...rootData}, dStr=date.toISOString().slice(0,10); const tasks = [];
     const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
-    const needsTodayHistory = calcs.some(c => c.rootDataDependencies.includes('history'));
-    const needsYesterdayHistory = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('history'));
-    const needsYesterdayInsights = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
-    const needsYesterdaySocial = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
-    
-    // --- MODIFIED: Be smarter about loading data ---
+    const needsTodayHistory       = calcs.some(c => c.rootDataDependencies.includes('history'));
+    const needsYesterdayHistory   = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('history'));
+    const needsYesterdayInsights  = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
+    const needsYesterdaySocial    = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
     if(needsYesterdayPortfolio) {
         tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10);
-            logger.log('INFO', `[PassRunner] Loading YESTERDAY portfolio data for ${prevStr}`);
-            updated.yesterdayPortfolios=await loadFullDayMap(config,deps,await getPortfolioPartRefs(config,deps,prevStr)); 
-        })());
-    }
+        logger.log('INFO', `[PassRunner] Loading YESTERDAY portfolio data for ${prevStr}`);
+        updated.yesterdayPortfolios=await loadFullDayMap(config,deps,await getPortfolioPartRefs(config,deps,prevStr)); })());}
     if(needsTodayHistory) {
         tasks.push((async()=>{ 
-            logger.log('INFO', `[PassRunner] Loading TODAY history data for ${dStr}`);
-            updated.todayHistoryData=await loadFullDayMap(config,deps,rootData.historyRefs); 
-        })());
-    }
+        logger.log('INFO', `[PassRunner] Loading TODAY history data for ${dStr}`);
+        updated.todayHistoryData=await loadFullDayMap(config,deps,rootData.historyRefs); })());}
     if(needsYesterdayHistory) {
         tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
-            logger.log('INFO', `[PassRunner] Loading YESTERDAY history data for ${prevStr}`);
-            updated.yesterdayHistoryData=await loadFullDayMap(config,deps,await getHistoryPartRefs(config,deps,prevStr)); 
-        })());
-    }
+        logger.log('INFO', `[PassRunner] Loading YESTERDAY history data for ${prevStr}`);
+        updated.yesterdayHistoryData=await loadFullDayMap(config,deps,await getHistoryPartRefs(config,deps,prevStr)); })());}
     if(needsYesterdayInsights) {
         tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
-            logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
-            updated.yesterdayInsights=await loadDailyInsights(config,deps,prevStr); 
-        })());
-    }
+        logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
+        updated.yesterdayInsights=await loadDailyInsights(config,deps,prevStr); })());}
     if(needsYesterdaySocial) {
         tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
-            logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
-            updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); 
-        })());
-    }
-
+        logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
+        updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); })());}
     await Promise.all(tasks); return updated;
 }
 
@@ -223,17 +154,10 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
  */
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData) { 
     const { logger, calculationUtils } = deps;
-    // --- MODIFIED: yesterdayInsights/Social are now loaded by loadHistoricalData ---
     const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, todayHistoryData, yesterdayHistoryData, yesterdayPortfolios } = rootData;
-    
-    // --- NEW: Check if streaming is even needed ---
     const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
-    
     const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
-    
-    // --- MODIFIED: Run non-streaming calcs first (social/insights) ---
-    // This allows them to run even if portfolio data is missing
-    let firstUser=true; // Used to run them only once
+    let firstUser=true; 
     for(const name in state){ 
         const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
         const cat=calc.manifest.category;
@@ -241,7 +165,6 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
             if (firstUser) {
                 logger.log('INFO', `[${passName}] Running non-streaming calc: ${name}`);
                 let args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData];
-                // Pass historical data if needed
                 if(calc.manifest.isHistorical) {
                     args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData];
                 }
@@ -249,15 +172,10 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
             }
         }
     }
-
-
     if (calcsThatStreamPortfolio.length === 0) {
         logger.log('INFO', `[${passName}] No portfolio-streaming calcs to run for ${dateStr}. Skipping stream.`);
-        return; // Exit stream function
-    }
-    
+        return; }
     logger.log('INFO', `[${passName}] Streaming portfolio data for ${calcsThatStreamPortfolio.length} calcs...`);
-    
     for await (const chunk of streamPortfolioData(config, deps, dateStr)) {
         for(const uid in chunk){ const p=chunk[uid]; if(!p) continue; 
             const userType=p.PublicPositions?'speculator':'normal'; 
@@ -265,18 +183,12 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
             for(const name in state){ 
                 const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
                 const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; 
-                
-                // --- MODIFIED: Skip social/insights here, they ran above ---
                 if(isSocialOrInsights) continue; 
-                
                 let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
-                
                 if(isHistorical){ 
-                    const pY=yesterdayPortfolios ? yesterdayPortfolios[uid] : null; // Check if yesterdayPortfolios exists
-                    // V3 behavioural calcs (like history aggregator) *can* run without pY
+                    const pY=yesterdayPortfolios ? yesterdayPortfolios[uid] : null;
                     if(!pY && (cat !== 'behavioural' && name !== 'historical-performance-aggregator')) continue; 
-                    args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
-                } 
+                    args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; } 
                 if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; 
                 try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} } 
             firstUser=false; 
@@ -287,20 +199,13 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
 /** Stage 9: Run standard computations */
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    // --- THIS IS THE CRITICAL CHANGE ---
-    // If 'calcs' is empty *because of the new filter*, this log won't even appear.
     if (calcs.length === 0) {
         logger.log('INFO', `[${passName}] No standard calcs to run for ${dStr} after filtering.`);
-        return; 
-    }
-    // This log now only appears if there is *actually* work to do.
+        return; }
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
-
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
     const state = initializeCalculators(calcs, logger);
-    
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot);
-    
     let success = 0;
     const standardWrites = [];
     const shardedWrites = {}; 
@@ -337,8 +242,8 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
     if (standardWrites.length > 0) {
         await commitBatchInChunks(config, deps, standardWrites, `${passName} Standard ${dStr}`);
     }
-    for (const docPath in shardedWrites) { // 'docPath' is the key, e.g., 'user_profile_history_shard_0' or 'social_.../history'
-        const docData = shardedWrites[docPath]; // 'docData' is the object to write, e.g., { profiles: ... }
+    for (const docPath in shardedWrites) { 
+        const docData = shardedWrites[docPath]; 
         const shardedDocWrites = [];
         let docRef;
         if (docPath.includes('/')) {
@@ -364,17 +269,12 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
 /** Stage 10: Run meta computations */
 async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    // --- THIS IS THE CRITICAL CHANGE ---
     if (calcs.length === 0) {
         logger.log('INFO', `[${passName}] No meta calcs to run for ${dStr} after filtering.`);
         return;
     }
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
-    
-    // --- NEW: Load historical data for meta calcs if needed ---
-    // (This might be redundant if standard pass ran, but meta-calcs can run standalone)
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
-
     let success = 0;
     const standardWrites = [];
     const shardedWrites = {}; 
@@ -383,7 +283,6 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
         if (typeof Cl !== 'function') {logger.log('ERROR', `Invalid class ${name}`);continue;}
         const inst = new Cl();
         try {
-            // --- MODIFIED: Pass fullRoot to meta calcs ---
             const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData: fullRoot }, config, fetchedDeps));
             if (result && Object.keys(result).length > 0) {const standardResult = {}; for (const key in result) {
                     if (key.startsWith('sharded_')) {const shardedData = result[key];for (const collectionName in shardedData) 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.143",
+  "version": "1.0.145",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [