bulltrackers-module 1.0.131 → 1.0.133

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -6,9 +6,18 @@
  * It reads its pass number from the config and executes only those calculations.
  * This file contains the high-level "manual" of steps. The "how-to" logic
  * is extracted into 'computation_system_utils.js'.
+ * --- MODIFIED: To use getEarliestDataDates and pass the date map to the orchestrator helpers. ---
+ * --- MODIFIED: To run date processing in parallel batches. ---
+ * --- MODIFIED: To fetch ALL existing results to enable incremental (skip) logic. ---
  */
-const { groupByPass, checkRootDataAvailability, fetchDependenciesForPass, filterCalculations, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
-const { getExpectedDateStrings, getFirstDateFromSourceData } = require('../utils/utils.js');
+
+// --- MODIFIED: Renamed fetchDependenciesForPass to fetchExistingResults ---
+const { groupByPass, checkRootDataAvailability, fetchExistingResults, filterCalculations, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
+// --- MODIFIED: Import getEarliestDataDates ---
+const { getExpectedDateStrings, getEarliestDataDates } = require('../utils/utils.js');
+
+// --- NEW: Parallel processing batch size ---
+const PARALLEL_BATCH_SIZE = 7; // Process a week at a time
 
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
@@ -17,7 +26,12 @@ async function runComputationPass(config, dependencies, computationManifest) {
 
     const yesterday = new Date(); yesterday.setUTCDate(yesterday.getUTCDate()-1); 
     const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
-    const firstDate = await getFirstDateFromSourceData(config, dependencies);
+    
+    // --- MODIFIED: Call new date function ---
+    const earliestDates = await getEarliestDataDates(config, dependencies);
+    const firstDate = earliestDates.absoluteEarliest; // Use the absolute earliest for the loop
+    // --- END MODIFICATION ---
+    
     const startDateUTC = firstDate ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) : new Date(config.earliestComputationDate+'T00:00:00Z');
     const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
 
@@ -27,21 +41,52 @@ async function runComputationPass(config, dependencies, computationManifest) {
     const standardCalcs = calcsInThisPass.filter(c => c.type==='standard'); 
     const metaCalcs = calcsInThisPass.filter(c => c.type==='meta');
 
-    for (const dateStr of allExpectedDates) {
+    // --- NEW: Helper function to process a single date ---
+    const processDate = async (dateStr) => {
         const dateToProcess = new Date(dateStr+'T00:00:00Z');
         try {
-            const rootData = await checkRootDataAvailability(dateStr, config, dependencies); if (!rootData) continue;
-            const fetchedDeps = await fetchDependenciesForPass(dateStr, calcsInThisPass, computationManifest, config, dependencies);
-            const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, fetchedDeps, passToRun, dateStr, logger);
+            // --- MODIFIED: Pass earliestDates map to checkRootDataAvailability ---
+            const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
+            if (!rootData) {
+                logger.log('WARN', `[PassRunner] Skipping ${dateStr} for Pass ${passToRun}: No root data.`);
+                return; // Use 'return' instead of 'continue' for a mapped function
+            }
+            // --- END MODIFICATION ---
+            
+            // --- MODIFIED: Fetch ALL existing results for this pass, not just dependencies ---
+            const existingResults = await fetchExistingResults(dateStr, calcsInThisPass, computationManifest, config, dependencies);
+            
+            // --- MODIFIED: Pass existingResults to filterCalculations ---
+            const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, existingResults, passToRun, dateStr, logger);
+            
+            if (standardCalcsToRun.length === 0 && metaCalcsToRun.length === 0) {
+                logger.log('INFO', `[PassRunner] All calcs for ${dateStr} Pass ${passToRun} are already complete. Skipping.`);
+                return;
+            }
+
+            // --- MODIFIED: Pass existingResults (as fetchedDeps) to meta pass ---
             if (standardCalcsToRun.length) await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
-            if (metaCalcsToRun.length) await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, fetchedDeps, rootData);
-            logger.log('SUCCESS', `[PassRunner] Completed Pass ${passToRun} for ${dateStr}.`);
+            if (metaCalcsToRun.length) await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);
+            
+            // Note: We no longer log "Completed" here, as the sub-functions do.
+            // logger.log('SUCCESS', `[PassRunner] Completed Pass ${passToRun} for ${dateStr}.`);
+
         } catch (err) {
             logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
         }
+    };
+    // --- END: Helper function ---
+
+    // --- NEW: Replace sequential loop with parallel batch loop ---
+    logger.log('INFO', `[PassRunner] Processing ${allExpectedDates.length} total dates in batches of ${PARALLEL_BATCH_SIZE}...`);
+    for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) {
+        const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
+        logger.log('INFO', `[PassRunner] Processing batch ${Math.floor(i / PARALLEL_BATCH_SIZE) + 1}/${Math.ceil(allExpectedDates.length / PARALLEL_BATCH_SIZE)} (Dates: ${batch[0]}...${batch[batch.length-1]})`);
+        await Promise.all(batch.map(dateStr => processDate(dateStr)));
     }
+    // --- END: Parallel batch loop ---
 
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 
-module.exports = { runComputationPass };
+module.exports = { runComputationPass };

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,60 +1,133 @@
 const { FieldPath } = require('@google-cloud/firestore');
-const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs } = require('../utils/data_loader.js');
+// --- MODIFIED: Import streamPortfolioData ---
+const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs, streamPortfolioData } = require('../utils/data_loader.js');
 const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
 
 /** Stage 1: Group manifest by pass number */
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
-/** Stage 2: Check root data dependencies for a calc */
-function checkRootDependencies(calcManifest, rootDataStatus) { if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) return true;
-    for (const dep of calcManifest.rootDataDependencies) if ((dep==='portfolio'&&!rootDataStatus.hasPortfolio)||(dep==='insights'&&!rootDataStatus.hasInsights)||(dep==='social'&&!rootDataStatus.hasSocial)||(dep==='history'&&!rootDataStatus.hasHistory)) return false;
-    return true;
+/** * --- MODIFIED: Returns detailed missing dependencies for logging ---
+ * Stage 2: Check root data dependencies for a calc 
+ */
+function checkRootDependencies(calcManifest, rootDataStatus) {
+    const missing = [];
+    if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) {
+        return { canRun: true, missing };}
+    for (const dep of calcManifest.rootDataDependencies) {
+        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
+        else if (dep === 'social' && !rootDataStatus.hasSocial) missing.push('social');
+        else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
+    }
+    return { canRun: missing.length === 0, missing };
 }
 
-/** Stage 3: Check root data availability for a date */
-// --- FIX: Passes the full 'dependencies' object down ---
-async function checkRootDataAvailability(dateStr, config, dependencies) {
-    const { logger } = dependencies; // Destructure logger for local use
+/** * --- MODIFIED: Uses earliestDates map to avoid unnecessary queries ---
+ * Stage 3: Check root data availability for a date 
+ */
+async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
+    const { logger } = dependencies;
     logger.log('INFO', `[PassRunner] Checking root data for ${dateStr}...`);
+    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+    let portfolioRefs = [], insightsData = null, socialData = null, historyRefs = [];
+    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
     try {
-        const [portfolioRefs, insightsData, socialData, historyRefs] = await Promise.all([
-            getPortfolioPartRefs(config, dependencies, dateStr),        // Pass full 'dependencies'
-            loadDailyInsights(config, dependencies, dateStr),         // Pass full 'dependencies'
-            loadDailySocialPostInsights(config, dependencies, dateStr), // Pass full 'dependencies'
-            getHistoryPartRefs(config, dependencies, dateStr)           // Pass full 'dependencies'
-        ]);
-        const hasPortfolio = !!(portfolioRefs?.length), hasInsights = !!insightsData, hasSocial = !!socialData, hasHistory = !!(historyRefs?.length);
-        if (!(hasPortfolio||hasInsights||hasSocial||hasHistory)) { logger.log('WARN', `[PassRunner] No root data for ${dateStr}.`); return null; }
-        return { portfolioRefs: portfolioRefs||[], insightsData: insightsData||null, socialData: socialData||null, historyRefs: historyRefs||[], status: { hasPortfolio, hasInsights, hasSocial, hasHistory } };
+        const tasks = [];
+        if (dateToProcess >= earliestDates.portfolio) 
+            {tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(res => {portfolioRefs = res;hasPortfolio = !!(res?.length);}));}
+        if (dateToProcess >= earliestDates.insights) {
+            tasks.push(loadDailyInsights(config, dependencies, dateStr).then(res => {insightsData = res;hasInsights = !!res;}));}
+        if (dateToProcess >= earliestDates.social) {
+            tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {socialData = res;hasSocial = !!res;}));}
+        if (dateToProcess >= earliestDates.history) {
+            tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(res => {historyRefs = res;hasHistory = !!(res?.length);}));}
+        await Promise.all(tasks);
+        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) {logger.log('WARN', `[PassRunner] No root data for ${dateStr}.`); return null;}
+        return {portfolioRefs, insightsData,socialData,historyRefs,status: { hasPortfolio, hasInsights, hasSocial, hasHistory }};
     } catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
 }
 
-/** Stage 4: Fetch computed dependencies from Firestore */
-async function fetchDependenciesForPass(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
+
+
+/** --- MODIFIED: Stage 4: Fetch ALL existing computed results for the pass ---
+ * This function now checks for *all* calcs in the pass, not just meta-dependencies,
+ * to enable skipping completed work.
+ */
+async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
-    const requiredDeps = new Set(calcsInPass.filter(c => c.type==='meta'&&c.dependencies).flatMap(c => c.dependencies.map(normalizeName)));
-    if (!requiredDeps.size) return {};
-    logger.log('INFO', `[PassRunner] Fetching ${requiredDeps.size} deps for ${dateStr}...`);
+    
+    // --- MODIFIED: Check all calcs in this pass, not just meta-deps ---
+    const allCalcsInPass = new Set(calcsInPass.map(c => normalizeName(c.name)));
+    if (!allCalcsInPass.size) return {};
+
+    logger.log('INFO', `[PassRunner] Checking for ${allCalcsInPass.size} existing results for ${dateStr}...`);
     const docRefs = [], depNames = [];
-    for (const calcName of requiredDeps) {
+    
+    for (const calcName of allCalcsInPass) {
         const calcManifest = manifestMap.get(calcName);
         if (!calcManifest) { logger.log('ERROR', `[PassRunner] Missing manifest for ${calcName}`); continue; }
         docRefs.push(db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection).doc(calcManifest.category||'unknown').collection(config.computationsSubcollection).doc(calcName));
         depNames.push(calcName);
     }
+    
     const fetched = {};
     if (docRefs.length) (await db.getAll(...docRefs)).forEach((doc,i)=>fetched[depNames[i]]=doc.exists?doc.data():null);
     return fetched;
 }
 
-/** Stage 5: Filter calculations based on available root data and dependencies */
-function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, fetchedDeps, passToRun, dateStr, logger) {
+/** --- MODIFIED: Stage 5: Filter calculations to skip completed work ---
+ */
+function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, logger) {
     const skipped = new Set();
-    const standardCalcsToRun = standardCalcs.filter(c => checkRootDependencies(c, rootDataStatus) || (logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} missing root data`), skipped.add(c.name), false));
-    const metaCalcsToRun = metaCalcs.filter(c => checkRootDependencies(c, rootDataStatus) && (c.dependencies||[]).every(d=>fetchedDeps[normalizeName(d)]) || (logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} missing dep`), skipped.add(c.name), false));
+
+    // Filter Standard Calcs
+    const standardCalcsToRun = standardCalcs.filter(c => {
+        // --- NEW: Skip if result already exists ---
+        if (existingResults[c.name]) {
+            logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Result already exists.`);
+            return false;
+        }
+        
+        const { canRun, missing } = checkRootDependencies(c, rootDataStatus);
+        if (canRun) return true;
+        
+        logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Missing root data: [${missing.join(', ')}]`);
+        skipped.add(c.name);
+        return false;
+    });
+
+    // Filter Meta Calcs
+    const metaCalcsToRun = metaCalcs.filter(c => {
+        // --- NEW: Skip if result already exists ---
+        if (existingResults[c.name]) {
+            logger.log('TRACE', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Result already exists.`);
+            skipped.add(c.name);
+            return false;
+        }
+
+        // 1. Check root data
+        const { canRun, missing: missingRoot } = checkRootDependencies(c, rootDataStatus);
+        if (!canRun) {
+            logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root data: [${missingRoot.join(', ')}]`);
+            skipped.add(c.name);
+            return false;
+        }
+        
+        // 2. Check computed dependencies
+        // --- MODIFIED: Check existingResults (was fetchedDeps) ---
+        const missingDeps = (c.dependencies || []).map(normalizeName).filter(d => !existingResults[d]);
+        if (missingDeps.length > 0) {
+            logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);
+            skipped.add(c.name);
+            return false;
+        }
+        return true; // All checks passed
+    });
+    
     return { standardCalcsToRun, metaCalcsToRun };
 }
 
+
 /** Stage 6: Initialize calculator instances */
 function initializeCalculators(calcs, logger) { const state = {}; for (const c of calcs) { const name=normalizeName(c.name), Cl=c.class; if(typeof Cl==='function') try { const inst=new Cl(); inst.manifest=c; state[name]=inst; } catch(e){logger.warn(`Init failed ${name}`,{errorMessage:e.message}); state[name]=null;} else {logger.warn(`Class missing ${name}`); state[name]=null;} } return state; }
 
@@ -66,21 +139,65 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) { const u
     await Promise.all(tasks); return updated;
 }
 
-/** Stage 8: Stream and process data for standard calculations */
-async function streamAndProcess(dateStr, todayRefs, state, passName, config, deps, rootData) { const { logger, calculationUtils } = deps;
+/** * --- MODIFIED: Stage 8: Stream and process data for standard calculations ---
+ * This function now uses an async generator to stream portfolio data
+ * instead of loading it all into memory.
+ */
+async function streamAndProcess(dateStr, state, passName, config, deps, rootData) { 
+    const { logger, calculationUtils } = deps;
     const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, todayHistoryData, yesterdayHistoryData, yesterdayPortfolios } = rootData;
-    const batchSize=config.partRefBatchSize||10; let firstUser=true;
+    
+    let firstUser=true;
+    // Load mappings once for all chunks
     const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
-    for(let i=0;i<todayRefs.length;i+=batchSize){ const batch=todayRefs.slice(i,i+batchSize); const chunk=await loadDataByRefs(config,deps,batch); for(const uid in chunk){ const p=chunk[uid]; if(!p) continue; const userType=p.PublicPositions?'speculator':'normal'; context.userType=userType; for(const name in state){ const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; let args=[p,null,uid,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; if(isSocialOrInsights&&!firstUser) continue; if(isHistorical){ const pY=yesterdayPortfolios[uid]; if(!pY) continue; args=[p,pY,uid,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; } if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} } firstUser=false; } }
+
+    // Use the new async generator stream
+    for await (const chunk of streamPortfolioData(config, deps, dateStr)) {
+        // This inner loop processes one chunk of users at a time
+        for(const uid in chunk){ 
+            const p=chunk[uid]; if(!p) continue; 
+            const userType=p.PublicPositions?'speculator':'normal'; 
+            context.userType=userType; 
+            for(const name in state){ 
+                const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
+                const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; 
+                // --- MODIFIED: Argument list now includes context ---
+                let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
+                
+                if(isSocialOrInsights&&!firstUser) continue; 
+                
+                if(isHistorical){ 
+                    const pY=yesterdayPortfolios[uid]; if(!pY) continue; 
+                    args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
+                } 
+                
+                if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; 
+                
+                try{ 
+                    // Pass context to all process functions.
+                    await Promise.resolve(calc.process(...args)); 
+                } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} 
+            } 
+            firstUser=false; 
+        } 
+        // Memory from 'chunk' is released here before the next iteration
+    }
 }
+// --- END MODIFICATION ---
 
 /** Stage 9: Run standard computations */
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
+    // --- MODIFIED: Log only if there are calcs to run ---
+    if (calcs.length === 0) return; 
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
+    
+    // --- MODIFIED: We no longer need portfolioRefs in fullRoot, as streamAndProcess handles it.
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
     const state = initializeCalculators(calcs, logger);
-    await streamAndProcess(dStr, fullRoot.portfolioRefs, state, passName, config, deps, fullRoot);
+    
+    // --- MODIFIED: Call to streamAndProcess no longer passes portfolioRefs ---
+    await streamAndProcess(dStr, state, passName, config, deps, fullRoot);
     
     // --- START: FULL COMMIT LOGIC ---
     let success = 0;
@@ -155,6 +272,8 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
 /** Stage 10: Run meta computations */
 async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
+    // --- MODIFIED: Log only if there are calcs to run ---
+    if (calcs.length === 0) return;
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
 
     // --- START: FULL COMMIT LOGIC ---
@@ -231,4 +350,5 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
 }
 
 
-module.exports = { groupByPass, checkRootDataAvailability, fetchDependenciesForPass, filterCalculations, runStandardComputationPass, runMetaComputationPass };
+// --- MODIFIED: Export new function name ---
+module.exports = { groupByPass, checkRootDataAvailability, fetchExistingResults, filterCalculations, runStandardComputationPass, runMetaComputationPass };

package/functions/computation-system/utils/data_loader.js

@@ -1,6 +1,7 @@
 /**
  * @fileoverview Data loader sub-pipes for the Computation System.
  * REFACTORED: Now stateless and receive dependencies.
+ * --- NEW: Added streamPortfolioData async generator ---
  */
 
 // <<< FIX: REMOVED all top-level 'require' and 'dependencies' lines >>>
@@ -165,6 +166,39 @@ async function getHistoryPartRefs(config, deps, dateString) {
     return allPartRefs;
 }
 
+// --- NEW: Stage 7: Stream portfolio data in chunks ---
+/**
+ * Streams portfolio data in chunks for a given date.
+ * This is an async generator.
+ * @param {object} config - The computation system configuration object.
+ * @param {object} deps - Contains db, logger, calculationUtils.
+ * @param {string} dateString - The date in YYYY-MM-DD format.
+ */
+async function* streamPortfolioData(config, deps, dateString) {
+    const { logger } = deps;
+    const refs = await getPortfolioPartRefs(config, deps, dateString);
+    if (refs.length === 0) {
+        logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`);
+        return;
+    }
+
+    // Use the same batch size as loadDataByRefs for consistency
+    const batchSize = config.partRefBatchSize || 50; 
+    
+    logger.log('INFO', `[streamPortfolioData] Streaming ${refs.length} portfolio parts in chunks of ${batchSize}...`);
+
+    for (let i = 0; i < refs.length; i += batchSize) {
+        const batchRefs = refs.slice(i, i + batchSize);
+        // Load one chunk of data
+        const data = await loadDataByRefs(config, deps, batchRefs);
+        // Yield it to the consumer, then release it from memory
+        yield data; 
+    }
+    logger.log('INFO', `[streamPortfolioData] Finished streaming for ${dateString}.`);
+}
+// --- END: Stage 7 ---
+
+
 module.exports = {
     getPortfolioPartRefs,
     loadDataByRefs,
@@ -172,4 +206,5 @@ module.exports = {
     loadDailyInsights,
     loadDailySocialPostInsights,
     getHistoryPartRefs,
+    streamPortfolioData, // <-- EXPORT NEW FUNCTION
 };

package/functions/computation-system/utils/utils.js

@@ -2,8 +2,10 @@
  * @fileoverview Computation system sub-pipes and utils.
  * REFACTORED: Now stateless and receive dependencies where needed.
  * DYNAMIC: Categorization logic is removed, replaced by manifest.
+ * --- MODIFIED: getFirstDateFromSourceData is now getEarliestDataDates
+ * and queries all data sources to build an availability map. ---
  */
-/** --- Computation System Sub-Pipes & Utils (Stateless) --- */
+/** --- Computation System Sub-Pipes & Utils (Stateless, Dependency-Injection) --- */
 
 const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 
@@ -46,13 +48,46 @@ function getExpectedDateStrings(startDate, endDate) {
     return dateStrings;
 }
 
-/** Stage 4: Get the earliest date in a collection */
+/**
+ * --- NEW HELPER ---
+ * Stage 4: Get the earliest date in a *flat* collection where doc IDs are dates.
+ */
+async function getFirstDateFromSimpleCollection(config, deps, collectionName) {
+    const { db, logger, calculationUtils } = deps;
+    const { withRetry } = calculationUtils;
+
+    try {
+        if (!collectionName) {
+            logger.log('WARN', `[Core Utils] Collection name not provided for simple date query.`);
+            return null;
+        }
+        const query = db.collection(collectionName)
+            .where(FieldPath.documentId(), '>=', '2000-01-01')
+            .orderBy(FieldPath.documentId(), 'asc')
+            .limit(1);
+        
+        const snapshot = await withRetry(() => query.get(), `GetEarliestDoc(${collectionName})`);
+        
+        if (!snapshot.empty && /^\d{4}-\d{2}-\d{2}$/.test(snapshot.docs[0].id)) {
+            return new Date(snapshot.docs[0].id + 'T00:00:00Z');
+        }
+    } catch (e) {
+        logger.log('ERROR', `GetFirstDate failed for ${collectionName}`, { errorMessage: e.message });
+    }
+    return null;
+}
+
+/** Stage 4: Get the earliest date in a sharded collection */
 async function getFirstDateFromCollection(config, deps, collectionName) {
     const { db, logger, calculationUtils } = deps;
     const { withRetry } = calculationUtils;
 
     let earliestDate = null;
     try {
+        if (!collectionName) {
+            logger.log('WARN', `[Core Utils] Collection name not provided for sharded date query.`);
+            return null;
+        }
         const blockDocRefs = await withRetry(() => db.collection(collectionName).listDocuments(), `GetBlocks(${collectionName})`);
         if (!blockDocRefs.length) { logger.log('WARN', `No block documents in collection: ${collectionName}`); return null; }
 
@@ -75,26 +110,69 @@ async function getFirstDateFromCollection(config, deps, collectionName) {
     return earliestDate;
 }
 
-/** Stage 5: Determine the earliest date from source data across both user types */
-async function getFirstDateFromSourceData(config, deps) {
+/** * --- MODIFIED FUNCTION ---
+ * Stage 5: Determine the earliest date from *all* source data.
+ */
+async function getEarliestDataDates(config, deps) {
     const { logger } = deps;
-    logger.log('INFO', 'Querying for earliest date from source portfolio data...');
-
-    const investorDate = await getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection);
-    const speculatorDate = await getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection);
-
-    let earliestDate;
-    if (investorDate && speculatorDate) earliestDate = investorDate < speculatorDate ? investorDate : speculatorDate;
-    else earliestDate = investorDate || speculatorDate;
-
-    if (earliestDate) {
-        logger.log('INFO', `Found earliest source data date: ${earliestDate.toISOString().slice(0, 10)}`);
-        return earliestDate;
-    } else {
-        const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
-        logger.log('WARN', `No source data found. Defaulting first date to: ${fallbackDate.toISOString().slice(0, 10)}`);
-        return fallbackDate;
-    }
+    logger.log('INFO', 'Querying for earliest date from ALL source data collections...');
+
+    // These return null on error or if empty
+    const [
+        investorDate,
+        speculatorDate,
+        investorHistoryDate,
+        speculatorHistoryDate,
+        insightsDate,
+        socialDate
+    ] = await Promise.all([
+        getFirstDateFromCollection(config, deps, config.normalUserPortfolioCollection),
+        getFirstDateFromCollection(config, deps, config.speculatorPortfolioCollection),
+        getFirstDateFromCollection(config, deps, config.normalUserHistoryCollection),
+        getFirstDateFromCollection(config, deps, config.speculatorHistoryCollection),
+        getFirstDateFromSimpleCollection(config, deps, config.insightsCollectionName),
+        getFirstDateFromSimpleCollection(config, deps, config.socialInsightsCollectionName)
+    ]);
+    
+    // Helper to find the minimum (earliest) of a set of dates
+    const getMinDate = (...dates) => {
+        const validDates = dates.filter(Boolean); // Filter out nulls
+        if (validDates.length === 0) return null;
+        return new Date(Math.min(...validDates));
+    };
+
+    const earliestPortfolioDate = getMinDate(investorDate, speculatorDate);
+    const earliestHistoryDate = getMinDate(investorHistoryDate, speculatorHistoryDate);
+    const earliestInsightsDate = getMinDate(insightsDate); // Already a single date
+    const earliestSocialDate = getMinDate(socialDate); // Already a single date
+
+    const absoluteEarliest = getMinDate(
+        earliestPortfolioDate,
+        earliestHistoryDate,
+        earliestInsightsDate,
+        earliestSocialDate
+    );
+
+    // Fallback date
+    const fallbackDate = new Date(config.earliestComputationDate + 'T00:00:00Z' || '2023-01-01T00:00:00Z');
+
+    const result = {
+        portfolio: earliestPortfolioDate || new Date('2999-12-31'), // Use a 'far future' date if null
+        history: earliestHistoryDate || new Date('2999-12-31'),
+        insights: earliestInsightsDate || new Date('2999-12-31'),
+        social: earliestSocialDate || new Date('2999-12-31'),
+        absoluteEarliest: absoluteEarliest || fallbackDate // Use fallback for the main loop
+    };
+
+    logger.log('INFO', 'Earliest data availability map built:', {
+        portfolio: result.portfolio.toISOString().slice(0, 10),
+        history: result.history.toISOString().slice(0, 10),
+        insights: result.insights.toISOString().slice(0, 10),
+        social: result.social.toISOString().slice(0, 10),
+        absoluteEarliest: result.absoluteEarliest.toISOString().slice(0, 10)
+    });
+
+    return result;
 }
 
 module.exports = {
@@ -103,5 +181,6 @@ module.exports = {
     normalizeName,
     commitBatchInChunks,
     getExpectedDateStrings,
-    getFirstDateFromSourceData,
-};
+    // getFirstDateFromSourceData, // This is replaced
+    getEarliestDataDates, // <-- EXPORT NEW FUNCTION
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.131",
+  "version": "1.0.133",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [