bulltrackers-module 1.0.132 → 1.0.134

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -7,50 +7,50 @@
  * This file contains the high-level "manual" of steps. The "how-to" logic
  * is extracted into 'computation_system_utils.js'.
  * --- MODIFIED: To use getEarliestDataDates and pass the date map to the orchestrator helpers. ---
+ * --- MODIFIED: To run date processing in parallel batches. ---
+ * --- MODIFIED: To fetch ALL existing results to enable incremental (skip) logic. ---
  */
-const { groupByPass, checkRootDataAvailability, fetchDependenciesForPass, filterCalculations, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
+
+// --- MODIFIED: Renamed fetchDependenciesForPass to fetchExistingResults ---
+const { groupByPass, checkRootDataAvailability, fetchExistingResults, filterCalculations, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
 // --- MODIFIED: Import getEarliestDataDates ---
 const { getExpectedDateStrings, getEarliestDataDates } = require('../utils/utils.js');
 
+// --- NEW: Parallel processing batch size ---
+const PARALLEL_BATCH_SIZE = 7; // Process a week at a time
+
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
     const passToRun = String(config.COMPUTATION_PASS_TO_RUN); if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
     logger.log('INFO', `🚀 Starting PASS ${passToRun}...`);
-
     const yesterday = new Date(); yesterday.setUTCDate(yesterday.getUTCDate()-1); 
     const endDateUTC = new Date(Date.UTC(yesterday.getUTCFullYear(), yesterday.getUTCMonth(), yesterday.getUTCDate()));
-    
-    // --- MODIFIED: Call new date function ---
     const earliestDates = await getEarliestDataDates(config, dependencies);
-    const firstDate = earliestDates.absoluteEarliest; // Use the absolute earliest for the loop
-    // --- END MODIFICATION ---
-    
+    const firstDate = earliestDates.absoluteEarliest; 
     const startDateUTC = firstDate ? new Date(Date.UTC(firstDate.getUTCFullYear(), firstDate.getUTCMonth(), firstDate.getUTCDate())) : new Date(config.earliestComputationDate+'T00:00:00Z');
     const allExpectedDates = getExpectedDateStrings(startDateUTC, endDateUTC);
-
     const passes = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || []; if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
-
     const standardCalcs = calcsInThisPass.filter(c => c.type==='standard'); 
     const metaCalcs = calcsInThisPass.filter(c => c.type==='meta');
-
-    for (const dateStr of allExpectedDates) {
+    const processDate = async (dateStr) => {
         const dateToProcess = new Date(dateStr+'T00:00:00Z');
         try {
-            // --- MODIFIED: Pass earliestDates map to checkRootDataAvailability ---
-            const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates); if (!rootData) continue;
-            // --- END MODIFICATION ---
-            
-            const fetchedDeps = await fetchDependenciesForPass(dateStr, calcsInThisPass, computationManifest, config, dependencies);
-            const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, fetchedDeps, passToRun, dateStr, logger);
+            const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
+            if (!rootData) { logger.log('WARN', `[PassRunner] Skipping ${dateStr} for Pass ${passToRun}: No root data.`);return;}
+            const existingResults = await fetchExistingResults(dateStr, calcsInThisPass, computationManifest, config, dependencies);
+            const { standardCalcsToRun, metaCalcsToRun } = filterCalculations(standardCalcs, metaCalcs, rootData.status, existingResults, passToRun, dateStr, logger);
+            if (standardCalcsToRun.length === 0 && metaCalcsToRun.length === 0) {logger.log('INFO', `[PassRunner] All calcs for ${dateStr} Pass ${passToRun} are already complete. Skipping.`);return;}
             if (standardCalcsToRun.length) await runStandardComputationPass(dateToProcess, standardCalcsToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData);
-            if (metaCalcsToRun.length) await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, fetchedDeps, rootData);
-            logger.log('SUCCESS', `[PassRunner] Completed Pass ${passToRun} for ${dateStr}.`);
-        } catch (err) {
-            logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });
-        }
+            if (metaCalcsToRun.length) await runMetaComputationPass(dateToProcess, metaCalcsToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);
+        } catch (err) {logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack });}
+    };
+    logger.log('INFO', `[PassRunner] Processing ${allExpectedDates.length} total dates in batches of ${PARALLEL_BATCH_SIZE}...`);
+    for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) {
+        const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
+        logger.log('INFO', `[PassRunner] Processing batch ${Math.floor(i / PARALLEL_BATCH_SIZE) + 1}/${Math.ceil(allExpectedDates.length / PARALLEL_BATCH_SIZE)} (Dates: ${batch[0]}...${batch[batch.length-1]})`);
+        await Promise.all(batch.map(dateStr => processDate(dateStr)));
     }
-
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,5 +1,6 @@
 const { FieldPath } = require('@google-cloud/firestore');
-const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs } = require('../utils/data_loader.js');
+// --- MODIFIED: Import streamPortfolioData ---
+const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs, streamPortfolioData } = require('../utils/data_loader.js');
 const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
 
 /** Stage 1: Group manifest by pass number */
@@ -11,8 +12,7 @@ function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[ca
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) {
-        return { canRun: true, missing };
-    }
+        return { canRun: true, missing };}
     for (const dep of calcManifest.rootDataDependencies) {
         if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
         else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
@@ -28,81 +28,38 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger } = dependencies;
     logger.log('INFO', `[PassRunner] Checking root data for ${dateStr}...`);
-
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
     let portfolioRefs = [], insightsData = null, socialData = null, historyRefs = [];
     let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
-
     try {
         const tasks = [];
-
-        if (dateToProcess >= earliestDates.portfolio) {
-            tasks.push(
-                getPortfolioPartRefs(config, dependencies, dateStr).then(res => {
-                    portfolioRefs = res;
-                    hasPortfolio = !!(res?.length);
-                })
-            );
-        }
-
+        if (dateToProcess >= earliestDates.portfolio) 
+            {tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(res => {portfolioRefs = res;hasPortfolio = !!(res?.length);}));}
         if (dateToProcess >= earliestDates.insights) {
-            tasks.push(
-                loadDailyInsights(config, dependencies, dateStr).then(res => {
-                    insightsData = res;
-                    hasInsights = !!res;
-                })
-            );
-        }
-
+            tasks.push(loadDailyInsights(config, dependencies, dateStr).then(res => {insightsData = res;hasInsights = !!res;}));}
         if (dateToProcess >= earliestDates.social) {
-            tasks.push(
-                loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {
-                    socialData = res;
-                    hasSocial = !!res;
-                })
-            );
-        }
-
+            tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {socialData = res;hasSocial = !!res;}));}
         if (dateToProcess >= earliestDates.history) {
-            tasks.push(
-                getHistoryPartRefs(config, dependencies, dateStr).then(res => {
-                    historyRefs = res;
-                    hasHistory = !!(res?.length);
-                })
-            );
-        }
-
+            tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(res => {historyRefs = res;hasHistory = !!(res?.length);}));}
         await Promise.all(tasks);
-
-        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) {
-            logger.log('WARN', `[PassRunner] No root data for ${dateStr}.`);
-            return null;
-        }
-
-        return {
-            portfolioRefs,
-            insightsData,
-            socialData,
-            historyRefs,
-            status: { hasPortfolio, hasInsights, hasSocial, hasHistory }
-        };
-
-    } catch (err) {
-        logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message });
-        return null;
-    }
+        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) {logger.log('WARN', `[PassRunner] No root data for ${dateStr}.`); return null;}
+        return {portfolioRefs, insightsData,socialData,historyRefs,status: { hasPortfolio, hasInsights, hasSocial, hasHistory }};
+    } catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
 }
 
 
 
-/** Stage 4: Fetch computed dependencies from Firestore */
-async function fetchDependenciesForPass(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
+/** --- MODIFIED: Stage 4: Fetch ALL existing computed results for the pass ---
+ * This function now checks for *all* calcs in the pass, not just meta-dependencies,
+ * to enable skipping completed work.
+ */
+async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db, logger }) {
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
-    const requiredDeps = new Set(calcsInPass.filter(c => c.type==='meta'&&c.dependencies).flatMap(c => c.dependencies.map(normalizeName)));
-    if (!requiredDeps.size) return {};
-    logger.log('INFO', `[PassRunner] Fetching ${requiredDeps.size} deps for ${dateStr}...`);
+    const allCalcsInPass = new Set(calcsInPass.map(c => normalizeName(c.name)));
+    if (!allCalcsInPass.size) return {};
+    logger.log('INFO', `[PassRunner] Checking for ${allCalcsInPass.size} existing results for ${dateStr}...`);
     const docRefs = [], depNames = [];
-    for (const calcName of requiredDeps) {
+    for (const calcName of allCalcsInPass) {
         const calcManifest = manifestMap.get(calcName);
         if (!calcManifest) { logger.log('ERROR', `[PassRunner] Missing manifest for ${calcName}`); continue; }
         docRefs.push(db.collection(config.resultsCollection).doc(dateStr).collection(config.resultsSubcollection).doc(calcManifest.category||'unknown').collection(config.computationsSubcollection).doc(calcName));
@@ -113,43 +70,28 @@ async function fetchDependenciesForPass(dateStr, calcsInPass, fullManifest, conf
     return fetched;
 }
 
-/** * --- MODIFIED: Added detailed logging ---
- * Stage 5: Filter calculations based on available root data and dependencies 
+/** --- MODIFIED: Stage 5: Filter calculations to skip completed work ---
  */
-function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, fetchedDeps, passToRun, dateStr, logger) {
+function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, logger) {
     const skipped = new Set();
-
     // Filter Standard Calcs
     const standardCalcsToRun = standardCalcs.filter(c => {
+        if (existingResults[c.name]) {logger.log('TRACE', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Result already exists.`);return false;}
         const { canRun, missing } = checkRootDependencies(c, rootDataStatus);
         if (canRun) return true;
-        
         logger.log('INFO', `[Pass ${passToRun}] Skipping ${c.name} for ${dateStr}. Missing root data: [${missing.join(', ')}]`);
         skipped.add(c.name);
         return false;
     });
-
     // Filter Meta Calcs
     const metaCalcsToRun = metaCalcs.filter(c => {
+        if (existingResults[c.name]) {logger.log('TRACE', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Result already exists.`);skipped.add(c.name);return false;}
         // 1. Check root data
         const { canRun, missing: missingRoot } = checkRootDependencies(c, rootDataStatus);
-        if (!canRun) {
-            logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root data: [${missingRoot.join(', ')}]`);
-            skipped.add(c.name);
-            return false;
-        }
-        
+        if (!canRun) {logger.log('INFO', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing root data: [${missingRoot.join(', ')}]`);skipped.add(c.name);return false;}
         // 2. Check computed dependencies
-        const missingDeps = (c.dependencies || []).map(normalizeName).filter(d => !fetchedDeps[d]);
-        if (missingDeps.length > 0) {
-            logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);
-            skipped.add(c.name);
-            return false;
-        }
-        
-        return true; // All checks passed
-    });
-    
+        const missingDeps = (c.dependencies || []).map(normalizeName).filter(d => !existingResults[d]);
+        if (missingDeps.length > 0) {logger.log('WARN', `[Pass ${passToRun} Meta] Skipping ${c.name} for ${dateStr}. Missing computed deps: [${missingDeps.join(', ')}]`);skipped.add(c.name);return false;} return true;});
     return { standardCalcsToRun, metaCalcsToRun };
 }
 
@@ -165,54 +107,60 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) { const u
     await Promise.all(tasks); return updated;
 }
 
-/** Stage 8: Stream and process data for standard calculations */
-async function streamAndProcess(dateStr, todayRefs, state, passName, config, deps, rootData) { const { logger, calculationUtils } = deps;
+/** * --- MODIFIED: Stage 8: Stream and process data for standard calculations ---
+ * This function now uses an async generator to stream portfolio data
+ * instead of loading it all into memory.
+ */
+async function streamAndProcess(dateStr, state, passName, config, deps, rootData) { 
+    const { logger, calculationUtils } = deps;
     const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights, todayHistoryData, yesterdayHistoryData, yesterdayPortfolios } = rootData;
-    const batchSize=config.partRefBatchSize||10; let firstUser=true;
+    let firstUser=true;
     const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
-    for(let i=0;i<todayRefs.length;i+=batchSize){ const batch=todayRefs.slice(i,i+batchSize); const chunk=await loadDataByRefs(config,deps,batch); for(const uid in chunk){ const p=chunk[uid]; if(!p) continue; const userType=p.PublicPositions?'speculator':'normal'; context.userType=userType; for(const name in state){ const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; let args=[p,null,uid,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; if(isSocialOrInsights&&!firstUser) continue; if(isHistorical){ const pY=yesterdayPortfolios[uid]; if(!pY) continue; args=[p,pY,uid,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; } if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} } firstUser=false; } }
+    for await (const chunk of streamPortfolioData(config, deps, dateStr)) {
+        for(const uid in chunk){ const p=chunk[uid]; if(!p) continue; 
+            const userType=p.PublicPositions?'speculator':'normal'; 
+            context.userType=userType; 
+            for(const name in state){ 
+                const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
+                const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; 
+                let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; 
+                if(isSocialOrInsights&&!firstUser) continue; 
+                if(isHistorical){ const pY=yesterdayPortfolios[uid]; if(!pY) continue; args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,todayHistoryData,yesterdayHistoryData]; } 
+                if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; 
+                try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} } 
+            firstUser=false; 
+        } 
+    }
 }
 
 /** Stage 9: Run standard computations */
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
+    if (calcs.length === 0) return; 
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
     const state = initializeCalculators(calcs, logger);
-    await streamAndProcess(dStr, fullRoot.portfolioRefs, state, passName, config, deps, fullRoot);
-    
-    // --- START: FULL COMMIT LOGIC ---
+    await streamAndProcess(dStr, state, passName, config, deps, fullRoot);
     let success = 0;
     const standardWrites = [];
-    const shardedWrites = {}; // Format: { [collectionName]: { [docId]: data } }
-
+    const shardedWrites = {}; 
     for (const name in state) {
         const calc = state[name];
         if (!calc || typeof calc.getResult !== 'function') continue;
-        
-        try {
-            const result = await Promise.resolve(calc.getResult());
+        try {const result = await Promise.resolve(calc.getResult());
             if (result && Object.keys(result).length > 0) {
-                
-                // Separate sharded data from standard data
                 const standardResult = {};
                 for (const key in result) {
                     if (key.startsWith('sharded_')) {
-                        // This is sharded data, e.g., sharded_user_profitability
-                        // The value is expected to be: { "collection_name": { "doc1": {...}, "doc2": {...} } }
                         const shardedData = result[key];
                         for (const collectionName in shardedData) {
                             if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
-                            // Merge doc data (e.g., combining data for "user_profitability_shard_1")
                             Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
                         }
                     } else {
-                        // This is a standard, single-doc result
                         standardResult[key] = result[key];
                     }
                 }
-
-                // Add standard result to the batch
                 if (Object.keys(standardResult).length > 0) {
                     const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
                         .collection(config.resultsSubcollection).doc(calc.manifest.category)
@@ -220,80 +168,70 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
                     
                     standardWrites.push({ ref: docRef, data: standardResult });
                 }
-                success++; // Mark as success even if only sharded data was produced
+                success++;
             }
         } catch (e) {
             logger.log('ERROR', `getResult failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
         }
     }
-    
-    // Commit standard (non-sharded) writes in chunks
     if (standardWrites.length > 0) {
         await commitBatchInChunks(config, deps, standardWrites, `${passName} Standard ${dStr}`);
     }
-
-    // Commit all sharded writes
-    for (const collectionName in shardedWrites) {
-        const docs = shardedWrites[collectionName];
+    for (const docPath in shardedWrites) { // 'docPath' is the key, e.g., 'user_profile_history_shard_0' or 'social_.../history'
+        const docData = shardedWrites[docPath]; // 'docData' is the object to write, e.g., { profiles: ... }
         const shardedDocWrites = [];
-        for (const docId in docs) {
-            // This assumes docId is the full path for sharded docs, or just the doc ID
-            // Based on user_profitability_tracker, it's just the doc ID.
-            const docRef = deps.db.collection(collectionName).doc(docId);
-            shardedDocWrites.push({ ref: docRef, data: docs[docId] });
+        let docRef;
+        if (docPath.includes('/')) {
+            // Path is absolute, e.g., 'social_prediction_regime_state/history'
+            docRef = deps.db.doc(docPath);
+        } else {
+            // Path is a docId, e.g., 'user_profile_history_shard_0'
+            // We must infer its collection from config.
+            const collection = (docPath.startsWith('user_profile_history')) 
+                ? config.shardedUserProfileCollection // 'user_profile_history'
+                : config.shardedProfitabilityCollection; // Fallback
+            docRef = deps.db.collection(collection).doc(docPath);
+        }
+        // Ensure data is a valid object before pushing
+        if (docData && typeof docData === 'object' && !Array.isArray(docData)) {
+             shardedDocWrites.push({ ref: docRef, data: docData });
+        } else {
+             logger.log('ERROR', `[${passName}] Invalid sharded document data for ${docPath}. Not an object.`, { data: docData });
         }
+        // Commit this single document write (or small batch if logic is changed later)
         if (shardedDocWrites.length > 0) {
-            await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`);
+            // Use the docPath in the operation name for clearer logging
+            await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${docPath} ${dStr}`);
         }
     }
-    // --- END: FULL COMMIT LOGIC ---
-
     logger.log(success === calcs.length ? 'SUCCESS' : 'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
 }
 
 /** Stage 10: Run meta computations */
 async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
+    if (calcs.length === 0) return;
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
-
-    // --- START: FULL COMMIT LOGIC ---
     let success = 0;
     const standardWrites = [];
-    const shardedWrites = {}; // Format: { [collectionName]: { [docId]: data } }
-
+    const shardedWrites = {}; 
     for (const mCalc of calcs) {
         const name = normalizeName(mCalc.name), Cl = mCalc.class;
-        if (typeof Cl !== 'function') {
-            logger.log('ERROR', `Invalid class ${name}`);
-            continue;
-        }
+        if (typeof Cl !== 'function') {logger.log('ERROR', `Invalid class ${name}`);continue;}
         const inst = new Cl();
         try {
-            // Pass the full dependencies object to process()
             const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData }, config, fetchedDeps));
-            
-            if (result && Object.keys(result).length > 0) {
-                
-                // Separate sharded data from standard data
-                const standardResult = {};
-                for (const key in result) {
-                    if (key.startsWith('sharded_')) {
-                        const shardedData = result[key];
-                        for (const collectionName in shardedData) {
-                            if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
-                            Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
-                        }
+            if (result && Object.keys(result).length > 0) {const standardResult = {}; for (const key in result) {
+                    if (key.startsWith('sharded_')) {const shardedData = result[key];for (const collectionName in shardedData) 
+                        {if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};Object.assign(shardedWrites[collectionName], shardedData[collectionName]);}
                     } else {
                         standardResult[key] = result[key];
                     }
                 }
-
-                // Add standard result to the batch
                 if (Object.keys(standardResult).length > 0) {
                     const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
                         .collection(config.resultsSubcollection).doc(mCalc.category)
                         .collection(config.computationsSubcollection).doc(name);
-                    
                     standardWrites.push({ ref: docRef, data: standardResult });
                 }
                 success++;
@@ -302,32 +240,23 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
             logger.log('ERROR', `Meta-calc failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
         }
     }
-
-    // Commit standard (non-sharded) writes in chunks
     if (standardWrites.length > 0) {
         await commitBatchInChunks(config, deps, standardWrites, `${passName} Meta ${dStr}`);
     }
-
-    // Commit all sharded writes
     for (const collectionName in shardedWrites) {
         const docs = shardedWrites[collectionName];
         const shardedDocWrites = [];
         for (const docId in docs) {
-            // Special case for stateful meta-calcs that write to a specific path
             const docRef = docId.includes('/')
-                ? deps.db.doc(docId) // docId is a full path
-                : deps.db.collection(collectionName).doc(docId); // docId is just an ID
-            
+                ? deps.db.doc(docId) 
+                : deps.db.collection(collectionName).doc(docId);
             shardedDocWrites.push({ ref: docRef, data: docs[docId] });
         }
         if (shardedDocWrites.length > 0) {
             await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`);
         }
-    }
-    // --- END: FULL COMMIT LOGIC ---
-    
+    }    
     logger.log(success === calcs.length ? 'SUCCESS' : 'WARN', `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`);
 }
 
-
-module.exports = { groupByPass, checkRootDataAvailability, fetchDependenciesForPass, filterCalculations, runStandardComputationPass, runMetaComputationPass };
+module.exports = { groupByPass, checkRootDataAvailability, fetchExistingResults, filterCalculations, runStandardComputationPass, runMetaComputationPass };

package/functions/computation-system/utils/data_loader.js

@@ -1,6 +1,7 @@
 /**
  * @fileoverview Data loader sub-pipes for the Computation System.
  * REFACTORED: Now stateless and receive dependencies.
+ * --- NEW: Added streamPortfolioData async generator ---
  */
 
 // <<< FIX: REMOVED all top-level 'require' and 'dependencies' lines >>>
@@ -165,6 +166,39 @@ async function getHistoryPartRefs(config, deps, dateString) {
     return allPartRefs;
 }
 
+// --- NEW: Stage 7: Stream portfolio data in chunks ---
+/**
+ * Streams portfolio data in chunks for a given date.
+ * This is an async generator.
+ * @param {object} config - The computation system configuration object.
+ * @param {object} deps - Contains db, logger, calculationUtils.
+ * @param {string} dateString - The date in YYYY-MM-DD format.
+ */
+async function* streamPortfolioData(config, deps, dateString) {
+    const { logger } = deps;
+    const refs = await getPortfolioPartRefs(config, deps, dateString);
+    if (refs.length === 0) {
+        logger.log('WARN', `[streamPortfolioData] No portfolio refs found for ${dateString}. Stream is empty.`);
+        return;
+    }
+
+    // Use the same batch size as loadDataByRefs for consistency
+    const batchSize = config.partRefBatchSize || 50; 
+    
+    logger.log('INFO', `[streamPortfolioData] Streaming ${refs.length} portfolio parts in chunks of ${batchSize}...`);
+
+    for (let i = 0; i < refs.length; i += batchSize) {
+        const batchRefs = refs.slice(i, i + batchSize);
+        // Load one chunk of data
+        const data = await loadDataByRefs(config, deps, batchRefs);
+        // Yield it to the consumer, then release it from memory
+        yield data; 
+    }
+    logger.log('INFO', `[streamPortfolioData] Finished streaming for ${dateString}.`);
+}
+// --- END: Stage 7 ---
+
+
 module.exports = {
     getPortfolioPartRefs,
     loadDataByRefs,
@@ -172,4 +206,5 @@ module.exports = {
     loadDailyInsights,
     loadDailySocialPostInsights,
     getHistoryPartRefs,
+    streamPortfolioData, // <-- EXPORT NEW FUNCTION
 };

package/functions/core/utils/firestore_utils.js

@@ -17,28 +17,17 @@ const { FieldValue, FieldPath } = require('@google-cloud/firestore');
 async function getLatestNormalUserPortfolios(dependencies, config) {
     const { db, logger } = dependencies;
     const { normalUserCollectionName, snapshotsSubCollectionName, partsSubCollectionName } = config;
-
     logger.log('INFO', `[Core Utils] Fetching latest portfolios from ${normalUserCollectionName}...`);
     const allPortfolios = {};
-
     const yesterday = new Date();
     yesterday.setDate(yesterday.getDate() - 1);
     const dateString = yesterday.toISOString().slice(0, 10);
-
     const blockDocs = await db.collection(normalUserCollectionName).listDocuments();
-
     for (const blockDoc of blockDocs) {
         const snapshotDocRef = blockDoc.collection(snapshotsSubCollectionName).doc(dateString);
         const partsCollectionRef = snapshotDocRef.collection(partsSubCollectionName);
         const partsSnapshot = await partsCollectionRef.get();
-
-        if (!partsSnapshot.empty) {
-            partsSnapshot.forEach(partDoc => {
-                Object.assign(allPortfolios, partDoc.data());
-            });
-        }
-    }
-
+        if (!partsSnapshot.empty) {partsSnapshot.forEach(partDoc => {Object.assign(allPortfolios, partDoc.data());});}}
     logger.log('INFO', `[Core Utils] Found ${Object.keys(allPortfolios).length} user portfolios from ${dateString}'s snapshot.`);
     return allPortfolios;
 }
@@ -53,27 +42,14 @@ async function getLatestNormalUserPortfolios(dependencies, config) {
 async function resetProxyLocks(dependencies, config) {
     const { db, logger } = dependencies;
     const { proxyPerformanceDocPath } = config;
-
     logger.log('INFO','[Core Utils] Resetting proxy locks...');
-    try {
-        if (!proxyPerformanceDocPath) {
-            logger.log('ERROR', '[Core Utils] Missing proxyPerformanceDocPath. Cannot reset locks.');
-            return;
-        }
-        
+    try {if (!proxyPerformanceDocPath) {logger.log('ERROR', '[Core Utils] Missing proxyPerformanceDocPath. Cannot reset locks.');return;}
         const perfDocRef = db.doc(proxyPerformanceDocPath);
-
-        await perfDocRef.update({
-            locks: FieldValue.delete() 
-        });
-
+        await perfDocRef.update({locks: FieldValue.delete() });
         logger.log('INFO',`[Core Utils] Proxy locks map reset in ${proxyPerformanceDocPath}.`);
     } catch (error) {
-        if (error.code === 5) { // 5 = NOT_FOUND
-             logger.log('WARN',`[Core Utils] Proxy performance doc or 'locks' field not found at ${proxyPerformanceDocPath}. No locks to reset.`);
-        } else {
-            logger.log('ERROR','[Core Utils] Error resetting proxy locks', { errorMessage: error.message, path: proxyPerformanceDocPath });
-        }
+        if (error.code === 5) { logger.log('WARN',`[Core Utils] Proxy performance doc or 'locks' field not found at ${proxyPerformanceDocPath}. No locks to reset.`);
+        } else {logger.log('ERROR','[Core Utils] Error resetting proxy locks', { errorMessage: error.message, path: proxyPerformanceDocPath });}
     }
 }
 
@@ -89,29 +65,15 @@ async function resetProxyLocks(dependencies, config) {
 async function getBlockCapacities(dependencies, config, userType) {
     const { db, logger } = dependencies;
     const { speculatorBlockCountsDocPath, normalBlockCountsDocPath } = config;
-
     logger.log('INFO',`[Core Utils] Getting block capacities for ${userType}...`);
     try {
-        const docPath = userType === 'speculator'
-            ? speculatorBlockCountsDocPath
-            : normalBlockCountsDocPath;
-            
-        if (!docPath) {
-             logger.log('ERROR', `[Core Utils] Missing block counts document path for ${userType}.`);
-             return {};
-        }
-
+        const docPath = userType === 'speculator'? speculatorBlockCountsDocPath : normalBlockCountsDocPath;
+        if (!docPath) {logger.log('ERROR', `[Core Utils] Missing block counts document path for ${userType}.`);return {};}
         const countsRef = db.doc(docPath);
         const countsDoc = await countsRef.get();
-        if (!countsDoc.exists) {
-            logger.log('WARN',`[Core Utils] Block counts document not found for ${userType} at ${docPath}. Returning empty.`);
-            return {};
-        }
+        if (!countsDoc.exists) {logger.log('WARN',`[Core Utils] Block counts document not found for ${userType} at ${docPath}. Returning empty.`);return {};}
         return countsDoc.data().counts || {};
-    } catch (error) {
-        logger.log('ERROR',`[Core Utils] Error getting block capacities for ${userType}`, { errorMessage: error.message });
-        throw error;
-    }
+    } catch (error) {logger.log('ERROR',`[Core Utils] Error getting block capacities for ${userType}`, { errorMessage: error.message });throw error;}
 }
 
 /**
@@ -127,63 +89,30 @@ async function getBlockCapacities(dependencies, config, userType) {
  */
 async function getExclusionIds(dependencies, config, userType) {
     const { db, logger } = dependencies;
-    const { 
-        specBlocksCollection, 
-        pendingSpecCollection, 
-        invalidSpecCollection,
-        existingNormalUserIds // Get the pre-fetched IDs
-    } = config;
-    
+    const { specBlocksCollection, pendingSpecCollection, invalidSpecCollection,existingNormalUserIds } = config;
     logger.log('INFO',`[Core Utils] Getting exclusion IDs for ${userType} discovery...`);
-    
     const exclusionIds = new Set(existingNormalUserIds);
     logger.log('TRACE', `[Core Utils] Loaded ${exclusionIds.size} existing normal user IDs for exclusion.`);
-    
     const promises = [];
-
     try {
         // 1. Existing Speculators
         const specBlocksRef = db.collection(specBlocksCollection);
-        promises.push(specBlocksRef.get().then(snapshot => {
-            snapshot.forEach(doc => {
-                 const users = doc.data().users || {};
-                 Object.keys(users).forEach(key => exclusionIds.add(key.split('.')[1]));
-            });
-            logger.log('TRACE','[Core Utils] Fetched existing speculator IDs for exclusion.');
-        }));
-
+        promises.push(specBlocksRef.get().then(snapshot => {snapshot.forEach(doc => {const users = doc.data().users || {}; Object.keys(users).forEach(key => exclusionIds.add(key.split('.')[1])); });
+        logger.log('TRACE','[Core Utils] Fetched existing speculator IDs for exclusion.');}));
         // 2. Pending Speculators
-        if (userType === 'speculator') {
-            const pendingRef = db.collection(pendingSpecCollection);
-             promises.push(pendingRef.get().then(snapshot => {
-                snapshot.forEach(doc => {
-                    Object.keys(doc.data().users || {}).forEach(cid => exclusionIds.add(cid));
-                });
-                logger.log('TRACE','[Core Utils] Fetched pending speculator IDs for exclusion.');
-             }));
-        }
-
+        if (userType === 'speculator') {const pendingRef = db.collection(pendingSpecCollection);
+        promises.push(pendingRef.get().then(snapshot => {snapshot.forEach(doc => {Object.keys(doc.data().users || {}).forEach(cid => exclusionIds.add(cid));});
+        logger.log('TRACE','[Core Utils] Fetched pending speculator IDs for exclusion.');})); }
         // 3. Invalid Speculators
         const invalidRef = db.collection(invalidSpecCollection);
-         promises.push(invalidRef.get().then(snapshot => {
-             snapshot.forEach(doc => {
-                 const data = doc.data();
-                 if (data) {
-                    Object.keys(data.users || {}).forEach(cid => exclusionIds.add(cid));
-                 }
-             });
-              logger.log('TRACE','[Core Utils] Fetched invalid speculator IDs for exclusion.');
-         }));
-
+        promises.push(invalidRef.get().then(snapshot => { snapshot.forEach(doc => {const data = doc.data();if (data) {Object.keys(data.users || {}).forEach(cid => exclusionIds.add(cid));}});
+        logger.log('TRACE','[Core Utils] Fetched invalid speculator IDs for exclusion.');}));
         await Promise.all(promises);
         logger.log('INFO',`[Core Utils] Total unique exclusion IDs found: ${exclusionIds.size}`);
         return exclusionIds;
-
-    } catch (error)
- {
+    } catch (error){
         logger.log('ERROR','[Core Utils] Error getting exclusion IDs', { errorMessage: error.message });
-        throw error;
-    }
+        throw error; }
 }
 
 /**
@@ -198,20 +127,12 @@ async function getPrioritizedSpeculators(dependencies, exclusionIds, speculatorI
     const { logger } = dependencies;
     logger.log('INFO','[Core Utils] Scanning normal users for prioritized speculators...');
     const candidates = new Set();
-
     try {
         for (const userId in latestNormalPortfolios) {
             if (exclusionIds.has(userId)) continue;
-
             const portfolio = latestNormalPortfolios[userId];
-            const holdsSpeculatorAsset = portfolio?.AggregatedPositions?.some(p =>
-                speculatorInstrumentSet.has(p.InstrumentID)
-            );
-
-            if (holdsSpeculatorAsset) {
-                candidates.add(userId);
-            }
-        }
+            const holdsSpeculatorAsset = portfolio?.AggregatedPositions?.some(p => speculatorInstrumentSet.has(p.InstrumentID));
+            if (holdsSpeculatorAsset) { candidates.add(userId);}}
         logger.log('INFO',`[Core Utils] Found ${candidates.size} potential prioritized speculators.`);
         return Array.from(candidates);
     } catch (error) {
@@ -235,23 +156,15 @@ async function clearCollection(dependencies, collectionPath, maxBatchSize = 400)
         let query = collectionRef.limit(maxBatchSize);
         let snapshot;
         let deleteCount = 0;
-
         while (true) {
             snapshot = await query.get();
-            if (snapshot.size === 0) {
-                break; 
-            }
-
+            if (snapshot.size === 0) { break; }
             const batch = db.batch();
             snapshot.docs.forEach(doc => batch.delete(doc.ref));
             await batch.commit();
             deleteCount += snapshot.size;
-
-            if (snapshot.size < maxBatchSize) {
-                break; 
-            }
-             query = collectionRef.limit(maxBatchSize); 
-        }
+            if (snapshot.size < maxBatchSize) {break; }
+             query = collectionRef.limit(maxBatchSize); }
         logger.log('SUCCESS', `[Core Utils] Scorched earth complete. Deleted ${deleteCount} documents from ${collectionPath}.`);
     } catch (error) {
         logger.log('ERROR', `[Core Utils] Error clearing collection ${collectionPath}`, { errorMessage: error.message });
@@ -272,17 +185,9 @@ async function clearCollection(dependencies, collectionPath, maxBatchSize = 400)
  */
 async function batchWriteShardedIds(dependencies, config) {
     const { db, logger } = dependencies;
-    const {
-        collectionPath,
-        items,
-        timestamp,
-        maxFieldsPerDoc,
-        maxWritesPerBatch
-    } = config;
-
+    const {collectionPath, items, timestamp, maxFieldsPerDoc, maxWritesPerBatch } = config;
     logger.log('INFO', `[Core Utils] Batch writing ${items.length} IDs to sharded path: ${collectionPath} (max ${maxFieldsPerDoc}/doc, ${maxWritesPerBatch} docs/batch)...`);
     if (items.length === 0) return;
-
     try {
         const collectionRef = db.collection(collectionPath);
         let batch = db.batch();
@@ -290,27 +195,19 @@ async function batchWriteShardedIds(dependencies, config) {
         let currentFieldCount = 0;
         let batchWriteCount = 0;
         let docCounter = 0;
-
         for (let i = 0; i < items.length; i++) {
             const userId = items[i];
             const key = `users.${userId}`; 
             currentDocFields[key] = timestamp;
             currentFieldCount++;
-
             if (currentFieldCount >= maxFieldsPerDoc || i === items.length - 1) {
                 const docRef = collectionRef.doc(`pending_${docCounter}_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`);
                 batch.set(docRef, currentDocFields); 
                 batchWriteCount++;
-
                 currentDocFields = {};
                 currentFieldCount = 0;
                 docCounter++;
-
-                if (batchWriteCount >= maxWritesPerBatch || i === items.length - 1) {
-                    await batch.commit();
-                    batch = db.batch(); 
-                    batchWriteCount = 0;
-                }
+                if (batchWriteCount >= maxWritesPerBatch || i === items.length - 1) { await batch.commit(); batch = db.batch(); batchWriteCount = 0; }
             }
         }
          logger.log('SUCCESS', `[Core Utils] Sharded write complete for ${collectionPath}. Created ${docCounter} documents.`);
@@ -336,24 +233,11 @@ async function getNormalUsersToUpdate(dependencies, config) {
     logger.log('INFO','[Core Utils] Getting normal users to update...');
     const usersToUpdate = [];
     try {
-        const timestampDocRef = db.collection(normalUserCollectionName)
-                                 .doc('timestamps')
-                                 .collection('users')
-                                 .doc(normalUserTimestampsDocId); 
+        const timestampDocRef = db.collection(normalUserCollectionName) .doc('timestamps') .collection('users') .doc(normalUserTimestampsDocId); 
         const timestampDoc = await timestampDocRef.get();
-
-        if (!timestampDoc.exists) {
-            logger.log('WARN',`[Core Utils] Normal user timestamp document not found at ${timestampDocRef.path}.`);
-            return [];
-        }
-
+        if (!timestampDoc.exists) { logger.log('WARN',`[Core Utils] Normal user timestamp document not found at ${timestampDocRef.path}.`); return []; }
         const timestamps = timestampDoc.data().users || {};
-        for (const userId in timestamps) {
-            const lastProcessed = timestamps[userId]?.toDate ? timestamps[userId].toDate() : new Date(0);
-            if (lastProcessed < dateThreshold) {
-                usersToUpdate.push(userId);
-            }
-        }
+        for (const userId in timestamps) { const lastProcessed = timestamps[userId]?.toDate ? timestamps[userId].toDate() : new Date(0); if (lastProcessed < dateThreshold) { usersToUpdate.push(userId); }}
         logger.log('INFO',`[Core Utils] Found ${usersToUpdate.length} normal users to update.`);
         return usersToUpdate;
     } catch (error) {
@@ -374,44 +258,26 @@ async function getNormalUsersToUpdate(dependencies, config) {
 async function getSpeculatorsToUpdate(dependencies, config) {
     const { db, logger } = dependencies;
     const { dateThreshold, gracePeriodThreshold, speculatorBlocksCollectionName } = config;
-
     logger.log('INFO','[Core Utils] Getting speculators to update...');
     const updates = [];
     try {
         const blocksRef = db.collection(speculatorBlocksCollectionName);
         const snapshot = await blocksRef.get();
-
-        if (snapshot.empty) {
-            logger.log('INFO','[Core Utils] No speculator blocks found.');
-            return [];
-        }
-
+        if (snapshot.empty) { logger.log('INFO','[Core Utils] No speculator blocks found.'); return []; }
         snapshot.forEach(doc => {
             const blockData = doc.data();
-
-            // Iterate over the document's top-level keys
             for (const key in blockData) {
-                 // Filter for keys that match the 'users.CID' format
                  if (!key.startsWith('users.')) continue; 
-
                  const userId = key.split('.')[1];
-                 if (!userId) continue; // Safety check
-
-                 const userData = blockData[key]; // Get the user's map
-
+                 if (!userId) continue; 
+                 const userData = blockData[key]; 
                  const lastVerified = userData.lastVerified?.toDate ? userData.lastVerified.toDate() : new Date(0);
                  const lastHeld = userData.lastHeldSpeculatorAsset?.toDate ? userData.lastHeldSpeculatorAsset.toDate() : new Date(0);
-
-                 if (lastVerified < dateThreshold && lastHeld > gracePeriodThreshold) {
-                     if (userData.instruments && Array.isArray(userData.instruments)) {
-                         userData.instruments.forEach(instrumentId => {
-                             updates.push({ userId, instrumentId });
-                         });
+                 if (lastVerified < dateThreshold && lastHeld > gracePeriodThreshold) { if (userData.instruments && Array.isArray(userData.instruments)) { userData.instruments.forEach(instrumentId => { updates.push({ userId, instrumentId }); });
                      }
                  }
             }
         });
-
         logger.log('INFO',`[Core Utils] Found ${updates.length} speculator user/instrument pairs to update.`);
         return updates;
     } catch (error) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.132",
+  "version": "1.0.134",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [