bulltrackers-module 1.0.151 → 1.0.153

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,26 +1,22 @@
 const { FieldPath } = require('@google-cloud/firestore');
-// --- MODIFIED: Import streamPortfolioData ---
 const { getPortfolioPartRefs, loadFullDayMap, loadDataByRefs, loadDailyInsights, loadDailySocialPostInsights, getHistoryPartRefs, streamPortfolioData, streamHistoryData } = require('../utils/data_loader.js');
 const { normalizeName, commitBatchInChunks } = require('../utils/utils.js');
-// --- CHANGED: We only need batchStoreSchemas now ---
-const { batchStoreSchemas } = require('../utils/schema_capture');
+const { batchStoreSchemas } = require('../utils/schema_capture.js');
 
 /** Stage 1: Group manifest by pass number */
 function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
 /** * --- MODIFIED: Returns detailed missing dependencies for logging ---
  * Stage 2: Check root data dependencies for a calc
- * --- THIS FUNCTION IS NOW MORE GRANULAR ---
  */
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     if (!calcManifest.rootDataDependencies || !calcManifest.rootDataDependencies.length) { return { canRun: true, missing };}
     for (const dep of calcManifest.rootDataDependencies) {
-        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
-        else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
-        else if (dep === 'social' && !rootDataStatus.hasSocial) missing.push('social');
-        else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
-    }
+        if (dep === 'portfolio'     && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        else if (dep === 'insights' && !rootDataStatus.hasInsights)  missing.push('insights');
+        else if (dep === 'social'   && !rootDataStatus.hasSocial)    missing.push('social');
+        else if (dep === 'history'  && !rootDataStatus.hasHistory)   missing.push('history'); }
     return { canRun: missing.length === 0, missing };
 }   
 
@@ -40,7 +36,7 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
         if (dateToProcess >= earliestDates.social)    {tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(res => {socialData = res;hasSocial = !!res;}));}
         if (dateToProcess >= earliestDates.history)   {tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(res => {historyRefs = res;hasHistory = !!(res?.length);}));}
         await Promise.all(tasks);
-        logger.log('INFO', `[PassRunner] Data availability for ${dateStr}: P:${hasPortfolio}, I:${insightsData}, S:${socialData}, H:${hasHistory}`);
+        logger.log('INFO', `[PassRunner] Data availability for ${dateStr}: P:${hasPortfolio}, I:${hasInsights}, S:${hasSocial}, H:${hasHistory}`); 
         if (!(hasPortfolio || hasInsights || hasSocial || hasHistory)) { logger.log('WARN', `[PassRunner] No root data at all for ${dateStr}.`); return null; }
         return { portfolioRefs, todayInsights: insightsData, todaySocialPostInsights: socialData, historyRefs, status: { hasPortfolio, hasInsights, hasSocial, hasHistory } };
     } catch (err) { logger.log('ERROR', `[PassRunner] Error checking data for ${dateStr}`, { errorMessage: err.message }); return null; }
@@ -77,9 +73,9 @@ function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingRe
         const dependencies = calc.rootDataDependencies || [];
         for (const dep of dependencies) {
             if (dep === 'portfolio' && earliestDates.portfolio > earliestRunDate) earliestRunDate = earliestDates.portfolio;
-            if (dep === 'history' && earliestDates.history > earliestRunDate) earliestRunDate = earliestDates.history;
-            if (dep === 'social' && earliestDates.social > earliestRunDate) earliestRunDate = earliestDates.social;
-            if (dep === 'insights' && earliestDates.insights > earliestRunDate) earliestRunDate = earliestDates.insights;
+            if (dep === 'history'   && earliestDates.history   > earliestRunDate) earliestRunDate = earliestDates.history;
+            if (dep === 'social'    && earliestDates.social    > earliestRunDate) earliestRunDate = earliestDates.social;
+            if (dep === 'insights'  && earliestDates.insights  > earliestRunDate) earliestRunDate = earliestDates.insights;
         }
         if (calc.isHistorical && earliestRunDate.getTime() > 0) { earliestRunDate.setUTCDate(earliestRunDate.getUTCDate() + 1); }
         return earliestRunDate;
@@ -103,14 +99,16 @@ function initializeCalculators(calcs, logger) { const state = {}; for (const c o
 
 /** * Stage 7: Load historical data required for calculations 
  */
-// --- MODIFIED: Stage 7: Load ONLY non-streaming historical data ---
 async function loadHistoricalData(date, calcs, config, deps, rootData) { 
     const { logger }              = deps;
     const updated                 = {...rootData};
     const tasks                   = [];
     const needsYesterdayInsights  = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('insights'));
     const needsYesterdaySocial    = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('social'));
-
+    const needsYesterdayPortfolio = calcs.some(c => c.isHistorical && c.rootDataDependencies.includes('portfolio'));
+    const prev = new Date(date);
+    prev.setUTCDate(prev.getUTCDate() - 1);
+    const prevStr = prev.toISOString().slice(0, 10);
     if(needsYesterdayInsights) {
         tasks.push((async()=>{ const prev=new Date(date); prev.setUTCDate(prev.getUTCDate()-1); const prevStr=prev.toISOString().slice(0,10); 
         logger.log('INFO', `[PassRunner] Loading YESTERDAY insights data for ${prevStr}`);
@@ -120,240 +118,120 @@ async function loadHistoricalData(date, calcs, config, deps, rootData) {
         logger.log('INFO', `[PassRunner] Loading YESTERDAY social data for ${prevStr}`);
         updated.yesterdaySocialPostInsights=await loadDailySocialPostInsights(config,deps,prevStr); })());}
     
+    if(needsYesterdayPortfolio) {
+        tasks.push((async()=>{ 
+            logger.log('INFO', `[PassRunner] Getting YESTERDAY portfolio refs for ${prevStr}`);
+            updated.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr); 
+        })());
+    }
     await Promise.all(tasks); 
     return updated; 
 }
 
 /**
  * --- REFACTORED: Stage 8: Stream and process data for standard calculations ---
- * This function now streams today's portfolios, yesterday's portfolios, 
- * and today's history data in parallel to avoid OOM errors.
- * It loads chunks of all three streams, processes UIDs found in the
- * main (today's portfolio) stream, and then deletes processed users
- * from the historical maps to free memory.
  */
-async function streamAndProcess(dateStr, state, passName, config, deps, rootData) { 
+async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs) { 
     const { logger, calculationUtils } = deps;
     const { todayInsights, yesterdayInsights, todaySocialPostInsights, yesterdaySocialPostInsights } = rootData;
     const calcsThatStreamPortfolio = Object.values(state).filter(calc => calc && calc.manifest && (calc.manifest.rootDataDependencies.includes('portfolio') || calc.manifest.category === 'speculators'));
     const context={instrumentMappings:(await calculationUtils.loadInstrumentMappings()).instrumentToTicker, sectorMapping:(await calculationUtils.loadInstrumentMappings()).instrumentToSector, todayDateStr:dateStr, dependencies:deps, config};
     let firstUser=true; 
-
-    for(const name in state){ 
-        const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
-        const cat=calc.manifest.category;
-        if(cat==='socialPosts'||cat==='insights') {
-            if (firstUser) {
-                logger.log('INFO', `[${passName}] Running non-streaming calc: ${name}`);
-                let args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,null,null];
-                if(calc.manifest.isHistorical) {
-                    args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,null,null];
-                }
-                try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} (non-stream)`,{err:e.message});}
-            }
-        }
-    }
-
-    if (calcsThatStreamPortfolio.length === 0) {
-        logger.log('INFO', `[${passName}] No portfolio-streaming calcs to run for ${dateStr}. Skipping stream.`);
-        return; 
-    }
-
+    for(const name in state){  const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
+    const cat=calc.manifest.category;
+    if(cat==='socialPosts'||cat==='insights') {
+    if (firstUser) {
+    logger.log('INFO', `[${passName}] Running non-streaming calc: ${name}`);
+    let args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,null,null];
+    if(calc.manifest.isHistorical) { args=[null,null,null,{...context, userType: 'n/a'},todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,null,null]; }
+    try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} (non-stream)`,{err:e.message});} } } }
+    if (calcsThatStreamPortfolio.length === 0) { logger.log('INFO', `[${passName}] No portfolio-streaming calcs to run for ${dateStr}. Skipping stream.`); return;  }
     logger.log('INFO', `[${passName}] Streaming portfolio & historical data for ${calcsThatStreamPortfolio.length} calcs...`);
-
     const prevDate = new Date(dateStr + 'T00:00:00Z');
     prevDate.setUTCDate(prevDate.getUTCDate() - 1);
     const prevDateStr = prevDate.toISOString().slice(0, 10);
-
     const needsYesterdayPortfolio = Object.values(state).some(c => c && c.manifest.isHistorical && c.manifest.rootDataDependencies.includes('portfolio'));
     const needsTodayHistory = Object.values(state).some(c => c && c.manifest.rootDataDependencies.includes('history'));
-
-    const yP_iterator = needsYesterdayPortfolio ? streamPortfolioData(config, deps, prevDateStr) : null;
-    const hT_iterator = needsTodayHistory ? streamHistoryData(config, deps, dateStr) : null;
-
+    const yP_iterator = needsYesterdayPortfolio ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
+    const hT_iterator = needsTodayHistory ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
     let yesterdayPortfolios = {};
     let todayHistoryData = {};
-
-    if (yP_iterator) { 
-        Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); 
-        logger.log('INFO', `[${passName}] Loaded first chunk of yesterday's portfolios.`);
-    }
-    if (hT_iterator) { 
-        Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); 
-        logger.log('INFO', `[${passName}] Loaded first chunk of today's history.`);
-    }
-
-    for await (const chunk of streamPortfolioData(config, deps, dateStr)) {
-        
-        if (yP_iterator) { 
-            Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); 
-        }
-        if (hT_iterator) { 
-            Object.assign(todayHistoryData, (await hT_iterator.next()).value || {}); 
-        }
-
-        for(const uid in chunk){ 
-            const p = chunk[uid]; if(!p) continue; 
-            const userType=p.PublicPositions?'speculator':'normal'; 
-            context.userType=userType; 
-
-            const pY = yesterdayPortfolios[uid] || null; // Yesterday's Portfolio
-            const hT = todayHistoryData[uid] || null;    // Today's History
-            // (Note: yesterdayHistoryData (hY) would require another stream if needed)
-
-            for(const name in state){ 
-                const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
-                const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; 
-                
-                if(isSocialOrInsights) continue; // Skip non-streaming calcs
-
-                let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,hT,null]; 
-                
-                if(isHistorical){ 
-                    if(!pY && (cat !== 'behavioural' && name !== 'historical-performance-aggregator')) continue; 
-                    args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,hT,null]; 
-                } 
-
-                if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; 
-                
-                try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});} 
-            } 
-            firstUser=false; 
-
-            if (pY) { delete yesterdayPortfolios[uid]; }
-            if (hT) { delete todayHistoryData[uid]; }
-        } 
-    }
+    if (yP_iterator) {  Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {});  logger.log('INFO', `[${passName}] Loaded first chunk of yesterday's portfolios.`); }
+    if (hT_iterator) {  Object.assign(todayHistoryData, (await hT_iterator.next()).value || {});  logger.log('INFO', `[${passName}] Loaded first chunk of today's history.`); }
+    for await (const chunk of streamPortfolioData(config, deps, dateStr, portfolioRefs)) {
+    if (yP_iterator) {  Object.assign(yesterdayPortfolios, (await yP_iterator.next()).value || {}); }
+    if (hT_iterator) {  Object.assign(todayHistoryData, (await hT_iterator.next()).value || {});  }
+    for(const uid in chunk){ 
+    const p = chunk[uid]; if(!p) continue; 
+    const userType=p.PublicPositions?'speculator':'normal'; 
+    context.userType=userType; 
+    const pY = yesterdayPortfolios[uid] || null; 
+    const hT = todayHistoryData[uid] || null;   
+    for(const name in state){ 
+    const calc=state[name]; if(!calc||typeof calc.process!=='function') continue; 
+    const cat=calc.manifest.category, isSocialOrInsights=cat==='socialPosts'||cat==='insights', isHistorical=calc.manifest.isHistorical, isSpec=cat==='speculators'; 
+    if(isSocialOrInsights) continue; 
+    let args=[p,null,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,hT,null]; 
+    if(isHistorical){  if(!pY && (cat !== 'behavioural' && name !== 'historical-performance-aggregator')) continue;  args=[p,pY,uid,context,todayInsights,yesterdayInsights,todaySocialPostInsights,yesterdaySocialPostInsights,hT,null]; } 
+    if((userType==='normal'&&isSpec)||(userType==='speculator'&&!isSpec&&name!=='users-processed')) continue; 
+    try{ await Promise.resolve(calc.process(...args)); } catch(e){logger.log('WARN',`Process error ${name} for ${uid}`,{err:e.message});}  } 
+    firstUser=false; 
+    if (pY) { delete yesterdayPortfolios[uid]; }
+    if (hT) { delete todayHistoryData[uid]; } }  }
     logger.log('INFO', `[${passName}] Finished streaming data for ${dateStr}.`);
 }
 
 /** Stage 9: Run standard computations */
 async function runStandardComputationPass(date, calcs, passName, config, deps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    if (calcs.length === 0) {
-        logger.log('INFO', `[${passName}] No standard calcs to run for ${dStr} after filtering.`);
-        return;
-    }
+    if (calcs.length === 0) { logger.log('INFO', `[${passName}] No standard calcs to run for ${dStr} after filtering.`); return; }
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
     const state = initializeCalculators(calcs, logger);
-    await streamAndProcess(dStr, state, passName, config, deps, fullRoot);
-        
+    await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs);
     let success = 0;
     const failedCalcs = [];
     const standardWrites = [];
     const shardedWrites = {}; 
-    
-    // === NEW: Collect schemas ===
     const schemasToStore = [];
-    
-    for (const name in state) {
-        const calc = state[name];
-        if (!calc || typeof calc.getResult !== 'function') continue;
-        try {
-            const result = await Promise.resolve(calc.getResult());
-            if (result && Object.keys(result).length > 0) {
-                const standardResult = {};
-                for (const key in result) {
-                    if (key.startsWith('sharded_')) {
-                        const shardedData = result[key];
-                        for (const collectionName in shardedData) {
-                            if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
-                            Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
-                        }
-                    } else {
-                        standardResult[key] = result[key];
-                    }
-                }
-                if (Object.keys(standardResult).length > 0) {
-                    const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
-                        .collection(config.resultsSubcollection).doc(calc.manifest.category)
-                        .collection(config.computationsSubcollection).doc(name);
-                    standardWrites.push({ ref: docRef, data: standardResult });
-                }
-                
-                // === CHANGED: Capture static schema ===
-                const calcClass = calc.manifest.class;
-                let staticSchema = null;
-                if (calcClass && typeof calcClass.getSchema === 'function') {
-                    try {
-                        staticSchema = calcClass.getSchema();
-                    } catch (e) {
-                        logger.log('WARN', `[SchemaCapture] Failed to get static schema for ${name}`, { err: e.message });
-                    }
-                } else {
-                    logger.log('TRACE', `[SchemaCapture] No static schema found for ${name}. Skipping manifest entry.`);
-                }
-
-                if (staticSchema) {
-                    schemasToStore.push({
-                        name,
-                        category: calc.manifest.category,
-                        schema: staticSchema, // <-- Use the static schema
-                        metadata: {
-                            isHistorical: calc.manifest.isHistorical || false,
-                            dependencies: calc.manifest.dependencies || [],
-                            rootDataDependencies: calc.manifest.rootDataDependencies || [],
-                            pass: calc.manifest.pass,
-                            type: calc.manifest.type || 'standard'
-                        }
-                    });
-                }
-                // === END CHANGED SECTION ===
-                
-                success++;
-            }
-        } catch (e) {
-            logger.log('ERROR', `getResult failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
-            failedCalcs.push(name);
-        }
-    }
-    
-    // === NEW: Store schemas asynchronously (don't block computation) ===
-    if (schemasToStore.length > 0) {
-        // This function is now imported from the simplified schema_capture.js
-        batchStoreSchemas(deps, config, schemasToStore).catch(err => {
-            logger.log('WARN', '[SchemaCapture] Non-blocking schema storage failed', {
-                errorMessage: err.message
-            });
-        });
-    }
-        
-    if (standardWrites.length > 0) {
-        await commitBatchInChunks(config, deps, standardWrites, `${passName} Standard ${dStr}`);
-    }
-    
+    for (const name in state) { const calc = state[name];
+    if (!calc || typeof calc.getResult !== 'function') continue;
+    try { const result = await Promise.resolve(calc.getResult());
+    if (result && Object.keys(result).length > 0) {
+    const standardResult = {};
+    for (const key in result) {
+    if (key.startsWith('sharded_')) {
+    const shardedData = result[key];
+    for (const collectionName in shardedData) {
+    if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
+    Object.assign(shardedWrites[collectionName], shardedData[collectionName]); }
+    } else { standardResult[key] = result[key]; }}
+    if (Object.keys(standardResult).length > 0) {
+    const docRef = deps.db.collection(config.resultsCollection).doc(dStr) .collection(config.resultsSubcollection).doc(calc.manifest.category) .collection(config.computationsSubcollection).doc(name);
+    standardWrites.push({ ref: docRef, data: standardResult });}
+    const calcClass = calc.manifest.class;
+    let staticSchema = null;
+    if (calcClass && typeof calcClass.getSchema === 'function') {
+    try { staticSchema = calcClass.getSchema(); } catch (e) { logger.log('WARN', `[SchemaCapture] Failed to get static schema for ${name}`, { err: e.message }); }
+    } else { logger.log('TRACE', `[SchemaCapture] No static schema found for ${name}. Skipping manifest entry.`); }
+    if (staticSchema) {
+    schemasToStore.push({ name, category: calc.manifest.category, schema: staticSchema, metadata: { isHistorical: calc.manifest.isHistorical || false, dependencies: calc.manifest.dependencies || [], rootDataDependencies: calc.manifest.rootDataDependencies || [], pass: calc.manifest.pass, type: calc.manifest.type || 'standard' } }); }
+    success++; } } catch (e) { logger.log('ERROR', `getResult failed ${name} for ${dStr}`, { err: e.message, stack: e.stack }); failedCalcs.push(name); } }
+    if (schemasToStore.length > 0) { batchStoreSchemas(deps, config, schemasToStore).catch(err => { logger.log('WARN', '[SchemaCapture] Non-blocking schema storage failed', { errorMessage: err.message }); });}
+    if (standardWrites.length > 0) { await commitBatchInChunks(config, deps, standardWrites, `${passName} Standard ${dStr}`); }
     for (const docPath in shardedWrites) { 
-        const docData = shardedWrites[docPath]; 
-        const shardedDocWrites = [];
-        let docRef;
-        if (docPath.includes('/')) {
-            docRef = deps.db.doc(docPath);
-        } else {
-            const collection = (docPath.startsWith('user_profile_history')) 
-                ? config.shardedUserProfileCollection
-                : config.shardedProfitabilityCollection;
-            docRef = deps.db.collection(collection).doc(docPath);
-        }
-        if (docData && typeof docData === 'object' && !Array.isArray(docData)) {
-             shardedDocWrites.push({ ref: docRef, data: docData });
-        } else {
-             logger.log('ERROR', `[${passName}] Invalid sharded document data for ${docPath}. Not an object.`, { data: docData });
-        }
-        if (shardedDocWrites.length > 0) {
-            await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${docPath} ${dStr}`);
-        }
-    }
-    
+    const docData = shardedWrites[docPath]; 
+    const shardedDocWrites = [];
+    let docRef;
+    if (docPath.includes('/')) { docRef = deps.db.doc(docPath); } else {
+    const collection = (docPath.startsWith('user_profile_history'))  ? config.shardedUserProfileCollection : config.shardedProfitabilityCollection;
+    docRef = deps.db.collection(collection).doc(docPath); }
+    if (docData && typeof docData === 'object' && !Array.isArray(docData)) {shardedDocWrites.push({ ref: docRef, data: docData });
+    } else { logger.log('ERROR', `[${passName}] Invalid sharded document data for ${docPath}. Not an object.`, { data: docData }); }
+    if (shardedDocWrites.length > 0) { await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${docPath} ${dStr}`); } }
     const logMetadata = {};
-    if (failedCalcs.length > 0) {
-        logMetadata.failedComputations = failedCalcs;
-    }
-    logger.log(
-        success === calcs.length ? 'SUCCESS' : 'WARN', 
-        `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`,
-        logMetadata
-    );
+    if (failedCalcs.length > 0) { logMetadata.failedComputations = failedCalcs; }
+    logger.log(success === calcs.length ? 'SUCCESS' : 'WARN',  `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`, logMetadata );
 }
 
 /**
@@ -361,116 +239,44 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
  */
 async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
     const dStr = date.toISOString().slice(0, 10), logger = deps.logger;
-    if (calcs.length === 0) {
-        logger.log('INFO', `[${passName}] No meta calcs to run for ${dStr} after filtering.`);
-        return;
-    }
+    if (calcs.length === 0) { logger.log('INFO', `[${passName}] No meta calcs to run for ${dStr} after filtering.`); return; }
     logger.log('INFO', `[${passName}] Running ${dStr} with ${calcs.length} calcs.`);
     const fullRoot = await loadHistoricalData(date, calcs, config, deps, rootData);
-        
     let success = 0;
     const failedCalcs = [];
     const standardWrites = [];
     const shardedWrites = {}; 
-    
-    // === NEW: Collect schemas ===
     const schemasToStore = [];
-    
     for (const mCalc of calcs) {
-        const name = normalizeName(mCalc.name), Cl = mCalc.class;
-        if (typeof Cl !== 'function') {
-            logger.log('ERROR', `Invalid class ${name}`);
-            failedCalcs.push(name);
-            continue;
-        }
-        const inst = new Cl();
-        try {
-            const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData: fullRoot }, config, fetchedDeps));
-            if (result && Object.keys(result).length > 0) {
-                const standardResult = {};
-                for (const key in result) {
-                    if (key.startsWith('sharded_')) {
-                        const shardedData = result[key];
-                        for (const collectionName in shardedData) {
-                            if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {};
-                            Object.assign(shardedWrites[collectionName], shardedData[collectionName]);
-                        }
-                    } else {
-                        standardResult[key] = result[key];
-                    }
-                }
-                if (Object.keys(standardResult).length > 0) {
-                    const docRef = deps.db.collection(config.resultsCollection).doc(dStr)
-                        .collection(config.resultsSubcollection).doc(mCalc.category)
-                        .collection(config.computationsSubcollection).doc(name);
-                    standardWrites.push({ ref: docRef, data: standardResult });
-                }
-                
-                // === CHANGED: Capture static schema ===
-                const calcClass = mCalc.class;
-                let staticSchema = null;
-                if (calcClass && typeof calcClass.getSchema === 'function') {
-                    try {
-                        staticSchema = calcClass.getSchema();
-                    } catch (e) {
-                        logger.log('WARN', `[SchemaCapture] Failed to get static schema for ${name}`, { err: e.message });
-                    }
-                } else {
-                    logger.log('TRACE', `[SchemaCapture] No static schema found for ${name}. Skipping manifest entry.`);
-                }
-
-                if (staticSchema) {
-                    schemasToStore.push({
-                        name,
-                        category: mCalc.category,
-                        schema: staticSchema, // <-- Use the static schema
-                        metadata: {
-                            isHistorical: mCalc.isHistorical || false,
-                            dependencies: mCalc.dependencies || [],
-                            rootDataDependencies: mCalc.rootDataDependencies || [],
-                            pass: mCalc.pass,
-                            type: 'meta'
-                        }
-                    });
-                }
-                // === END CHANGED SECTION ===
-                
-                success++;
-            }
-        } catch (e) {
-            logger.log('ERROR', `Meta-calc failed ${name} for ${dStr}`, { err: e.message, stack: e.stack });
-            failedCalcs.push(name);
-        }
-    }
-    
-    // === NEW: Store schemas asynchronously ===
-    if (schemasToStore.length > 0) {
-        // This function is now imported from the simplified schema_capture.js
-        batchStoreSchemas(deps, config, schemasToStore).catch(err => {
-            logger.log('WARN', '[SchemaCapture] Non-blocking schema storage failed', {
-                errorMessage: err.message
-            });
-        });
-    }
-    
-    if (standardWrites.length > 0) {
-        await commitBatchInChunks(config, deps, standardWrites, `${passName} Meta ${dStr}`);
-    }
-    
+    const name = normalizeName(mCalc.name), Cl = mCalc.class;
+    if (typeof Cl !== 'function') { logger.log('ERROR', `Invalid class ${name}`); failedCalcs.push(name); continue; }
+    const inst = new Cl();
+    try { const result = await Promise.resolve(inst.process(dStr, { ...deps, rootData: fullRoot }, config, fetchedDeps));
+    if (result && Object.keys(result).length > 0) {
+    const standardResult = {};
+    for (const key in result) {
+    if (key.startsWith('sharded_')) { const shardedData = result[key]; for (const collectionName in shardedData) {
+    if (!shardedWrites[collectionName]) shardedWrites[collectionName] = {}; Object.assign(shardedWrites[collectionName], shardedData[collectionName]); }
+    } else { standardResult[key] = result[key]; } }
+    if (Object.keys(standardResult).length > 0) {
+    const docRef = deps.db.collection(config.resultsCollection).doc(dStr) .collection(config.resultsSubcollection).doc(mCalc.category) .collection(config.computationsSubcollection).doc(name);
+    standardWrites.push({ ref: docRef, data: standardResult }); }
+    const calcClass = mCalc.class;
+    let staticSchema = null;
+    if (calcClass && typeof calcClass.getSchema === 'function') {
+    try { staticSchema = calcClass.getSchema();
+    } catch (e) { logger.log('WARN', `[SchemaCapture] Failed to get static schema for ${name}`, { err: e.message }); }
+    } else { logger.log('TRACE', `[SchemaCapture] No static schema found for ${name}. Skipping manifest entry.`); }
+    if (staticSchema) { schemasToStore.push({ name, category: mCalc.category, schema: staticSchema, metadata: { isHistorical: mCalc.isHistorical || false, dependencies: mCalc.dependencies || [], rootDataDependencies: mCalc.rootDataDependencies || [], pass: mCalc.pass, type: 'meta' } }); }
+    success++; }
+    } catch (e) { logger.log('ERROR', `Meta-calc failed ${name} for ${dStr}`, { err: e.message, stack: e.stack }); failedCalcs.push(name); } }
+    if (schemasToStore.length > 0) { batchStoreSchemas(deps, config, schemasToStore).catch(err => { logger.log('WARN', '[SchemaCapture] Non-blocking schema storage failed', { errorMessage: err.message }); }); }
+    if (standardWrites.length > 0) { await commitBatchInChunks(config, deps, standardWrites, `${passName} Meta ${dStr}`);}
     for (const collectionName in shardedWrites) {
-        const docs = shardedWrites[collectionName];
-        const shardedDocWrites = [];
-        for (const docId in docs) {
-            const docRef = docId.includes('/')
-                ? deps.db.doc(docId) 
-                : deps.db.collection(collectionName).doc(docId);
-            shardedDocWrites.push({ ref: docRef, data: docs[docId] });
-        }
-        if (shardedDocWrites.length > 0) {
-            await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`);
-        }
-    }    
-
+    const docs = shardedWrites[collectionName];
+    const shardedDocWrites = [];
+    for (const docId in docs) { const docRef = docId.includes('/') ? deps.db.doc(docId) : deps.db.collection(collectionName).doc(docId); shardedDocWrites.push({ ref: docRef, data: docs[docId] }); }
+    if (shardedDocWrites.length > 0) { await commitBatchInChunks(config, deps, shardedDocWrites, `${passName} Sharded ${collectionName} ${dStr}`); } }    
     const logMetadata = {};
     if (failedCalcs.length > 0) { logMetadata.failedComputations = failedCalcs; }
     logger.log( success === calcs.length ? 'SUCCESS' : 'WARN',  `[${passName}] Completed ${dStr}. Success: ${success}/${calcs.length}`, logMetadata );