bulltrackers-module 1.0.172 → 1.0.174

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,8 +1,6 @@
 /**
  * FIXED: computation_pass_runner.js
- * Now calculates earliest date PER CALCULATION, not per pass
- * --- MODIFIED: Passes 'existingResults' to 'runStandardComputationPass'
- * to replicate the test harness's 7-argument signature. ---
+ * V3: Fetches Previous Day's Results to enable State Persistence.
  */
 
 const { groupByPass, checkRootDataAvailability, fetchExistingResults, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
@@ -11,63 +9,98 @@ const PARALLEL_BATCH_SIZE = 7;
 
 async function runComputationPass(config, dependencies, computationManifest) {
     const { logger } = dependencies;
-    const passToRun = String(config.COMPUTATION_PASS_TO_RUN);
-    if (!passToRun) return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
+    const passToRun  = String(config.COMPUTATION_PASS_TO_RUN);
+    if (!passToRun) 
+        return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
+    
     logger.log('INFO', `🚀 Starting PASS ${passToRun}...`);
 
-    const earliestDates = { portfolio: new Date('2025-09-25T00:00:00Z'), history:   new Date('2025-11-05T00:00:00Z'), social:    new Date('2025-10-30T00:00:00Z'), insights:  new Date('2025-08-26T00:00:00Z') };
+    const earliestDates            = { portfolio: new Date('2025-09-25T00:00:00Z'), history: new Date('2025-11-05T00:00:00Z'), social: new Date('2025-10-30T00:00:00Z'), insights: new Date('2025-08-26T00:00:00Z') };
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
-    logger.log('INFO', `Hardcoded earliest dates: ${Object.entries(earliestDates).map(([k,v]) => `${k}=${v.toISOString().slice(0,10)}`).join(', ')}`);
 
-    const passes = groupByPass(computationManifest);
+    const passes          = groupByPass(computationManifest);
     const calcsInThisPass = passes[passToRun] || [];
-    if (!calcsInThisPass.length) return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
+
+    if (!calcsInThisPass.length) 
+        return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
+    
     const calcEarliestDates = new Map();
-    for (const calc of calcsInThisPass) { const deps = calc.rootDataDependencies || []; if (!deps.length) { calcEarliestDates.set(calc.name, earliestDates.absoluteEarliest); continue; }
-    const latestDep = new Date(Math.max(...deps.map(d => earliestDates[d]?.getTime() || 0)));
-    const calcDate = calc.isHistorical ? new Date(latestDep.getTime() + 86400000) : latestDep; // +1 day if historical
-    calcEarliestDates.set(calc.name, calcDate);
-    logger.log('TRACE', `[PassRunner] ${calc.name}: earliest=${calcDate.toISOString().slice(0,10)}`); }
+
+    for (const calc of calcsInThisPass) { 
+        const deps = calc.rootDataDependencies || []; 
+
+        if (!deps.length) 
+            { calcEarliestDates.set(calc.name, earliestDates.absoluteEarliest); continue; }
+
+        const latestDep = new Date(Math.max(...deps.map(d => earliestDates[d]?.getTime() || 0)));
+        const calcDate = calc.isHistorical 
+            ? new Date(latestDep.getTime() + 86400000) 
+            : latestDep; 
+        
+            calcEarliestDates.set(calc.name, calcDate);
+    }
 
     const passEarliestDate = new Date(Math.min(...Array.from(calcEarliestDates.values()).map(d => d.getTime())));
-    logger.log('INFO', `[PassRunner] Pass ${passToRun} analysis: Total calcs=${calcsInThisPass.length}, can start from ${passEarliestDate.toISOString().slice(0,10)}`);
-    const endDateUTC = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
+    const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
-    const standardCalcs = calcsInThisPass.filter(c => c.type === 'standard');
-    const metaCalcs = calcsInThisPass.filter(c => c.type === 'meta');
-    const checkDeps = (calc, rootData, existingResults, dateToProcess) => { if (existingResults[calc.name]) return false;
-    const earliest = calcEarliestDates.get(calc.name);
-    if (earliest && dateToProcess < earliest) return false;
+    const standardCalcs    = calcsInThisPass.filter(c => c.type === 'standard');
+    const metaCalcs        = calcsInThisPass.filter(c => c.type === 'meta');
+
+    const checkDeps = (calc, rootData, existingResults, dateToProcess) => { 
+        if (existingResults[calc.name]) 
+            return false;
 
-    const missingRoot = (calc.rootDataDependencies || []).filter(dep => !rootData.status[`has${dep[0].toUpperCase() + dep.slice(1)}`]);
-    if (missingRoot.length) return false;
+        const earliest = calcEarliestDates.get(calc.name);
 
-    if (calc.type === 'meta') { const missingComputed = (calc.dependencies || []).filter(d => !existingResults[d]); if (missingComputed.length) return false; } return true; };
+        if (earliest && dateToProcess < earliest) 
+            return false;
+
+        const missingRoot = (calc.rootDataDependencies || []).filter(dep => !rootData.status[`has${dep[0].toUpperCase() + dep.slice(1)}`]);
+
+        if (missingRoot.length) 
+            return false;
+
+        if (calc.type === 'meta') 
+            { const missingComputed = (calc.dependencies || []).filter(d => !existingResults[d]);
+                 if (missingComputed.length)
+                     return false; } 
+        return true; 
+    };
 
     const processDate = async (dateStr) => {
         const dateToProcess = new Date(dateStr + 'T00:00:00Z');
         try {
             const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
             if (!rootData) return logger.log('WARN', `[PassRunner] Skipping ${dateStr}: No root data.`);
-            const existingResults = await fetchExistingResults(dateStr, calcsInThisPass, computationManifest, config, dependencies);
+            
+            // 1. Fetch TODAY'S results (Check what is already done)
+            const existingResults = await fetchExistingResults(dateStr, calcsInThisPass, computationManifest, config, dependencies, false);
+            
+            // 2. Fetch YESTERDAY'S results (For State Persistence) We calculate T-1 date string
+            const prevDate    = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+            const prevDateStr = prevDate.toISOString().slice(0, 10);
+
+            // We pass 'true' to includeSelf, because we need the calc's OWN history (e.g. mimetic-latency needs previous mimetic-latency)
+            const previousResults = await fetchExistingResults(prevDateStr, calcsInThisPass, computationManifest, config, dependencies, true);
+
             const standardToRun = standardCalcs.filter(c => checkDeps(c, rootData, existingResults, dateToProcess));
             const metaToRun     = metaCalcs.filter(c => checkDeps(c, rootData, existingResults, dateToProcess));
+            
             if (!standardToRun.length && !metaToRun.length) return logger.log('INFO', `[PassRunner] All calcs complete for ${dateStr}. Skipping.`);
+            
             logger.log('INFO', `[PassRunner] Running ${dateStr}: ${standardToRun.length} standard, ${metaToRun.length} meta`);
             
-            // --- THIS IS THE CHANGE ---
-            // 'existingResults' (which is the test harness's 'precomputedDependencies')
-            // is now passed to 'runStandardComputationPass'.
-            if (standardToRun.length) await runStandardComputationPass(dateToProcess, standardToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData, existingResults);
-            // --- END CHANGE ---
-
-            if (metaToRun.length) await runMetaComputationPass(dateToProcess, metaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, rootData);
+            // Pass 'previousResults' to the runners
+            if (standardToRun.length) await runStandardComputationPass(dateToProcess, standardToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData, existingResults, previousResults);
+            if (metaToRun.length)     await runMetaComputationPass(dateToProcess, metaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData);
+            
         } catch (err) { logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack }); }
     };
 
-    for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) { const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
-    logger.log('INFO', `[PassRunner] Processing batch ${Math.floor(i / PARALLEL_BATCH_SIZE) + 1}/${Math.ceil(allExpectedDates.length / PARALLEL_BATCH_SIZE)} (Dates: ${batch[0]}...${batch[batch.length-1]})`);
-    await Promise.all(batch.map(processDate)); }
+    for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) { 
+        const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
+        await Promise.all(batch.map(processDate)); 
+    }
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,6 +1,7 @@
 /**
- * @fileoverview Orchestration Helpers (V2 Refactor)
- * Delegates all execution logic to the ComputationController.
+ * FIXED: orchestration_helpers.js
+ * V3.2: Enabled streaming of Trading History data (tH_iter) for computations
+ * that require it, alongside Portfolio data.
  */
 
 const { ComputationController } = require('../controllers/computation_controller');
@@ -11,7 +12,7 @@ const {
 const { batchStoreSchemas } = require('../utils/schema_capture');
 const { normalizeName, commitBatchInChunks } = require('../utils/utils');
 
-// --- Helpers (Unchanged) ---
+// --- Helpers ---
 
 function groupByPass(manifest) { 
     return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); 
@@ -21,10 +22,10 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
     for (const dep of calcManifest.rootDataDependencies) {
-        if (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
-        else if (dep === 'insights' && !rootDataStatus.hasInsights) missing.push('insights');
-        else if (dep === 'social' && !rootDataStatus.hasSocial) missing.push('social');
-        else if (dep === 'history' && !rootDataStatus.hasHistory) missing.push('history');
+        if      (dep === 'portfolio' && !rootDataStatus.hasPortfolio) missing.push('portfolio');
+        else if (dep === 'insights'  && !rootDataStatus.hasInsights)  missing.push('insights');
+        else if (dep === 'social'    && !rootDataStatus.hasSocial)    missing.push('social');
+        else if (dep === 'history'   && !rootDataStatus.hasHistory)   missing.push('history');
     }
     return { canRun: missing.length === 0, missing };
 }
@@ -33,8 +34,6 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     const { logger } = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
     
-    // Quick check for existence of required root collections
-    // (Implementation preserved from original for consistency)
     let portfolioRefs = [], historyRefs = [];
     let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
     let insightsData = null, socialData = null;
@@ -65,13 +64,20 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     }
 }
 
-async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }) {
-    // (Implementation preserved: loads dependencies for computations)
+// --- OPTIMIZED FETCH ---
+async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     const calcsToFetch = new Set();
+    
     for (const calc of calcsInPass) {
-        if (calc.dependencies) calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d)));
+        if (calc.dependencies) {
+            calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d)));
+        }
+        if (includeSelf && calc.isHistorical) {
+            calcsToFetch.add(normalizeName(calc.name));
+        }
     }
+    
     if (!calcsToFetch.size) return {};
     
     const fetched = {};
@@ -90,36 +96,35 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
     
     if (docRefs.length) {
         const snaps = await db.getAll(...docRefs);
-        snaps.forEach((doc, i) => { if(doc.exists && doc.data()._completed) fetched[names[i]] = doc.data(); });
+        snaps.forEach((doc, i) => { 
+            if(doc.exists && doc.data()._completed) {
+                fetched[names[i]] = doc.data(); 
+            }
+        });
     }
     return fetched;
 }
 
-function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates) {
-    // (Implementation preserved)
-    const filter = (c) => {
-        if (existingResults[c.name]) return false;
-        // Date check
+function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates) { // TODO passtorun is unused, why? 
+    const filter = (c) => {                                                                                                 // Since we are using toplogical sorting for deps, surely the pass being run is implicit
+        if (existingResults[c.name]) return false;                                                                          // Am a bit confused here. Feel this may be a bug.
         let earliest = new Date('1970-01-01');
         (c.rootDataDependencies || []).forEach(d => { if(earliestDates[d] > earliest) earliest = earliestDates[d]; });
         if (c.isHistorical) earliest.setUTCDate(earliest.getUTCDate() + 1);
         if (new Date(dateStr) < earliest) return false;
-        // Data check
         if (!checkRootDependencies(c, rootDataStatus).canRun) return false;
-        // Dependency check
         if (c.type === 'meta' && c.dependencies && c.dependencies.some(d => !existingResults[normalizeName(d)])) return false;
         return true;
     };
     return { standardCalcsToRun: standardCalcs.filter(filter), metaCalcsToRun: metaCalcs.filter(filter) };
 }
 
-// --- NEW: Execution Delegates ---
+// --- EXECUTION DELEGATES ---
 
-async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps) {
+async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
     const { logger } = deps;
     const controller = new ComputationController(config, deps);
 
-    // 1. Filter for Standard Calculations (that need streaming)
     const calcs = Object.values(state).filter(c => c && c.manifest);
     const streamingCalcs = calcs.filter(c => 
         c.manifest.rootDataDependencies.includes('portfolio') || 
@@ -130,50 +135,55 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
 
     logger.log('INFO', `[${passName}] Streaming for ${streamingCalcs.length} computations...`);
 
-    // 2. Prepare Streams
-    await controller.loader.loadMappings(); // Pre-cache mappings
+    await controller.loader.loadMappings();
     const prevDate = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
     const prevDateStr = prevDate.toISOString().slice(0, 10);
 
+    // 1. Today's Portfolio Stream
     const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
-    // Only stream yesterday if we have refs and computations need historical data
-    const needsHistory = streamingCalcs.some(c => c.manifest.isHistorical);
-    const yP_iter = (needsHistory && rootData.yesterdayPortfolioRefs) 
+    
+    // 2. Yesterday's Portfolio Stream (for 'isHistorical' calcs)
+    const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
+    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) 
         ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) 
         : null;
 
+    // 3. Today's History Stream (NEW)
+    const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
+    const tH_iter = (needsTradingHistory && historyRefs)
+        ? streamHistoryData(config, deps, dateStr, historyRefs)
+        : null;
+
     let yP_chunk = {};
+    let tH_chunk = {};
 
-    // 3. Stream & Execute
     for await (const tP_chunk of tP_iter) {
         if (yP_iter) yP_chunk = (await yP_iter.next()).value || {};
+        if (tH_iter) tH_chunk = (await tH_iter.next()).value || {};
 
-        // Execute Batch
         const promises = streamingCalcs.map(calc => 
             controller.executor.executePerUser(
                 calc,
                 calc.manifest,
                 dateStr,
-                tP_chunk,       // Today's Portfolio
-                yP_chunk,       // Yesterday's Portfolio
-                fetchedDeps
+                tP_chunk,
+                yP_chunk, // Yesterday Portfolio
+                tH_chunk, // Today History (NEW ARGUMENT)
+                fetchedDeps,
+                previousFetchedDeps
             )
         );
         await Promise.all(promises);
     }
-
     logger.log('INFO', `[${passName}] Streaming complete.`);
 }
 
-// --- Pass Runners ---
+// --- RUNNERS ---
 
-async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps) {
+async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
     const dStr = date.toISOString().slice(0, 10);
     const logger = deps.logger;
     
-    // 1. Load additional historical context if needed (e.g. previous day's insights)
-    // Note: Portfolio/History streams are handled inside streamAndProcess
-    // We just need to attach references if they exist
     const fullRoot = { ...rootData };
     if (calcs.some(c => c.isHistorical)) {
         const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
@@ -181,7 +191,6 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
         fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
     }
 
-    // 2. Initialize Calculation Instances
     const state = {};
     for (const c of calcs) {
         try {
@@ -191,14 +200,11 @@ async function runStandardComputationPass(date, calcs, passName, config, deps, r
         } catch(e) { logger.log('WARN', `Failed to init ${c.name}`); }
     }
 
-    // 3. Execute (Delegated to Controller via Helper)
-    await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps);
-
-    // 4. Commit Results
+    await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
     await commitResults(state, dStr, passName, config, deps);
 }
 
-async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, rootData) {
+async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData) {
     const controller = new ComputationController(config, deps);
     const dStr = date.toISOString().slice(0, 10);
     const state = {};
@@ -207,36 +213,28 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
         try {
             const inst = new mCalc.class();
             inst.manifest = mCalc;
-            
-            await controller.executor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps);
-            
+            await controller.executor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps);
             state[normalizeName(mCalc.name)] = inst;
-        } catch (e) {
-            deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`);
-        }
+        } catch (e) { deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); }
     }
 
     await commitResults(state, dStr, passName, config, deps);
 }
 
-// --- Commit Helper (Standardized) ---
 async function commitResults(stateObj, dStr, passName, config, deps) {
     const writes = [], schemas = [], sharded = {};
-    
     for (const name in stateObj) {
         const calc = stateObj[name];
         try {
             const result = await calc.getResult(); 
             if (!result) continue;
-
             const standardRes = {};
             for (const key in result) {
-                if (key.startsWith('sharded_')) {
+                if (key.startsWith('sharded_')) { // TODO - This should iedally become redundant, computations themselves should NEVER return an object so large it requires sharding...
                     const sData = result[key];
                     for (const c in sData) { sharded[c] = sharded[c] || {}; Object.assign(sharded[c], sData[c]); }
                 } else standardRes[key] = result[key];
             }
-
             if (Object.keys(standardRes).length) {
                 standardRes._completed = true;
                 writes.push({
@@ -246,9 +244,16 @@ async function commitResults(stateObj, dStr, passName, config, deps) {
                     data: standardRes
                 });
             }
-
             if (calc.manifest.class.getSchema) {
-                schemas.push({ name, category: calc.manifest.category, schema: calc.manifest.class.getSchema(), metadata: calc.manifest });
+                // FIX: Remove the 'class' property (function) because Firestore cannot store it. (We were literally submitting the entire JS class to firestore...)
+                const { class: _cls, ...safeMetadata } = calc.manifest;
+                
+                schemas.push({ 
+                    name, 
+                    category: calc.manifest.category, 
+                    schema: calc.manifest.class.getSchema(), 
+                    metadata: safeMetadata 
+                });
             }
         } catch (e) { deps.logger.log('ERROR', `Commit failed ${name}: ${e.message}`); }
     }