bulltrackers-module 1.0.174 → 1.0.175

package/functions/computation-system/helpers/computation_pass_runner.js

@@ -1,9 +1,21 @@
 /**
  * FIXED: computation_pass_runner.js
- * V3: Fetches Previous Day's Results to enable State Persistence.
+ * V3.4: Optimized Orchestration using Status Document (Single Source of Truth).
+ * - Reads one 'computation_status' doc per day to decide what to run.
+ * - Reduces Firestore reads significantly.
+ * - explicitly marks failures as false in the status doc.
  */
 
-const { groupByPass, checkRootDataAvailability, fetchExistingResults, runStandardComputationPass, runMetaComputationPass } = require('./orchestration_helpers.js');
+const { 
+    groupByPass, 
+    checkRootDataAvailability, 
+    fetchExistingResults, 
+    fetchComputationStatus, 
+    updateComputationStatus,
+    runStandardComputationPass, 
+    runMetaComputationPass,
+    checkRootDependencies
+} = require('./orchestration_helpers.js');
 const { getExpectedDateStrings } = require('../utils/utils.js');
 const PARALLEL_BATCH_SIZE = 7;
 
@@ -13,8 +25,9 @@ async function runComputationPass(config, dependencies, computationManifest) {
     if (!passToRun) 
         return logger.log('ERROR', '[PassRunner] No pass defined. Aborting.');
     
-    logger.log('INFO', `🚀 Starting PASS ${passToRun}...`);
+    logger.log('INFO', `🚀 Starting PASS ${passToRun} with Optimized Status Check...`);
 
+    // Hardcoded earliest dates for global availability checks
     const earliestDates            = { portfolio: new Date('2025-09-25T00:00:00Z'), history: new Date('2025-11-05T00:00:00Z'), social: new Date('2025-10-30T00:00:00Z'), insights: new Date('2025-08-26T00:00:00Z') };
     earliestDates.absoluteEarliest = Object.values(earliestDates).reduce((a,b) => a < b ? a : b);
 
@@ -24,79 +37,96 @@ async function runComputationPass(config, dependencies, computationManifest) {
     if (!calcsInThisPass.length) 
         return logger.log('WARN', `[PassRunner] No calcs for Pass ${passToRun}. Exiting.`);
     
-    const calcEarliestDates = new Map();
-
-    for (const calc of calcsInThisPass) { 
-        const deps = calc.rootDataDependencies || []; 
-
-        if (!deps.length) 
-            { calcEarliestDates.set(calc.name, earliestDates.absoluteEarliest); continue; }
-
-        const latestDep = new Date(Math.max(...deps.map(d => earliestDates[d]?.getTime() || 0)));
-        const calcDate = calc.isHistorical 
-            ? new Date(latestDep.getTime() + 86400000) 
-            : latestDep; 
-        
-            calcEarliestDates.set(calc.name, calcDate);
-    }
-
-    const passEarliestDate = new Date(Math.min(...Array.from(calcEarliestDates.values()).map(d => d.getTime())));
+    // Determine date range to process
+    // We no longer need complex per-calc date derivation since the status doc handles "done-ness".
+    // We just iterate from absolute earliest to yesterday.
+    const passEarliestDate = earliestDates.absoluteEarliest; 
     const endDateUTC       = new Date(Date.UTC(new Date().getUTCFullYear(), new Date().getUTCMonth(), new Date().getUTCDate() - 1));
     const allExpectedDates = getExpectedDateStrings(passEarliestDate, endDateUTC);
+
     const standardCalcs    = calcsInThisPass.filter(c => c.type === 'standard');
     const metaCalcs        = calcsInThisPass.filter(c => c.type === 'meta');
 
-    const checkDeps = (calc, rootData, existingResults, dateToProcess) => { 
-        if (existingResults[calc.name]) 
-            return false;
-
-        const earliest = calcEarliestDates.get(calc.name);
-
-        if (earliest && dateToProcess < earliest) 
-            return false;
-
-        const missingRoot = (calc.rootDataDependencies || []).filter(dep => !rootData.status[`has${dep[0].toUpperCase() + dep.slice(1)}`]);
+    // Helper: Decide if a calculation should run based on the status document
+    const shouldRun = (calc, statusData) => {
+        // 1. If explicitly TRUE, it ran fine. Ignore.
+        if (statusData[calc.name] === true) return false;
 
-        if (missingRoot.length) 
-            return false;
+        // 2. If missing or FALSE, it needs to run.
+        //    But first, check if its dependencies are ready (if any).
+        //    Note: Dependencies from previous passes must be TRUE.
+        if (calc.dependencies && calc.dependencies.length > 0) {
+            const depsMet = calc.dependencies.every(depName => statusData[depName] === true);
+            if (!depsMet) return false; // Dependency not ready yet
+        }
 
-        if (calc.type === 'meta') 
-            { const missingComputed = (calc.dependencies || []).filter(d => !existingResults[d]);
-                 if (missingComputed.length)
-                     return false; } 
-        return true; 
+        return true;
     };
 
     const processDate = async (dateStr) => {
+        // 1. Fetch the Single Status Document (Cheap Read)
+        const statusData = await fetchComputationStatus(dateStr, config, dependencies);
         const dateToProcess = new Date(dateStr + 'T00:00:00Z');
+
+        // 2. Filter calculations based on Status Doc
+        const standardToRun = standardCalcs.filter(c => shouldRun(c, statusData));
+        const metaToRun     = metaCalcs.filter(c => shouldRun(c, statusData));
+        
+        // Optimization: If nothing needs to run, stop here. 
+        // No checking root data, no loading results.
+        if (!standardToRun.length && !metaToRun.length) {
+             return; // logger.log('INFO', `[PassRunner] ${dateStr} complete or waiting for deps.`);
+        }
+
+        // 3. Check Root Data Availability (Only done if we have work to do)
+        // We filter standardToRun further based on root data requirements
+        // (e.g. if calc needs history but history isn't ready for this date)
+        const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
+        
+        if (!rootData) {
+            // If root data is completely missing for the day, we can't run anything.
+            // We do NOT mark as false, because it's not a failure, just data unavailability.
+            // We just skip.
+            return;
+        }
+
+        // Further filter based on specific root data needs (e.g. checks "hasHistory")
+        const finalStandardToRun = standardToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
+        const finalMetaToRun     = metaToRun.filter(c => checkRootDependencies(c, rootData.status).canRun);
+
+        if (!finalStandardToRun.length && !finalMetaToRun.length) return;
+
+        logger.log('INFO', `[PassRunner] Running ${dateStr}: ${finalStandardToRun.length} standard, ${finalMetaToRun.length} meta`);
+
         try {
-            const rootData = await checkRootDataAvailability(dateStr, config, dependencies, earliestDates);
-            if (!rootData) return logger.log('WARN', `[PassRunner] Skipping ${dateStr}: No root data.`);
-            
-            // 1. Fetch TODAY'S results (Check what is already done)
-            const existingResults = await fetchExistingResults(dateStr, calcsInThisPass, computationManifest, config, dependencies, false);
+            // 4. Fetch Data for Execution (Dependencies)
+            // We only fetch data required by the calcs that are actually running.
+            const calcsRunning = [...finalStandardToRun, ...finalMetaToRun];
+            const existingResults = await fetchExistingResults(dateStr, calcsRunning, computationManifest, config, dependencies, false);
             
-            // 2. Fetch YESTERDAY'S results (For State Persistence) We calculate T-1 date string
+            // Fetch Previous Day's Results (for State Persistence)
             const prevDate    = new Date(dateToProcess); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
             const prevDateStr = prevDate.toISOString().slice(0, 10);
+            const previousResults = await fetchExistingResults(prevDateStr, calcsRunning, computationManifest, config, dependencies, true);
 
-            // We pass 'true' to includeSelf, because we need the calc's OWN history (e.g. mimetic-latency needs previous mimetic-latency)
-            const previousResults = await fetchExistingResults(prevDateStr, calcsInThisPass, computationManifest, config, dependencies, true);
-
-            const standardToRun = standardCalcs.filter(c => checkDeps(c, rootData, existingResults, dateToProcess));
-            const metaToRun     = metaCalcs.filter(c => checkDeps(c, rootData, existingResults, dateToProcess));
-            
-            if (!standardToRun.length && !metaToRun.length) return logger.log('INFO', `[PassRunner] All calcs complete for ${dateStr}. Skipping.`);
+            // 5. Execute
+            if (finalStandardToRun.length) 
+                await runStandardComputationPass(dateToProcess, finalStandardToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData, existingResults, previousResults);
             
-            logger.log('INFO', `[PassRunner] Running ${dateStr}: ${standardToRun.length} standard, ${metaToRun.length} meta`);
+            if (finalMetaToRun.length)     
+                await runMetaComputationPass(dateToProcess, finalMetaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData);
             
-            // Pass 'previousResults' to the runners
-            if (standardToRun.length) await runStandardComputationPass(dateToProcess, standardToRun, `Pass ${passToRun} (Standard)`, config, dependencies, rootData, existingResults, previousResults);
-            if (metaToRun.length)     await runMetaComputationPass(dateToProcess, metaToRun, `Pass ${passToRun} (Meta)`, config, dependencies, existingResults, previousResults, rootData);
+        } catch (err) { 
+            logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message });
             
-        } catch (err) { logger.log('ERROR', `[PassRunner] FAILED Pass ${passToRun} for ${dateStr}`, { errorMessage: err.message, stack: err.stack }); }
+            // 6. Explicitly Mark Failures as FALSE
+            const failedUpdates = {};
+            [...finalStandardToRun, ...finalMetaToRun].forEach(c => failedUpdates[c.name] = false);
+            await updateComputationStatus(dateStr, failedUpdates, config, dependencies);
+        }
     };
 
+    // Batch process dates
     for (let i = 0; i < allExpectedDates.length; i += PARALLEL_BATCH_SIZE) { 
         const batch = allExpectedDates.slice(i, i + PARALLEL_BATCH_SIZE);
         await Promise.all(batch.map(processDate)); 
@@ -104,4 +134,4 @@ async function runComputationPass(config, dependencies, computationManifest) {
     logger.log('INFO', `[PassRunner] Pass ${passToRun} orchestration finished.`);
 }
 
-module.exports = { runComputationPass };
+module.exports = { runComputationPass };

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,7 +1,6 @@
 /**
  * FIXED: orchestration_helpers.js
- * V3.2: Enabled streaming of Trading History data (tH_iter) for computations
- * that require it, alongside Portfolio data.
+ * V3.3: Added Status Document logic (Single Source of Truth for Run Status).
  */
 
 const { ComputationController } = require('../controllers/computation_controller');
@@ -64,7 +63,25 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
     }
 }
 
+// --- NEW: Status Document Helpers ---
+
+async function fetchComputationStatus(dateStr, config, { db }) {
+    const collection = config.computationStatusCollection || 'computation_status';
+    const docRef = db.collection(collection).doc(dateStr);
+    const snap = await docRef.get();
+    return snap.exists ? snap.data() : {};
+}
+
+async function updateComputationStatus(dateStr, updates, config, { db }) {
+    if (!updates || Object.keys(updates).length === 0) return;
+    const collection = config.computationStatusCollection || 'computation_status';
+    const docRef = db.collection(collection).doc(dateStr);
+    // Merge the new statuses (true/false) into the daily tracking document
+    await docRef.set(updates, { merge: true });
+}
+
 // --- OPTIMIZED FETCH ---
+// Now strictly used for fetching DATA (results), not for checking if something ran.
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     const calcsToFetch = new Set();
@@ -105,18 +122,10 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
     return fetched;
 }
 
-function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates) { // TODO passtorun is unused, why? 
-    const filter = (c) => {                                                                                                 // Since we are using toplogical sorting for deps, surely the pass being run is implicit
-        if (existingResults[c.name]) return false;                                                                          // Am a bit confused here. Feel this may be a bug.
-        let earliest = new Date('1970-01-01');
-        (c.rootDataDependencies || []).forEach(d => { if(earliestDates[d] > earliest) earliest = earliestDates[d]; });
-        if (c.isHistorical) earliest.setUTCDate(earliest.getUTCDate() + 1);
-        if (new Date(dateStr) < earliest) return false;
-        if (!checkRootDependencies(c, rootDataStatus).canRun) return false;
-        if (c.type === 'meta' && c.dependencies && c.dependencies.some(d => !existingResults[normalizeName(d)])) return false;
-        return true;
-    };
-    return { standardCalcsToRun: standardCalcs.filter(filter), metaCalcsToRun: metaCalcs.filter(filter) };
+function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates) { 
+    // DEPRECATED in favor of Status Document logic in Pass Runner.
+    // Kept for backward compatibility if needed, but effectively replaced.
+    return { standardCalcsToRun: standardCalcs, metaCalcsToRun: metaCalcs };
 }
 
 // --- EXECUTION DELEGATES ---
@@ -148,7 +157,7 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
         ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) 
         : null;
 
-    // 3. Today's History Stream (NEW)
+    // 3. Today's History Stream
     const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
     const tH_iter = (needsTradingHistory && historyRefs)
         ? streamHistoryData(config, deps, dateStr, historyRefs)
@@ -167,8 +176,8 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
                 calc.manifest,
                 dateStr,
                 tP_chunk,
-                yP_chunk, // Yesterday Portfolio
-                tH_chunk, // Today History (NEW ARGUMENT)
+                yP_chunk, 
+                tH_chunk, 
                 fetchedDeps,
                 previousFetchedDeps
             )
@@ -223,6 +232,8 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
 
 async function commitResults(stateObj, dStr, passName, config, deps) {
     const writes = [], schemas = [], sharded = {};
+    const successUpdates = {}; // Track which calcs finished successfully
+
     for (const name in stateObj) {
         const calc = stateObj[name];
         try {
@@ -230,7 +241,7 @@ async function commitResults(stateObj, dStr, passName, config, deps) {
             if (!result) continue;
             const standardRes = {};
             for (const key in result) {
-                if (key.startsWith('sharded_')) { // TODO - This should iedally become redundant, computations themselves should NEVER return an object so large it requires sharding...
+                if (key.startsWith('sharded_')) {
                     const sData = result[key];
                     for (const c in sData) { sharded[c] = sharded[c] || {}; Object.assign(sharded[c], sData[c]); }
                 } else standardRes[key] = result[key];
@@ -245,9 +256,7 @@ async function commitResults(stateObj, dStr, passName, config, deps) {
                 });
             }
             if (calc.manifest.class.getSchema) {
-                // FIX: Remove the 'class' property (function) because Firestore cannot store it. (We were literally submitting the entire JS class to firestore...)
                 const { class: _cls, ...safeMetadata } = calc.manifest;
-                
                 schemas.push({ 
                     name, 
                     category: calc.manifest.category, 
@@ -255,12 +264,20 @@ async function commitResults(stateObj, dStr, passName, config, deps) {
                     metadata: safeMetadata 
                 });
             }
+            
+            // Mark as successful in our local tracker
+            successUpdates[name] = true;
+
         } catch (e) { deps.logger.log('ERROR', `Commit failed ${name}: ${e.message}`); }
     }
 
+    // 1. Store Schemas
     if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(()=>{});
+    
+    // 2. Write Results
     if (writes.length) await commitBatchInChunks(config, deps, writes, `${passName} Results`);
     
+    // 3. Write Sharded Results
     for (const col in sharded) {
         const sWrites = [];
         for (const id in sharded[col]) {
@@ -269,13 +286,21 @@ async function commitResults(stateObj, dStr, passName, config, deps) {
         }
         if (sWrites.length) await commitBatchInChunks(config, deps, sWrites, `${passName} Sharded ${col}`);
     }
+
+    // 4. Update Status Document (Single Source of Truth)
+    if (Object.keys(successUpdates).length > 0) {
+        await updateComputationStatus(dStr, successUpdates, config, deps);
+        deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} computations.`);
+    }
 }
 
 module.exports = {
     groupByPass,
+    checkRootDependencies,
     checkRootDataAvailability,
     fetchExistingResults,
-    filterCalculations,
+    fetchComputationStatus, // Exported for Pass Runner
+    updateComputationStatus, // Exported for Pass Runner (error handling)
     runStandardComputationPass,
     runMetaComputationPass
-};
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.174",
+  "version": "1.0.175",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [