bulltrackers-module 1.0.174 → 1.0.176

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,23 +1,24 @@
 /**
- * FIXED: orchestration_helpers.js
- * V3.2: Enabled streaming of Trading History data (tH_iter) for computations
- * that require it, alongside Portfolio data.
+ * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
  */
 
-const { ComputationController } = require('../controllers/computation_controller');
+const { ComputationController }              = require('../controllers/computation_controller');
+const { batchStoreSchemas }                  = require('../utils/schema_capture');
+const { normalizeName, commitBatchInChunks } = require('../utils/utils');
 const { 
     getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights, 
     getHistoryPartRefs, streamPortfolioData, streamHistoryData 
 } = require('../utils/data_loader');
-const { batchStoreSchemas } = require('../utils/schema_capture');
-const { normalizeName, commitBatchInChunks } = require('../utils/utils');
 
-// --- Helpers ---
 
-function groupByPass(manifest) { 
-    return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); 
-}
+/**
+ * Groups calculations from a manifest by their 'pass' property.
+ */
+function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
+/**
+ * Checks if all root data dependencies for a given calculation are met.
+ */
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
@@ -30,24 +31,22 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
     return { canRun: missing.length === 0, missing };
 }
 
+/**
+ * Checks for the availability of all required root data for a specific date.
+ */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
-    const { logger } = dependencies;
+    // ... [Unchanged content of checkRootDataAvailability] ...
+    const { logger }    = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-    
-    let portfolioRefs = [], historyRefs = [];
-    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
-    let insightsData = null, socialData = null;
+    let portfolioRefs   = [],    historyRefs = [];
+    let hasPortfolio    = false, hasInsights = false, hasSocial = false, hasHistory = false, insightsData = null , socialData  = null;
 
     try {
         const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio) 
-            tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights)
-            tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
-        if (dateToProcess >= earliestDates.social)
-            tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
-        if (dateToProcess >= earliestDates.history)
-            tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
+        if (dateToProcess >= earliestDates.portfolio)  tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r =>        { portfolioRefs = r; hasPortfolio = !!r.length; }));
+        if (dateToProcess >= earliestDates.insights)   tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r =>           { insightsData  = r; hasInsights  = !!r; }));
+        if (dateToProcess >= earliestDates.social)     tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData    = r; hasSocial    = !!r; }));
+        if (dateToProcess >= earliestDates.history)    tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r =>          { historyRefs   = r; hasHistory   = !!r.length; }));
         
         await Promise.all(tasks);
         
@@ -58,73 +57,114 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
             todayInsights: insightsData, todaySocialPostInsights: socialData, 
             status: { hasPortfolio, hasInsights, hasSocial, hasHistory } 
         };
+
     } catch (err) {
         logger.log('ERROR', `Error checking data: ${err.message}`);
         return null;
     }
 }
 
-// --- OPTIMIZED FETCH ---
-async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
-    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
-    const calcsToFetch = new Set();
+/**
+ * --- DEPRECATED: Old per-date fetch ---
+ * Keeps compatibility but logic moves to fetchGlobalComputationStatus
+ */
+async function fetchComputationStatus(dateStr, config, { db }) {
+    const collection = config.computationStatusCollection || 'computation_status';
+    const docRef     = db.collection(collection).doc(dateStr);
+    const snap       = await docRef.get();
+    return snap.exists ? snap.data() : {};
+}
+
+/**
+ * --- NEW: Fetches the SINGLE GLOBAL status document ---
+ * Loads the entire history of statuses in one read.
+ */
+async function fetchGlobalComputationStatus(config, { db }) {
+    const collection = config.computationStatusCollection || 'computation_status';
+    const docRef = db.collection(collection).doc('global_status');
+    const snap = await docRef.get();
+    return snap.exists ? snap.data() : {};
+}
+
+/**
+ * --- DEPRECATED: Old per-date update ---
+ */
+async function updateComputationStatus(dateStr, updates, config, { db }) {
+    if (!updates || Object.keys(updates).length === 0) return;
+    const collection = config.computationStatusCollection || 'computation_status';
+    const docRef     = db.collection(collection).doc(dateStr);
+    await docRef.set(updates, { merge: true }); 
+}
+
+/**
+ * --- NEW: Batch Updates to Global Document ---
+ * Accepts a map of { "YYYY-MM-DD": { calcName: true, ... } }
+ * and writes them using dot notation to avoid overwriting other dates.
+ */
+async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
+    if (!updatesByDate || Object.keys(updatesByDate).length === 0) return;
+    const collection = config.computationStatusCollection || 'computation_status';
+    const docRef = db.collection(collection).doc('global_status');
     
-    for (const calc of calcsInPass) {
-        if (calc.dependencies) {
-            calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d)));
-        }
-        if (includeSelf && calc.isHistorical) {
-            calcsToFetch.add(normalizeName(calc.name));
+    // Flatten to dot notation for Firestore update: "2023-10-27.calcName": true
+    const flattenUpdates = {};
+    for (const [date, statuses] of Object.entries(updatesByDate)) {
+        for (const [calc, status] of Object.entries(statuses)) {
+            flattenUpdates[`${date}.${calc}`] = status;
         }
     }
     
+    try {
+        await docRef.update(flattenUpdates);
+    } catch (err) {
+        // If doc doesn't exist (first run), update fails. Use set({merge:true}).
+        if (err.code === 5) { // NOT_FOUND
+             const deepObj = {};
+             for (const [date, statuses] of Object.entries(updatesByDate)) {
+                 deepObj[date] = statuses;
+             }
+             await docRef.set(deepObj, { merge: true });
+        } else {
+             throw err;
+        }
+    }
+}
+
+/**
+ * --- UPDATED: fetchExistingResults ---
+ * (Unchanged, keeps fetching results per date as this is heavy data)
+ */
+async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
+    const manifestMap  = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
+    const calcsToFetch = new Set();
+    for (const calc of calcsInPass) {
+        if (calc.dependencies)                { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); }
+        if (includeSelf && calc.isHistorical) { calcsToFetch.add(normalizeName(calc.name)); }
+    }
     if (!calcsToFetch.size) return {};
-    
     const fetched = {};
     const docRefs = [];
-    const names = [];
-    
+    const names   = [];
     for (const name of calcsToFetch) {
         const m = manifestMap.get(name);
-        if (m) {
-            docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
-                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
-                .collection(config.computationsSubcollection).doc(name));
-            names.push(name);
-        }
-    }
-    
+        if (m) { docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
+                                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
+                                .collection(config.computationsSubcollection).doc(name));
+                                names.push(name); } }
     if (docRefs.length) {
         const snaps = await db.getAll(...docRefs);
-        snaps.forEach((doc, i) => { 
-            if(doc.exists && doc.data()._completed) {
-                fetched[names[i]] = doc.data(); 
-            }
-        });
+        snaps.forEach((doc, i) => {  if(doc.exists && doc.data()._completed) { fetched[names[i]] = doc.data();  } });
     }
     return fetched;
 }
 
-function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates) { // TODO passtorun is unused, why? 
-    const filter = (c) => {                                                                                                 // Since we are using toplogical sorting for deps, surely the pass being run is implicit
-        if (existingResults[c.name]) return false;                                                                          // Am a bit confused here. Feel this may be a bug.
-        let earliest = new Date('1970-01-01');
-        (c.rootDataDependencies || []).forEach(d => { if(earliestDates[d] > earliest) earliest = earliestDates[d]; });
-        if (c.isHistorical) earliest.setUTCDate(earliest.getUTCDate() + 1);
-        if (new Date(dateStr) < earliest) return false;
-        if (!checkRootDependencies(c, rootDataStatus).canRun) return false;
-        if (c.type === 'meta' && c.dependencies && c.dependencies.some(d => !existingResults[normalizeName(d)])) return false;
-        return true;
-    };
-    return { standardCalcsToRun: standardCalcs.filter(filter), metaCalcsToRun: metaCalcs.filter(filter) };
-}
-
-// --- EXECUTION DELEGATES ---
-
+/**
+ * --- UPDATED: streamAndProcess ---
+ * (Unchanged)
+ */
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
     const { logger } = deps;
     const controller = new ComputationController(config, deps);
-
     const calcs = Object.values(state).filter(c => c && c.manifest);
     const streamingCalcs = calcs.filter(c => 
         c.manifest.rootDataDependencies.includes('portfolio') || 
@@ -139,20 +179,13 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     const prevDate = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
     const prevDateStr = prevDate.toISOString().slice(0, 10);
 
-    // 1. Today's Portfolio Stream
     const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
     
-    // 2. Yesterday's Portfolio Stream (for 'isHistorical' calcs)
     const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) 
-        ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) 
-        : null;
+    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs)  : null;
 
-    // 3. Today's History Stream (NEW)
     const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-    const tH_iter = (needsTradingHistory && historyRefs)
-        ? streamHistoryData(config, deps, dateStr, historyRefs)
-        : null;
+    const tH_iter = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
 
     let yP_chunk = {};
     let tH_chunk = {};
@@ -167,8 +200,8 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
                 calc.manifest,
                 dateStr,
                 tP_chunk,
-                yP_chunk, // Yesterday Portfolio
-                tH_chunk, // Today History (NEW ARGUMENT)
+                yP_chunk, 
+                tH_chunk, 
                 fetchedDeps,
                 previousFetchedDeps
             )
@@ -178,36 +211,47 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     logger.log('INFO', `[${passName}] Streaming complete.`);
 }
 
-// --- RUNNERS ---
-
-async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
+/**
+ * --- UPDATED: runStandardComputationPass ---
+ * Now accepts `skipStatusWrite` and returns `successUpdates`
+ */
+async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
     const dStr = date.toISOString().slice(0, 10);
     const logger = deps.logger;
     
     const fullRoot = { ...rootData };
     if (calcs.some(c => c.isHistorical)) {
-        const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
+        const prev    = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
         const prevStr = prev.toISOString().slice(0, 10);
         fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
     }
 
     const state = {};
     for (const c of calcs) {
-        try {
-            const inst = new c.class();
-            inst.manifest = c;
-            state[normalizeName(c.name)] = inst;
-        } catch(e) { logger.log('WARN', `Failed to init ${c.name}`); }
+        try { 
+            const inst = new c.class(); 
+            inst.manifest = c; 
+            state[normalizeName(c.name)] = inst; 
+        } 
+        catch(e)  { 
+            logger.log('WARN', `Failed to init ${c.name}`); 
+        }
     }
 
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
-    await commitResults(state, dStr, passName, config, deps);
+    
+    // Return the updates instead of just writing them
+    return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
 
-async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData) {
+/**
+ * --- UPDATED: runMetaComputationPass ---
+ * Now accepts `skipStatusWrite` and returns `successUpdates`
+ */
+async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData, skipStatusWrite = false) {
     const controller = new ComputationController(config, deps);
-    const dStr = date.toISOString().slice(0, 10);
-    const state = {};
+    const dStr       = date.toISOString().slice(0, 10);
+    const state      = {};
 
     for (const mCalc of calcs) {
         try {
@@ -218,11 +262,17 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
         } catch (e) { deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); }
     }
 
-    await commitResults(state, dStr, passName, config, deps);
+    return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
 
-async function commitResults(stateObj, dStr, passName, config, deps) {
+/**
+ * --- UPDATED: commitResults ---
+ * Added `skipStatusWrite` parameter. Returns `successUpdates`.
+ */
+async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
     const writes = [], schemas = [], sharded = {};
+    const successUpdates = {}; 
+
     for (const name in stateObj) {
         const calc = stateObj[name];
         try {
@@ -230,7 +280,7 @@ async function commitResults(stateObj, dStr, passName, config, deps) {
             if (!result) continue;
             const standardRes = {};
             for (const key in result) {
-                if (key.startsWith('sharded_')) { // TODO - This should iedally become redundant, computations themselves should NEVER return an object so large it requires sharding...
+                if (key.startsWith('sharded_')) {
                     const sData = result[key];
                     for (const c in sData) { sharded[c] = sharded[c] || {}; Object.assign(sharded[c], sData[c]); }
                 } else standardRes[key] = result[key];
@@ -245,9 +295,7 @@ async function commitResults(stateObj, dStr, passName, config, deps) {
                 });
             }
             if (calc.manifest.class.getSchema) {
-                // FIX: Remove the 'class' property (function) because Firestore cannot store it. (We were literally submitting the entire JS class to firestore...)
                 const { class: _cls, ...safeMetadata } = calc.manifest;
-                
                 schemas.push({ 
                     name, 
                     category: calc.manifest.category, 
@@ -255,6 +303,9 @@ async function commitResults(stateObj, dStr, passName, config, deps) {
                     metadata: safeMetadata 
                 });
             }
+            
+            successUpdates[name] = true;
+
         } catch (e) { deps.logger.log('ERROR', `Commit failed ${name}: ${e.message}`); }
     }
 
@@ -269,13 +320,24 @@ async function commitResults(stateObj, dStr, passName, config, deps) {
         }
         if (sWrites.length) await commitBatchInChunks(config, deps, sWrites, `${passName} Sharded ${col}`);
     }
+
+    if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
+        await updateComputationStatus(dStr, successUpdates, config, deps);
+        deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} computations.`);
+    }
+    
+    return successUpdates;
 }
 
 module.exports = {
     groupByPass,
+    checkRootDependencies,
     checkRootDataAvailability,
     fetchExistingResults,
-    filterCalculations,
+    fetchComputationStatus,
+    fetchGlobalComputationStatus,   
+    updateComputationStatus, 
+    updateGlobalComputationStatus, 
     runStandardComputationPass,
     runMetaComputationPass
 };