bulltrackers-module 1.0.175 → 1.0.176

package/functions/computation-system/helpers/orchestration_helpers.js

@@ -1,22 +1,24 @@
 /**
- * FIXED: orchestration_helpers.js
- * V3.3: Added Status Document logic (Single Source of Truth for Run Status).
+ * FILENAME: bulltrackers-module/functions/computation-system/helpers/orchestration_helpers.js
  */
 
-const { ComputationController } = require('../controllers/computation_controller');
+const { ComputationController }              = require('../controllers/computation_controller');
+const { batchStoreSchemas }                  = require('../utils/schema_capture');
+const { normalizeName, commitBatchInChunks } = require('../utils/utils');
 const { 
     getPortfolioPartRefs, loadDailyInsights, loadDailySocialPostInsights, 
     getHistoryPartRefs, streamPortfolioData, streamHistoryData 
 } = require('../utils/data_loader');
-const { batchStoreSchemas } = require('../utils/schema_capture');
-const { normalizeName, commitBatchInChunks } = require('../utils/utils');
 
-// --- Helpers ---
 
-function groupByPass(manifest) { 
-    return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); 
-}
+/**
+ * Groups calculations from a manifest by their 'pass' property.
+ */
+function groupByPass(manifest) { return manifest.reduce((acc, calc) => { (acc[calc.pass] = acc[calc.pass] || []).push(calc); return acc; }, {}); }
 
+/**
+ * Checks if all root data dependencies for a given calculation are met.
+ */
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
@@ -29,24 +31,22 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
     return { canRun: missing.length === 0, missing };
 }
 
+/**
+ * Checks for the availability of all required root data for a specific date.
+ */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
-    const { logger } = dependencies;
+    // ... [Unchanged content of checkRootDataAvailability] ...
+    const { logger }    = dependencies;
     const dateToProcess = new Date(dateStr + 'T00:00:00Z');
-    
-    let portfolioRefs = [], historyRefs = [];
-    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false;
-    let insightsData = null, socialData = null;
+    let portfolioRefs   = [],    historyRefs = [];
+    let hasPortfolio    = false, hasInsights = false, hasSocial = false, hasHistory = false, insightsData = null , socialData  = null;
 
     try {
         const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio) 
-            tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
-        if (dateToProcess >= earliestDates.insights)
-            tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
-        if (dateToProcess >= earliestDates.social)
-            tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
-        if (dateToProcess >= earliestDates.history)
-            tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
+        if (dateToProcess >= earliestDates.portfolio)  tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r =>        { portfolioRefs = r; hasPortfolio = !!r.length; }));
+        if (dateToProcess >= earliestDates.insights)   tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r =>           { insightsData  = r; hasInsights  = !!r; }));
+        if (dateToProcess >= earliestDates.social)     tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData    = r; hasSocial    = !!r; }));
+        if (dateToProcess >= earliestDates.history)    tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r =>          { historyRefs   = r; hasHistory   = !!r.length; }));
         
         await Promise.all(tasks);
         
@@ -57,83 +57,114 @@ async function checkRootDataAvailability(dateStr, config, dependencies, earliest
             todayInsights: insightsData, todaySocialPostInsights: socialData, 
             status: { hasPortfolio, hasInsights, hasSocial, hasHistory } 
         };
+
     } catch (err) {
         logger.log('ERROR', `Error checking data: ${err.message}`);
         return null;
     }
 }
 
-// --- NEW: Status Document Helpers ---
-
+/**
+ * --- DEPRECATED: Old per-date fetch ---
+ * Keeps compatibility but logic moves to fetchGlobalComputationStatus
+ */
 async function fetchComputationStatus(dateStr, config, { db }) {
     const collection = config.computationStatusCollection || 'computation_status';
-    const docRef = db.collection(collection).doc(dateStr);
+    const docRef     = db.collection(collection).doc(dateStr);
+    const snap       = await docRef.get();
+    return snap.exists ? snap.data() : {};
+}
+
+/**
+ * --- NEW: Fetches the SINGLE GLOBAL status document ---
+ * Loads the entire history of statuses in one read.
+ */
+async function fetchGlobalComputationStatus(config, { db }) {
+    const collection = config.computationStatusCollection || 'computation_status';
+    const docRef = db.collection(collection).doc('global_status');
     const snap = await docRef.get();
     return snap.exists ? snap.data() : {};
 }
 
+/**
+ * --- DEPRECATED: Old per-date update ---
+ */
 async function updateComputationStatus(dateStr, updates, config, { db }) {
     if (!updates || Object.keys(updates).length === 0) return;
     const collection = config.computationStatusCollection || 'computation_status';
-    const docRef = db.collection(collection).doc(dateStr);
-    // Merge the new statuses (true/false) into the daily tracking document
-    await docRef.set(updates, { merge: true });
+    const docRef     = db.collection(collection).doc(dateStr);
+    await docRef.set(updates, { merge: true }); 
 }
 
-// --- OPTIMIZED FETCH ---
-// Now strictly used for fetching DATA (results), not for checking if something ran.
-async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
-    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
-    const calcsToFetch = new Set();
+/**
+ * --- NEW: Batch Updates to Global Document ---
+ * Accepts a map of { "YYYY-MM-DD": { calcName: true, ... } }
+ * and writes them using dot notation to avoid overwriting other dates.
+ */
+async function updateGlobalComputationStatus(updatesByDate, config, { db }) {
+    if (!updatesByDate || Object.keys(updatesByDate).length === 0) return;
+    const collection = config.computationStatusCollection || 'computation_status';
+    const docRef = db.collection(collection).doc('global_status');
     
-    for (const calc of calcsInPass) {
-        if (calc.dependencies) {
-            calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d)));
-        }
-        if (includeSelf && calc.isHistorical) {
-            calcsToFetch.add(normalizeName(calc.name));
+    // Flatten to dot notation for Firestore update: "2023-10-27.calcName": true
+    const flattenUpdates = {};
+    for (const [date, statuses] of Object.entries(updatesByDate)) {
+        for (const [calc, status] of Object.entries(statuses)) {
+            flattenUpdates[`${date}.${calc}`] = status;
         }
     }
     
+    try {
+        await docRef.update(flattenUpdates);
+    } catch (err) {
+        // If doc doesn't exist (first run), update fails. Use set({merge:true}).
+        if (err.code === 5) { // NOT_FOUND
+             const deepObj = {};
+             for (const [date, statuses] of Object.entries(updatesByDate)) {
+                 deepObj[date] = statuses;
+             }
+             await docRef.set(deepObj, { merge: true });
+        } else {
+             throw err;
+        }
+    }
+}
+
+/**
+ * --- UPDATED: fetchExistingResults ---
+ * (Unchanged, keeps fetching results per date as this is heavy data)
+ */
+async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
+    const manifestMap  = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
+    const calcsToFetch = new Set();
+    for (const calc of calcsInPass) {
+        if (calc.dependencies)                { calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); }
+        if (includeSelf && calc.isHistorical) { calcsToFetch.add(normalizeName(calc.name)); }
+    }
     if (!calcsToFetch.size) return {};
-    
     const fetched = {};
     const docRefs = [];
-    const names = [];
-    
+    const names   = [];
     for (const name of calcsToFetch) {
         const m = manifestMap.get(name);
-        if (m) {
-            docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
-                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
-                .collection(config.computationsSubcollection).doc(name));
-            names.push(name);
-        }
-    }
-    
+        if (m) { docRefs.push(db.collection(config.resultsCollection).doc(dateStr)
+                                .collection(config.resultsSubcollection).doc(m.category || 'unknown')
+                                .collection(config.computationsSubcollection).doc(name));
+                                names.push(name); } }
     if (docRefs.length) {
         const snaps = await db.getAll(...docRefs);
-        snaps.forEach((doc, i) => { 
-            if(doc.exists && doc.data()._completed) {
-                fetched[names[i]] = doc.data(); 
-            }
-        });
+        snaps.forEach((doc, i) => {  if(doc.exists && doc.data()._completed) { fetched[names[i]] = doc.data();  } });
     }
     return fetched;
 }
 
-function filterCalculations(standardCalcs, metaCalcs, rootDataStatus, existingResults, passToRun, dateStr, earliestDates) { 
-    // DEPRECATED in favor of Status Document logic in Pass Runner.
-    // Kept for backward compatibility if needed, but effectively replaced.
-    return { standardCalcsToRun: standardCalcs, metaCalcsToRun: metaCalcs };
-}
-
-// --- EXECUTION DELEGATES ---
-
+/**
+ * --- UPDATED: streamAndProcess ---
+ * (Unchanged)
+ */
 async function streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps) {
     const { logger } = deps;
     const controller = new ComputationController(config, deps);
-
     const calcs = Object.values(state).filter(c => c && c.manifest);
     const streamingCalcs = calcs.filter(c => 
         c.manifest.rootDataDependencies.includes('portfolio') || 
@@ -148,20 +179,13 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     const prevDate = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
     const prevDateStr = prevDate.toISOString().slice(0, 10);
 
-    // 1. Today's Portfolio Stream
     const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
     
-    // 2. Yesterday's Portfolio Stream (for 'isHistorical' calcs)
     const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs) 
-        ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) 
-        : null;
+    const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs)  : null;
 
-    // 3. Today's History Stream
     const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-    const tH_iter = (needsTradingHistory && historyRefs)
-        ? streamHistoryData(config, deps, dateStr, historyRefs)
-        : null;
+    const tH_iter = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
 
     let yP_chunk = {};
     let tH_chunk = {};
@@ -187,36 +211,47 @@ async function streamAndProcess(dateStr, state, passName, config, deps, rootData
     logger.log('INFO', `[${passName}] Streaming complete.`);
 }
 
-// --- RUNNERS ---
-
-async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
+/**
+ * --- UPDATED: runStandardComputationPass ---
+ * Now accepts `skipStatusWrite` and returns `successUpdates`
+ */
+async function runStandardComputationPass(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps, skipStatusWrite = false) {
     const dStr = date.toISOString().slice(0, 10);
     const logger = deps.logger;
     
     const fullRoot = { ...rootData };
     if (calcs.some(c => c.isHistorical)) {
-        const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
+        const prev    = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
         const prevStr = prev.toISOString().slice(0, 10);
         fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
     }
 
     const state = {};
     for (const c of calcs) {
-        try {
-            const inst = new c.class();
-            inst.manifest = c;
-            state[normalizeName(c.name)] = inst;
-        } catch(e) { logger.log('WARN', `Failed to init ${c.name}`); }
+        try { 
+            const inst = new c.class(); 
+            inst.manifest = c; 
+            state[normalizeName(c.name)] = inst; 
+        } 
+        catch(e)  { 
+            logger.log('WARN', `Failed to init ${c.name}`); 
+        }
     }
 
     await streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps);
-    await commitResults(state, dStr, passName, config, deps);
+    
+    // Return the updates instead of just writing them
+    return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
 
-async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData) {
+/**
+ * --- UPDATED: runMetaComputationPass ---
+ * Now accepts `skipStatusWrite` and returns `successUpdates`
+ */
+async function runMetaComputationPass(date, calcs, passName, config, deps, fetchedDeps, previousFetchedDeps, rootData, skipStatusWrite = false) {
     const controller = new ComputationController(config, deps);
-    const dStr = date.toISOString().slice(0, 10);
-    const state = {};
+    const dStr       = date.toISOString().slice(0, 10);
+    const state      = {};
 
     for (const mCalc of calcs) {
         try {
@@ -227,12 +262,16 @@ async function runMetaComputationPass(date, calcs, passName, config, deps, fetch
         } catch (e) { deps.logger.log('ERROR', `Meta calc failed ${mCalc.name}: ${e.message}`); }
     }
 
-    await commitResults(state, dStr, passName, config, deps);
+    return await commitResults(state, dStr, passName, config, deps, skipStatusWrite);
 }
 
-async function commitResults(stateObj, dStr, passName, config, deps) {
+/**
+ * --- UPDATED: commitResults ---
+ * Added `skipStatusWrite` parameter. Returns `successUpdates`.
+ */
+async function commitResults(stateObj, dStr, passName, config, deps, skipStatusWrite = false) {
     const writes = [], schemas = [], sharded = {};
-    const successUpdates = {}; // Track which calcs finished successfully
+    const successUpdates = {}; 
 
     for (const name in stateObj) {
         const calc = stateObj[name];
@@ -265,19 +304,14 @@ async function commitResults(stateObj, dStr, passName, config, deps) {
                 });
             }
             
-            // Mark as successful in our local tracker
             successUpdates[name] = true;
 
         } catch (e) { deps.logger.log('ERROR', `Commit failed ${name}: ${e.message}`); }
     }
 
-    // 1. Store Schemas
     if (schemas.length) batchStoreSchemas(deps, config, schemas).catch(()=>{});
-    
-    // 2. Write Results
     if (writes.length) await commitBatchInChunks(config, deps, writes, `${passName} Results`);
     
-    // 3. Write Sharded Results
     for (const col in sharded) {
         const sWrites = [];
         for (const id in sharded[col]) {
@@ -287,11 +321,12 @@ async function commitResults(stateObj, dStr, passName, config, deps) {
         if (sWrites.length) await commitBatchInChunks(config, deps, sWrites, `${passName} Sharded ${col}`);
     }
 
-    // 4. Update Status Document (Single Source of Truth)
-    if (Object.keys(successUpdates).length > 0) {
+    if (!skipStatusWrite && Object.keys(successUpdates).length > 0) {
         await updateComputationStatus(dStr, successUpdates, config, deps);
         deps.logger.log('INFO', `[${passName}] Updated status document for ${Object.keys(successUpdates).length} computations.`);
     }
+    
+    return successUpdates;
 }
 
 module.exports = {
@@ -299,8 +334,10 @@ module.exports = {
     checkRootDependencies,
     checkRootDataAvailability,
     fetchExistingResults,
-    fetchComputationStatus, // Exported for Pass Runner
-    updateComputationStatus, // Exported for Pass Runner (error handling)
+    fetchComputationStatus,
+    fetchGlobalComputationStatus,   
+    updateComputationStatus, 
+    updateGlobalComputationStatus, 
     runStandardComputationPass,
     runMetaComputationPass
-};
+};