bulltrackers-module 1.0.247 → 1.0.249

package/functions/computation-system/WorkflowOrchestrator.js

@@ -1,6 +1,6 @@
 /**
  * @fileoverview Main Orchestrator. Coordinates the topological execution.
- * UPDATED: Implements State Simulation for accurate single-pass reporting.
+ * UPDATED: Implements State Simulation and handles Indexer-based null references.
  */
 const { normalizeName, DEFINITIVE_EARLIEST_DATES }         = require('./utils/utils');
 const { checkRootDataAvailability }                        = require('./data/AvailabilityChecker');
@@ -21,76 +21,49 @@ function groupByPass(manifest) {
 
 /**
  * Analyzes whether calculations should run, be skipped, or are blocked.
- * NOW WITH SIMULATION: Updates a local status map as it progresses to ensure
- * downstream dependencies 'see' the decisions made by upstream calculations.
  */
 function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus = null) {
-    const report = {
-        runnable: [],
-        blocked: [],          
-        impossible: [],       
-        failedDependency: [], 
-        reRuns: [],           
-        skipped: []           
-    };
-
-    // [SIMULATION STATE] Clone the initial DB status. 
-    // We will update this locally as we make decisions, allowing 'future' calcs 
-    // in this list to see the predicted state of their dependencies.
+    const report = { runnable: [], blocked: [], impossible: [], failedDependency: [], reRuns: [], skipped: [] };
     const simulationStatus = { ...dailyStatus };
-
     const isTargetToday = (dateStr === new Date().toISOString().slice(0, 10));
 
     const isDepSatisfied = (depName, currentStatusMap, manifestMap) => {
         const norm = normalizeName(depName);
         const stored = currentStatusMap[norm]; 
         const depManifest = manifestMap.get(norm);
-
         if (!stored) return false;
         if (stored.hash === STATUS_IMPOSSIBLE) return false;
         if (!depManifest) return false;
         if (stored.hash !== depManifest.hash) return false;
-
         return true;
     };
 
     for (const calc of calcsInPass) {
         const cName = normalizeName(calc.name);
-        
-        // Use simulationStatus instead of dailyStatus
         const stored = simulationStatus[cName];
-        
         const storedHash = stored ? stored.hash : null;
         const storedCategory = stored ? stored.category : null;
         const currentHash = calc.hash;
 
-        // Decision Helpers
         const markImpossible = (reason) => {
             report.impossible.push({ name: cName, reason });
-            // UPDATE SIMULATION: Downstream deps will now see this as IMPOSSIBLE
             simulationStatus[cName] = { hash: STATUS_IMPOSSIBLE, category: calc.category };
         };
 
         const markRunnable = (isReRun = false, reRunDetails = null) => {
             if (isReRun) report.reRuns.push(reRunDetails);
             else report.runnable.push(calc);
-            // UPDATE SIMULATION: Downstream deps will see this as SUCCESS (matching hash)
             simulationStatus[cName] = { hash: currentHash, category: calc.category };
         };
 
         let migrationOldCategory = null;
-        if (storedCategory && storedCategory !== calc.category) {
-            migrationOldCategory = storedCategory;
-        }
+        if (storedCategory && storedCategory !== calc.category) { migrationOldCategory = storedCategory; }
 
-        // 1. Check Impossible (Previously recorded)
         if (storedHash === STATUS_IMPOSSIBLE) {
             report.skipped.push({ name: cName, reason: 'Permanently Impossible' });
-            // Simulation state remains IMPOSSIBLE
             continue;
         }
 
-        // 2. Root Data Check
         const missingRoots = [];
         if (calc.rootDataDependencies) {
             for (const dep of calc.rootDataDependencies) {
@@ -104,91 +77,44 @@ function analyzeDateExecution(dateStr, calcsInPass, rootDataStatus, dailyStatus,
 
         if (missingRoots.length > 0) {
             if (!isTargetToday) {
-                // If it's a past date and root data is missing, it's permanently impossible.
                 markImpossible(`Missing Root Data: ${missingRoots.join(', ')} (Historical)`);
             } else {
-                // If it's today, we might just be early. Block, don't Impossible.
                 report.blocked.push({ name: cName, reason: `Missing Root Data: ${missingRoots.join(', ')} (Waiting)` });
-                // We DO NOT update simulationStatus here because it's not permanently dead, just waiting.
             }
             continue;
         }
 
-        // 3. Dependency Check (Using Simulation Status)
         let dependencyIsImpossible = false;
         const missingDeps = [];
-        
         if (calc.dependencies) {
             for (const dep of calc.dependencies) {
                 const normDep = normalizeName(dep);
-                
-                // LOOK AT SIMULATION STATUS, NOT DB SNAPSHOT
                 const depStored = simulationStatus[normDep];
-                
-                if (depStored && depStored.hash === STATUS_IMPOSSIBLE) {
-                    dependencyIsImpossible = true;
-                    break; 
-                }
-
-                if (!isDepSatisfied(dep, simulationStatus, manifestMap)) {
-                    missingDeps.push(dep);
-                }
+                if (depStored && depStored.hash === STATUS_IMPOSSIBLE) { dependencyIsImpossible = true; break; }
+                if (!isDepSatisfied(dep, simulationStatus, manifestMap)) { missingDeps.push(dep); }
             }
         }
 
-        if (dependencyIsImpossible) {
-            markImpossible('Dependency is Impossible');
-            continue;
-        }
+        if (dependencyIsImpossible) { markImpossible('Dependency is Impossible'); continue; }
+        if (missingDeps.length > 0) { report.failedDependency.push({ name: cName, missing: missingDeps }); continue; }
 
-        if (missingDeps.length > 0) {
-            report.failedDependency.push({ name: cName, missing: missingDeps });
-            // Do not update simulation status; downstream will see this as 'missing' (Blocked)
-            continue;
-        }
-
-        // 4. Strict Historical Consistency
         if (calc.isHistorical && prevDailyStatus) {
             const yesterday = new Date(dateStr + 'T00:00:00Z');
             yesterday.setUTCDate(yesterday.getUTCDate() - 1);
-            
             if (yesterday >= DEFINITIVE_EARLIEST_DATES.absoluteEarliest) {
                 const prevStored = prevDailyStatus[cName];
-                
                 if (!prevStored || prevStored.hash !== currentHash) {
-                    report.blocked.push({ 
-                        name: cName, 
-                        reason: `Waiting for historical continuity (Yesterday ${!prevStored ? 'Missing' : 'Hash Mismatch'})` 
-                    });
+                    report.blocked.push({ name: cName, reason: `Waiting for historical continuity (Yesterday ${!prevStored ? 'Missing' : 'Hash Mismatch'})` });
                     continue;
                 }
             }
         }
 
-        // 5. Runnable Decision
-        if (!storedHash) { 
-            markRunnable();
-        } else if (storedHash !== currentHash) {
-            markRunnable(true, { 
-                name: cName, 
-                oldHash: storedHash, 
-                newHash: currentHash,
-                previousCategory: migrationOldCategory 
-            });
-        } else if (migrationOldCategory) {
-            markRunnable(true, { 
-                name: cName, 
-                reason: 'Category Migration', 
-                previousCategory: migrationOldCategory, 
-                newCategory: calc.category 
-            });
-        } else {
-            report.skipped.push({ name: cName });
-            // Even if skipped, ensure simulation status is fresh/set (it usually is from clone)
-            simulationStatus[cName] = { hash: currentHash, category: calc.category };
-        }
+        if (!storedHash) { markRunnable(); } 
+        else if (storedHash !== currentHash) { markRunnable(true, { name: cName, oldHash: storedHash, newHash: currentHash, previousCategory: migrationOldCategory }); } 
+        else if (migrationOldCategory) { markRunnable(true, { name: cName, reason: 'Category Migration', previousCategory: migrationOldCategory, newCategory: calc.category }); } 
+        else { report.skipped.push({ name: cName }); simulationStatus[cName] = { hash: currentHash, category: calc.category }; }
     }
-
     return report;
 }
 
@@ -204,15 +130,14 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
     const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
     const calcManifest = manifestMap.get(normalizeName(targetComputation));
     
-    if (!calcManifest) {
-        throw new Error(`Calculation '${targetComputation}' not found in manifest.`);
-    }
+    if (!calcManifest) { throw new Error(`Calculation '${targetComputation}' not found in manifest.`); }
 
-    // 2. Fetch Root Data References (Required for execution streaming)
+    // 2. Fetch Root Data Availability
+    // Note: this returns { status: {...}, portfolioRefs: null, historyRefs: null, ... }
     const rootData = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
     
     if (!rootData) {
-        logger.log('ERROR', `[Executor] FATAL: Root data missing for ${targetComputation} on ${dateStr}. Dispatcher desync?`);
+        logger.log('ERROR', `[Executor] FATAL: Root data check failed for ${targetComputation} on ${dateStr}. Index might be missing.`);
         return;
     }
 
@@ -230,53 +155,23 @@ async function executeDispatchTask(dateStr, pass, targetComputation, config, dep
 
     // 4. Execute
     logger.log('INFO', `[Executor] Running ${calcManifest.name} for ${dateStr}`, { processId: pid });
-    
     let resultUpdates = {};
 
     try {
         if (calcManifest.type === 'standard') {
-             resultUpdates = await StandardExecutor.run(
-                new Date(dateStr + 'T00:00:00Z'), 
-                [calcManifest], 
-                `Pass ${pass}`, 
-                config, 
-                dependencies, 
-                rootData, 
-                existingResults, 
-                previousResults
-            );
+             // StandardExecutor handles the null refs in rootData by fetching on demand
+             resultUpdates = await StandardExecutor.run(new Date(dateStr + 'T00:00:00Z'), [calcManifest], `Pass ${pass}`, config, dependencies, rootData, existingResults, previousResults);
         } else if (calcManifest.type === 'meta') {
-            resultUpdates = await MetaExecutor.run(
-                new Date(dateStr + 'T00:00:00Z'), 
-                [calcManifest], 
-                `Pass ${pass}`, 
-                config, 
-                dependencies, 
-                existingResults, 
-                previousResults, 
-                rootData
-            );
+            resultUpdates = await MetaExecutor.run(new Date(dateStr + 'T00:00:00Z'), [calcManifest], `Pass ${pass}`, config, dependencies, existingResults, previousResults, rootData);
         }
-        
         logger.log('INFO', `[Executor] Success: ${calcManifest.name} for ${dateStr}`);
         return { date: dateStr, updates: resultUpdates };
-
     } catch (err) {
         logger.log('ERROR', `[Executor] Failed ${calcManifest.name}: ${err.message}`, { processId: pid, stack: err.stack });
-        throw err; // Trigger retry
+        throw err;
     }
 }
 
-/**
- * Legacy/Orchestrator Mode execution (Performs analysis).
- */
-async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) {
-    // Legacy support logic...
-}
+async function runDateComputation(dateStr, passToRun, calcsInThisPass, config, dependencies, computationManifest) { /* Legacy support stub */ }
 
-module.exports = { 
-    runDateComputation, 
-    executeDispatchTask, 
-    groupByPass, 
-    analyzeDateExecution 
-};
+module.exports = { runDateComputation, executeDispatchTask, groupByPass, analyzeDateExecution };

package/functions/computation-system/data/AvailabilityChecker.js

@@ -1,14 +1,12 @@
 /**
- * @fileoverview Checks availability of root data (Portfolios, Prices, etc).
+ * @fileoverview Checks availability of root data via the Root Data Index.
+ * REFACTORED: Now relies on the centralized 'system_root_data_index' map.
  */
-const { 
-    getPortfolioPartRefs, 
-    loadDailyInsights, 
-    loadDailySocialPostInsights, 
-    getHistoryPartRefs
-} = require('../utils/data_loader');
 const { normalizeName } = require('../utils/utils');
 
+// Config for the index location (matches rootDataIndexer_config)
+const INDEX_COLLECTION = process.env.ROOT_DATA_AVAILABILITY_COLLECTION || 'system_root_data_index';
+
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     if (!calcManifest.rootDataDependencies) return { canRun: true, missing };
@@ -22,28 +20,14 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
     return { canRun: missing.length === 0, missing };
 }
 
-/**
- * Filters candidates to only those that are strictly "viable" to run.
- * A calculation is Viable if:
- * 1. All required Root Data is present.
- * 2. All required Dependencies are present AND their stored hash matches their current code hash.
- * * @param {Array} candidates - Calculations attempting to run in this pass.
- * @param {Array} fullManifest - The complete manifest (to lookup dependency current hashes).
- * @param {Object} rootDataStatus - { hasPortfolio: bool, hasPrices: bool... }
- * @param {Object} dailyStatus - Map of { "calc-name": "hash" } for completed items.
- */
 function getViableCalculations(candidates, fullManifest, rootDataStatus, dailyStatus) {
     const viable = [];
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     
     for (const calc of candidates) {
-        // 1. Check Root Data
         const rootCheck = checkRootDependencies(calc, rootDataStatus);
-        if (!rootCheck.canRun) {
-            continue; // Root data missing -> Impossible.
-        }
+        if (!rootCheck.canRun) continue;
 
-        // 2. Check Dependencies (Strict Hash Verification)
         let dependenciesMet = true;
         if (calc.dependencies && calc.dependencies.length > 0) {
             for (const depName of calc.dependencies) {
@@ -51,15 +35,9 @@ function getViableCalculations(candidates, fullManifest, rootDataStatus, dailySt
                 const storedHash = dailyStatus[normDep];
                 const depManifest = manifestMap.get(normDep);
                 
-                // If dependency is missing from manifest, we can't verify it (shouldn't happen)
                 if (!depManifest) { dependenciesMet = false; break; }
-
-                // CHECK: Does the dependency exist in DB?
                 if (!storedHash) { dependenciesMet = false; break; }
-
-                // CHECK: Does the stored hash match the current code hash?
-                // This prevents running on stale data if a dependency failed to update.
-                if (storedHash !== depManifest.hash) { dependenciesMet = false; break; }
+                if (storedHash.hash !== depManifest.hash) { dependenciesMet = false; break; }
             }
         }
 
@@ -69,56 +47,49 @@ function getViableCalculations(candidates, fullManifest, rootDataStatus, dailySt
     return viable;
 }
 
+/**
+ * Checks data availability by reading the centralized index.
+ * Only falls back to raw checks if explicitly configured or index is missing.
+ */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger, db } = dependencies;
-    const dateToProcess = new Date(dateStr + 'T00:00:00Z');
     
-    let portfolioRefs = [], historyRefs = [];
-    let hasPortfolio = false, hasInsights = false, hasSocial = false, hasHistory = false, hasPrices = false;
-    let insightsData = null, socialData = null;
-
     try {
-        const tasks = [];
-        if (dateToProcess >= earliestDates.portfolio) {
-            tasks.push(getPortfolioPartRefs(config, dependencies, dateStr).then(r => { portfolioRefs = r; hasPortfolio = !!r.length; }));
-        }
-        if (dateToProcess >= earliestDates.insights) {
-            tasks.push(loadDailyInsights(config, dependencies, dateStr).then(r => { insightsData = r; hasInsights = !!r; }));
-        }
-        if (dateToProcess >= earliestDates.social) {
-            tasks.push(loadDailySocialPostInsights(config, dependencies, dateStr).then(r => { socialData = r; hasSocial = !!r; }));
-        }
-        if (dateToProcess >= earliestDates.history) {
-            tasks.push(getHistoryPartRefs(config, dependencies, dateStr).then(r => { historyRefs = r; hasHistory = !!r.length; }));
-        }
-        if (dateToProcess >= earliestDates.price) {   
-            tasks.push(checkPriceAvailability(config, db).then(r => { hasPrices = r; })); 
+        // 1. Try reading the Index
+        const indexDoc = await db.collection(INDEX_COLLECTION).doc(dateStr).get();
+        
+        if (indexDoc.exists) {
+            const data = indexDoc.data();
+            // Return status based on the map
+            // Note: We return null references. The data loaders in streamPortfolioData
+            // have logic to fetch refs if providedRefs is null (which they are here).
+            return {
+                status: {
+                    hasPortfolio: !!data.hasPortfolio,
+                    hasHistory:   !!data.hasHistory,
+                    hasSocial:    !!data.hasSocial,
+                    hasInsights:  !!data.hasInsights,
+                    hasPrices:    !!data.hasPrices
+                },
+                portfolioRefs: null, 
+                historyRefs: null,
+                todayInsights: null,
+                todaySocialPostInsights: null,
+                yesterdayPortfolioRefs: null
+            };
+        } else {
+            // Index missing: implies data hasn't been indexed yet or doesn't exist.
+            // For safety in this strict model, we assume MISSING.
+            logger.log('WARN', `[Availability] Index not found for ${dateStr}. Assuming NO data.`);
+            return {
+                status: { hasPortfolio: false, hasHistory: false, hasSocial: false, hasInsights: false, hasPrices: false }
+            };
         }
 
-        await Promise.all(tasks);
-
-        if (!(hasPortfolio || hasInsights || hasSocial || hasHistory || hasPrices)) return null;
-
-        return { 
-            portfolioRefs, 
-            historyRefs, 
-            todayInsights: insightsData, 
-            todaySocialPostInsights: socialData, 
-            status: { hasPortfolio, hasInsights, hasSocial, hasHistory, hasPrices }, 
-            yesterdayPortfolioRefs: null 
-        };
     } catch (err) { 
-        logger.log('ERROR', `Error checking data: ${err.message}`); 
+        logger.log('ERROR', `Error checking availability index: ${err.message}`); 
         return null; 
     }
 }
 
-async function checkPriceAvailability(config, db) { 
-    try { 
-        const collection = config.priceCollection || 'asset_prices';
-        const snapshot = await db.collection(collection).limit(1).get(); 
-        return !snapshot.empty; 
-    } catch (e) { return false; } 
-}
-
 module.exports = { checkRootDependencies, checkRootDataAvailability, getViableCalculations };

package/functions/computation-system/executors/MetaExecutor.js

@@ -1,5 +1,6 @@
 /**
  * @fileoverview Executor for "Meta" (global) calculations.
+ * UPDATED: Uses CachedDataLoader for all data access.
  */
 const { normalizeName }    = require('../utils/utils');
 const { CachedDataLoader } = require('../data/CachedDataLoader');
@@ -18,6 +19,8 @@ class MetaExecutor {
                 const inst = new mCalc.class(); 
                 inst.manifest = mCalc;
                 
+                // We do not pass 'rootData' (which has null refs) to execution. 
+                // The Executor fetches its own data via loader.
                 await MetaExecutor.executeOncePerDay(inst, mCalc, dStr, fetchedDeps, previousFetchedDeps, config, deps, cachedLoader);
                 state[normalizeName(mCalc.name)] = inst;
             } catch (e) { 
@@ -30,6 +33,8 @@ class MetaExecutor {
     static async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps, config, deps, loader) {
         const mappings = await loader.loadMappings();
         const { logger } = deps;
+        
+        // Lazy fetch insights/social using the loader
         const insights   = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
         const social     = metadata.rootDataDependencies?.includes('social')   ? { today: await loader.loadSocial(dateStr)   } : null;
         

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,5 +1,6 @@
 /**
  * @fileoverview Executor for "Standard" (per-user) calculations.
+ * UPDATED: Handles lazy loading of data references (accepts null refs from Indexer).
  */
 const { normalizeName }                                                = require('../utils/utils');
 const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs } = require('../utils/data_loader');
@@ -18,6 +19,7 @@ class StandardExecutor {
         if (calcs.some(c => c.isHistorical)) {
             const prev                      = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
             const prevStr                   = prev.toISOString().slice(0, 10);
+            // Explicitly fetch yesterday's refs as they aren't provided by the daily indexer
             fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr);
         }
 
@@ -56,12 +58,16 @@ class StandardExecutor {
         const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
         const prevDateStr = prevDate.toISOString().slice(0, 10);
         
-        const tP_iter                 = streamPortfolioData(config, deps, dateStr, portfolioRefs);
+        // [FIX] pass null if portfolioRefs is null; streamPortfolioData handles the fetch
+        const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs);
+        
         const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-        const yP_iter                 = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
+        // yesterdayPortfolioRefs are manually fetched in run(), so they are usually populated
+        const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
         
         const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-        const tH_iter = (needsTradingHistory && historyRefs) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
+        // [FIX] Removed '&& historyRefs' check. We pass null to streamHistoryData if refs are missing, allowing it to fetch them.
+        const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, historyRefs) : null;
 
         let yP_chunk = {}, tH_chunk = {};
         

package/functions/computation-system/tools/BuildReporter.js

@@ -11,7 +11,7 @@ const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
 const { FieldValue } = require('@google-cloud/firestore');
 const pLimit = require('p-limit');
 const path = require('path');
-const packageJson = require(path.join(__dirname, '..', '..', 'package.json'));
+const packageJson = require(path.join(__dirname, '..', '..', '..', 'package.json'));
 const packageVersion = packageJson.version;
 
 /**

package/index.js

@@ -52,6 +52,10 @@ const { runBackfillAssetPrices }                          = require('./functions
 // Proxy
 const { handlePost }                                      = require('./functions/appscript-api/index');
 
+// NEW
+
+const { runRootDataIndexer } = require('./functions/root-data-indexer/index'); // <--- IMPORT
+
 const core = {
   IntelligentHeaderManager,
   IntelligentProxyManager,
@@ -108,6 +112,7 @@ const maintenance = {
   runSocialOrchestrator,
   handleSocialTask,
   runBackfillAssetPrices,
+  runRootDataIndexer, // <--- EXPORT
 };
 
 const proxy = { handlePost };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.247",
+  "version": "1.0.249",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [