bulltrackers-module 1.0.698 → 1.0.700

package/functions/computation-system/data/AvailabilityChecker.js

@@ -11,7 +11,13 @@ const INDEX_COLLECTION = process.env.ROOT_DATA_AVAILABILITY_COLLECTION || 'syste
 // CONFIGURATION: Dependency Mappings
 // =============================================================================
 
-// Dependencies that map directly to a single status flag
+// Global dependencies that are NOT date-specific and are always considered available.
+// These are stored at fixed paths and don't rely on daily availability index.
+const GLOBAL_DEPS = new Set([
+    'piMasterList'  // Stored at /system_state/popular_investor_master_list (global CID->username mapping)
+]);
+
+// Dependencies that map directly to a single status flag (date-specific)
 const SIMPLE_DEP_MAP = {
     rankings:     'piRankings',
     verification: 'signedInUserVerification',
@@ -21,7 +27,7 @@ const SIMPLE_DEP_MAP = {
     pageViews:    'piPageViews',
     watchlist:    'watchlistMembership',
     alerts:       'piAlertHistory',
-    piMasterList: 'piMasterList' // <--- ADD THIS LINE
+    piMasterList: 'piMasterList' // Kept for normalizeStatus compatibility, but checked via GLOBAL_DEPS
 };
 
 // Dependencies that vary based on the userType of the calculation
@@ -68,8 +74,12 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
         let isAvailable = false;
         let missingKey = dep;
 
-        // 1. Resolve Status Key
-        if (SIMPLE_DEP_MAP[dep]) {
+        // 0. Check for Global Dependencies (always available, not date-specific)
+        if (GLOBAL_DEPS.has(dep)) {
+            isAvailable = true;
+        }
+        // 1. Resolve Status Key for date-specific dependencies
+        else if (SIMPLE_DEP_MAP[dep]) {
             const key = SIMPLE_DEP_MAP[dep];
             if (rootDataStatus[key]) isAvailable = true;
             else missingKey = key;

package/functions/computation-system/data/DependencyFetcher.js

@@ -216,6 +216,12 @@ async function fetchDependencies(date, calcs, config, deps, manifestLookup = {},
 
 /**
  * Fetches result series (Historical data) for lookbacks.
+ * @param {string} endDateStr - The end date for the series
+ * @param {string[]} calcNames - Original (case-sensitive) computation names
+ * @param {Object} manifestLookup - Map of normalizedName -> category
+ * @param {Object} config - Configuration object
+ * @param {Object} deps - Dependencies (db, logger)
+ * @param {number} lookbackDays - Number of days to look back
  */
 async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config, deps, lookbackDays) {
     const { db, logger } = deps;
@@ -224,7 +230,7 @@ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config,
     // Initialize results structure
     calcNames.forEach(n => results[normalizeName(n)] = {});
 
-    // Generate Date List
+    // Generate Date List (going backwards from endDate)
     const dates = [];
     const d = new Date(endDateStr);
     for (let i = 0; i < lookbackDays; i++) {
@@ -238,6 +244,7 @@ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config,
         for (const rawName of calcNames) {
             const norm = normalizeName(rawName);
             const category = manifestLookup[norm] || 'analytics';
+            
             ops.push(async () => {
                 const val = await fetchSingleResult(db, { ...config, logger }, dateStr, rawName, category);
                 if (val && !isDataEmpty(val)) {

package/functions/computation-system/executors/MetaExecutor.js

@@ -9,6 +9,16 @@ const { commitResults }     = require('../persistence/ResultCommitter');
 const { fetchResultSeries } = require('../data/DependencyFetcher');
 const { getManifest }       = require('../topology/ManifestLoader');
 
+// [FIX] Load calculations package directly for manifest building during force runs
+let _calculations = null;
+function getCalculations() {
+    if (!_calculations) {
+        try { _calculations = require('aiden-shared-calculations-unified').calculations; } 
+        catch (e) { _calculations = {}; }
+    }
+    return _calculations;
+}
+
 class MetaExecutor {
 
     // =========================================================================
@@ -94,13 +104,17 @@ class MetaExecutor {
         const { logger } = deps;
         const calcs = [metadata]; // Treat single as list for helpers
 
+        // [FIX] Build manifestLookup using the actual calculations package (not config.calculationsDirectory)
+        const allManifests = getManifest([], getCalculations(), deps);
+        const manifestLookup = Object.fromEntries(allManifests.map(m => [normalizeName(m.name), m.category]));
+
         // 1. Load Data using Shared Helpers
         const [mappings, rankings, variableRoots, seriesData, piMasterList] = await Promise.all([
             loader.loadMappings(),
             loader.loadRankings(dateStr),
             loadVariableRootData(loader, dateStr, calcs, logger),
-            loadSeriesData(loader, dateStr, calcs, {}, config, deps),
-            loader.loadPIMasterList() // <--- ADDED LOAD
+            loadSeriesData(loader, dateStr, calcs, manifestLookup, config, deps),
+            loader.loadPIMasterList()
         ]);
 
         let rankingsYesterday = null;
@@ -211,9 +225,9 @@ async function loadVariableRootData(loader, dateStr, calcs, logger) {
  */
 async function loadSeriesData(loader, dateStr, calcs, manifestLookup, config, deps) {
     const rootRequests = {};
-    const depRequests = {};
+    const depRequests = {};        // norm -> { days, originalName }
 
-    // 1. Aggregate Lookback Depths
+    // 1. Aggregate Lookback Depths from dependencySeries config
     for (const c of calcs) {
         if (c.rootDataSeries) {
             Object.entries(c.rootDataSeries).forEach(([type, val]) => {
@@ -225,7 +239,9 @@ async function loadSeriesData(loader, dateStr, calcs, manifestLookup, config, de
             Object.entries(c.dependencySeries).forEach(([name, val]) => {
                 const days = typeof val === 'object' ? val.lookback : val;
                 const norm = normalizeName(name);
-                depRequests[norm] = Math.max(depRequests[norm] || 0, days);
+                if (!depRequests[norm] || depRequests[norm].days < days) {
+                    depRequests[norm] = { days, originalName: name };
+                }
             });
         }
     }
@@ -249,12 +265,14 @@ async function loadSeriesData(loader, dateStr, calcs, manifestLookup, config, de
         }
     });
 
-    // 3. Fetch Dependency Series
-    const depNames = Object.keys(depRequests);
-    if (depNames.length > 0) {
-        const maxDays = Math.max(...Object.values(depRequests));
-        deps.logger.log('INFO', `[MetaExecutor] Loading up to ${maxDays}-day series for Dependencies: ${depNames.join(', ')}`);
-        seriesData.results = await fetchResultSeries(dateStr, depNames, manifestLookup, config, deps, maxDays);
+    // 3. Fetch Dependency Series (category comes from manifestLookup)
+    const depEntries = Object.values(depRequests);
+    if (depEntries.length > 0) {
+        const depOriginalNames = depEntries.map(e => e.originalName);
+        const maxDays = Math.max(...depEntries.map(e => e.days));
+        
+        deps.logger.log('INFO', `[MetaExecutor] Loading up to ${maxDays}-day series for Dependencies: ${depOriginalNames.join(', ')}`);
+        seriesData.results = await fetchResultSeries(dateStr, depOriginalNames, manifestLookup, config, deps, maxDays);
     }
 
     await Promise.all(rootPromises);

package/functions/computation-system/executors/StandardExecutor.js

@@ -13,6 +13,16 @@ const mathLayer = require('../layers/index');
 const { performance } = require('perf_hooks');
 const v8 = require('v8');
 
+// [FIX] Load calculations package directly for manifest building
+let _calculations = null;
+function getCalculations() {
+    if (!_calculations) {
+        try { _calculations = require('aiden-shared-calculations-unified').calculations; } 
+        catch (e) { _calculations = {}; }
+    }
+    return _calculations;
+}
+
 class StandardExecutor {
 
     // =========================================================================
@@ -383,14 +393,20 @@ async function loadGlobalRoots(loader, dateStr, calcs, deps) {
 
 async function loadSeriesData(loader, dateStr, calcs, config, deps) {
     const rootReqs = {};
-    const depReqs = {};
+    const depReqs = {};  // norm -> { days, originalName, category }
 
     calcs.forEach(c => {
         if (c.manifest.rootDataSeries) {
             Object.entries(c.manifest.rootDataSeries).forEach(([k, v]) => rootReqs[k] = Math.max(rootReqs[k]||0, v.lookback||v));
         }
         if (c.manifest.dependencySeries) {
-            Object.entries(c.manifest.dependencySeries).forEach(([k, v]) => depReqs[normalizeName(k)] = Math.max(depReqs[normalizeName(k)]||0, v.lookback||v));
+            Object.entries(c.manifest.dependencySeries).forEach(([k, v]) => {
+                const norm = normalizeName(k);
+                const days = typeof v === 'object' ? v.lookback : v;
+                if (!depReqs[norm] || depReqs[norm].days < days) {
+                    depReqs[norm] = { days, originalName: k };
+                }
+            });
         }
     });
 
@@ -404,12 +420,17 @@ async function loadSeriesData(loader, dateStr, calcs, config, deps) {
         if (rootMap[key]) series.root[key] = (await loader.loadSeries(rootMap[key], dateStr, days)).data;
     }));
 
-    const depNames = Object.keys(depReqs);
-    if (depNames.length) {
-        // Construct manifest lookup on the fly for fetched names
-        const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps);
+    // [FIX] Use getCalculations() to load ALL computations for proper category lookup
+    const depEntries = Object.values(depReqs);
+    if (depEntries.length) {
+        const depOriginalNames = depEntries.map(e => e.originalName);
+        const maxDays = Math.max(...depEntries.map(e => e.days));
+        
+        // Build lookup from ALL computations (empty productLines = load all)
+        const allManifests = getManifest([], getCalculations(), deps);
         const lookup = Object.fromEntries(allManifests.map(m => [normalizeName(m.name), m.category]));
-        series.results = await fetchResultSeries(dateStr, depNames, lookup, config, deps, Math.max(...Object.values(depReqs)));
+        
+        series.results = await fetchResultSeries(dateStr, depOriginalNames, lookup, config, deps, maxDays);
     }
 
     return series;

package/functions/computation-system/legacy/AvailabilityCheckerOld.js

@@ -189,6 +189,12 @@ function checkRootDependencies(calcManifest, rootDataStatus) {
                 }
             }
         }
+        // [GLOBAL] piMasterList is always available (not date-specific)
+        // Stored at /system_state/popular_investor_master_list
+        else if (dep === 'piMasterList') {
+            isAvailable = true;
+            available.push('piMasterList');
+        }
     }
 
     // [NEW] Enforce Mandatory Roots (defined by computation)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.698",
+  "version": "1.0.700",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [