bulltrackers-module 1.0.632 → 1.0.633

package/functions/computation-system/data/DependencyFetcher.js

@@ -1,206 +1,153 @@
 /**
- * @fileoverview Fetches results from previous computations, handling auto-sharding and decompression.
- * UPDATED: Implemented 'Batched Series Fetching' to reduce Firestore read operations by ~98% for time-series lookups.
+ * @fileoverview Fetches dependencies for computations.
+ * UPDATED: Uses 'manifestLookup' to resolve the correct category (Core vs Non-Core).
+ * UPDATED: Supports automatic reassembly of sharded results (_shards subcollection).
+ * UPDATED: Supports decompression of zipped results.
  */
 const { normalizeName } = require('../utils/utils');
-const zlib = require('zlib'); 
-const pLimit = require('p-limit');
+const zlib = require('zlib');
 
-async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
-    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
-    const calcsToFetch = new Set();
+/**
+ * Fetches dependencies for a specific date (Standard pass).
+ * @param {Date} date - The target date.
+ * @param {Array} calcs - The computations requiring dependencies.
+ * @param {Object} config - System config.
+ * @param {Object} deps - System dependencies (db, logger).
+ * @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
+ */
+async function fetchDependencies(date, calcs, config, deps, manifestLookup = {}) {
+    const { db, logger } = deps;
+    const dStr = date.toISOString().slice(0, 10);
     
-    for (const calc of calcsInPass) { 
-        if (calc.dependencies) calc.dependencies.forEach(d => calcsToFetch.add(normalizeName(d))); 
-        if (includeSelf && calc.isHistorical) calcsToFetch.add(normalizeName(calc.name)); 
-    }
+    // 1. Identify unique dependencies needed
+    const needed = new Set();
+    calcs.forEach(c => {
+        if (c.getDependencies) {
+            const reqs = c.getDependencies();
+            reqs.forEach(r => needed.add(normalizeName(r)));
+        }
+    });
     
-    if (!calcsToFetch.size) return {};
+    if (needed.size === 0) return {};
     
-    const fetched = {};
-    const docRefs = [];
-    const names = [];
-
-    for (const name of calcsToFetch) { 
-        const m = manifestMap.get(name); 
-        if (m) { 
-            docRefs.push(db.collection(config.resultsCollection)
-                .doc(dateStr)
-                .collection(config.resultsSubcollection)
-                .doc(m.category || 'unknown')
-                .collection(config.computationsSubcollection)
-                .doc(name)); 
-            names.push(name); 
-        } 
-    }
-
-    if (docRefs.length) { 
-        const snaps = await db.getAll(...docRefs); 
-        const hydrationPromises = [];
-        
-        snaps.forEach((doc, i) => { 
-            const name = names[i]; 
-            if (!doc.exists) return; 
-            const data = doc.data(); 
-
-            // Handle Decompression
-            if (data._compressed === true && data.payload) {
-                try {
-                    const unzipped = zlib.gunzipSync(data.payload);
-                    fetched[name] = JSON.parse(unzipped.toString());
-                } catch (e) {
-                    console.error(`[Hydration] Failed to decompress ${name}:`, e);
-                    fetched[name] = {};
-                }
-            }
-            // Handle Sharding
-            else if (data._sharded === true) { 
-                hydrationPromises.push(hydrateAutoShardedResult(doc.ref, name)); 
-            } 
-            // Standard
-            else if (data._completed) { 
-                fetched[name] = data; 
-            } 
-        });
-
-        if (hydrationPromises.length > 0) { 
-            const hydratedResults = await Promise.all(hydrationPromises); 
-            hydratedResults.forEach(res => { fetched[res.name] = res.data; }); 
-        } 
-    }
-    return fetched;
-}
-
-async function hydrateAutoShardedResult(docRef, resultName) {
-    const shardsCol = docRef.collection('_shards');
-    const snapshot = await shardsCol.get();
-    const assembledData = { _completed: true }; 
-    snapshot.forEach(doc => { 
-        const chunk = doc.data(); 
-        // [FIX] Ensure we don't merge metadata fields that might corrupt the object
-        const { _expireAt, ...safeChunk } = chunk;
-        Object.assign(assembledData, safeChunk); 
+    logger.log('INFO', `[DependencyFetcher] Fetching ${needed.size} dependencies for ${dStr}`);
+    
+    const results = {};
+    const promises = Array.from(needed).map(async (name) => {
+        try {
+            // Resolve Category from Lookup, default to 'analytics' if unknown
+            const category = manifestLookup[name] || 'analytics'; 
+            const data = await fetchSingleResult(db, config, dStr, name, category);
+            if (data) results[name] = data;
+        } catch (e) {
+            logger.log('WARN', `[DependencyFetcher] Failed to load dependency ${name}: ${e.message}`);
+        }
     });
-    delete assembledData._sharded;
-    delete assembledData._completed;
-    return { name: resultName, data: assembledData };
+    
+    await Promise.all(promises);
+    return results;
 }
 
 /**
- * [OPTIMIZED] Fetch Result Series using Batch Read
- * Reduces N x M reads to a single (or chunked) getAll operation.
+ * Fetches result series (Historical data) for lookbacks.
+ * @param {string} endDateStr - The most recent date.
+ * @param {Array} calcNames - Names of computations to fetch.
+ * @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
  */
-async function fetchResultSeries(dateStr, calcsToFetchNames, fullManifest, config, deps, lookbackDays) {
-    const { db } = deps;
-    const results = {}; // Structure: { [date]: { [calcName]: data } }
-    const endDate = new Date(dateStr);
+async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config, deps, lookbackDays) {
+    const { db, logger } = deps;
+    const results = {};
+    const dates = [];
     
-    // 1. Build Manifest Map for quick lookups
-    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
+    // Generate date list (starting from yesterday relative to endDateStr)
+    const d = new Date(endDateStr);
+    for (let i = 0; i < lookbackDays; i++) {
+        d.setUTCDate(d.getUTCDate() - 1); 
+        dates.push(d.toISOString().slice(0, 10));
+    }
     
-    // 2. Pre-calculate all Document References needed
-    const batchRequest = [];
+    // Initialize structure
+    calcNames.forEach(name => { results[normalizeName(name)] = {}; });
     
-    for (let i = 0; i < lookbackDays; i++) {
-        const d = new Date(endDate);
-        d.setUTCDate(d.getUTCDate() - i);
-        const dString = d.toISOString().slice(0, 10);
-        
-        for (const name of calcsToFetchNames) {
-            const normName = normalizeName(name);
-            const m = manifestMap.get(normName);
-            if (!m) continue;
+    logger.log('INFO', `[DependencyFetcher] Loading series for ${calcNames.length} calcs over ${lookbackDays} days.`);
 
-            const ref = db.collection(config.resultsCollection)
-                .doc(dString)
-                .collection(config.resultsSubcollection)
-                .doc(m.category || 'unknown')
-                .collection(config.computationsSubcollection)
-                .doc(normName);
-                
-            batchRequest.push({ date: dString, name: normName, ref });
+    const fetchOps = [];
+    
+    for (const dateStr of dates) {
+        for (const rawName of calcNames) {
+            const normName = normalizeName(rawName);
+            const category = manifestLookup[normName] || 'analytics';
+            
+            fetchOps.push(async () => {
+                const val = await fetchSingleResult(db, config, dateStr, rawName, category);
+                if (val) {
+                    if (!results[normName]) results[normName] = {};
+                    results[normName][dateStr] = val;
+                }
+            });
         }
     }
+    
+    // Limited concurrency batch execution (Batch size 20)
+    const BATCH_SIZE = 20;
+    for (let i = 0; i < fetchOps.length; i += BATCH_SIZE) {
+        await Promise.all(fetchOps.slice(i, i + BATCH_SIZE).map(fn => fn()));
+    }
+    
+    return results;
+}
 
-    if (batchRequest.length === 0) return {};
-
-    // 3. Batch Fetch (Chunked to respect Firestore limits, usually 100-500 is safe)
-    const BATCH_SIZE = 100;
-    const hydrationTasks = [];
+/**
+ * Core Helper: Fetches a single result, handles Sharding & Compression.
+ */
+async function fetchSingleResult(db, config, dateStr, name, category) {
+    const docRef = db.collection(config.resultsCollection)
+                     .doc(dateStr)
+                     .collection(config.resultsSubcollection)
+                     .doc(category)
+                     .collection(config.computationsSubcollection)
+                     .doc(name);
+                     
+    const snap = await docRef.get();
+    if (!snap.exists) return null;
     
-    // Helper to process a batch of snapshots
-    const processBatch = async (items) => {
-        const refs = items.map(i => i.ref);
-        let snapshots;
+    let data = snap.data();
+    
+    // 1. Handle Compression
+    if (data._compressed && data.payload) {
         try {
-            snapshots = await db.getAll(...refs);
+            const buffer = (data.payload instanceof Buffer) ? data.payload : data.payload.toDate();
+            const decompressed = zlib.gunzipSync(buffer);
+            const jsonStr = decompressed.toString('utf8');
+            const realData = JSON.parse(jsonStr);
+            // Merge decompressed data
+            data = { ...data, ...realData }; 
+            delete data.payload; 
         } catch (e) {
-            console.warn(`[DependencyFetcher] Batch read failed: ${e.message}. Skipping batch.`);
-            return;
+            console.warn(`[DependencyFetcher] Decompression failed for ${name}: ${e.message}`);
+            return null;
         }
-
-        for (let i = 0; i < snapshots.length; i++) {
-            const doc = snapshots[i];
-            const meta = items[i];
-            
-            if (!doc.exists) continue;
-            
-            const data = doc.data();
-            let finalData = null;
-
-            // A. Compressed
-            if (data._compressed === true && data.payload) {
-                try {
-                    const unzipped = zlib.gunzipSync(data.payload);
-                    finalData = JSON.parse(unzipped.toString());
-                } catch (e) {
-                    console.error(`[Hydration] Failed to decompress ${meta.name} for ${meta.date}`, e);
-                }
-            }
-            // B. Sharded (Defer hydration to avoid blocking the loop)
-            else if (data._sharded === true) {
-                hydrationTasks.push({
-                    date: meta.date,
-                    name: meta.name,
-                    ref: doc.ref
+    }
+    
+    // 2. Handle Sharding
+    if (data._sharded) {
+        const shardCol = docRef.collection('_shards');
+        const shardSnaps = await shardCol.get();
+        
+        if (!shardSnaps.empty) {
+            shardSnaps.forEach(shard => {
+                const shardData = shard.data();
+                // Merge shard contents, ignoring internal metadata if it clashes
+                Object.entries(shardData).forEach(([k, v]) => {
+                    if (!k.startsWith('_')) {
+                        data[k] = v;
+                    }
                 });
-                continue; // Skip immediate assignment
-            }
-            // C. Standard
-            else if (data._completed) {
-                finalData = data;
-            }
-
-            // Assign if we have data
-            if (finalData) {
-                if (!results[meta.date]) results[meta.date] = {};
-                results[meta.date][meta.name] = finalData;
-            }
+            });
         }
-    };
-
-    // Execute batches
-    for (let i = 0; i < batchRequest.length; i += BATCH_SIZE) {
-        const chunk = batchRequest.slice(i, i + BATCH_SIZE);
-        await processBatch(chunk);
-    }
-
-    // 4. Handle Sharded Results (Parallel Hydration)
-    if (hydrationTasks.length > 0) {
-        // Limit concurrency for shard fetching to avoid overwhelming the client
-        const limit = pLimit(20); 
-        await Promise.all(hydrationTasks.map(task => limit(async () => {
-            try {
-                const res = await hydrateAutoShardedResult(task.ref, task.name);
-                if (!results[task.date]) results[task.date] = {};
-                results[task.date][task.name] = res.data;
-            } catch (e) {
-                console.warn(`[DependencyFetcher] Failed to hydrate shards for ${task.name}/${task.date}: ${e.message}`);
-            }
-        })));
     }
-
-    return results;
+    
+    return data;
 }
 
-module.exports = { fetchExistingResults, fetchResultSeries };
+module.exports = { fetchDependencies, fetchResultSeries };

package/functions/computation-system/executors/MetaExecutor.js

@@ -5,12 +5,14 @@
  * UPDATED: Sends 'isInitialWrite: true' for robust cleanup.
  * UPDATED: Support for historical rankings in Meta Context.
  * UPDATED: Added support for loading Series Data (Root & Results) for lookbacks.
+ * UPDATED: Builds Manifest Lookup for DependencyFetcher.
  */
 const { normalizeName }     = require('../utils/utils');
 const { CachedDataLoader }  = require('../data/CachedDataLoader');
 const { ContextFactory }    = require('../context/ContextFactory');
 const { commitResults }     = require('../persistence/ResultCommitter');
-const { fetchResultSeries } = require('../data/DependencyFetcher');
+const { fetchResultSeries, fetchDependencies } = require('../data/DependencyFetcher');
+const { getManifest }    = require('../topology/ManifestLoader');
 
 class MetaExecutor {
     static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
@@ -18,6 +20,12 @@ class MetaExecutor {
         const { logger, db } = deps;
         const loader = new CachedDataLoader(config, deps);
 
+        // --- [FIXED] Build Global Manifest Lookup using getManifest ---
+        // getManifest is typically cached, so this is cheap.
+        const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps); 
+        const manifestLookup = {};
+        allManifests.forEach(m => { manifestLookup[normalizeName(m.name)] = m.category; });
+
         // [FIX] Check if any meta calculation needs history
         const needsHistory = calcs.some(c => c.isHistorical);
         let rankingsYesterday = null;
@@ -51,16 +59,11 @@ class MetaExecutor {
             // Ratings
             if (calcs.some(c => c.rootDataDependencies?.includes('ratings'))) {
                 loadPromises.push(loader.loadRatings(dStr).then(r => { ratings = r; }).catch(e => {
-                    // Only catch if ALL calcs allow missing roots
                     const allAllowMissing = calcs.every(c => {
                         const needsRatings = c.rootDataDependencies?.includes('ratings');
                         return !needsRatings || c.canHaveMissingRoots === true;
                     });
-                    if (allAllowMissing) {
-                        ratings = null;
-                    } else {
-                        throw e; // Re-throw if any calc requires this data
-                    }
+                    if (allAllowMissing) { ratings = null; } else { throw e; }
                 }));
             }
             
@@ -71,11 +74,7 @@ class MetaExecutor {
                         const needsPageViews = c.rootDataDependencies?.includes('pageViews');
                         return !needsPageViews || c.canHaveMissingRoots === true;
                     });
-                    if (allAllowMissing) {
-                        pageViews = null;
-                    } else {
-                        throw e;
-                    }
+                    if (allAllowMissing) { pageViews = null; } else { throw e; }
                 }));
             }
             
@@ -86,11 +85,7 @@ class MetaExecutor {
                         const needsWatchlist = c.rootDataDependencies?.includes('watchlist');
                         return !needsWatchlist || c.canHaveMissingRoots === true;
                     });
-                    if (allAllowMissing) {
-                        watchlistMembership = null;
-                    } else {
-                        throw e;
-                    }
+                    if (allAllowMissing) { watchlistMembership = null; } else { throw e; }
                 }));
             }
             
@@ -101,11 +96,7 @@ class MetaExecutor {
                         const needsAlerts = c.rootDataDependencies?.includes('alerts');
                         return !needsAlerts || c.canHaveMissingRoots === true;
                     });
-                    if (allAllowMissing) {
-                        alertHistory = null;
-                    } else {
-                        throw e;
-                    }
+                    if (allAllowMissing) { alertHistory = null; } else { throw e; }
                 }));
             }
             
@@ -115,23 +106,13 @@ class MetaExecutor {
             for (const c of calcs) {
                 const deps = c.rootDataDependencies || [];
                 const canSkip = c.canHaveMissingRoots === true;
-                
-                // Helper to check if a specific root is missing
                 const isMissing = (key, val) => deps.includes(key) && (val === null || val === undefined);
                 
                 if (!canSkip) {
-                    if (isMissing('ratings', ratings)) {
-                        throw new Error(`[MetaExecutor] Missing required root 'ratings' for ${c.name}`);
-                    }
-                    if (isMissing('pageViews', pageViews)) {
-                        throw new Error(`[MetaExecutor] Missing required root 'pageViews' for ${c.name}`);
-                    }
-                    if (isMissing('watchlist', watchlistMembership)) {
-                        throw new Error(`[MetaExecutor] Missing required root 'watchlist' for ${c.name}`);
-                    }
-                    if (isMissing('alerts', alertHistory)) {
-                        throw new Error(`[MetaExecutor] Missing required root 'alerts' for ${c.name}`);
-                    }
+                    if (isMissing('ratings', ratings)) throw new Error(`[MetaExecutor] Missing required root 'ratings' for ${c.name}`);
+                    if (isMissing('pageViews', pageViews)) throw new Error(`[MetaExecutor] Missing required root 'pageViews' for ${c.name}`);
+                    if (isMissing('watchlist', watchlistMembership)) throw new Error(`[MetaExecutor] Missing required root 'watchlist' for ${c.name}`);
+                    if (isMissing('alerts', alertHistory)) throw new Error(`[MetaExecutor] Missing required root 'alerts' for ${c.name}`);
                 }
             }
         }
@@ -142,7 +123,6 @@ class MetaExecutor {
 
         // 1. Identify requirements from manifests
         calcs.forEach(c => {
-            // Check for Root Data Series
             if (c.rootDataSeries) {
                 Object.entries(c.rootDataSeries).forEach(([type, conf]) => {
                     const days = typeof conf === 'object' ? conf.lookback : conf;
@@ -151,7 +131,6 @@ class MetaExecutor {
                     }
                 });
             }
-            // Check for Computation Result Series
             if (c.dependencySeries) {
                 Object.entries(c.dependencySeries).forEach(([depName, conf]) => {
                     const days = typeof conf === 'object' ? conf.lookback : conf;
@@ -177,9 +156,8 @@ class MetaExecutor {
 
             if (loaderMethod) {
                 logger.log('INFO', `[MetaExecutor] Loading ${days}-day series for Root Data '${type}'...`);
-                // Assume CachedDataLoader has loadSeries method (added in previous step)
                 const series = await loader.loadSeries(loaderMethod, dStr, days);
-                seriesData.root[type] = series.data; // map of date -> data
+                seriesData.root[type] = series.data;
             }
         }
 
@@ -189,11 +167,10 @@ class MetaExecutor {
             const maxDays = Math.max(...Object.values(dependencySeriesRequests));
             logger.log('INFO', `[MetaExecutor] Loading up to ${maxDays}-day series for Dependencies: ${calcNamesToFetch.join(', ')}`);
             
-            // We pass the list of manifests (calcs) so the fetcher knows details if needed
-            const resultsSeries = await fetchResultSeries(dStr, calcNamesToFetch, calcs, config, deps, maxDays);
-            seriesData.results = resultsSeries; // map of date -> { calcName: data }
+            // Pass manifestLookup to fetcher
+            const resultsSeries = await fetchResultSeries(dStr, calcNamesToFetch, manifestLookup, config, deps, maxDays);
+            seriesData.results = resultsSeries; 
         }
-        // ---------------------------------------------
 
         const state = {};
         for (const c of calcs) {
@@ -210,21 +187,17 @@ class MetaExecutor {
                 previousComputedDependencies: previousFetchedDeps,
                 config, deps,
                 allRankings: rankings,
-                allRankingsYesterday: rankingsYesterday, // [FIX] Injected
+                allRankingsYesterday: rankingsYesterday,
                 allVerifications: verifications,
-                // [NEW] Pass New Root Data Types for Profile Metrics
                 ratings: ratings || {},
                 pageViews: pageViews || {},
                 watchlistMembership: watchlistMembership || {},
                 alertHistory: alertHistory || {},
-                // [NEW] Pass Series Data
                 seriesData
             });
 
             try {
                 const result = await inst.process(context);
-                // Meta results are usually wrapped in a global key or just the result object
-                // The structure below implies we store it under the date key
                 inst.results = { [dStr]: { global: result } };
                 state[c.name] = inst;
             } catch (e) {
@@ -244,7 +217,6 @@ class MetaExecutor {
         const insights   = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
         const social     = metadata.rootDataDependencies?.includes('social')   ? { today: await loader.loadSocial(dateStr)   } : null;
         
-        // [FIX] Historical support for Batch/OncePerDay execution
         let rankingsYesterday = null;
         if (metadata.isHistorical) {
              const prevDate = new Date(dateStr);
@@ -253,67 +225,37 @@ class MetaExecutor {
              rankingsYesterday = await loader.loadRankings(prevStr);
         }
 
-        // Load current rankings (often needed for ContextFactory.buildMetaContext)
         const rankings = await loader.loadRankings(dateStr);
-        
-        // [NEW] Load New Root Data Types for Profile Metrics
-        // [FIX] Enforce canHaveMissingRoots
         const allowMissing = metadata.canHaveMissingRoots === true;
         
         let ratings = null;
         if (metadata.rootDataDependencies?.includes('ratings')) {
-            try {
-                ratings = await loader.loadRatings(dateStr);
-            } catch (e) {
-                if (!allowMissing) throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'ratings' failed to load for ${metadata.name}: ${e.message}`);
-                ratings = null;
-            }
-            if (!ratings && !allowMissing) {
-                throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'ratings' is missing for ${metadata.name}`);
-            }
+            try { ratings = await loader.loadRatings(dateStr); } 
+            catch (e) { if (!allowMissing) throw e; ratings = null; }
+            if (!ratings && !allowMissing) throw new Error(`Missing ratings`);
         }
         
         let pageViews = null;
         if (metadata.rootDataDependencies?.includes('pageViews')) {
-            try {
-                pageViews = await loader.loadPageViews(dateStr);
-            } catch (e) {
-                if (!allowMissing) throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'pageViews' failed to load for ${metadata.name}: ${e.message}`);
-                pageViews = null;
-            }
-            if (!pageViews && !allowMissing) {
-                throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'pageViews' is missing for ${metadata.name}`);
-            }
+            try { pageViews = await loader.loadPageViews(dateStr); } 
+            catch (e) { if (!allowMissing) throw e; pageViews = null; }
+            if (!pageViews && !allowMissing) throw new Error(`Missing pageViews`);
         }
         
         let watchlistMembership = null;
         if (metadata.rootDataDependencies?.includes('watchlist')) {
-            try {
-                watchlistMembership = await loader.loadWatchlistMembership(dateStr);
-            } catch (e) {
-                if (!allowMissing) throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'watchlist' failed to load for ${metadata.name}: ${e.message}`);
-                watchlistMembership = null;
-            }
-            if (!watchlistMembership && !allowMissing) {
-                throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'watchlist' is missing for ${metadata.name}`);
-            }
+            try { watchlistMembership = await loader.loadWatchlistMembership(dateStr); } 
+            catch (e) { if (!allowMissing) throw e; watchlistMembership = null; }
+            if (!watchlistMembership && !allowMissing) throw new Error(`Missing watchlist`);
         }
         
         let alertHistory = null;
         if (metadata.rootDataDependencies?.includes('alerts')) {
-            try {
-                alertHistory = await loader.loadAlertHistory(dateStr);
-            } catch (e) {
-                if (!allowMissing) throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'alerts' failed to load for ${metadata.name}: ${e.message}`);
-                alertHistory = null;
-            }
-            if (!alertHistory && !allowMissing) {
-                throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'alerts' is missing for ${metadata.name}`);
-            }
+            try { alertHistory = await loader.loadAlertHistory(dateStr); } 
+            catch (e) { if (!allowMissing) throw e; alertHistory = null; }
+            if (!alertHistory && !allowMissing) throw new Error(`Missing alerts`);
         }
 
-        // [NOTE] "executeOncePerDay" is typically for sharded price/batch jobs. 
-        // We initialize empty series data to maintain compatibility.
         const seriesData = { root: {}, results: {} }; 
 
         if (metadata.rootDataDependencies?.includes('price')) {
@@ -330,12 +272,10 @@ class MetaExecutor {
                     previousComputedDependencies: prevDeps, config, deps,
                     allRankings: rankings,
                     allRankingsYesterday: rankingsYesterday,
-                    // [NEW] Pass New Root Data Types for Profile Metrics
                     ratings: ratings || {},
                     pageViews: pageViews || {},
                     watchlistMembership: watchlistMembership || {},
                     alertHistory: alertHistory || {},
-                    // [NEW] Pass Series Data
                     seriesData
                 });
                 
@@ -357,12 +297,10 @@ class MetaExecutor {
                 previousComputedDependencies: prevDeps, config, deps,
                 allRankings: rankings,
                 allRankingsYesterday: rankingsYesterday,
-                // [NEW] Pass New Root Data Types for Profile Metrics
                 ratings: ratings || {},
                 pageViews: pageViews || {},
                 watchlistMembership: watchlistMembership || {},
                 alertHistory: alertHistory || {},
-                // [NEW] Pass Series Data
                 seriesData
             });
             const res = await calcInstance.process(context);

package/functions/computation-system/executors/StandardExecutor.js

@@ -4,6 +4,7 @@ const { CachedDataLoader }                                             = require
 const { ContextFactory }                                               = require('../context/ContextFactory');
 const { commitResults }                                                = require('../persistence/ResultCommitter');
 const { fetchResultSeries }                                            = require('../data/DependencyFetcher');
+const { getManifest }                                                  = require('../topology/ManifestLoader'); 
 const mathLayer                                                        = require('../layers/index');
 const { performance }                                                  = require('perf_hooks');
 const v8                                                               = require('v8'); 
@@ -71,6 +72,11 @@ class StandardExecutor {
         
         if (streamingCalcs.length === 0) return { successUpdates: {}, failureReport: [] };
 
+        // --- 0. [NEW] Build Global Manifest Lookup ---
+        const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps); 
+        const manifestLookup = {};
+        allManifests.forEach(m => { manifestLookup[normalizeName(m.name)] = m.category; });
+
         // --- 1. Resolve and Filter Portfolio Refs (Today) ---
         let effectivePortfolioRefs = portfolioRefs;
         if (!effectivePortfolioRefs) {
@@ -177,22 +183,8 @@ class StandardExecutor {
             const maxDays = Math.max(...Object.values(dependencySeriesRequests));
             logger.log('INFO', `[StandardExecutor] Loading up to ${maxDays}-day series for Dependencies: ${calcNamesToFetch.join(', ')}`);
             
-            const allManifests = streamingCalcs.map(c => c.manifest); 
-            const resultsSeries = await fetchResultSeries(dateStr, calcNamesToFetch, allManifests, config, deps, maxDays);
-            
-            // [NEW] Validate Series Availability
-            streamingCalcs.forEach(c => {
-                if (c.manifest.dependencySeries && !c.manifest.canHaveMissingSeries) {
-                    Object.keys(c.manifest.dependencySeries).forEach(depName => {
-                        const normDep = normalizeName(depName);
-                        // If series data is completely missing for a required dependency, verify logic
-                        if (!resultsSeries[normDep] || Object.keys(resultsSeries[normDep]).length === 0) {
-                            logger.log('WARN', `[StandardExecutor] Missing dependency series '${depName}' for '${c.name}' (and canHaveMissingSeries=false). This may cause calculation errors.`);
-                        }
-                    });
-                }
-            });
-
+            // Pass manifestLookup to fetcher
+            const resultsSeries = await fetchResultSeries(dateStr, calcNamesToFetch, manifestLookup, config, deps, maxDays);
             seriesData.results = resultsSeries; 
         }
         // ---------------------------------------------

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.632",
+  "version": "1.0.633",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [