bulltrackers-module 1.0.631 → 1.0.633

package/functions/computation-system/executors/MetaExecutor.js

@@ -5,12 +5,14 @@
  * UPDATED: Sends 'isInitialWrite: true' for robust cleanup.
  * UPDATED: Support for historical rankings in Meta Context.
  * UPDATED: Added support for loading Series Data (Root & Results) for lookbacks.
+ * UPDATED: Builds Manifest Lookup for DependencyFetcher.
  */
 const { normalizeName }     = require('../utils/utils');
 const { CachedDataLoader }  = require('../data/CachedDataLoader');
 const { ContextFactory }    = require('../context/ContextFactory');
 const { commitResults }     = require('../persistence/ResultCommitter');
-const { fetchResultSeries } = require('../data/DependencyFetcher'); // [NEW] Import series fetcher
+const { fetchResultSeries, fetchDependencies } = require('../data/DependencyFetcher');
+const { getManifest }    = require('../topology/ManifestLoader');
 
 class MetaExecutor {
     static async run(date, calcs, passName, config, deps, rootData, fetchedDeps, previousFetchedDeps) {
@@ -18,6 +20,12 @@ class MetaExecutor {
         const { logger, db } = deps;
         const loader = new CachedDataLoader(config, deps);
 
+        // --- [FIXED] Build Global Manifest Lookup using getManifest ---
+        // getManifest is typically cached, so this is cheap.
+        const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps); 
+        const manifestLookup = {};
+        allManifests.forEach(m => { manifestLookup[normalizeName(m.name)] = m.category; });
+
         // [FIX] Check if any meta calculation needs history
         const needsHistory = calcs.some(c => c.isHistorical);
         let rankingsYesterday = null;
@@ -47,82 +55,64 @@ class MetaExecutor {
         let ratings = null, pageViews = null, watchlistMembership = null, alertHistory = null;
         if (needsNewRootData) {
             const loadPromises = [];
+            
+            // Ratings
             if (calcs.some(c => c.rootDataDependencies?.includes('ratings'))) {
                 loadPromises.push(loader.loadRatings(dStr).then(r => { ratings = r; }).catch(e => {
-                    // Only catch if ALL calcs allow missing roots
                     const allAllowMissing = calcs.every(c => {
                         const needsRatings = c.rootDataDependencies?.includes('ratings');
                         return !needsRatings || c.canHaveMissingRoots === true;
                     });
-                    if (allAllowMissing) {
-                        ratings = null;
-                    } else {
-                        throw e; // Re-throw if any calc requires this data
-                    }
+                    if (allAllowMissing) { ratings = null; } else { throw e; }
                 }));
             }
+            
+            // PageViews
             if (calcs.some(c => c.rootDataDependencies?.includes('pageViews'))) {
                 loadPromises.push(loader.loadPageViews(dStr).then(pv => { pageViews = pv; }).catch(e => {
                     const allAllowMissing = calcs.every(c => {
                         const needsPageViews = c.rootDataDependencies?.includes('pageViews');
                         return !needsPageViews || c.canHaveMissingRoots === true;
                     });
-                    if (allAllowMissing) {
-                        pageViews = null;
-                    } else {
-                        throw e;
-                    }
+                    if (allAllowMissing) { pageViews = null; } else { throw e; }
                 }));
             }
+            
+            // Watchlist
             if (calcs.some(c => c.rootDataDependencies?.includes('watchlist'))) {
                 loadPromises.push(loader.loadWatchlistMembership(dStr).then(w => { watchlistMembership = w; }).catch(e => {
                     const allAllowMissing = calcs.every(c => {
                         const needsWatchlist = c.rootDataDependencies?.includes('watchlist');
                         return !needsWatchlist || c.canHaveMissingRoots === true;
                     });
-                    if (allAllowMissing) {
-                        watchlistMembership = null;
-                    } else {
-                        throw e;
-                    }
+                    if (allAllowMissing) { watchlistMembership = null; } else { throw e; }
                 }));
             }
+            
+            // Alerts
             if (calcs.some(c => c.rootDataDependencies?.includes('alerts'))) {
                 loadPromises.push(loader.loadAlertHistory(dStr).then(a => { alertHistory = a; }).catch(e => {
                     const allAllowMissing = calcs.every(c => {
                         const needsAlerts = c.rootDataDependencies?.includes('alerts');
                         return !needsAlerts || c.canHaveMissingRoots === true;
                     });
-                    if (allAllowMissing) {
-                        alertHistory = null;
-                    } else {
-                        throw e;
-                    }
+                    if (allAllowMissing) { alertHistory = null; } else { throw e; }
                 }));
             }
+            
             await Promise.all(loadPromises);
             
             // [FIX] Enforce canHaveMissingRoots - validate after loading
             for (const c of calcs) {
                 const deps = c.rootDataDependencies || [];
                 const canSkip = c.canHaveMissingRoots === true;
-                
-                // Helper to check if a specific root is missing
                 const isMissing = (key, val) => deps.includes(key) && (val === null || val === undefined);
                 
                 if (!canSkip) {
-                    if (isMissing('ratings', ratings)) {
-                        throw new Error(`[MetaExecutor] Missing required root 'ratings' for ${c.name}`);
-                    }
-                    if (isMissing('pageViews', pageViews)) {
-                        throw new Error(`[MetaExecutor] Missing required root 'pageViews' for ${c.name}`);
-                    }
-                    if (isMissing('watchlist', watchlistMembership)) {
-                        throw new Error(`[MetaExecutor] Missing required root 'watchlist' for ${c.name}`);
-                    }
-                    if (isMissing('alerts', alertHistory)) {
-                        throw new Error(`[MetaExecutor] Missing required root 'alerts' for ${c.name}`);
-                    }
+                    if (isMissing('ratings', ratings)) throw new Error(`[MetaExecutor] Missing required root 'ratings' for ${c.name}`);
+                    if (isMissing('pageViews', pageViews)) throw new Error(`[MetaExecutor] Missing required root 'pageViews' for ${c.name}`);
+                    if (isMissing('watchlist', watchlistMembership)) throw new Error(`[MetaExecutor] Missing required root 'watchlist' for ${c.name}`);
+                    if (isMissing('alerts', alertHistory)) throw new Error(`[MetaExecutor] Missing required root 'alerts' for ${c.name}`);
                 }
             }
         }
@@ -133,7 +123,6 @@ class MetaExecutor {
 
         // 1. Identify requirements from manifests
         calcs.forEach(c => {
-            // Check for Root Data Series
             if (c.rootDataSeries) {
                 Object.entries(c.rootDataSeries).forEach(([type, conf]) => {
                     const days = typeof conf === 'object' ? conf.lookback : conf;
@@ -142,7 +131,6 @@ class MetaExecutor {
                     }
                 });
             }
-            // Check for Computation Result Series
             if (c.dependencySeries) {
                 Object.entries(c.dependencySeries).forEach(([depName, conf]) => {
                     const days = typeof conf === 'object' ? conf.lookback : conf;
@@ -163,13 +151,13 @@ class MetaExecutor {
             else if (type === 'insights') loaderMethod = 'loadInsights';
             else if (type === 'ratings') loaderMethod = 'loadRatings';
             else if (type === 'watchlist') loaderMethod = 'loadWatchlistMembership';
-            // Add other root types if needed
+            // [CRITICAL UPDATE] Add rankings support for Meta lookbacks
+            else if (type === 'rankings') loaderMethod = 'loadRankings';
 
             if (loaderMethod) {
                 logger.log('INFO', `[MetaExecutor] Loading ${days}-day series for Root Data '${type}'...`);
-                // Assume CachedDataLoader has loadSeries method (added in previous step)
                 const series = await loader.loadSeries(loaderMethod, dStr, days);
-                seriesData.root[type] = series.data; // map of date -> data
+                seriesData.root[type] = series.data;
             }
         }
 
@@ -179,11 +167,10 @@ class MetaExecutor {
             const maxDays = Math.max(...Object.values(dependencySeriesRequests));
             logger.log('INFO', `[MetaExecutor] Loading up to ${maxDays}-day series for Dependencies: ${calcNamesToFetch.join(', ')}`);
             
-            // We pass the list of manifests (calcs) so the fetcher knows details if needed
-            const resultsSeries = await fetchResultSeries(dStr, calcNamesToFetch, calcs, config, deps, maxDays);
-            seriesData.results = resultsSeries; // map of date -> { calcName: data }
+            // Pass manifestLookup to fetcher
+            const resultsSeries = await fetchResultSeries(dStr, calcNamesToFetch, manifestLookup, config, deps, maxDays);
+            seriesData.results = resultsSeries; 
         }
-        // ---------------------------------------------
 
         const state = {};
         for (const c of calcs) {
@@ -200,21 +187,17 @@ class MetaExecutor {
                 previousComputedDependencies: previousFetchedDeps,
                 config, deps,
                 allRankings: rankings,
-                allRankingsYesterday: rankingsYesterday, // [FIX] Injected
+                allRankingsYesterday: rankingsYesterday,
                 allVerifications: verifications,
-                // [NEW] Pass New Root Data Types for Profile Metrics
                 ratings: ratings || {},
                 pageViews: pageViews || {},
                 watchlistMembership: watchlistMembership || {},
                 alertHistory: alertHistory || {},
-                // [NEW] Pass Series Data
                 seriesData
             });
 
             try {
                 const result = await inst.process(context);
-                // Meta results are usually wrapped in a global key or just the result object
-                // The structure below implies we store it under the date key
                 inst.results = { [dStr]: { global: result } };
                 state[c.name] = inst;
             } catch (e) {
@@ -222,7 +205,8 @@ class MetaExecutor {
             }
         }
 
-        return await commitResults(state, dStr, passName, config, deps);
+        // CRITICAL FIX: Pass 'isInitialWrite: true' to ensure proper cleanup of old meta data
+        return await commitResults(state, dStr, passName, config, deps, false, { isInitialWrite: true });
     }
 
     static async executeOncePerDay(calcInstance, metadata, dateStr, computedDeps, prevDeps, config, deps, loader) {
@@ -233,7 +217,6 @@ class MetaExecutor {
         const insights   = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
         const social     = metadata.rootDataDependencies?.includes('social')   ? { today: await loader.loadSocial(dateStr)   } : null;
         
-        // [FIX] Historical support for Batch/OncePerDay execution
         let rankingsYesterday = null;
         if (metadata.isHistorical) {
              const prevDate = new Date(dateStr);
@@ -242,67 +225,37 @@ class MetaExecutor {
              rankingsYesterday = await loader.loadRankings(prevStr);
         }
 
-        // Load current rankings (often needed for ContextFactory.buildMetaContext)
         const rankings = await loader.loadRankings(dateStr);
-        
-        // [NEW] Load New Root Data Types for Profile Metrics
-        // [FIX] Enforce canHaveMissingRoots
         const allowMissing = metadata.canHaveMissingRoots === true;
         
         let ratings = null;
         if (metadata.rootDataDependencies?.includes('ratings')) {
-            try {
-                ratings = await loader.loadRatings(dateStr);
-            } catch (e) {
-                if (!allowMissing) throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'ratings' failed to load for ${metadata.name}: ${e.message}`);
-                ratings = null;
-            }
-            if (!ratings && !allowMissing) {
-                throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'ratings' is missing for ${metadata.name}`);
-            }
+            try { ratings = await loader.loadRatings(dateStr); } 
+            catch (e) { if (!allowMissing) throw e; ratings = null; }
+            if (!ratings && !allowMissing) throw new Error(`Missing ratings`);
         }
         
         let pageViews = null;
         if (metadata.rootDataDependencies?.includes('pageViews')) {
-            try {
-                pageViews = await loader.loadPageViews(dateStr);
-            } catch (e) {
-                if (!allowMissing) throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'pageViews' failed to load for ${metadata.name}: ${e.message}`);
-                pageViews = null;
-            }
-            if (!pageViews && !allowMissing) {
-                throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'pageViews' is missing for ${metadata.name}`);
-            }
+            try { pageViews = await loader.loadPageViews(dateStr); } 
+            catch (e) { if (!allowMissing) throw e; pageViews = null; }
+            if (!pageViews && !allowMissing) throw new Error(`Missing pageViews`);
         }
         
         let watchlistMembership = null;
         if (metadata.rootDataDependencies?.includes('watchlist')) {
-            try {
-                watchlistMembership = await loader.loadWatchlistMembership(dateStr);
-            } catch (e) {
-                if (!allowMissing) throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'watchlist' failed to load for ${metadata.name}: ${e.message}`);
-                watchlistMembership = null;
-            }
-            if (!watchlistMembership && !allowMissing) {
-                throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'watchlist' is missing for ${metadata.name}`);
-            }
+            try { watchlistMembership = await loader.loadWatchlistMembership(dateStr); } 
+            catch (e) { if (!allowMissing) throw e; watchlistMembership = null; }
+            if (!watchlistMembership && !allowMissing) throw new Error(`Missing watchlist`);
         }
         
         let alertHistory = null;
         if (metadata.rootDataDependencies?.includes('alerts')) {
-            try {
-                alertHistory = await loader.loadAlertHistory(dateStr);
-            } catch (e) {
-                if (!allowMissing) throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'alerts' failed to load for ${metadata.name}: ${e.message}`);
-                alertHistory = null;
-            }
-            if (!alertHistory && !allowMissing) {
-                throw new Error(`[MetaExecutor.executeOncePerDay] Required root 'alerts' is missing for ${metadata.name}`);
-            }
+            try { alertHistory = await loader.loadAlertHistory(dateStr); } 
+            catch (e) { if (!allowMissing) throw e; alertHistory = null; }
+            if (!alertHistory && !allowMissing) throw new Error(`Missing alerts`);
         }
 
-        // [NOTE] "executeOncePerDay" is typically for sharded price/batch jobs. 
-        // We initialize empty series data to maintain compatibility.
         const seriesData = { root: {}, results: {} }; 
 
         if (metadata.rootDataDependencies?.includes('price')) {
@@ -319,12 +272,10 @@ class MetaExecutor {
                     previousComputedDependencies: prevDeps, config, deps,
                     allRankings: rankings,
                     allRankingsYesterday: rankingsYesterday,
-                    // [NEW] Pass New Root Data Types for Profile Metrics
                     ratings: ratings || {},
                     pageViews: pageViews || {},
                     watchlistMembership: watchlistMembership || {},
                     alertHistory: alertHistory || {},
-                    // [NEW] Pass Series Data
                     seriesData
                 });
                 
@@ -346,12 +297,10 @@ class MetaExecutor {
                 previousComputedDependencies: prevDeps, config, deps,
                 allRankings: rankings,
                 allRankingsYesterday: rankingsYesterday,
-                // [NEW] Pass New Root Data Types for Profile Metrics
                 ratings: ratings || {},
                 pageViews: pageViews || {},
                 watchlistMembership: watchlistMembership || {},
                 alertHistory: alertHistory || {},
-                // [NEW] Pass Series Data
                 seriesData
             });
             const res = await calcInstance.process(context);

package/functions/computation-system/executors/StandardExecutor.js

@@ -1,16 +1,10 @@
-/**
- * {
- * type: uploaded file
- * fileName: computation-system/executors/StandardExecutor.js
- * }
- */
 const { normalizeName, getEarliestDataDates }                          = require('../utils/utils');
 const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
 const { CachedDataLoader }                                             = require('../data/CachedDataLoader');
 const { ContextFactory }                                               = require('../context/ContextFactory');
 const { commitResults }                                                = require('../persistence/ResultCommitter');
-// [NEW] Import series fetcher for computation results
 const { fetchResultSeries }                                            = require('../data/DependencyFetcher');
+const { getManifest }                                                  = require('../topology/ManifestLoader'); 
 const mathLayer                                                        = require('../layers/index');
 const { performance }                                                  = require('perf_hooks');
 const v8                                                               = require('v8'); 
@@ -78,6 +72,11 @@ class StandardExecutor {
         
         if (streamingCalcs.length === 0) return { successUpdates: {}, failureReport: [] };
 
+        // --- 0. [NEW] Build Global Manifest Lookup ---
+        const allManifests = getManifest(config.productLines, config.calculationsDirectory, deps); 
+        const manifestLookup = {};
+        allManifests.forEach(m => { manifestLookup[normalizeName(m.name)] = m.category; });
+
         // --- 1. Resolve and Filter Portfolio Refs (Today) ---
         let effectivePortfolioRefs = portfolioRefs;
         if (!effectivePortfolioRefs) {
@@ -167,7 +166,8 @@ class StandardExecutor {
             else if (type === 'insights') loaderMethod = 'loadInsights';
             else if (type === 'ratings') loaderMethod = 'loadRatings';
             else if (type === 'watchlist') loaderMethod = 'loadWatchlistMembership';
-            // Add other root types as needed...
+            // [CRITICAL UPDATE] Add rankings support for AUM lookback
+            else if (type === 'rankings') loaderMethod = 'loadRankings';
 
             if (loaderMethod) {
                 logger.log('INFO', `[StandardExecutor] Loading ${days}-day series for Root Data '${type}'...`);
@@ -183,22 +183,8 @@ class StandardExecutor {
             const maxDays = Math.max(...Object.values(dependencySeriesRequests));
             logger.log('INFO', `[StandardExecutor] Loading up to ${maxDays}-day series for Dependencies: ${calcNamesToFetch.join(', ')}`);
             
-            const allManifests = streamingCalcs.map(c => c.manifest); 
-            const resultsSeries = await fetchResultSeries(dateStr, calcNamesToFetch, allManifests, config, deps, maxDays);
-            
-            // [NEW] Validate Series Availability
-            streamingCalcs.forEach(c => {
-                if (c.manifest.dependencySeries && !c.manifest.canHaveMissingSeries) {
-                    Object.keys(c.manifest.dependencySeries).forEach(depName => {
-                        const normDep = normalizeName(depName);
-                        // If series data is completely missing for a required dependency, verify logic
-                        if (!resultsSeries[normDep] || Object.keys(resultsSeries[normDep]).length === 0) {
-                            logger.log('WARN', `[StandardExecutor] Missing dependency series '${depName}' for '${c.name}' (and canHaveMissingSeries=false). This may cause calculation errors.`);
-                        }
-                    });
-                }
-            });
-
+            // Pass manifestLookup to fetcher
+            const resultsSeries = await fetchResultSeries(dateStr, calcNamesToFetch, manifestLookup, config, deps, maxDays);
             seriesData.results = resultsSeries; 
         }
         // ---------------------------------------------
@@ -487,4 +473,4 @@ class StandardExecutor {
     }
 }
 
-module.exports = { StandardExecutor };
+module.exports = { StandardExecutor };

package/functions/computation-system/persistence/ResultCommitter.js

@@ -4,6 +4,7 @@
  * UPDATED: Added support for 'isPage' mode to store per-user data in subcollections.
  * UPDATED: Implemented TTL retention policy. Defaults to 90 days from the computation date.
  * UPDATED: Fixed issue where switching to 'isPage' mode didn't clean up old sharded/raw data.
+ * CRITICAL FIX: Fixed sharding logic to prevent wiping existing shards during INTERMEDIATE flushes.
  */
 const { commitBatchInChunks, generateDataHash, FieldValue } = require('../utils/utils')
 const { updateComputationStatus }          = require('./StatusRepository');
@@ -138,8 +139,6 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 continue; 
             }
 
-            // [NEW] Page Computation Logic (Fan-Out) with TTL
-            // Bypasses standard compression/sharding to write per-user documents
             // [NEW] Page Computation Logic (Fan-Out) with TTL
             // Bypasses standard compression/sharding to write per-user documents
             if (isPageComputation && !isEmpty) {
@@ -429,7 +428,9 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
     let finalStats = { totalSize: 0, isSharded: false, shardCount: 1, nextShardIndex: startShardIndex };
     let rootMergeOption = !isInitialWrite; 
 
-    let shouldWipeShards = wasSharded; 
+    // CRITICAL FIX: Only wipe existing shards if this is the INITIAL write for this batch run.
+    // If we are flushing intermediate chunks, we should NOT wipe the shards created by previous chunks!
+    let shouldWipeShards = wasSharded && isInitialWrite; 
 
     for (let attempt = 0; attempt < strategies.length; attempt++) {
         if (committed) break;