bulltrackers-module 1.0.653 → 1.0.655

package/functions/api-v2/helpers/data-fetchers/firestore.js

@@ -1222,7 +1222,7 @@ const getComputationResults = async (db, computationName, dateStr, userId = null
             const shardCount = pointerData._shardCount || 0;
             console.log(`[Computation] Reassembling ${shardCount} shards for ${computationName}`);
             
-            if (shardCount === 0) return []; // Or empty object
+            if (shardCount === 0) return {}; // Return empty object for object-based results
 
             // Create an array of promises to fetch all shards in parallel
             const shardPromises = [];
@@ -1233,10 +1233,9 @@ const getComputationResults = async (db, computationName, dateStr, userId = null
 
             const shardSnaps = await Promise.all(shardPromises);
             
-            // Reassemble data. 
-            // Assuming shards contain lists of data that need concatenation.
-            // Adjust logic if shards contain maps that need merging.
-            let reassembledData = [];
+            // Reassemble data by merging objects (for object-based results like GlobalAumPerAsset30D)
+            // Shards contain partial objects that need to be merged together
+            let reassembledData = {};
             
             shardSnaps.forEach((snap, index) => {
                 if (snap.exists) {
@@ -1244,9 +1243,14 @@ const getComputationResults = async (db, computationName, dateStr, userId = null
                     // If the shard itself is compressed (common in big data), decompress it
                     const content = (data._compressed) ? tryDecompress(data) : data;
                     
-                    // Push to master list. Assuming standard structure { items: [...] } or Array
-                    // If specific structure isn't known, we push the whole object.
-                    reassembledData.push(content);
+                    // Merge shard contents, ignoring internal metadata fields
+                    if (content && typeof content === 'object') {
+                        Object.entries(content).forEach(([k, v]) => {
+                            if (!k.startsWith('_')) {
+                                reassembledData[k] = v;
+                            }
+                        });
+                    }
                 } else {
                     console.warn(`[Computation] Missing shard_${index} for ${computationName}`);
                 }
@@ -1273,6 +1277,62 @@ const getComputationResults = async (db, computationName, dateStr, userId = null
     }
 };
 
+// 10.5. Fetch GlobalAumPerAsset30D with 7-day lookback
+// Returns the last 7 days of successfully stored data for a specific ticker or all assets
+const fetchGlobalAumPerAssetWithLookback = async (db, dateStr, ticker = null, lookbackDays = 7) => {
+    try {
+        const endDate = new Date(dateStr);
+        const results = [];
+        
+        // Fetch data for the last 7 days, handling missing data gracefully
+        for (let i = 0; i < lookbackDays; i++) {
+            const checkDate = new Date(endDate);
+            checkDate.setDate(endDate.getDate() - i);
+            const dateKey = checkDate.toISOString().split('T')[0];
+            
+            try {
+                const computationData = await getComputationResults(db, 'GlobalAumPerAsset30D', dateKey);
+                
+                // If we got valid data (not null/empty), add it to results
+                if (computationData && typeof computationData === 'object' && Object.keys(computationData).length > 0) {
+                    if (ticker) {
+                        // If searching for a specific ticker, only include if it exists
+                        const tickerUpper = ticker.toUpperCase();
+                        if (computationData[tickerUpper] !== undefined) {
+                            results.push({
+                                date: dateKey,
+                                aum: computationData[tickerUpper],
+                                ticker: tickerUpper
+                            });
+                        }
+                    } else {
+                        // Return all assets for this date
+                        results.push({
+                            date: dateKey,
+                            data: computationData
+                        });
+                    }
+                }
+            } catch (error) {
+                // Missing data for this date - skip it (graceful handling)
+                console.log(`[GlobalAum] No data found for ${dateKey}, skipping`);
+            }
+        }
+        
+        // Sort by date descending (newest first)
+        results.sort((a, b) => {
+            const dateA = new Date(a.date);
+            const dateB = new Date(b.date);
+            return dateB.getTime() - dateA.getTime();
+        });
+        
+        return results;
+    } catch (error) {
+        console.error(`[GlobalAum] Error fetching with lookback:`, error);
+        throw error;
+    }
+};
+
 // 11. Fetch User Notifications
 const fetchNotifications = async (firestore, userId, options = {}) => {
     const { limit = 20, unreadOnly = false, excludeTypes = ['watchlistAlerts'] } = options;
@@ -2913,6 +2973,8 @@ const subscribeToAllWatchlistPIs = async (db, userId, watchlistId, alertTypes =
 module.exports = {
     latestUserCentricSnapshot,
     pageCollection,
+    fetchGlobalAumPerAssetWithLookback,
+    getComputationResults,
     fetchPopularInvestorMasterList,
     isDeveloper,
     lookupCidByEmail,

package/functions/api-v2/routes/popular_investors.js

@@ -8,10 +8,12 @@ const {
     getComputationResults, 
     requestPopularInvestorAddition,
     trackPopularInvestorView,
-    pageCollection
+    pageCollection,
+    fetchGlobalAumPerAssetWithLookback
 } = require('../helpers/data-fetchers/firestore.js');
 const { sanitizeCid } = require('../helpers/security_utils.js');
 const { timeouts } = require('../helpers/timeout_utils.js');
+const { requireFirebaseAuth } = require('../middleware/firebase_auth_middleware.js');
 
 const router = express.Router();
 
@@ -255,5 +257,93 @@ router.get('/:piId/analytics', async (req, res) => {
     }
 });
 
+/**
+ * GLOBAL AUM PER ASSET - Public for all signed-in users
+ * 
+ * These routes show how much $ is invested in each asset based on AUM totals of all Popular Investors.
+ * - Uses GlobalAumPerAsset30D computation
+ * - Access: Public (any signed-in user can view)
+ * - Requires Firebase authentication
+ */
+
+// GET /popular-investors/global-aum - Get top assets by AUM (default: top 10)
+router.get('/global-aum', requireFirebaseAuth, async (req, res, next) => {
+    try {
+        const { db } = req.dependencies;
+        const { date, limit = 10 } = req.query;
+        
+        // Default to today if no date provided
+        const targetDate = date || new Date().toISOString().split('T')[0];
+        
+        // Fetch the latest computation result
+        const computationData = await getComputationResults(db, 'GlobalAumPerAsset30D', targetDate);
+        
+        if (!computationData || typeof computationData !== 'object' || Object.keys(computationData).length === 0) {
+            return res.status(404).json({ 
+                success: false,
+                error: 'No AUM data available for the specified date' 
+            });
+        }
+        
+        // Convert to array and sort by AUM (descending)
+        const assetsArray = Object.entries(computationData)
+            .map(([ticker, aum]) => ({ ticker, aum: Number(aum) }))
+            .sort((a, b) => b.aum - a.aum)
+            .slice(0, parseInt(limit) || 10);
+        
+        res.json({ 
+            success: true, 
+            date: targetDate,
+            count: assetsArray.length,
+            data: assetsArray
+        });
+    } catch (error) {
+        next(error);
+    }
+});
+
+// GET /popular-investors/global-aum/:ticker - Get 7-day AUM history for a specific ticker
+router.get('/global-aum/:ticker', requireFirebaseAuth, async (req, res, next) => {
+    try {
+        const { db } = req.dependencies;
+        const { ticker } = req.params;
+        const { date, lookback = 7 } = req.query;
+        
+        // Validate ticker
+        if (!ticker || typeof ticker !== 'string' || ticker.length === 0) {
+            return res.status(400).json({ 
+                success: false,
+                error: 'Invalid ticker symbol' 
+            });
+        }
+        
+        // Default to today if no date provided
+        const targetDate = date || new Date().toISOString().split('T')[0];
+        
+        // Validate lookback parameter
+        const lookbackDays = Math.min(Math.max(parseInt(lookback) || 7, 1), 30); // Between 1 and 30 days
+        
+        // Fetch 7-day lookback data for the ticker
+        const results = await fetchGlobalAumPerAssetWithLookback(db, targetDate, ticker, lookbackDays);
+        
+        if (results.length === 0) {
+            return res.status(404).json({ 
+                success: false,
+                error: `No AUM data found for ticker ${ticker.toUpperCase()} in the last ${lookbackDays} days` 
+            });
+        }
+        
+        res.json({ 
+            success: true, 
+            ticker: ticker.toUpperCase(),
+            lookbackDays: lookbackDays,
+            count: results.length,
+            data: results
+        });
+    } catch (error) {
+        next(error);
+    }
+});
+
 
 module.exports = router;

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -5,16 +5,14 @@
  * UPDATED: Enforces Strict One-Shot Policy (Standard -> HighMem -> Dead Letter).
  * UPDATED: Generates Google Cloud Trace Context (traceId/spanId) for end-to-end monitoring.
  * UPDATED: Added Schedule Awareness (Daily, Weekly, Monthly) to filter tasks by date.
- * UPDATED: Force Run now validates Root Data Availability before dispatching.
  */
 
 const { getExpectedDateStrings, getEarliestDataDates, normalizeName, DEFINITIVE_EARLIEST_DATES } = require('../utils/utils.js');
 const { groupByPass, analyzeDateExecution } = require('../WorkflowOrchestrator.js');
 const { PubSubUtils } = require('../../core/utils/pubsub_utils');
 const { fetchComputationStatus } = require('../persistence/StatusRepository');
-// [UPDATED] Imported getAvailabilityWindow and checkRootDependencies
-const { checkRootDataAvailability, getAvailabilityWindow, checkRootDependencies } = require('../data/AvailabilityChecker');
-const { runFinalSweepCheck } = require('../tools/FinalSweepReporter'); 
+const { checkRootDataAvailability } = require('../data/AvailabilityChecker');
+const { runFinalSweepCheck } = require('../tools/FinalSweepReporter'); // [NEW]
 const crypto = require('crypto');
 
 const OOM_THRESHOLD_MB = 1500; // Unused
@@ -232,71 +230,92 @@ async function handleForceRun(config, dependencies, computationManifest, reqBody
     }
 
     // 2. Determine Target Dates
-    let targetDates = [];
+    let candidateDates = [];
     if (dateInput) {
         // Single Date Mode
-        // We still perform the check, but for a single item array
-        if (Array.isArray(dateInput)) {
-            targetDates = dateInput;
-        } else {
-            targetDates = [dateInput];
-        }
+        candidateDates = [dateInput];
     } else {
         // All Dates Mode (Backfill)
         logger.log('INFO', `[ForceRun] No date provided. Calculating date range for ${computationName}...`);
         const earliestDates = await getEarliestDataDates(config, dependencies);
         // Calculate from system start until today
-        targetDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date());
+        candidateDates = getExpectedDateStrings(earliestDates.absoluteEarliest, new Date());
     }
     
-    // [NEW] 3. Filter Impossible Dates (Availability Check)
-    logger.log('INFO', `[ForceRun] 🔍 Validating data availability for ${targetDates.length} candidate dates...`);
-    
-    // Sort dates to get efficient min/max for range query
-    targetDates.sort();
-    const startDate = targetDates[0];
-    const endDate = targetDates[targetDates.length - 1];
+    logger.log('INFO', `[ForceRun] Checking ${candidateDates.length} candidate dates for runnability...`);
 
-    const availabilityMap = await getAvailabilityWindow(dependencies, startDate, endDate);
-    const validDates = [];
-    const skippedStats = {};
-
-    for (const date of targetDates) {
-        const status = availabilityMap.get(date);
+    // 3. Filter to only runnable dates using analyzeDateExecution
+    const manifestMap = new Map(computationManifest.map(c => [normalizeName(c.name), c]));
+    const calcsInPass = groupByPass(computationManifest, manifestItem.pass || "1");
+    const targetComputationNormalized = normalizeName(computationName);
+    
+    // Filter to only the target computation
+    const targetCalcs = calcsInPass.filter(c => normalizeName(c.name) === targetComputationNormalized);
+    
+    if (targetCalcs.length === 0) {
+        throw new Error(`Computation '${computationName}' not found in pass ${manifestItem.pass || "1"}`);
+    }
+    
+    const runnableDates = [];
+    const skippedDates = [];
+    
+    for (const dateStr of candidateDates) {
+        // Check root data availability
+        const rootDataStatus = await checkRootDataAvailability(dateStr, config, dependencies, DEFINITIVE_EARLIEST_DATES);
+        
+        // Get computation status for this date
+        const dailyStatus = await fetchComputationStatus(dateStr, config, dependencies);
         
-        // If no index exists, we pass an empty object (AvailabilityChecker will fail all checks)
-        // This effectively filters out dates where we have NO knowledge of data
-        const effectiveStatus = status || {}; 
+        // Check previous day status if needed
+        let prevDailyStatus = null;
+        if (targetCalcs.some(c => c.isHistorical)) {
+            const prevDate = new Date(dateStr + 'T00:00:00Z');
+            prevDate.setUTCDate(prevDate.getUTCDate() - 1);
+            prevDailyStatus = await fetchComputationStatus(prevDate.toISOString().slice(0, 10), config, dependencies);
+        }
+        
+        // Analyze if this computation can run on this date
+        const report = analyzeDateExecution(dateStr, targetCalcs, rootDataStatus, dailyStatus, manifestMap, prevDailyStatus);
         
-        const check = checkRootDependencies(manifestItem, effectiveStatus);
+        // Check if the target computation is runnable, needs re-run, or has failed dependencies
+        const isRunnable = report.runnable.some(t => normalizeName(t.name) === targetComputationNormalized);
+        const needsReRun = report.reRuns.some(t => normalizeName(t.name) === targetComputationNormalized);
+        const hasFailedDep = report.failedDependency.some(t => normalizeName(t.name) === targetComputationNormalized);
+        const isImpossible = report.impossible.some(t => normalizeName(t.name) === targetComputationNormalized);
+        const isBlocked = report.blocked.some(t => normalizeName(t.name) === targetComputationNormalized);
         
-        if (check.canRun) {
-            validDates.push(date);
+        if (isRunnable || needsReRun || hasFailedDep) {
+            runnableDates.push(dateStr);
+        } else if (isImpossible) {
+            skippedDates.push({ date: dateStr, reason: report.impossible.find(t => normalizeName(t.name) === targetComputationNormalized)?.reason || 'Impossible' });
+        } else if (isBlocked) {
+            skippedDates.push({ date: dateStr, reason: report.blocked.find(t => normalizeName(t.name) === targetComputationNormalized)?.reason || 'Blocked' });
         } else {
-            const reason = check.missing.length > 0 ? `Missing: ${check.missing.join(', ')}` : 'Dependencies not met';
-            skippedStats[reason] = (skippedStats[reason] || 0) + 1;
+            skippedDates.push({ date: dateStr, reason: 'Not runnable (unknown reason)' });
         }
     }
-
-    if (validDates.length === 0) {
-        logger.log('WARN', `[ForceRun] 🛑 ABORTING: No valid dates found for ${computationName} out of ${targetDates.length} requested.`);
+    
+    logger.log('INFO', `[ForceRun] ✅ Found ${runnableDates.length} runnable dates out of ${candidateDates.length} candidates`);
+    if (skippedDates.length > 0) {
+        logger.log('INFO', `[ForceRun] ⏭️ Skipped ${skippedDates.length} dates: ${skippedDates.slice(0, 5).map(s => `${s.date} (${s.reason})`).join(', ')}${skippedDates.length > 5 ? '...' : ''}`);
+    }
+    
+    if (runnableDates.length === 0) {
         return {
-            status: 'ABORTED',
+            status: 'NO_RUNNABLE_DATES',
             computation: computationName,
-            reason: 'NO_DATA_AVAILABLE',
-            skippedDetails: skippedStats
+            mode: dateInput ? 'SINGLE_DATE' : 'ALL_DATES',
+            datesChecked: candidateDates.length,
+            datesRunnable: 0,
+            skippedReasons: skippedDates.slice(0, 10)
         };
     }
+    
+    logger.log('WARN', `[ForceRun] 🚨 MANUALLY Triggering ${computationName} for ${runnableDates.length} runnable dates. Pass: ${manifestItem.pass}`);
 
-    if (validDates.length < targetDates.length) {
-        logger.log('INFO', `[ForceRun] ⚠️ Filtered impossible dates: ${targetDates.length} requested -> ${validDates.length} valid.`, { skippedStats });
-    }
-
-    logger.log('WARN', `[ForceRun] 🚨 MANUALLY Triggering ${computationName} for ${validDates.length} VALID dates. Pass: ${manifestItem.pass}`);
-
-    // 4. Construct Tasks
+    // 4. Construct Tasks (only for runnable dates)
     const dispatchId = crypto.randomUUID();
-    const tasks = validDates.map(date => {
+    const tasks = runnableDates.map(date => {
         const traceId = crypto.randomBytes(16).toString('hex');
         const spanId = crypto.randomBytes(8).toString('hex');
         return {
@@ -312,7 +331,7 @@ async function handleForceRun(config, dependencies, computationManifest, reqBody
         };
     });
 
-    // 5. Batch Publish (Chunked to stay under Pub/Sub limits)
+    // 4. Batch Publish (Chunked to stay under Pub/Sub limits)
     const CHUNK_SIZE = 250; // Safe batch size
     const topic = (reqBody.resources === 'high-mem') 
         ? (config.computationTopicHighMem || 'computation-tasks-highmem')
@@ -337,10 +356,11 @@ async function handleForceRun(config, dependencies, computationManifest, reqBody
     return { 
         status: 'FORCED',
         computation: computationName,
-        mode: dateInput ? 'SINGLE/ARRAY_DATE' : 'ALL_DATES',
-        datesRequested: targetDates.length,
+        mode: dateInput ? 'SINGLE_DATE' : 'ALL_DATES',
+        datesChecked: candidateDates.length,
+        datesRunnable: runnableDates.length,
         datesTriggered: dispatchedCount,
-        skippedImpossible: targetDates.length - dispatchedCount,
+        skippedCount: skippedDates.length,
         targetTopic: topic
     };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.653",
+  "version": "1.0.655",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [