bulltrackers-module 1.0.536 → 1.0.538

package/functions/computation-system/executors/StandardExecutor.js

@@ -5,7 +5,7 @@
  * }
  */
 const { normalizeName, getEarliestDataDates }                          = require('../utils/utils');
-const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs } = require('../utils/data_loader');
+const { streamPortfolioData, streamHistoryData, getPortfolioPartRefs, getHistoryPartRefs } = require('../utils/data_loader');
 const { CachedDataLoader }                                             = require('../data/CachedDataLoader');
 const { ContextFactory }                                               = require('../context/ContextFactory');
 const { commitResults }                                                = require('../persistence/ResultCommitter');
@@ -24,15 +24,31 @@ class StandardExecutor {
             const type = (c.userType || 'ALL').toUpperCase();
             requiredUserTypes.add(type);
         });
-        // If any calc requires 'ALL' (or has no userType), we fetch everything
         const userTypeArray = requiredUserTypes.has('ALL') ? null : Array.from(requiredUserTypes);
         
+        // [OPTIMIZATION] Check for Target CID in manifests (On-Demand Optimization)
+        // If present, we will filter all data streams to strictly this user
+        const targetCid = calcs.find(c => c.targetCid)?.targetCid || calcs.find(c => c.manifest?.targetCid)?.manifest?.targetCid;
+        if (targetCid) {
+            logger.log('INFO', `[StandardExecutor] Running in Targeted Mode for CID: ${targetCid}`);
+        }
+
         const fullRoot = { ...rootData };
         if (calcs.some(c => c.isHistorical)) {
             const prev = new Date(date); prev.setUTCDate(prev.getUTCDate() - 1);
             const prevStr = prev.toISOString().slice(0, 10);
-            // [FIX] Pass userTypeArray to filter yesterday's data loading
-            fullRoot.yesterdayPortfolioRefs = await getPortfolioPartRefs(config, deps, prevStr, userTypeArray);
+            
+            // Fetch yesterday's refs
+            let yRefs = await getPortfolioPartRefs(config, deps, prevStr, userTypeArray);
+            
+            // [OPTIMIZATION] Filter Yesterday's Refs if targetCid is set
+            if (targetCid && yRefs) {
+                const originalCount = yRefs.length;
+                yRefs = yRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
+                logger.log('INFO', `[StandardExecutor] Filtered Yesterday's Refs: ${originalCount} -> ${yRefs.length}`);
+            }
+            
+            fullRoot.yesterdayPortfolioRefs = yRefs;
         }
 
         const state = {};
@@ -46,18 +62,48 @@ class StandardExecutor {
             } catch (e) { logger.log('WARN', `Failed to init ${c.name}`); } 
         }
 
-        return await StandardExecutor.streamAndProcess(dStr, state, passName, config, deps, fullRoot, rootData.portfolioRefs, rootData.historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, userTypeArray);
+        // Pass targetCid to streamAndProcess
+        return await StandardExecutor.streamAndProcess(
+            dStr, state, passName, config, deps, fullRoot, 
+            rootData.portfolioRefs, rootData.historyRefs, 
+            fetchedDeps, previousFetchedDeps, skipStatusWrite, 
+            userTypeArray, targetCid
+        );
     }
 
-    // [UPDATED] Accepts requiredUserTypes param
-    static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, requiredUserTypes = null) {
+    // [UPDATED] Added targetCid param
+    static async streamAndProcess(dateStr, state, passName, config, deps, rootData, portfolioRefs, historyRefs, fetchedDeps, previousFetchedDeps, skipStatusWrite, requiredUserTypes = null, targetCid = null) {
         const { logger } = deps;
         const calcs = Object.values(state).filter(c => c && c.manifest);
         const streamingCalcs = calcs.filter(c => c.manifest.rootDataDependencies.includes('portfolio') || c.manifest.rootDataDependencies.includes('history'));
         
         if (streamingCalcs.length === 0) return { successUpdates: {}, failureReport: [] };
+
+        // --- 1. Resolve and Filter Portfolio Refs (Today) ---
+        let effectivePortfolioRefs = portfolioRefs;
+        if (!effectivePortfolioRefs) {
+            // If refs weren't provided by AvailabilityChecker, fetch them now
+            effectivePortfolioRefs = await getPortfolioPartRefs(config, deps, dateStr, requiredUserTypes);
+        }
+        if (targetCid && effectivePortfolioRefs) {
+            // Filter: Keep only refs that match the CID (or Legacy refs without CID)
+            effectivePortfolioRefs = effectivePortfolioRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
+        }
+
+        // --- 2. Resolve and Filter History Refs ---
+        let effectiveHistoryRefs = historyRefs;
+        const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
+        
+        if (needsTradingHistory) {
+            if (!effectiveHistoryRefs) {
+                effectiveHistoryRefs = await getHistoryPartRefs(config, deps, dateStr, requiredUserTypes);
+            }
+            if (targetCid && effectiveHistoryRefs) {
+                effectiveHistoryRefs = effectiveHistoryRefs.filter(r => !r.cid || String(r.cid) === String(targetCid));
+            }
+        }
         
-        let totalReadOps = (portfolioRefs?.length || 0) + (historyRefs?.length || 0);
+        let totalReadOps = (effectivePortfolioRefs?.length || 0) + (effectiveHistoryRefs?.length || 0);
         if (rootData.yesterdayPortfolioRefs) totalReadOps += rootData.yesterdayPortfolioRefs.length;
         totalReadOps += 2; 
 
@@ -83,6 +129,7 @@ class StandardExecutor {
         const setupDuration = performance.now() - startSetup;
         Object.keys(executionStats).forEach(name => executionStats[name].timings.setup += setupDuration);
 
+        // Yesterday's Refs are already filtered in run()
         const prevDate    = new Date(dateStr + 'T00:00:00Z'); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
         const prevDateStr = prevDate.toISOString().slice(0, 10);
         
@@ -91,17 +138,13 @@ class StandardExecutor {
             earliestDates = await getEarliestDataDates(config, deps);
         }
 
-        // [FIX] Pass requiredUserTypes to streamPortfolioData
-        // Note: portfolioRefs (for today) might be null if not pre-fetched, allowing streamPortfolioData to fetch filtered refs
-        const tP_iter = streamPortfolioData(config, deps, dateStr, portfolioRefs, requiredUserTypes);
+        // [FIX] Use effective/filtered refs
+        const tP_iter = streamPortfolioData(config, deps, dateStr, effectivePortfolioRefs, requiredUserTypes);
         
         const needsYesterdayPortfolio = streamingCalcs.some(c => c.manifest.isHistorical);
-        // Yesterday's refs were already filtered in run(), so we pass them directly
         const yP_iter = (needsYesterdayPortfolio && rootData.yesterdayPortfolioRefs)  ? streamPortfolioData(config, deps, prevDateStr, rootData.yesterdayPortfolioRefs) : null;
         
-        const needsTradingHistory = streamingCalcs.some(c => c.manifest.rootDataDependencies.includes('history'));
-        // [FIX] Pass requiredUserTypes to streamHistoryData
-        const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, historyRefs, requiredUserTypes) : null;
+        const tH_iter = (needsTradingHistory) ? streamHistoryData(config, deps, dateStr, effectiveHistoryRefs, requiredUserTypes) : null;
 
         let yP_chunk = {}, tH_chunk = {};
         let usersSinceLastFlush = 0;
@@ -159,7 +202,7 @@ class StandardExecutor {
     
     // ... rest of the file (flushBuffer, mergeReports, executePerUser) ...
     static async flushBuffer(state, dateStr, passName, config, deps, shardIndexMap, executionStats, mode, skipStatusWrite, isInitialWrite = false) {
-        // ... (No changes to flushBuffer)
+        const { logger } = deps;
         const transformedState = {};
         for (const [name, inst] of Object.entries(state)) {
             const rawResult = inst.results || {};
@@ -197,7 +240,6 @@ class StandardExecutor {
     }
 
     static mergeReports(successAcc, failureAcc, newResult) {
-        // ... (No changes to mergeReports)
         if (!newResult) return;
         for (const [name, update] of Object.entries(newResult.successUpdates)) {
             if (!successAcc[name]) {
@@ -232,24 +274,23 @@ class StandardExecutor {
         const targetUserType = metadata.userType; 
         // [NEW] Always load Global Helpers
         const mappings = await loader.loadMappings();
-        const piMasterList = await loader.loadPIMasterList(); // [NEW] Loaded globally
+        const piMasterList = await loader.loadPopularInvestorMasterList(config, deps); // [NEW] Loaded globally
         const SCHEMAS = mathLayer.SCHEMAS;
 
         // 1. Load Root Data
         const insights = metadata.rootDataDependencies?.includes('insights') ? { today: await loader.loadInsights(dateStr) } : null;
-        const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerifications() : null;
-        const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadRankings(dateStr) : null;
+        const verifications = metadata.rootDataDependencies?.includes('verification') ? await loader.loadVerificationProfiles(config, deps) : null;
+        const rankings = metadata.rootDataDependencies?.includes('rankings') ? await loader.loadPopularInvestorRankings(config, deps, dateStr) : null;
         
         // [FIX] Load Yesterday's Rankings if isHistorical is true
         let yesterdayRankings = null;
         if (metadata.rootDataDependencies?.includes('rankings') && metadata.isHistorical) {
              const prevDate = new Date(dateStr); prevDate.setUTCDate(prevDate.getUTCDate() - 1);
              const prevStr = prevDate.toISOString().slice(0, 10);
-             // Assuming CachedDataLoader handles caching for efficiency
-             yesterdayRankings = await loader.loadRankings(prevStr); 
+             yesterdayRankings = await loader.loadPopularInvestorRankings(config, deps, prevStr); 
         }
 
-        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadSocial(dateStr) : null;
+        const socialContainer = metadata.rootDataDependencies?.includes('social') ? await loader.loadDailySocialPostInsights(config, deps, dateStr) : null;
         
         // [NEW] Load New Root Data Types for Profile Metrics
         // [FIX] Enforce canHaveMissingRoots
@@ -258,14 +299,13 @@ class StandardExecutor {
         let ratings = null;
         if (metadata.rootDataDependencies?.includes('ratings')) {
             try {
-                ratings = await loader.loadRatings(dateStr);
+                ratings = await loader.loadPIRatings(config, deps, dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'ratings' failed to load for ${metadata.name}: ${e.message}`);
                 }
                 ratings = null;
             }
-            // Check if result is null (clean fetch but empty/missing doc)
             if (!ratings && !allowMissing) {
                 throw new Error(`[StandardExecutor] Required root 'ratings' is missing for ${metadata.name}`);
             }
@@ -274,7 +314,7 @@ class StandardExecutor {
         let pageViews = null;
         if (metadata.rootDataDependencies?.includes('pageViews')) {
             try {
-                pageViews = await loader.loadPageViews(dateStr);
+                pageViews = await loader.loadPIPageViews(config, deps, dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'pageViews' failed to load for ${metadata.name}: ${e.message}`);
@@ -289,7 +329,7 @@ class StandardExecutor {
         let watchlistMembership = null;
         if (metadata.rootDataDependencies?.includes('watchlist')) {
             try {
-                watchlistMembership = await loader.loadWatchlistMembership(dateStr);
+                watchlistMembership = await loader.loadWatchlistMembership(config, deps, dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'watchlist' failed to load for ${metadata.name}: ${e.message}`);
@@ -304,7 +344,7 @@ class StandardExecutor {
         let alertHistory = null;
         if (metadata.rootDataDependencies?.includes('alerts')) {
             try {
-                alertHistory = await loader.loadAlertHistory(dateStr);
+                alertHistory = await loader.loadPIAlertHistory(config, deps, dateStr);
             } catch (e) {
                 if (!allowMissing) {
                     throw new Error(`[StandardExecutor] Required root 'alerts' failed to load for ${metadata.name}: ${e.message}`);
@@ -350,7 +390,6 @@ class StandardExecutor {
             
             const userVerification = verifications ? verifications[userId] : null;
             
-            // [FIX] Extract current AND yesterday's rank entry for this user
             const userRanking = rankings ? (rankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
             const userRankingYesterday = yesterdayRankings ? (yesterdayRankings.find(r => String(r.CustomerId) === String(userId)) || null) : null;
 
@@ -373,23 +412,19 @@ class StandardExecutor {
                 config, deps,
                 verification: userVerification, 
                 
-                // [FIX] Pass both ranking entries
                 rankings: userRanking,
                 yesterdayRankings: userRankingYesterday, 
                 
-                // [FIX] Pass both global lists
                 allRankings: rankings,
                 allRankingsYesterday: yesterdayRankings,
                 
                 allVerifications: verifications,
                 
-                // [NEW] Pass New Root Data Types for Profile Metrics
                 ratings: ratings || {},
                 pageViews: pageViews || {},
                 watchlistMembership: watchlistMembership || {},
                 alertHistory: alertHistory || {},
 
-                // [NEW] Pass Master List
                 piMasterList,
             });
 

package/functions/task-engine/helpers/root_data_indexer_helpers.js

@@ -5,17 +5,62 @@
 
 const { FieldValue } = require('@google-cloud/firestore');
 
+/**
+ * Get the detail field name for a data type
+ * @param {string} dataType - Data type identifier
+ * @returns {string} - Detail field name in the index document
+ */
+function getDetailFieldName(dataType) {
+    const dataTypeMap = {
+        'signedInUserPortfolio': 'signedInUserPortfolio',
+        'signedInUserHistory': 'signedInUserHistory',
+        'signedInUserSocial': 'signedInSocial',
+        'piPortfolios': 'piPortfolios',
+        'piHistory': 'piHistory',
+        'piSocial': 'piSocial',
+        'normalPortfolios': 'normalPortfolio',
+        'normalHistory': 'normalHistory',
+        'speculatorPortfolios': 'speculatorPortfolio',
+        'speculatorHistory': 'speculatorHistory'
+    };
+    return dataTypeMap[dataType] || dataType;
+}
+
 /**
  * Check if root data is already indexed for a specific date
  * @param {Firestore} db - Firestore instance
  * @param {string} dateStr - Date string (YYYY-MM-DD)
  * @param {string} availabilityCollection - Collection name for root data index
- * @returns {Promise<boolean>} - True if already indexed
+ * @param {Array<string>} dataTypesRun - Optional array of data types to verify are indexed (e.g., ['signedInUserPortfolio', 'piSocial'])
+ * @returns {Promise<boolean>} - True if already indexed (and all requested data types are indexed if provided)
  */
-async function isRootDataIndexed(db, dateStr, availabilityCollection = 'system_root_data_index') {
+async function isRootDataIndexed(db, dateStr, availabilityCollection = 'system_root_data_index', dataTypesRun = []) {
     try {
         const indexDoc = await db.collection(availabilityCollection).doc(dateStr).get();
-        return indexDoc.exists;
+        if (!indexDoc.exists) {
+            return false;
+        }
+        
+        // If specific data types were requested, verify they are actually indexed
+        if (dataTypesRun && dataTypesRun.length > 0) {
+            const indexData = indexDoc.data();
+            const details = indexData?.details || {};
+            
+            // Check if all requested data types are indexed (have true values)
+            // If any are false or missing, return false (needs indexing)
+            for (const dataType of dataTypesRun) {
+                const detailField = getDetailFieldName(dataType);
+                if (details[detailField] !== true) {
+                    // At least one requested data type is not indexed
+                    return false;
+                }
+            }
+            // All requested data types are indexed as true
+            return true;
+        }
+        
+        // No specific data types requested - just check if document exists
+        return true;
     } catch (error) {
         // If we can't check, assume not indexed to be safe
         return false;
@@ -39,11 +84,43 @@ async function conditionallyRunRootDataIndexer({ db, logger, dateStr, rootDataIn
     const { runRootDataIndexer } = require('../../root-data-indexer/index');
     const availabilityCollection = rootDataIndexerConfig?.availabilityCollection || 'system_root_data_index';
     
-    // Check if already indexed
-    const alreadyIndexed = await isRootDataIndexed(db, dateStr, availabilityCollection);
-    if (alreadyIndexed) {
-        logger.log('INFO', `[RootDataIndexer] Root data already indexed for ${dateStr}, skipping`);
-        return false;
+    // IMPORTANT: Only index data types that were successfully stored (dataTypesRun contains only successful operations)
+    // If a write failed, it won't be in dataTypesRun, so we won't try to index it
+    // However, we still run the indexer to check/update the types that did succeed
+    
+    // Check if already indexed (verify specific data types if provided)
+    // Only check the data types that were actually run - if they're all already true, skip indexing (never overwrite true values)
+    if (dataTypesRun.length > 0) {
+        const alreadyIndexed = await isRootDataIndexed(db, dateStr, availabilityCollection, dataTypesRun);
+        if (alreadyIndexed) {
+            logger.log('INFO', `[RootDataIndexer] All requested data types (${dataTypesRun.join(', ')}) are already indexed as true for ${dateStr}, skipping (never overwrite true values)`);
+            return false;
+        }
+        
+        // If document exists but data types aren't indexed, log it
+        try {
+            const indexDoc = await db.collection(availabilityCollection).doc(dateStr).get();
+            if (indexDoc.exists) {
+                const indexData = indexDoc.data();
+                const details = indexData?.details || {};
+                const missingTypes = dataTypesRun.filter(dt => {
+                    const detailField = getDetailFieldName(dt);
+                    return details[detailField] !== true;
+                });
+                if (missingTypes.length > 0) {
+                    logger.log('INFO', `[RootDataIndexer] Index document exists for ${dateStr} but data types (${missingTypes.join(', ')}) are not indexed. Re-indexing...`);
+                }
+            }
+        } catch (e) {
+            // Ignore errors checking document
+        }
+    } else {
+        // No specific data types - check if document exists (legacy behavior)
+        const alreadyIndexed = await isRootDataIndexed(db, dateStr, availabilityCollection, []);
+        if (alreadyIndexed) {
+            logger.log('INFO', `[RootDataIndexer] Root data already indexed for ${dateStr}, skipping`);
+            return false;
+        }
     }
     
     // Use Firestore transaction to ensure only one instance runs the indexer

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.536",
+  "version": "1.0.538",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [