bulltrackers-module 1.0.658 → 1.0.659

package/functions/computation-system/data/AvailabilityChecker.js

@@ -1,239 +1,186 @@
 /**
  * @fileoverview Checks availability of root data via the Root Data Index.
- * REFACTORED: Fully supports granular flags for PI, Signed-In Users, Rankings, and Verification.
- * UPDATED: Enforces 'mandatoryRoots' metadata to override permissive flags.
- * NEW: Added 'getAvailabilityWindow' for efficient batch availability lookups using range queries.
+ * REFACTORED: Config-driven dependency checking and shared status normalization.
  */
 const { normalizeName } = require('../utils/utils');
 const { FieldPath } = require('@google-cloud/firestore');
 
 const INDEX_COLLECTION = process.env.ROOT_DATA_AVAILABILITY_COLLECTION || 'system_root_data_index';
 
+// =============================================================================
+// CONFIGURATION: Dependency Mappings
+// =============================================================================
+
+// Dependencies that map directly to a single status flag
+const SIMPLE_DEP_MAP = {
+    rankings:     'piRankings',
+    verification: 'signedInUserVerification',
+    insights:     'hasInsights',
+    price:        'hasPrices',
+    ratings:      'piRatings',
+    pageViews:    'piPageViews',
+    watchlist:    'watchlistMembership',
+    alerts:       'piAlertHistory'
+};
+
+// Dependencies that vary based on the userType of the calculation
+const COMPLEX_DEP_MAP = {
+    portfolio: {
+        speculator:       'speculatorPortfolio',
+        normal:           'normalPortfolio',
+        popular_investor: 'piPortfolios',
+        signed_in_user:   'signedInUserPortfolio',
+        _default:         'hasPortfolio'
+    },
+    history: {
+        speculator:       'speculatorHistory',
+        normal:           'normalHistory',
+        popular_investor: 'piHistory',
+        signed_in_user:   'signedInUserHistory',
+        _default:         'hasHistory'
+    },
+    social: {
+        popular_investor: 'hasPISocial',
+        signed_in_user:   'hasSignedInSocial',
+        _default:         'hasSocial'
+    }
+};
+
+// =============================================================================
+// LOGIC: Dependency Checking
+// =============================================================================
+
 /**
  * Checks if a specific calculation can run based on its dependencies and the current data status.
- * @param {Object} calcManifest - The calculation manifest.
- * @param {Object} rootDataStatus - The availability status object.
- * @returns {Object} { canRun: boolean, missing: Array, available: Array }
  */
 function checkRootDependencies(calcManifest, rootDataStatus) {
     const missing = [];
     const available = [];
-    
-    if (!calcManifest.rootDataDependencies) return { canRun: true, missing, available };
+    const deps = calcManifest.rootDataDependencies || [];
 
-    // Check if computation can run with missing rootdata
-    const canHaveMissingRoots = calcManifest.canHaveMissingRoots === true;
+    if (!deps.length) return { canRun: true, missing, available };
 
-    // Normalize userType to lowercase for comparison (computations use uppercase)
     const userType = (calcManifest.userType || 'all').toLowerCase();
+    const canHaveMissing = calcManifest.canHaveMissingRoots === true;
 
-    for (const dep of calcManifest.rootDataDependencies) {
+    for (const dep of deps) {
         let isAvailable = false;
-        
-        if (dep === 'portfolio') {
-            if (userType === 'speculator'            && rootDataStatus.speculatorPortfolio) isAvailable = true;
-            else if (userType === 'normal'           && rootDataStatus.normalPortfolio) isAvailable = true;
-            else if (userType === 'popular_investor' && rootDataStatus.piPortfolios) isAvailable = true;
-            else if (userType === 'signed_in_user'   && rootDataStatus.signedInUserPortfolio) isAvailable = true;
-            else if (userType === 'all'              && rootDataStatus.hasPortfolio) isAvailable = true;
-            
-            if (!isAvailable) {
-                // [OPTIMIZATION] Optimistic Series Check
-                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
-                    available.push('portfolio');
-                    continue; 
-                }
-
-                if (userType === 'speculator') missing.push('speculatorPortfolio');
-                else if (userType === 'normal') missing.push('normalPortfolio');
-                else if (userType === 'popular_investor') missing.push('piPortfolios');
-                else if (userType === 'signed_in_user') missing.push('signedInUserPortfolio');
-                else missing.push('portfolio');
-            } else {
-                available.push('portfolio');
-            }
+        let missingKey = dep;
+
+        // 1. Resolve Status Key
+        if (SIMPLE_DEP_MAP[dep]) {
+            const key = SIMPLE_DEP_MAP[dep];
+            if (rootDataStatus[key]) isAvailable = true;
+            else missingKey = key;
+        } 
+        else if (COMPLEX_DEP_MAP[dep]) {
+            const map = COMPLEX_DEP_MAP[dep];
+            const key = map[userType] || map._default;
+            if (rootDataStatus[key]) isAvailable = true;
+            else missingKey = key;
         }
-        else if (dep === 'history') {
-            if (userType === 'speculator' && rootDataStatus.speculatorHistory) isAvailable = true;
-            else if (userType === 'normal' && rootDataStatus.normalHistory) isAvailable = true;
-            else if (userType === 'popular_investor' && rootDataStatus.piHistory) isAvailable = true;
-            else if (userType === 'signed_in_user' && rootDataStatus.signedInUserHistory) isAvailable = true;
-            else if (userType === 'all' && rootDataStatus.hasHistory) isAvailable = true;
-            
-            if (!isAvailable) {
-                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
-                    available.push('history');
-                    continue; 
-                }
 
-                if (userType === 'speculator') missing.push('speculatorHistory');
-                else if (userType === 'normal') missing.push('normalHistory');
-                else if (userType === 'popular_investor') missing.push('piHistory');
-                else if (userType === 'signed_in_user') missing.push('signedInUserHistory');
-                else missing.push('history');
-            } else {
-                available.push('history');
-            }
-        }
-        else if (dep === 'rankings') {
-            if (rootDataStatus.piRankings) {
-                isAvailable = true;
-                available.push('rankings');
-            } else {
-                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
-                    available.push('rankings');
-                } else {
-                    missing.push('piRankings');
-                }
-            }
-        }
-        else if (dep === 'verification') {
-            if (rootDataStatus.signedInUserVerification) {
-                isAvailable = true;
-                available.push('verification');
-            } else {
-                missing.push('signedInUserVerification');
-            }
-        }
-        else if (dep === 'insights') {
-            if (rootDataStatus.hasInsights) {
-                isAvailable = true;
-                available.push('insights');
-            } else {
-                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
-                    available.push('insights');
-                } else {
-                    missing.push('insights');
-                }
-            }
-        }
-        else if (dep === 'social') {
-            if (userType === 'popular_investor') {
-                if (rootDataStatus.hasPISocial) isAvailable = true;
-            } else if (userType === 'signed_in_user') {
-                if (rootDataStatus.hasSignedInSocial) isAvailable = true;
-            } else {
-                if (rootDataStatus.hasSocial) isAvailable = true;
-            }
-
-            if (isAvailable) {
-                available.push('social');
-            } else {
-                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
-                    available.push('social');
-                    continue; 
-                }
-
-                if (userType === 'popular_investor') missing.push('piSocial');
-                else if (userType === 'signed_in_user') missing.push('signedInSocial');
-                else missing.push('social');
-            }
-        }
-        else if (dep === 'price') {
-            if (rootDataStatus.hasPrices) {
-                isAvailable = true;
-                available.push('price');
-            } else {
-                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
-                    available.push('price');
-                } else {
-                    missing.push('price');
-                }
-            }
-        }
-        else if (dep === 'ratings') {
-            if (rootDataStatus.piRatings) {
-                isAvailable = true;
-                available.push('ratings');
-            } else {
-                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
-                    available.push('ratings');
-                } else {
-                    missing.push('piRatings');
-                }
-            }
-        }
-        else if (dep === 'pageViews') {
-            if (rootDataStatus.piPageViews) {
-                isAvailable = true;
-                available.push('pageViews');
-            } else {
-                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
-                    available.push('pageViews');
-                } else {
-                    missing.push('piPageViews');
-                }
-            }
-        }
-        else if (dep === 'watchlist') {
-            if (rootDataStatus.watchlistMembership) {
-                isAvailable = true;
-                available.push('watchlist');
-            } else {
-                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
-                    available.push('watchlist');
-                } else {
-                    missing.push('watchlistMembership');
-                }
-            }
-        }
-        else if (dep === 'alerts') {
-            if (rootDataStatus.piAlertHistory) {
-                isAvailable = true;
-                available.push('alerts');
+        // 2. Check Availability (with Optimistic Series Fallback)
+        if (isAvailable) {
+            available.push(dep);
+        } else {
+            // [OPTIMIZATION] If series data is requested, we can sometimes proceed without the daily snapshot
+            if (calcManifest.rootDataSeries?.[dep]) {
+                available.push(dep);
             } else {
-                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
-                    available.push('alerts');
-                } else {
-                    missing.push('piAlertHistory');
-                }
+                missing.push(missingKey);
             }
         }
     }
 
-    // [NEW] Enforce Mandatory Roots (defined by computation)
-    // This allows granular control: "I need portfolio, but ratings are optional"
-    if (calcManifest.mandatoryRoots && Array.isArray(calcManifest.mandatoryRoots) && calcManifest.mandatoryRoots.length > 0) {
-        const missingMandatory = calcManifest.mandatoryRoots.filter(r => !available.includes(r));
-        if (missingMandatory.length > 0) {
+    // 3. Enforce Mandatory Roots (Granular Override)
+    if (calcManifest.mandatoryRoots?.length) {
+        if (calcManifest.mandatoryRoots.some(r => !available.includes(r))) {
             return { canRun: false, missing, available };
         }
     }
-    
-    if (canHaveMissingRoots) {
-        const canRun = available.length > 0;
-        return { canRun, missing, available };
-    }
-    
-    // Strict mode: all required rootdata must be available
-    return { canRun: missing.length === 0, missing, available };
+
+    return { 
+        canRun: canHaveMissing ? available.length > 0 : missing.length === 0, 
+        missing, 
+        available 
+    };
 }
 
+/**
+ * filters a list of calculations to those that can run given the current data.
+ */
 function getViableCalculations(candidates, fullManifest, rootDataStatus, dailyStatus) {
-    const viable = [];
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
     
-    for (const calc of candidates) {
-        const rootCheck = checkRootDependencies(calc, rootDataStatus);
-        if (!rootCheck.canRun) continue;
-
-        let dependenciesMet = true;
-        if (calc.dependencies && calc.dependencies.length > 0) {
-            for (const depName of calc.dependencies) {
-                const normDep = normalizeName(depName);
-                const storedHash = dailyStatus[normDep];
-                const depManifest = manifestMap.get(normDep);
-                
-                if (!depManifest) { dependenciesMet = false; break; }
-                if (!storedHash) { dependenciesMet = false; break; }
-                if (storedHash.hash !== depManifest.hash) { dependenciesMet = false; break; }
-            }
+    return candidates.filter(calc => {
+        // 1. Check Root Data
+        const { canRun } = checkRootDependencies(calc, rootDataStatus);
+        if (!canRun) return false;
+
+        // 2. Check Computed Dependencies (Hashes must match)
+        if (calc.dependencies?.length) {
+            return calc.dependencies.every(depName => {
+                const norm = normalizeName(depName);
+                const stored = dailyStatus[norm];
+                const ref = manifestMap.get(norm);
+                // Dependency must exist, have run, and match the current manifest hash
+                return ref && stored && stored.hash === ref.hash;
+            });
         }
+        return true;
+    });
+}
 
-        if (dependenciesMet) { viable.push(calc); }
-    }
+// =============================================================================
+// DATA ACCESS: Status Normalization
+// =============================================================================
 
-    return viable;
+/**
+ * Standardizes the Firestore document into a flat status object.
+ * Handles fallbacks and merges 'details' fields.
+ */
+function normalizeStatus(data) {
+    // If data is null/undefined, return object with all false
+    const d = data || {};
+    const det = d.details || {};
+    
+    // Helper to check both root level and details object
+    const val = (key, rootFallback) => !!det[key] || (rootFallback ? !!d[rootFallback] : false);
+
+    return {
+        // Core Flags
+        hasPortfolio: !!d.hasPortfolio,
+        hasHistory:   !!d.hasHistory,
+        hasSocial:    !!d.hasSocial,
+        hasInsights:  !!d.hasInsights,
+        hasPrices:    !!d.hasPrices,
+        
+        // Granular Portfolio/History
+        speculatorPortfolio: !!det.speculatorPortfolio,
+        normalPortfolio:     !!det.normalPortfolio,
+        piPortfolios:        !!det.piPortfolios,
+        piDeepPortfolios:    !!det.piDeepPortfolios,
+        signedInUserPortfolio: !!det.signedInUserPortfolio,
+        
+        speculatorHistory:   !!det.speculatorHistory,
+        normalHistory:       !!det.normalHistory,
+        piHistory:           !!det.piHistory,
+        signedInUserHistory: !!det.signedInUserHistory,
+
+        // Meta Types
+        piRankings:               !!det.piRankings,
+        signedInUserVerification: !!det.signedInUserVerification,
+        hasPISocial:              val('hasPISocial', 'hasPISocial'),
+        hasSignedInSocial:        val('hasSignedInSocial', 'hasSignedInSocial'),
+        
+        // Extended Root Types
+        piRatings:           !!det.piRatings,
+        piPageViews:         !!det.piPageViews,
+        watchlistMembership: !!det.watchlistMembership,
+        piAlertHistory:      !!det.piAlertHistory,
+        piMasterList:        !!det.piMasterList
+    };
 }
 
 /**
@@ -241,137 +188,36 @@ function getViableCalculations(candidates, fullManifest, rootDataStatus, dailySt
  */
 async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
     const { logger, db } = dependencies;
-    
     try {
-        const indexDoc = await db.collection(INDEX_COLLECTION).doc(dateStr).get();
-        
-        if (indexDoc.exists) {
-            const data = indexDoc.data();
-            const details = data.details || {};
-
-            return {
-                status: {
-                    hasPortfolio: !!data.hasPortfolio,
-                    hasHistory:   !!data.hasHistory,
-                    hasSocial:    !!data.hasSocial, 
-                    hasInsights:  !!data.hasInsights,
-                    hasPrices:    !!data.hasPrices,
-                    speculatorPortfolio: !!details.speculatorPortfolio,
-                    normalPortfolio:     !!details.normalPortfolio,
-                    speculatorHistory:   !!details.speculatorHistory,
-                    normalHistory:       !!details.normalHistory,
-                    piRankings:               !!details.piRankings,
-                    piPortfolios:             !!details.piPortfolios,
-                    piDeepPortfolios:         !!details.piDeepPortfolios,
-                    piHistory:                !!details.piHistory,
-                    signedInUserPortfolio:    !!details.signedInUserPortfolio,
-                    signedInUserHistory:      !!details.signedInUserHistory,
-                    signedInUserVerification: !!details.signedInUserVerification,
-                    hasPISocial:       !!details.hasPISocial || !!data.hasPISocial,
-                    hasSignedInSocial: !!details.hasSignedInSocial || !!data.hasSignedInSocial,
-                    piRatings:          !!details.piRatings,
-                    piPageViews:        !!details.piPageViews,
-                    watchlistMembership: !!details.watchlistMembership,
-                    piAlertHistory:     !!details.piAlertHistory,
-                    piMasterList: !!details.piMasterList
-                },
-                portfolioRefs: null, 
-                historyRefs: null,
-                todayInsights: null,
-                todaySocialPostInsights: null,
-                yesterdayPortfolioRefs: null
-            };
-        } else {
-            logger.log('WARN', `[Availability] Index not found for ${dateStr}. Assuming NO data.`);
-            return {
-                status: { 
-                    hasPortfolio: false, 
-                    hasHistory: false, 
-                    hasSocial: false, 
-                    hasInsights: false, 
-                    hasPrices: false,
-                    speculatorPortfolio: false, 
-                    normalPortfolio: false, 
-                    speculatorHistory: false, 
-                    normalHistory: false,
-                    piRankings: false, 
-                    piPortfolios: false, 
-                    piDeepPortfolios: false, 
-                    piHistory: false,
-                    signedInUserPortfolio: false, 
-                    signedInUserHistory: false, 
-                    signedInUserVerification: false,
-                    hasPISocial: false, 
-                    hasSignedInSocial: false,
-                    piRatings: false, 
-                    piPageViews: false, 
-                    watchlistMembership: false, 
-                    piAlertHistory: false
-                }
-            };
+        const doc = await db.collection(INDEX_COLLECTION).doc(dateStr).get();
+        if (!doc.exists) {
+            logger.log('WARN', `[Availability] Index not found for ${dateStr}.`);
+            return { status: normalizeStatus(null) };
         }
-
-    } catch (err) { 
-        logger.log('ERROR', `Error checking availability index: ${err.message}`); 
-        return null; 
+        return { 
+            status: normalizeStatus(doc.data()),
+            // Legacy placeholders preserved for compatibility
+            portfolioRefs: null, historyRefs: null, todayInsights: null, todaySocialPostInsights: null, yesterdayPortfolioRefs: null
+        };
+    } catch (err) {
+        logger.log('ERROR', `Error checking availability index: ${err.message}`);
+        return null;
     }
 }
 
 /**
- * [NEW] Fetches availability status for a range of dates.
- * Uses a range query to only retrieve indices that actually exist, preventing wasted reads on empty days.
- * @param {Object} deps - Dependencies (must include db)
- * @param {string} startDateStr - ISO Date string (YYYY-MM-DD) inclusive start
- * @param {string} endDateStr - ISO Date string (YYYY-MM-DD) inclusive end
- * @returns {Promise<Map<string, Object>>} Map of dateStr -> status object
+ * Fetches availability status for a range of dates.
  */
 async function getAvailabilityWindow(deps, startDateStr, endDateStr) {
     const { db } = deps;
-    
-    // Perform Range Query on Document ID (Date String)
     const snapshot = await db.collection(INDEX_COLLECTION)
         .where(FieldPath.documentId(), '>=', startDateStr)
         .where(FieldPath.documentId(), '<=', endDateStr)
         .get();
 
-    const availabilityMap = new Map();
-    
-    snapshot.forEach(doc => {
-        const data = doc.data();
-        const details = data.details || {};
-        const dateStr = doc.id;
-
-        // Construct status object matching checkRootDataAvailability structure
-        const status = {
-            hasPortfolio: !!data.hasPortfolio,
-            hasHistory:   !!data.hasHistory,
-            hasSocial:    !!data.hasSocial, 
-            hasInsights:  !!data.hasInsights,
-            hasPrices:    !!data.hasPrices,
-            speculatorPortfolio: !!details.speculatorPortfolio,
-            normalPortfolio:     !!details.normalPortfolio,
-            speculatorHistory:   !!details.speculatorHistory,
-            normalHistory:       !!details.normalHistory,
-            piRankings:               !!details.piRankings,
-            piPortfolios:             !!details.piPortfolios,
-            piDeepPortfolios:         !!details.piDeepPortfolios,
-            piHistory:                !!details.piHistory,
-            signedInUserPortfolio:    !!details.signedInUserPortfolio,
-            signedInUserHistory:      !!details.signedInUserHistory,
-            signedInUserVerification: !!details.signedInUserVerification,
-            hasPISocial:       !!details.hasPISocial || !!data.hasPISocial,
-            hasSignedInSocial: !!details.hasSignedInSocial || !!data.hasSignedInSocial,
-            piRatings:          !!details.piRatings,
-            piPageViews:        !!details.piPageViews,
-            watchlistMembership: !!details.watchlistMembership,
-            piAlertHistory:     !!details.piAlertHistory,
-            piMasterList:       !!details.piMasterList
-        };
-        
-        availabilityMap.set(dateStr, status);
-    });
-
-    return availabilityMap;
+    const map = new Map();
+    snapshot.forEach(doc => map.set(doc.id, normalizeStatus(doc.data())));
+    return map;
 }
 
 module.exports = { 
@@ -379,4 +225,4 @@ module.exports = {
     checkRootDataAvailability, 
     getViableCalculations,
     getAvailabilityWindow 
-};
+};