bulltrackers-module 1.0.658 → 1.0.659

package/functions/computation-system/helpers/computation_worker.js

@@ -249,10 +249,11 @@ async function handleComputationTask(message, config, dependencies) {
 
     // [FIX] Declare manifest here so it is accessible in both try and catch blocks
     let manifest;
+    const taskStartTime = Date.now(); // Declare outside try/catch for access in error handler
 
     try {
         manifest = getManifest(config.activeProductLines || [], calculations, runDeps);
-        const startTime = Date.now();
+        const startTime = taskStartTime;
         
         const result = await executeDispatchTask(
             date, pass, computation, config, runDeps, 
@@ -336,7 +337,7 @@ async function handleComputationTask(message, config, dependencies) {
                 resourceTier: resourceTier 
             }, { merge: true });
 
-            await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'FATAL' }, { peakMemoryMB: heartbeats.getPeak() }, triggerReason, resourceTier);
+            await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'FATAL' }, { durationMs: Date.now() - taskStartTime, peakMemoryMB: heartbeats.getPeak() }, triggerReason, resourceTier);
             
             // Send error notification if this was an on-demand computation
             if (metadata?.onDemand && metadata?.requestId && metadata?.requestingUserCid) {

package/functions/computation-system/legacy/AvailabilityCheckerOld.js

@@ -0,0 +1,382 @@
+/**
+ * @fileoverview Checks availability of root data via the Root Data Index.
+ * REFACTORED: Fully supports granular flags for PI, Signed-In Users, Rankings, and Verification.
+ * UPDATED: Enforces 'mandatoryRoots' metadata to override permissive flags.
+ * NEW: Added 'getAvailabilityWindow' for efficient batch availability lookups using range queries.
+ */
+const { normalizeName } = require('../utils/utils');
+const { FieldPath } = require('@google-cloud/firestore');
+
+const INDEX_COLLECTION = process.env.ROOT_DATA_AVAILABILITY_COLLECTION || 'system_root_data_index';
+
+/**
+ * Checks if a specific calculation can run based on its dependencies and the current data status.
+ * @param {Object} calcManifest - The calculation manifest.
+ * @param {Object} rootDataStatus - The availability status object.
+ * @returns {Object} { canRun: boolean, missing: Array, available: Array }
+ */
+function checkRootDependencies(calcManifest, rootDataStatus) {
+    const missing = [];
+    const available = [];
+    
+    if (!calcManifest.rootDataDependencies) return { canRun: true, missing, available };
+
+    // Check if computation can run with missing rootdata
+    const canHaveMissingRoots = calcManifest.canHaveMissingRoots === true;
+
+    // Normalize userType to lowercase for comparison (computations use uppercase)
+    const userType = (calcManifest.userType || 'all').toLowerCase();
+
+    for (const dep of calcManifest.rootDataDependencies) {
+        let isAvailable = false;
+        
+        if (dep === 'portfolio') {
+            if (userType === 'speculator'            && rootDataStatus.speculatorPortfolio) isAvailable = true;
+            else if (userType === 'normal'           && rootDataStatus.normalPortfolio) isAvailable = true;
+            else if (userType === 'popular_investor' && rootDataStatus.piPortfolios) isAvailable = true;
+            else if (userType === 'signed_in_user'   && rootDataStatus.signedInUserPortfolio) isAvailable = true;
+            else if (userType === 'all'              && rootDataStatus.hasPortfolio) isAvailable = true;
+            
+            if (!isAvailable) {
+                // [OPTIMIZATION] Optimistic Series Check
+                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
+                    available.push('portfolio');
+                    continue; 
+                }
+
+                if (userType === 'speculator') missing.push('speculatorPortfolio');
+                else if (userType === 'normal') missing.push('normalPortfolio');
+                else if (userType === 'popular_investor') missing.push('piPortfolios');
+                else if (userType === 'signed_in_user') missing.push('signedInUserPortfolio');
+                else missing.push('portfolio');
+            } else {
+                available.push('portfolio');
+            }
+        }
+        else if (dep === 'history') {
+            if (userType === 'speculator' && rootDataStatus.speculatorHistory) isAvailable = true;
+            else if (userType === 'normal' && rootDataStatus.normalHistory) isAvailable = true;
+            else if (userType === 'popular_investor' && rootDataStatus.piHistory) isAvailable = true;
+            else if (userType === 'signed_in_user' && rootDataStatus.signedInUserHistory) isAvailable = true;
+            else if (userType === 'all' && rootDataStatus.hasHistory) isAvailable = true;
+            
+            if (!isAvailable) {
+                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
+                    available.push('history');
+                    continue; 
+                }
+
+                if (userType === 'speculator') missing.push('speculatorHistory');
+                else if (userType === 'normal') missing.push('normalHistory');
+                else if (userType === 'popular_investor') missing.push('piHistory');
+                else if (userType === 'signed_in_user') missing.push('signedInUserHistory');
+                else missing.push('history');
+            } else {
+                available.push('history');
+            }
+        }
+        else if (dep === 'rankings') {
+            if (rootDataStatus.piRankings) {
+                isAvailable = true;
+                available.push('rankings');
+            } else {
+                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
+                    available.push('rankings');
+                } else {
+                    missing.push('piRankings');
+                }
+            }
+        }
+        else if (dep === 'verification') {
+            if (rootDataStatus.signedInUserVerification) {
+                isAvailable = true;
+                available.push('verification');
+            } else {
+                missing.push('signedInUserVerification');
+            }
+        }
+        else if (dep === 'insights') {
+            if (rootDataStatus.hasInsights) {
+                isAvailable = true;
+                available.push('insights');
+            } else {
+                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
+                    available.push('insights');
+                } else {
+                    missing.push('insights');
+                }
+            }
+        }
+        else if (dep === 'social') {
+            if (userType === 'popular_investor') {
+                if (rootDataStatus.hasPISocial) isAvailable = true;
+            } else if (userType === 'signed_in_user') {
+                if (rootDataStatus.hasSignedInSocial) isAvailable = true;
+            } else {
+                if (rootDataStatus.hasSocial) isAvailable = true;
+            }
+
+            if (isAvailable) {
+                available.push('social');
+            } else {
+                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
+                    available.push('social');
+                    continue; 
+                }
+
+                if (userType === 'popular_investor') missing.push('piSocial');
+                else if (userType === 'signed_in_user') missing.push('signedInSocial');
+                else missing.push('social');
+            }
+        }
+        else if (dep === 'price') {
+            if (rootDataStatus.hasPrices) {
+                isAvailable = true;
+                available.push('price');
+            } else {
+                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
+                    available.push('price');
+                } else {
+                    missing.push('price');
+                }
+            }
+        }
+        else if (dep === 'ratings') {
+            if (rootDataStatus.piRatings) {
+                isAvailable = true;
+                available.push('ratings');
+            } else {
+                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
+                    available.push('ratings');
+                } else {
+                    missing.push('piRatings');
+                }
+            }
+        }
+        else if (dep === 'pageViews') {
+            if (rootDataStatus.piPageViews) {
+                isAvailable = true;
+                available.push('pageViews');
+            } else {
+                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
+                    available.push('pageViews');
+                } else {
+                    missing.push('piPageViews');
+                }
+            }
+        }
+        else if (dep === 'watchlist') {
+            if (rootDataStatus.watchlistMembership) {
+                isAvailable = true;
+                available.push('watchlist');
+            } else {
+                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
+                    available.push('watchlist');
+                } else {
+                    missing.push('watchlistMembership');
+                }
+            }
+        }
+        else if (dep === 'alerts') {
+            if (rootDataStatus.piAlertHistory) {
+                isAvailable = true;
+                available.push('alerts');
+            } else {
+                if (calcManifest.rootDataSeries && calcManifest.rootDataSeries[dep]) {
+                    available.push('alerts');
+                } else {
+                    missing.push('piAlertHistory');
+                }
+            }
+        }
+    }
+
+    // [NEW] Enforce Mandatory Roots (defined by computation)
+    // This allows granular control: "I need portfolio, but ratings are optional"
+    if (calcManifest.mandatoryRoots && Array.isArray(calcManifest.mandatoryRoots) && calcManifest.mandatoryRoots.length > 0) {
+        const missingMandatory = calcManifest.mandatoryRoots.filter(r => !available.includes(r));
+        if (missingMandatory.length > 0) {
+            return { canRun: false, missing, available };
+        }
+    }
+    
+    if (canHaveMissingRoots) {
+        const canRun = available.length > 0;
+        return { canRun, missing, available };
+    }
+    
+    // Strict mode: all required rootdata must be available
+    return { canRun: missing.length === 0, missing, available };
+}
+
+function getViableCalculations(candidates, fullManifest, rootDataStatus, dailyStatus) {
+    const viable = [];
+    const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
+    
+    for (const calc of candidates) {
+        const rootCheck = checkRootDependencies(calc, rootDataStatus);
+        if (!rootCheck.canRun) continue;
+
+        let dependenciesMet = true;
+        if (calc.dependencies && calc.dependencies.length > 0) {
+            for (const depName of calc.dependencies) {
+                const normDep = normalizeName(depName);
+                const storedHash = dailyStatus[normDep];
+                const depManifest = manifestMap.get(normDep);
+                
+                if (!depManifest) { dependenciesMet = false; break; }
+                if (!storedHash) { dependenciesMet = false; break; }
+                if (storedHash.hash !== depManifest.hash) { dependenciesMet = false; break; }
+            }
+        }
+
+        if (dependenciesMet) { viable.push(calc); }
+    }
+
+    return viable;
+}
+
+/**
+ * Checks root data availability for a single date.
+ */
+async function checkRootDataAvailability(dateStr, config, dependencies, earliestDates) {
+    const { logger, db } = dependencies;
+    
+    try {
+        const indexDoc = await db.collection(INDEX_COLLECTION).doc(dateStr).get();
+        
+        if (indexDoc.exists) {
+            const data = indexDoc.data();
+            const details = data.details || {};
+
+            return {
+                status: {
+                    hasPortfolio: !!data.hasPortfolio,
+                    hasHistory:   !!data.hasHistory,
+                    hasSocial:    !!data.hasSocial, 
+                    hasInsights:  !!data.hasInsights,
+                    hasPrices:    !!data.hasPrices,
+                    speculatorPortfolio: !!details.speculatorPortfolio,
+                    normalPortfolio:     !!details.normalPortfolio,
+                    speculatorHistory:   !!details.speculatorHistory,
+                    normalHistory:       !!details.normalHistory,
+                    piRankings:               !!details.piRankings,
+                    piPortfolios:             !!details.piPortfolios,
+                    piDeepPortfolios:         !!details.piDeepPortfolios,
+                    piHistory:                !!details.piHistory,
+                    signedInUserPortfolio:    !!details.signedInUserPortfolio,
+                    signedInUserHistory:      !!details.signedInUserHistory,
+                    signedInUserVerification: !!details.signedInUserVerification,
+                    hasPISocial:       !!details.hasPISocial || !!data.hasPISocial,
+                    hasSignedInSocial: !!details.hasSignedInSocial || !!data.hasSignedInSocial,
+                    piRatings:          !!details.piRatings,
+                    piPageViews:        !!details.piPageViews,
+                    watchlistMembership: !!details.watchlistMembership,
+                    piAlertHistory:     !!details.piAlertHistory,
+                    piMasterList: !!details.piMasterList
+                },
+                portfolioRefs: null, 
+                historyRefs: null,
+                todayInsights: null,
+                todaySocialPostInsights: null,
+                yesterdayPortfolioRefs: null
+            };
+        } else {
+            logger.log('WARN', `[Availability] Index not found for ${dateStr}. Assuming NO data.`);
+            return {
+                status: { 
+                    hasPortfolio: false, 
+                    hasHistory: false, 
+                    hasSocial: false, 
+                    hasInsights: false, 
+                    hasPrices: false,
+                    speculatorPortfolio: false, 
+                    normalPortfolio: false, 
+                    speculatorHistory: false, 
+                    normalHistory: false,
+                    piRankings: false, 
+                    piPortfolios: false, 
+                    piDeepPortfolios: false, 
+                    piHistory: false,
+                    signedInUserPortfolio: false, 
+                    signedInUserHistory: false, 
+                    signedInUserVerification: false,
+                    hasPISocial: false, 
+                    hasSignedInSocial: false,
+                    piRatings: false, 
+                    piPageViews: false, 
+                    watchlistMembership: false, 
+                    piAlertHistory: false
+                }
+            };
+        }
+
+    } catch (err) { 
+        logger.log('ERROR', `Error checking availability index: ${err.message}`); 
+        return null; 
+    }
+}
+
+/**
+ * [NEW] Fetches availability status for a range of dates.
+ * Uses a range query to only retrieve indices that actually exist, preventing wasted reads on empty days.
+ * @param {Object} deps - Dependencies (must include db)
+ * @param {string} startDateStr - ISO Date string (YYYY-MM-DD) inclusive start
+ * @param {string} endDateStr - ISO Date string (YYYY-MM-DD) inclusive end
+ * @returns {Promise<Map<string, Object>>} Map of dateStr -> status object
+ */
+async function getAvailabilityWindow(deps, startDateStr, endDateStr) {
+    const { db } = deps;
+    
+    // Perform Range Query on Document ID (Date String)
+    const snapshot = await db.collection(INDEX_COLLECTION)
+        .where(FieldPath.documentId(), '>=', startDateStr)
+        .where(FieldPath.documentId(), '<=', endDateStr)
+        .get();
+
+    const availabilityMap = new Map();
+    
+    snapshot.forEach(doc => {
+        const data = doc.data();
+        const details = data.details || {};
+        const dateStr = doc.id;
+
+        // Construct status object matching checkRootDataAvailability structure
+        const status = {
+            hasPortfolio: !!data.hasPortfolio,
+            hasHistory:   !!data.hasHistory,
+            hasSocial:    !!data.hasSocial, 
+            hasInsights:  !!data.hasInsights,
+            hasPrices:    !!data.hasPrices,
+            speculatorPortfolio: !!details.speculatorPortfolio,
+            normalPortfolio:     !!details.normalPortfolio,
+            speculatorHistory:   !!details.speculatorHistory,
+            normalHistory:       !!details.normalHistory,
+            piRankings:               !!details.piRankings,
+            piPortfolios:             !!details.piPortfolios,
+            piDeepPortfolios:         !!details.piDeepPortfolios,
+            piHistory:                !!details.piHistory,
+            signedInUserPortfolio:    !!details.signedInUserPortfolio,
+            signedInUserHistory:      !!details.signedInUserHistory,
+            signedInUserVerification: !!details.signedInUserVerification,
+            hasPISocial:       !!details.hasPISocial || !!data.hasPISocial,
+            hasSignedInSocial: !!details.hasSignedInSocial || !!data.hasSignedInSocial,
+            piRatings:          !!details.piRatings,
+            piPageViews:        !!details.piPageViews,
+            watchlistMembership: !!details.watchlistMembership,
+            piAlertHistory:     !!details.piAlertHistory,
+            piMasterList:       !!details.piMasterList
+        };
+        
+        availabilityMap.set(dateStr, status);
+    });
+
+    return availabilityMap;
+}
+
+module.exports = { 
+    checkRootDependencies, 
+    checkRootDataAvailability, 
+    getViableCalculations,
+    getAvailabilityWindow 
+};