bulltrackers-module 1.0.721 → 1.0.723

package/functions/computation-system/data/CachedDataLoader.js

@@ -1,32 +1,32 @@
 /**
- * {
- * type: uploaded file
- * fileName: computation-system/data/CachedDataLoader.js
- * }
- * REFACTORED: Unified Loader Configuration for DRY principles.
+ * @fileoverview Caching layer for data loading.
+ * REFACTORED:
+ * 1. Supports new BigQuery-backed loader functions.
+ * 2. correctly passes 'userTypes' to portfolio/history loaders.
+ * 3. Handles Virtual Shards for BigQuery prices.
  */
 const { 
     loadDailyInsights, 
     loadDailySocialPostInsights, 
-    getRelevantShardRefs, 
     getPriceShardRefs,
+    getRelevantShardRefs,
     loadVerificationProfiles,
     loadPopularInvestorRankings,
     loadPIRatings,
     loadPIPageViews,
-    loadWatchlistMembership: loadWatchlistMembershipData,
+    loadWatchlistMembership,
     loadPIAlertHistory,
     loadPIWatchlistData,
     loadPopularInvestorMasterList,
-    loadDailyPortfolios // <--- IMPORTED
+    loadDailyPortfolios,
+    loadDailyHistory
 } = require('../utils/data_loader');
 const { getAvailabilityWindow } = require('./AvailabilityChecker');
 const zlib = require('zlib'); 
 
 // =============================================================================
-// CONFIGURATION: Unified Loader Definitions
+// CONFIGURATION
 // =============================================================================
-// Centralizes config keys, defaults, loader functions, and availability flags.
 const LOADER_DEFINITIONS = {
     loadRankings: {
         cache: 'rankings',
@@ -38,28 +38,28 @@ const LOADER_DEFINITIONS = {
     loadRatings: {
         cache: 'ratings',
         configKey: 'piRatingsCollection',
-        defaultCol: 'PIRatingsData',
+        defaultCol: 'pi_ratings',
         fn: loadPIRatings,
         flag: 'piRatings'
     },
     loadPageViews: {
         cache: 'pageViews',
         configKey: 'piPageViewsCollection',
-        defaultCol: 'PIPageViewsData',
+        defaultCol: 'pi_page_views',
         fn: loadPIPageViews,
         flag: 'piPageViews'
     },
     loadWatchlistMembership: {
         cache: 'watchlistMembership',
         configKey: 'watchlistMembershipCollection',
-        defaultCol: 'WatchlistMembershipData',
-        fn: loadWatchlistMembershipData,
+        defaultCol: 'watchlist_membership',
+        fn: loadWatchlistMembership,
         flag: 'watchlistMembership'
     },
     loadAlertHistory: {
         cache: 'alertHistory',
         configKey: 'piAlertHistoryCollection',
-        defaultCol: 'PIAlertHistoryData',
+        defaultCol: 'pi_alert_history',
         fn: loadPIAlertHistory,
         flag: 'piAlertHistory'
     },
@@ -79,15 +79,18 @@ const LOADER_DEFINITIONS = {
     },
     loadPIWatchlistData: {
         cache: 'piWatchlistData',
-        // No collection key needed for direct implementation, but handled by fn
         fn: loadPIWatchlistData,
-        isIdBased: true // Uses ID instead of Date
+        isIdBased: true
     },
-    // <--- ADDED SUPPORT FOR PORTFOLIO SERIES
     loadPortfolios: {
         cache: 'portfolios',
-        fn: loadDailyPortfolios
-        // No configKey/flag ensures legacy loop (required for complex multi-collection fetch)
+        fn: loadDailyPortfolios,
+        flag: 'hasPortfolio' // Loose check, accurate check handles inside loader
+    },
+    loadHistory: {
+        cache: 'history',
+        fn: loadDailyHistory,
+        flag: 'hasHistory'
     }
 };
 
@@ -96,11 +99,12 @@ class CachedDataLoader {
         this.config = config; 
         this.deps = dependencies; 
         
-        // Initialize caches dynamically based on definitions + static extras
+        // Initialize caches
         this.cache = { 
             mappings: null, 
             verifications: null, 
             piMasterList: null,
+            prices: new Map(), // Dedicated cache for price shard data
             ...Object.values(LOADER_DEFINITIONS).reduce((acc, def) => {
                 acc[def.cache] = new Map();
                 return acc;
@@ -111,70 +115,66 @@ class CachedDataLoader {
     _tryDecompress(data) {
         if (data?._compressed === true && data.payload) {
             try { return JSON.parse(zlib.gunzipSync(data.payload).toString()); } 
-            catch (e) { console.error('[CachedDataLoader] Decompression failed', e); return {}; }
+            catch (e) { this.deps.logger?.log('WARN', '[CachedDataLoader] Decompression failed'); return {}; }
         }
         return data;
     }
 
     // =========================================================================
-    // GENERIC LOADER HELPER
+    // GENERIC LOADER
     // =========================================================================
-    // [FIX] Accepts ...args to pass down filters (like requiredUserTypes)
     async _loadGeneric(methodName, key, ...args) {
         const def = LOADER_DEFINITIONS[methodName];
         if (!def) throw new Error(`Unknown loader method: ${methodName}`);
 
+        // Construct a cache key that includes args (like userTypes) if present
+        // e.g. "2025-01-01::POPULAR_INVESTOR"
+        const cacheKey = args.length > 0 ? `${key}::${JSON.stringify(args)}` : key;
         const cacheMap = this.cache[def.cache];
-        if (cacheMap.has(key)) return cacheMap.get(key);
+        
+        if (cacheMap.has(cacheKey)) return cacheMap.get(cacheKey);
 
-        const collection = this.config[def.configKey] || def.defaultCol;
-        // Only log if we have a collection context (some ID-based loaders might differ)
-        if (def.configKey) {
-            this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading '${def.cache}' from: ${collection}/${key}`);
+        if (def.configKey && this.deps.logger) {
+            // Only log simple date-based loads to avoid spam
+            // this.deps.logger.log('INFO', `[CachedDataLoader] Loading ${def.cache} for ${key}`);
         }
 
         const promise = def.fn(this.config, this.deps, key, ...args);
-        cacheMap.set(key, promise);
+        cacheMap.set(cacheKey, promise);
         return promise;
     }
 
     // =========================================================================
-    // PUBLIC ACCESSORS (Generated Wrappers)
+    // PUBLIC METHODS
     // =========================================================================
-    
-    // Explicitly defined for IDE autocompletion / static analysis, 
-    // but internally they all delegate to _loadGeneric.
-    async loadInsights(dateStr)             { return this._loadGeneric('loadInsights'           , dateStr); }
-    async loadSocial(dateStr)               { return this._loadGeneric('loadSocial'             , dateStr); }
-    async loadRankings(dateStr)             { return this._loadGeneric('loadRankings'           , dateStr); }
-    async loadRatings(dateStr)              { return this._loadGeneric('loadRatings'            , dateStr); }
-    async loadPageViews(dateStr)            { return this._loadGeneric('loadPageViews'          , dateStr); }
+    async loadInsights(dateStr)             { return this._loadGeneric('loadInsights', dateStr); }
+    async loadSocial(dateStr, userTypes)    { return this._loadGeneric('loadSocial', dateStr, userTypes); }
+    async loadRankings(dateStr)             { return this._loadGeneric('loadRankings', dateStr); }
+    async loadRatings(dateStr)              { return this._loadGeneric('loadRatings', dateStr); }
+    async loadPageViews(dateStr)            { return this._loadGeneric('loadPageViews', dateStr); }
     async loadWatchlistMembership(dateStr)  { return this._loadGeneric('loadWatchlistMembership', dateStr); }
-    async loadAlertHistory(dateStr)         { return this._loadGeneric('loadAlertHistory'       , dateStr); }
-    async loadPIWatchlistData(piCid)        { return this._loadGeneric('loadPIWatchlistData'    , String(piCid)); }
-    // <--- ADDED PORTFOLIOS ACCESSOR with extra arg
+    async loadAlertHistory(dateStr)         { return this._loadGeneric('loadAlertHistory', dateStr); }
+    async loadPIWatchlistData(piCid)        { return this._loadGeneric('loadPIWatchlistData', String(piCid)); }
     async loadPortfolios(dateStr, userTypes) { return this._loadGeneric('loadPortfolios', dateStr, userTypes); }
+    async loadHistory(dateStr, userTypes)    { return this._loadGeneric('loadHistory', dateStr, userTypes); }
 
     // =========================================================================
-    // SPECIALIZED LOADERS (Non-Standard Patterns)
+    // SPECIALIZED METHODS
     // =========================================================================
-
+    
     async loadMappings() { 
         if (this.cache.mappings) return this.cache.mappings; 
         this.cache.mappings = await this.deps.calculationUtils.loadInstrumentMappings(); 
         return this.cache.mappings; 
     }
 
-    async loadVerifications(dateStr) { // <--- Added dateStr arg
+    async loadVerifications(dateStr) { 
+        // Verifications are mostly static/global, but we accept dateStr for consistency
         if (this.cache.verifications) return this.cache.verifications;
-        
-        const col = this.config.verificationsCollection || 'verification_profiles';
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading 'verifications' from: ${col} (Context: ${dateStr || 'Global'})`);
-        
-        // Pass dateStr so data_loader can check GCS snapshots
         this.cache.verifications = await loadVerificationProfiles(this.config, this.deps, dateStr); 
         return this.cache.verifications;
     }
+
     async loadPIMasterList() {
         if (this.cache.piMasterList) return this.cache.piMasterList;
         this.cache.piMasterList = await loadPopularInvestorMasterList(this.config, this.deps);
@@ -185,70 +185,63 @@ class CachedDataLoader {
         return getPriceShardRefs(this.config, this.deps); 
     }
 
-    async getSpecificPriceShardReferences(targetInstrumentIds) { 
-        return getRelevantShardRefs(this.config, this.deps, targetInstrumentIds); 
+    async getSpecificPriceShardReferences(ids) { 
+        return getRelevantShardRefs(this.config, this.deps, ids); 
     }
 
     async loadPriceShard(docRef) {
-        // Check if this is a BigQuery marker
+        // Handle BigQuery "Virtual Shard"
         if (docRef && docRef._bigquery === true) {
-            // Load all price data from BigQuery
-            try {
-                const { queryAssetPrices } = require('../../core/utils/bigquery_utils');
-                const priceData = await queryAssetPrices(null, null, null, this.deps.logger);
-                
-                if (priceData && Object.keys(priceData).length > 0) {
-                    this.deps.logger.log('INFO', `[CachedDataLoader] ✅ Loaded ${Object.keys(priceData).length} instruments from BigQuery`);
-                    return priceData;
-                }
-                
-                // If BigQuery returns empty, fallback to Firestore
-                this.deps.logger.log('WARN', `[CachedDataLoader] BigQuery returned no price data, falling back to Firestore`);
-            } catch (bqError) {
-                this.deps.logger.log('WARN', `[CachedDataLoader] BigQuery price load failed, falling back to Firestore: ${bqError.message}`);
-                // Fall through to Firestore
+            // Use a dedicated cache key for the ALL-PRICE blob
+            // If targetIds are specified, create a unique cache key
+            const targetIds = docRef.targetIds && docRef.targetIds.length > 0 ? docRef.targetIds : null;
+            const cacheKey = targetIds 
+                ? `BQ_PRICES_${targetIds.sort().join(',')}` 
+                : 'BQ_ALL_PRICES';
+            
+            if (this.cache.prices.has(cacheKey)) {
+                return this.cache.prices.get(cacheKey);
             }
+
+            const { queryAssetPrices } = require('../../core/utils/bigquery_utils');
+            // queryAssetPrices signature: (startDateStr, endDateStr, instrumentIds, logger)
+            const promise = queryAssetPrices(null, null, targetIds, this.deps.logger).then(data => {
+                const result = data || {};
+                this.cache.prices.set(cacheKey, result);
+                return result;
+            });
+            
+            return promise;
         }
-        
-        // Firestore fallback (original logic)
+
+        // Firestore Fallback
         try { 
             const snap = await docRef.get(); 
             return snap.exists ? this._tryDecompress(snap.data()) : {}; 
-        } catch (e) { 
-            console.error(`Error loading shard ${docRef.path}:`, e); 
-            return {}; 
-        } 
+        } catch (e) { return {}; } 
     }
 
     // =========================================================================
-    // BATCH SERIES LOADING
+    // SERIES LOADING (Batching)
     // =========================================================================
-    /**
-     * Optimistically loads data series using Batch Reads (db.getAll).
-     * Uses Availability Index to minimize costs.
-     * [FIX] Now accepts ...args to pass context (e.g. requiredUserTypes)
-     */
     async loadSeries(loaderMethod, dateStr, lookbackDays, ...args) {
         const def = LOADER_DEFINITIONS[loaderMethod];
         if (!def) throw new Error(`[CachedDataLoader] Unknown series method ${loaderMethod}`);
         
-        // Fallback to legacy loop if method isn't configured for batching (missing config/flag)
+        // If method doesn't support batching configuration, use legacy loop
         if (!def.configKey || !def.flag) return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays, ...args);
 
-        // 1. Calculate Date Range
+        // 1. Calculate Range
         const endDate = new Date(dateStr);
         const startDate = new Date(endDate);
         startDate.setUTCDate(startDate.getUTCDate() - (lookbackDays - 1));
         
-        // 2. Pre-flight: Check Availability
+        // 2. Check Availability
         let availabilityMap = new Map();
         try {
             availabilityMap = await getAvailabilityWindow(this.deps, startDate.toISOString().slice(0, 10), endDate.toISOString().slice(0, 10));
-        } catch (e) {
-            console.warn(`[CachedDataLoader] Availability check failed. Optimistic batching enabled.`);
-        }
+        } catch (e) {}
 
-        // 3. Construct Batch Refs
         const collectionName = this.config[def.configKey] || def.defaultCol;
         const batchRefs = [];
         const dateKeyMap = []; 
@@ -259,35 +252,38 @@ class CachedDataLoader {
             const dString = d.toISOString().slice(0, 10);
             
             const dayStatus = availabilityMap.get(dString);
-            // Fetch if index says data exists OR if index is missing (optimistic)
             if (!dayStatus || dayStatus[def.flag]) {
                 batchRefs.push(this.deps.db.collection(collectionName).doc(dString));
                 dateKeyMap.push(dString);
             }
         }
 
-        // 4. Execute Batch Read
+        // Note: Batch loading ONLY works for Firestore.
+        // If we are in BigQuery mode, we should NOT use Firestore batching for these types.
+        // However, this `loadSeries` method is mostly used for simple docs (Rankings, Ratings).
+        // Since `loadRankings` in `data_loader.js` now points to BigQuery, passing a Firestore Ref 
+        // won't work because `data_loader` isn't called here; we are accessing DB directly.
+        
+        // [CRITICAL FIX] 
+        // If we are using BigQuery, we cannot use `db.getAll` on non-existent Firestore docs.
+        // We must delegate back to the loader method (which handles BQ) in a loop (concurrently).
+        if (process.env.BIGQUERY_ENABLED !== 'false') {
+             return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays, ...args);
+        }
+
+        // Firestore Batch Logic (Only runs if BQ Disabled)
         const results = {};
         if (batchRefs.length > 0) {
-            this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Batch loading ${batchRefs.length} docs for '${loaderMethod}'`);
             try {
                 const snapshots = await this.deps.db.getAll(...batchRefs);
                 snapshots.forEach((snap, idx) => {
                     if (snap.exists) {
                         const raw = this._tryDecompress(snap.data());
-                        // Clean metadata if necessary
-                        if (['loadRatings', 'loadPageViews', 'loadWatchlistMembership', 'loadAlertHistory'].includes(loaderMethod)) {
-                            const { date, lastUpdated, ...clean } = raw;
-                            results[dateKeyMap[idx]] = clean;
-                        } else if (loaderMethod === 'loadRankings') {
-                            results[dateKeyMap[idx]] = raw.Items || [];
-                        } else {
-                            results[dateKeyMap[idx]] = raw;
-                        }
+                        if (loaderMethod === 'loadRankings') results[dateKeyMap[idx]] = raw.Items || [];
+                        else results[dateKeyMap[idx]] = raw;
                     }
                 });
             } catch (err) {
-                console.warn(`[CachedDataLoader] Batch failed: ${err.message}. Legacy fallback.`);
                 return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays, ...args);
             }
         }
@@ -309,8 +305,11 @@ class CachedDataLoader {
             const d = new Date(endDate);
             d.setUTCDate(d.getUTCDate() - i);
             const dStr = d.toISOString().slice(0, 10);
-            // [FIX] Pass args (e.g. requiredUserTypes) to the loader method
-            promises.push(this[loaderMethod](dStr, ...args).then(data => data ? results[dStr] = data : null).catch(() => null));
+            promises.push(
+                this[loaderMethod](dStr, ...args)
+                    .then(data => { if (data) results[dStr] = data; })
+                    .catch(() => {})
+            );
         }
 
         await Promise.all(promises);

package/functions/computation-system/data/DependencyFetcher.js

@@ -2,6 +2,7 @@
  * FILENAME: computation-system/data/DependencyFetcher.js
  * @fileoverview Fetches dependencies for computations.
  * REFACTORED: Unified fetch logic, streamlined decompression/sharding/GCS.
+ * UPDATED: Properly checks isPage/isAlert flags to determine BigQuery vs Firestore routing.
  */
 const { normalizeName } = require('../utils/utils');
 const zlib = require('zlib');
@@ -9,6 +10,39 @@ const { Storage } = require('@google-cloud/storage');
 
 const storage = new Storage(); // Singleton Client
 
+/**
+ * Helper to check if a computation is isPage or isAlert by looking up manifest
+ * @param {string} normalizedName - Normalized computation name
+ * @param {object} config - Config object that may contain manifest info
+ * @param {string} category - Computation category (fallback for alerts)
+ * @returns {Promise<{isPage: boolean, isAlert: boolean}>}
+ */
+async function checkComputationFlags(normalizedName, config, category) {
+    let isPage = false;
+    let isAlert = false;
+    
+    // Try to get manifest from config if available
+    if (config.getCalculations && typeof config.getCalculations === 'function') {
+        try {
+            const calculations = config.getCalculations(config);
+            const manifest = calculations.find(c => normalizeName(c.name) === normalizedName);
+            if (manifest) {
+                isPage = manifest.isPage === true;
+                isAlert = manifest.isAlertComputation === true;
+                return { isPage, isAlert };
+            }
+        } catch (e) {
+            // Fall through to category-based detection
+        }
+    }
+    
+    // Fallback: infer from category (alerts category = alert computation)
+    // Note: isPage cannot be inferred from category alone, so defaults to false
+    isAlert = category === 'alerts';
+    
+    return { isPage, isAlert };
+}
+
 // =============================================================================
 // HELPERS
 // =============================================================================
@@ -45,20 +79,24 @@ function tryDecompress(payload) {
 /**
  * Fetches, decompresses, and reassembles (if sharded or on GCS) a single result document.
  * NEW: For non-alert, non-page computations, tries BigQuery first (cheaper, no sharding/compression).
+ * @param {object} db - Firestore database instance
+ * @param {object} config - Configuration object (may contain logger, manifestLookup, etc.)
+ * @param {string} dateStr - Date string in YYYY-MM-DD format
+ * @param {string} name - Computation name (normalized)
+ * @param {string} category - Computation category
+ * @returns {Promise<object|null>} Result data or null if not found
  */
 async function fetchSingleResult(db, config, dateStr, name, category) {
     const { resultsCollection = 'computation_results', resultsSubcollection = 'results', computationsSubcollection = 'computations' } = config;
     const log = config.logger || console;
 
-    // NEW STRATEGY: Check if this is an alert or page computation
-    // We need to check the manifest to determine this, but we can infer from category
-    // For now, we'll try BigQuery first for all non-alert computations (alerts are in 'alerts' category)
-    const isAlertComputation = category === 'alerts';
-    // Page computations are typically in 'popular-investor' category but have isPage flag
-    // For now, we'll try BigQuery for all non-alert computations
+    // Check if this is an alert or page computation by looking up manifest
+    const { isPage: isPageComputation, isAlert: isAlertComputation } = await checkComputationFlags(name, config, category);
     
-    // Try BigQuery first for non-alert computations (reduces Firestore reads)
-    if (!isAlertComputation && process.env.BIGQUERY_ENABLED !== 'false') {
+    // Try BigQuery first for non-alert, non-page computations (reduces Firestore reads)
+    // isPage and isAlert computations are always written to Firestore (in addition to BigQuery),
+    // so we should check Firestore for them, but can also try BigQuery as a fallback
+    if (!isAlertComputation && !isPageComputation && process.env.BIGQUERY_ENABLED !== 'false') {
         try {
             const { queryComputationResult } = require('../../core/utils/bigquery_utils');
             const bigqueryResult = await queryComputationResult(name, category, dateStr, log);
@@ -71,6 +109,8 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
             log.log('WARN', `[DependencyFetcher] BigQuery fetch failed for ${name}, falling back to Firestore: ${bqError.message}`);
             // Fall through to Firestore
         }
+    } else if (isAlertComputation || isPageComputation) {
+        log.log('INFO', `[DependencyFetcher] 📄 Using Firestore for ${isAlertComputation ? 'alert' : 'page'} computation ${name} (${dateStr})`);
     }
 
     // Fallback to Firestore (for alerts, pages, or if BigQuery fails)