bulltrackers-module 1.0.658 → 1.0.659

package/functions/computation-system/data/CachedDataLoader.js

@@ -3,6 +3,7 @@
  * type: uploaded file
  * fileName: computation-system/data/CachedDataLoader.js
  * }
+ * REFACTORED: Unified Loader Configuration for DRY principles.
  */
 const { 
     loadDailyInsights, 
@@ -21,153 +22,153 @@ const {
 const { getAvailabilityWindow } = require('./AvailabilityChecker');
 const zlib = require('zlib'); 
 
-// [NEW] Mapping of Loader Methods to Availability Flags
-const LOADER_DEPENDENCY_MAP = {
-    'loadRankings': 'piRankings',
-    'loadRatings': 'piRatings',
-    'loadPageViews': 'piPageViews',
-    'loadWatchlistMembership': 'watchlistMembership',
-    'loadAlertHistory': 'piAlertHistory',
-    'loadInsights': 'hasInsights',
-    'loadSocial': 'hasSocial'
-};
-
-// [NEW] Mapping of Loader Methods to Collection Config Keys / Defaults
-// This allows us to construct references for Batch Reading without executing the opaque loader functions.
-const LOADER_COLLECTION_MAP = {
-    'loadRatings': { configKey: 'piRatingsCollection', default: 'PIRatingsData' },
-    'loadPageViews': { configKey: 'piPageViewsCollection', default: 'PIPageViewsData' },
-    'loadWatchlistMembership': { configKey: 'watchlistMembershipCollection', default: 'WatchlistMembershipData' },
-    'loadAlertHistory': { configKey: 'piAlertHistoryCollection', default: 'PIAlertHistoryData' },
-    'loadInsights': { configKey: 'insightsCollectionName', default: 'daily_instrument_insights' },
-    'loadRankings': { configKey: 'popularInvestorRankingsCollection', default: 'popular_investor_rankings' }
+// =============================================================================
+// CONFIGURATION: Unified Loader Definitions
+// =============================================================================
+// Centralizes config keys, defaults, loader functions, and availability flags.
+const LOADER_DEFINITIONS = {
+    loadRankings: {
+        cache: 'rankings',
+        configKey: 'popularInvestorRankingsCollection',
+        defaultCol: 'popular_investor_rankings',
+        fn: loadPopularInvestorRankings,
+        flag: 'piRankings'
+    },
+    loadRatings: {
+        cache: 'ratings',
+        configKey: 'piRatingsCollection',
+        defaultCol: 'PIRatingsData',
+        fn: loadPIRatings,
+        flag: 'piRatings'
+    },
+    loadPageViews: {
+        cache: 'pageViews',
+        configKey: 'piPageViewsCollection',
+        defaultCol: 'PIPageViewsData',
+        fn: loadPIPageViews,
+        flag: 'piPageViews'
+    },
+    loadWatchlistMembership: {
+        cache: 'watchlistMembership',
+        configKey: 'watchlistMembershipCollection',
+        defaultCol: 'WatchlistMembershipData',
+        fn: loadWatchlistMembershipData,
+        flag: 'watchlistMembership'
+    },
+    loadAlertHistory: {
+        cache: 'alertHistory',
+        configKey: 'piAlertHistoryCollection',
+        defaultCol: 'PIAlertHistoryData',
+        fn: loadPIAlertHistory,
+        flag: 'piAlertHistory'
+    },
+    loadInsights: {
+        cache: 'insights',
+        configKey: 'insightsCollectionName',
+        defaultCol: 'daily_instrument_insights',
+        fn: loadDailyInsights,
+        flag: 'hasInsights'
+    },
+    loadSocial: {
+        cache: 'social',
+        configKey: 'socialInsightsCollection',
+        defaultCol: 'daily_social_insights',
+        fn: loadDailySocialPostInsights,
+        flag: 'hasSocial'
+    },
+    loadPIWatchlistData: {
+        cache: 'piWatchlistData',
+        // No collection key needed for direct implementation, but handled by fn
+        fn: loadPIWatchlistData,
+        isIdBased: true // Uses ID instead of Date
+    }
 };
 
 class CachedDataLoader {
     constructor(config, dependencies) { 
         this.config = config; 
         this.deps = dependencies; 
+        
+        // Initialize caches dynamically based on definitions + static extras
         this.cache = { 
             mappings: null, 
-            insights: new Map(), 
-            social:   new Map(),
-            verifications: null,
-            rankings: new Map(),
-            ratings: new Map(),
-            pageViews: new Map(),
-            watchlistMembership: new Map(),
-            alertHistory: new Map(),
-            piWatchlistData: new Map(),
-            piMasterList: null
+            verifications: null, 
+            piMasterList: null,
+            ...Object.values(LOADER_DEFINITIONS).reduce((acc, def) => {
+                acc[def.cache] = new Map();
+                return acc;
+            }, {})
         }; 
     }
 
     _tryDecompress(data) {
-        if (data && data._compressed === true && data.payload) {
-            try {
-                return JSON.parse(zlib.gunzipSync(data.payload).toString());
-            } catch (e) {
-                console.error('[CachedDataLoader] Decompression failed', e);
-                return {};
-            }
+        if (data?._compressed === true && data.payload) {
+            try { return JSON.parse(zlib.gunzipSync(data.payload).toString()); } 
+            catch (e) { console.error('[CachedDataLoader] Decompression failed', e); return {}; }
         }
         return data;
     }
 
-    // ... [Existing single-day load methods remain unchanged] ...
-    async loadMappings() { 
-        if (this.cache.mappings) return this.cache.mappings; 
-        const { calculationUtils } = this.deps; 
-        this.cache.mappings = await calculationUtils.loadInstrumentMappings(); 
-        return this.cache.mappings; 
-    }
+    // =========================================================================
+    // GENERIC LOADER HELPER
+    // =========================================================================
+    async _loadGeneric(methodName, key) {
+        const def = LOADER_DEFINITIONS[methodName];
+        if (!def) throw new Error(`Unknown loader method: ${methodName}`);
 
-    async loadInsights(dateStr) { 
-        if (this.cache.insights.has(dateStr)) return this.cache.insights.get(dateStr); 
-        const collectionName = this.config.insightsCollectionName || 'daily_instrument_insights';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'insights' from: ${path}`);
-        const promise = loadDailyInsights(this.config, this.deps, dateStr); 
-        this.cache.insights.set(dateStr, promise); 
-        return promise; 
-    }
+        const cacheMap = this.cache[def.cache];
+        if (cacheMap.has(key)) return cacheMap.get(key);
 
-    async loadSocial(dateStr) { 
-        if (this.cache.social.has(dateStr)) return this.cache.social.get(dateStr); 
-        const collectionName = this.config.socialInsightsCollection || 'daily_social_insights';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'social' from: ${path}`);
-        const promise = loadDailySocialPostInsights(this.config, this.deps, dateStr); 
-        this.cache.social.set(dateStr, promise); 
-        return promise; 
-    }
-
-    async loadVerifications() {
-        if (this.cache.verifications) return this.cache.verifications;
-        const collectionName = this.config.verificationsCollection || 'verification_profiles';
-        const path = `${collectionName}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'verifications' from: ${path}`);
-        const verifications = await loadVerificationProfiles(this.config, this.deps);
-        this.cache.verifications = verifications;
-        return verifications;
-    }
+        const collection = this.config[def.configKey] || def.defaultCol;
+        // Only log if we have a collection context (some ID-based loaders might differ)
+        if (def.configKey) {
+            this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading '${def.cache}' from: ${collection}/${key}`);
+        }
 
-    async loadRankings(dateStr) {
-        if (this.cache.rankings.has(dateStr)) return this.cache.rankings.get(dateStr);
-        const collectionName = this.config.popularInvestorRankingsCollection || 'popular_investor_rankings';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'rankings' from: ${path}`);
-        const promise = loadPopularInvestorRankings(this.config, this.deps, dateStr);
-        this.cache.rankings.set(dateStr, promise);
+        const promise = def.fn(this.config, this.deps, key);
+        cacheMap.set(key, promise);
         return promise;
     }
 
-    async loadRatings(dateStr) {
-        if (this.cache.ratings.has(dateStr)) return this.cache.ratings.get(dateStr);
-        const collectionName = this.config.piRatingsCollection || 'PIRatingsData';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'ratings' from: ${path}`);
-        const promise = loadPIRatings(this.config, this.deps, dateStr);
-        this.cache.ratings.set(dateStr, promise);
-        return promise;
-    }
+    // =========================================================================
+    // PUBLIC ACCESSORS (Generated Wrappers)
+    // =========================================================================
+    
+    // Explicitly defined for IDE autocompletion / static analysis, 
+    // but internally they all delegate to _loadGeneric.
+    async loadInsights(dateStr)             { return this._loadGeneric('loadInsights'           , dateStr); }
+    async loadSocial(dateStr)               { return this._loadGeneric('loadSocial'             , dateStr); }
+    async loadRankings(dateStr)             { return this._loadGeneric('loadRankings'           , dateStr); }
+    async loadRatings(dateStr)              { return this._loadGeneric('loadRatings'            , dateStr); }
+    async loadPageViews(dateStr)            { return this._loadGeneric('loadPageViews'          , dateStr); }
+    async loadWatchlistMembership(dateStr)  { return this._loadGeneric('loadWatchlistMembership', dateStr); }
+    async loadAlertHistory(dateStr)         { return this._loadGeneric('loadAlertHistory'       , dateStr); }
+    async loadPIWatchlistData(piCid)        { return this._loadGeneric('loadPIWatchlistData'    , String(piCid)); }
+
+    // =========================================================================
+    // SPECIALIZED LOADERS (Non-Standard Patterns)
+    // =========================================================================
 
-    async loadPageViews(dateStr) {
-        if (this.cache.pageViews.has(dateStr)) return this.cache.pageViews.get(dateStr);
-        const collectionName = this.config.piPageViewsCollection || 'PIPageViewsData';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'pageViews' from: ${path}`);
-        const promise = loadPIPageViews(this.config, this.deps, dateStr);
-        this.cache.pageViews.set(dateStr, promise);
-        return promise;
-    }
-
-    async loadWatchlistMembership(dateStr) {
-        if (this.cache.watchlistMembership.has(dateStr)) return this.cache.watchlistMembership.get(dateStr);
-        const collectionName = this.config.watchlistMembershipCollection || 'WatchlistMembershipData';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'watchlistMembership' from: ${path}`);
-        const promise = loadWatchlistMembershipData(this.config, this.deps, dateStr);
-        this.cache.watchlistMembership.set(dateStr, promise);
-        return promise;
+    async loadMappings() { 
+        if (this.cache.mappings) return this.cache.mappings; 
+        this.cache.mappings = await this.deps.calculationUtils.loadInstrumentMappings(); 
+        return this.cache.mappings; 
     }
 
-    async loadAlertHistory(dateStr) {
-        if (this.cache.alertHistory.has(dateStr)) return this.cache.alertHistory.get(dateStr);
-        const collectionName = this.config.piAlertHistoryCollection || 'PIAlertHistoryData';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'alertHistory' from: ${path}`);
-        const promise = loadPIAlertHistory(this.config, this.deps, dateStr);
-        this.cache.alertHistory.set(dateStr, promise);
-        return promise;
+    async loadVerifications(dateStr) { // <--- Added dateStr arg
+        if (this.cache.verifications) return this.cache.verifications;
+        
+        const col = this.config.verificationsCollection || 'verification_profiles';
+        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading 'verifications' from: ${col} (Context: ${dateStr || 'Global'})`);
+        
+        // Pass dateStr so data_loader can check GCS snapshots
+        this.cache.verifications = await loadVerificationProfiles(this.config, this.deps, dateStr); 
+        return this.cache.verifications;
     }
-
-    async loadPIWatchlistData(piCid) {
-        const piCidStr = String(piCid);
-        if (this.cache.piWatchlistData.has(piCidStr)) return this.cache.piWatchlistData.get(piCidStr);
-        const promise = loadPIWatchlistData(this.config, this.deps, piCidStr);
-        this.cache.piWatchlistData.set(piCidStr, promise);
-        return promise;
+    async loadPIMasterList() {
+        if (this.cache.piMasterList) return this.cache.piMasterList;
+        this.cache.piMasterList = await loadPopularInvestorMasterList(this.config, this.deps);
+        return this.cache.piMasterList;
     }
 
     async getPriceShardReferences() { 
@@ -181,147 +182,92 @@ class CachedDataLoader {
     async loadPriceShard(docRef) { 
         try { 
             const snap = await docRef.get(); 
-            if (!snap.exists) return {}; 
-            return this._tryDecompress(snap.data()); 
+            return snap.exists ? this._tryDecompress(snap.data()) : {}; 
         } catch (e) { 
             console.error(`Error loading shard ${docRef.path}:`, e); 
             return {}; 
         } 
     }
 
-    async loadPIMasterList() {
-        if (this.cache.piMasterList) return this.cache.piMasterList;
-        const data = await loadPopularInvestorMasterList(this.config, this.deps);
-        this.cache.piMasterList = data;
-        return data;
-    }
-
-    // --- [UPDATED] Batched Series Loading Logic ---
+    // =========================================================================
+    // BATCH SERIES LOADING
+    // =========================================================================
     /**
-     * Optimistically loads a series of root data over a lookback period using Batch Reads.
-     * 1. Checks Availability Index (Range Query).
-     * 2. Constructs Refs for all existing dates.
-     * 3. Fetches all in ONE db.getAll() request.
+     * Optimistically loads data series using Batch Reads (db.getAll).
+     * Uses Availability Index to minimize costs.
      */
     async loadSeries(loaderMethod, dateStr, lookbackDays) {
-        if (!this[loaderMethod]) throw new Error(`[CachedDataLoader] Unknown method ${loaderMethod}`);
+        const def = LOADER_DEFINITIONS[loaderMethod];
+        if (!def) throw new Error(`[CachedDataLoader] Unknown series method ${loaderMethod}`);
         
+        // Fallback to legacy loop if method isn't configured for batching (missing config/flag)
+        if (!def.configKey || !def.flag) return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays);
+
         // 1. Calculate Date Range
         const endDate = new Date(dateStr);
         const startDate = new Date(endDate);
         startDate.setUTCDate(startDate.getUTCDate() - (lookbackDays - 1));
         
-        const startStr = startDate.toISOString().slice(0, 10);
-        const endStr = endDate.toISOString().slice(0, 10);
-
-        // 2. Pre-flight: Fetch Availability Window
+        // 2. Pre-flight: Check Availability
         let availabilityMap = new Map();
         try {
-            availabilityMap = await getAvailabilityWindow(this.deps, startStr, endStr);
+            availabilityMap = await getAvailabilityWindow(this.deps, startDate.toISOString().slice(0, 10), endDate.toISOString().slice(0, 10));
         } catch (e) {
-            console.warn(`[CachedDataLoader] Availability check failed for series. Falling back to optimistic batch fetch. Error: ${e.message}`);
-        }
-
-        // 3. Identify Collection & Required Flag
-        const collectionInfo = LOADER_COLLECTION_MAP[loaderMethod];
-        const requiredFlag = LOADER_DEPENDENCY_MAP[loaderMethod];
-        
-        if (!collectionInfo) {
-            // Fallback for methods not in the batch map (use legacy parallel loop)
-            return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays);
+            console.warn(`[CachedDataLoader] Availability check failed. Optimistic batching enabled.`);
         }
 
-        const collectionName = this.config[collectionInfo.configKey] || collectionInfo.default;
+        // 3. Construct Batch Refs
+        const collectionName = this.config[def.configKey] || def.defaultCol;
         const batchRefs = [];
-        const dateKeyMap = []; // Keep track of which date corresponds to which ref index
+        const dateKeyMap = []; 
 
-        // 4. Build Batch References
         for (let i = 0; i < lookbackDays; i++) {
             const d = new Date(endDate);
             d.setUTCDate(d.getUTCDate() - i);
             const dString = d.toISOString().slice(0, 10);
             
-            // Check Availability
             const dayStatus = availabilityMap.get(dString);
-            let shouldFetch = false;
-
-            if (availabilityMap.size > 0) {
-                // If index exists, trust it
-                if (dayStatus && (!requiredFlag || dayStatus[requiredFlag])) {
-                    shouldFetch = true;
-                }
-            } else {
-                // If index check failed/empty, try optimistically
-                shouldFetch = true;
-            }
-
-            if (shouldFetch) {
-                const ref = this.deps.db.collection(collectionName).doc(dString);
-                batchRefs.push(ref);
+            // Fetch if index says data exists OR if index is missing (optimistic)
+            if (!dayStatus || dayStatus[def.flag]) {
+                batchRefs.push(this.deps.db.collection(collectionName).doc(dString));
                 dateKeyMap.push(dString);
             }
         }
 
-        // 5. Execute Batch Read
+        // 4. Execute Batch Read
         const results = {};
-        let foundCount = 0;
-
         if (batchRefs.length > 0) {
-            // Log summary of all paths being loaded
-            const paths = batchRefs.map(ref => ref.path).join(', ');
-            this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Batch loading ${batchRefs.length} documents for '${loaderMethod}': ${paths}`);
-            
+            this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Batch loading ${batchRefs.length} docs for '${loaderMethod}'`);
             try {
                 const snapshots = await this.deps.db.getAll(...batchRefs);
-                
-                snapshots.forEach((snap, index) => {
+                snapshots.forEach((snap, idx) => {
                     if (snap.exists) {
-                        const dString = dateKeyMap[index];
-                        const rawData = snap.data();
-                        
-                        // Decompress and clean data
-                        const decompressed = this._tryDecompress(rawData);
-                        
-                        // Handle standard data shapes (removing metadata fields if necessary)
-                        // Most root data loaders return the full object, so we do too.
-                        // Specific logic from data_loader.js (like stripping 'date' key) is handled here generically
-                        // or by the consumer. For series data, returning the whole object is usually safer.
-                        
-                        // Special handling for cleaner output (mimicking data_loader.js logic)
-                        if (loaderMethod === 'loadRatings' || loaderMethod === 'loadPageViews' || 
-                            loaderMethod === 'loadWatchlistMembership' || loaderMethod === 'loadAlertHistory') {
-                            const { date, lastUpdated, ...cleanData } = decompressed;
-                            results[dString] = cleanData;
+                        const raw = this._tryDecompress(snap.data());
+                        // Clean metadata if necessary
+                        if (['loadRatings', 'loadPageViews', 'loadWatchlistMembership', 'loadAlertHistory'].includes(loaderMethod)) {
+                            const { date, lastUpdated, ...clean } = raw;
+                            results[dateKeyMap[idx]] = clean;
                         } else if (loaderMethod === 'loadRankings') {
-                            results[dString] = decompressed.Items || [];
+                            results[dateKeyMap[idx]] = raw.Items || [];
                         } else {
-                            results[dString] = decompressed;
+                            results[dateKeyMap[idx]] = raw;
                         }
-                        
-                        foundCount++;
                     }
                 });
             } catch (err) {
-                console.warn(`[CachedDataLoader] Batch fetch failed for ${loaderMethod}: ${err.message}. Falling back to individual fetches.`);
+                console.warn(`[CachedDataLoader] Batch failed: ${err.message}. Legacy fallback.`);
                 return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays);
             }
         }
 
-        const summary = {
+        return {
             dates: Object.keys(results).sort(),
             data: results,
-            found: foundCount,
+            found: Object.keys(results).length,
             requested: lookbackDays
         };
-        
-        this.deps.logger?.log('INFO', `[CachedDataLoader] ✅ Loaded ${foundCount}/${lookbackDays} dates for '${loaderMethod}' (found: ${summary.dates.join(', ') || 'none'})`);
-        
-        return summary;
     }
 
-    /**
-     * Legacy Fallback: Loads series using parallel promises (for custom/unmapped loaders)
-     */
     async _loadSeriesLegacy(loaderMethod, dateStr, lookbackDays) {
         const results = {};
         const promises = [];
@@ -330,21 +276,11 @@ class CachedDataLoader {
         for (let i = 0; i < lookbackDays; i++) {
             const d = new Date(endDate);
             d.setUTCDate(d.getUTCDate() - i);
-            const dString = d.toISOString().slice(0, 10);
-            
-            // Log path for legacy loader (will be logged by individual load methods)
-            promises.push(
-                this[loaderMethod](dString)
-                    .then(data => ({ date: dString, data }))
-                    .catch(() => ({ date: dString, data: null }))
-            );
+            const dStr = d.toISOString().slice(0, 10);
+            promises.push(this[loaderMethod](dStr).then(data => data ? results[dStr] = data : null).catch(() => null));
         }
 
-        const loaded = await Promise.all(promises);
-        loaded.forEach(({ date, data }) => {
-            if (data) results[date] = data;
-        });
-
+        await Promise.all(promises);
         return {
             dates: Object.keys(results).sort(),
             data: results,