bulltrackers-module 1.0.731 → 1.0.733

package/functions/computation-system/data/CachedDataLoader.js

@@ -1,325 +0,0 @@
-/**
- * @fileoverview Caching layer for data loading.
- * REFACTORED:
- * 1. Supports new BigQuery-backed loader functions.
- * 2. correctly passes 'userTypes' to portfolio/history loaders.
- * 3. Handles Virtual Shards for BigQuery prices.
- */
-const { 
-    loadDailyInsights, 
-    loadDailySocialPostInsights, 
-    getPriceShardRefs,
-    getRelevantShardRefs,
-    loadVerificationProfiles,
-    loadPopularInvestorRankings,
-    loadPIRatings,
-    loadPIPageViews,
-    loadWatchlistMembership,
-    loadPIAlertHistory,
-    loadPIWatchlistData,
-    loadPopularInvestorMasterList,
-    loadDailyPortfolios,
-    loadDailyHistory
-} = require('../utils/data_loader');
-const { getAvailabilityWindow } = require('./AvailabilityChecker');
-const zlib = require('zlib'); 
-
-// =============================================================================
-// CONFIGURATION
-// =============================================================================
-const LOADER_DEFINITIONS = {
-    loadRankings: {
-        cache: 'rankings',
-        configKey: 'popularInvestorRankingsCollection',
-        defaultCol: 'popular_investor_rankings',
-        fn: loadPopularInvestorRankings,
-        flag: 'piRankings'
-    },
-    loadRatings: {
-        cache: 'ratings',
-        configKey: 'piRatingsCollection',
-        defaultCol: 'pi_ratings',
-        fn: loadPIRatings,
-        flag: 'piRatings'
-    },
-    loadPageViews: {
-        cache: 'pageViews',
-        configKey: 'piPageViewsCollection',
-        defaultCol: 'pi_page_views',
-        fn: loadPIPageViews,
-        flag: 'piPageViews'
-    },
-    loadWatchlistMembership: {
-        cache: 'watchlistMembership',
-        configKey: 'watchlistMembershipCollection',
-        defaultCol: 'watchlist_membership',
-        fn: loadWatchlistMembership,
-        flag: 'watchlistMembership'
-    },
-    loadAlertHistory: {
-        cache: 'alertHistory',
-        configKey: 'piAlertHistoryCollection',
-        defaultCol: 'pi_alert_history',
-        fn: loadPIAlertHistory,
-        flag: 'piAlertHistory'
-    },
-    loadInsights: {
-        cache: 'insights',
-        configKey: 'insightsCollectionName',
-        defaultCol: 'daily_instrument_insights',
-        fn: loadDailyInsights,
-        flag: 'hasInsights'
-    },
-    loadSocial: {
-        cache: 'social',
-        configKey: 'socialInsightsCollection',
-        defaultCol: 'daily_social_insights',
-        fn: loadDailySocialPostInsights,
-        flag: 'hasSocial'
-    },
-    loadPIWatchlistData: {
-        cache: 'piWatchlistData',
-        fn: loadPIWatchlistData,
-        isIdBased: true
-    },
-    loadPortfolios: {
-        cache: 'portfolios',
-        fn: loadDailyPortfolios,
-        flag: 'hasPortfolio' // Loose check, accurate check handles inside loader
-    },
-    loadHistory: {
-        cache: 'history',
-        fn: loadDailyHistory,
-        flag: 'hasHistory'
-    }
-};
-
-class CachedDataLoader {
-    constructor(config, dependencies) { 
-        this.config = config; 
-        this.deps = dependencies; 
-        
-        // Initialize caches
-        this.cache = { 
-            mappings: null, 
-            verifications: null, 
-            piMasterList: null,
-            prices: new Map(), // Dedicated cache for price shard data
-            ...Object.values(LOADER_DEFINITIONS).reduce((acc, def) => {
-                acc[def.cache] = new Map();
-                return acc;
-            }, {})
-        }; 
-    }
-
-    _tryDecompress(data) {
-        if (data?._compressed === true && data.payload) {
-            try { return JSON.parse(zlib.gunzipSync(data.payload).toString()); } 
-            catch (e) { this.deps.logger?.log('WARN', '[CachedDataLoader] Decompression failed'); return {}; }
-        }
-        return data;
-    }
-
-    // =========================================================================
-    // GENERIC LOADER
-    // =========================================================================
-    async _loadGeneric(methodName, key, ...args) {
-        const def = LOADER_DEFINITIONS[methodName];
-        if (!def) throw new Error(`Unknown loader method: ${methodName}`);
-
-        // Construct a cache key that includes args (like userTypes) if present
-        // e.g. "2025-01-01::POPULAR_INVESTOR"
-        const cacheKey = args.length > 0 ? `${key}::${JSON.stringify(args)}` : key;
-        const cacheMap = this.cache[def.cache];
-        
-        if (cacheMap.has(cacheKey)) return cacheMap.get(cacheKey);
-
-        if (def.configKey && this.deps.logger) {
-            // Only log simple date-based loads to avoid spam
-            // this.deps.logger.log('INFO', `[CachedDataLoader] Loading ${def.cache} for ${key}`);
-        }
-
-        const promise = def.fn(this.config, this.deps, key, ...args);
-        cacheMap.set(cacheKey, promise);
-        return promise;
-    }
-
-    // =========================================================================
-    // PUBLIC METHODS
-    // =========================================================================
-    async loadInsights(dateStr)             { return this._loadGeneric('loadInsights', dateStr); }
-    async loadSocial(dateStr, userTypes)    { return this._loadGeneric('loadSocial', dateStr, userTypes); }
-    async loadRankings(dateStr)             { return this._loadGeneric('loadRankings', dateStr); }
-    async loadRatings(dateStr)              { return this._loadGeneric('loadRatings', dateStr); }
-    async loadPageViews(dateStr)            { return this._loadGeneric('loadPageViews', dateStr); }
-    async loadWatchlistMembership(dateStr)  { return this._loadGeneric('loadWatchlistMembership', dateStr); }
-    async loadAlertHistory(dateStr)         { return this._loadGeneric('loadAlertHistory', dateStr); }
-    async loadPIWatchlistData(piCid)        { return this._loadGeneric('loadPIWatchlistData', String(piCid)); }
-    async loadPortfolios(dateStr, userTypes) { return this._loadGeneric('loadPortfolios', dateStr, userTypes); }
-    async loadHistory(dateStr, userTypes)    { return this._loadGeneric('loadHistory', dateStr, userTypes); }
-
-    // =========================================================================
-    // SPECIALIZED METHODS
-    // =========================================================================
-    
-    async loadMappings() { 
-        if (this.cache.mappings) return this.cache.mappings; 
-        this.cache.mappings = await this.deps.calculationUtils.loadInstrumentMappings(); 
-        return this.cache.mappings; 
-    }
-
-    async loadVerifications(dateStr) { 
-        // Verifications are mostly static/global, but we accept dateStr for consistency
-        if (this.cache.verifications) return this.cache.verifications;
-        this.cache.verifications = await loadVerificationProfiles(this.config, this.deps, dateStr); 
-        return this.cache.verifications;
-    }
-
-    async loadPIMasterList() {
-        if (this.cache.piMasterList) return this.cache.piMasterList;
-        this.cache.piMasterList = await loadPopularInvestorMasterList(this.config, this.deps);
-        return this.cache.piMasterList;
-    }
-
-    async getPriceShardReferences() { 
-        return getPriceShardRefs(this.config, this.deps); 
-    }
-
-    async getSpecificPriceShardReferences(ids) { 
-        return getRelevantShardRefs(this.config, this.deps, ids); 
-    }
-
-    async loadPriceShard(docRef) {
-        // Handle BigQuery "Virtual Shard"
-        if (docRef && docRef._bigquery === true) {
-            // Use a dedicated cache key for the ALL-PRICE blob
-            // If targetIds are specified, create a unique cache key
-            const targetIds = docRef.targetIds && docRef.targetIds.length > 0 ? docRef.targetIds : null;
-            const cacheKey = targetIds 
-                ? `BQ_PRICES_${targetIds.sort().join(',')}` 
-                : 'BQ_ALL_PRICES';
-            
-            if (this.cache.prices.has(cacheKey)) {
-                return this.cache.prices.get(cacheKey);
-            }
-
-            const { queryAssetPrices } = require('../../core/utils/bigquery_utils');
-            // queryAssetPrices signature: (startDateStr, endDateStr, instrumentIds, logger)
-            const promise = queryAssetPrices(null, null, targetIds, this.deps.logger).then(data => {
-                const result = data || {};
-                this.cache.prices.set(cacheKey, result);
-                return result;
-            });
-            
-            return promise;
-        }
-
-        // Firestore Fallback
-        try { 
-            const snap = await docRef.get(); 
-            return snap.exists ? this._tryDecompress(snap.data()) : {}; 
-        } catch (e) { return {}; } 
-    }
-
-    // =========================================================================
-    // SERIES LOADING (Batching)
-    // =========================================================================
-    async loadSeries(loaderMethod, dateStr, lookbackDays, ...args) {
-        const def = LOADER_DEFINITIONS[loaderMethod];
-        if (!def) throw new Error(`[CachedDataLoader] Unknown series method ${loaderMethod}`);
-        
-        // If method doesn't support batching configuration, use legacy loop
-        if (!def.configKey || !def.flag) return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays, ...args);
-
-        // 1. Calculate Range
-        const endDate = new Date(dateStr);
-        const startDate = new Date(endDate);
-        startDate.setUTCDate(startDate.getUTCDate() - (lookbackDays - 1));
-        
-        // 2. Check Availability
-        let availabilityMap = new Map();
-        try {
-            availabilityMap = await getAvailabilityWindow(this.deps, startDate.toISOString().slice(0, 10), endDate.toISOString().slice(0, 10));
-        } catch (e) {}
-
-        const collectionName = this.config[def.configKey] || def.defaultCol;
-        const batchRefs = [];
-        const dateKeyMap = []; 
-
-        for (let i = 0; i < lookbackDays; i++) {
-            const d = new Date(endDate);
-            d.setUTCDate(d.getUTCDate() - i);
-            const dString = d.toISOString().slice(0, 10);
-            
-            const dayStatus = availabilityMap.get(dString);
-            if (!dayStatus || dayStatus[def.flag]) {
-                batchRefs.push(this.deps.db.collection(collectionName).doc(dString));
-                dateKeyMap.push(dString);
-            }
-        }
-
-        // Note: Batch loading ONLY works for Firestore.
-        // If we are in BigQuery mode, we should NOT use Firestore batching for these types.
-        // However, this `loadSeries` method is mostly used for simple docs (Rankings, Ratings).
-        // Since `loadRankings` in `data_loader.js` now points to BigQuery, passing a Firestore Ref 
-        // won't work because `data_loader` isn't called here; we are accessing DB directly.
-        
-        // [CRITICAL FIX] 
-        // If we are using BigQuery, we cannot use `db.getAll` on non-existent Firestore docs.
-        // We must delegate back to the loader method (which handles BQ) in a loop (concurrently).
-        if (process.env.BIGQUERY_ENABLED !== 'false') {
-             return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays, ...args);
-        }
-
-        // Firestore Batch Logic (Only runs if BQ Disabled)
-        const results = {};
-        if (batchRefs.length > 0) {
-            try {
-                const snapshots = await this.deps.db.getAll(...batchRefs);
-                snapshots.forEach((snap, idx) => {
-                    if (snap.exists) {
-                        const raw = this._tryDecompress(snap.data());
-                        if (loaderMethod === 'loadRankings') results[dateKeyMap[idx]] = raw.Items || [];
-                        else results[dateKeyMap[idx]] = raw;
-                    }
-                });
-            } catch (err) {
-                return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays, ...args);
-            }
-        }
-
-        return {
-            dates: Object.keys(results).sort(),
-            data: results,
-            found: Object.keys(results).length,
-            requested: lookbackDays
-        };
-    }
-
-    async _loadSeriesLegacy(loaderMethod, dateStr, lookbackDays, ...args) {
-        const results = {};
-        const promises = [];
-        const endDate = new Date(dateStr);
-
-        for (let i = 0; i < lookbackDays; i++) {
-            const d = new Date(endDate);
-            d.setUTCDate(d.getUTCDate() - i);
-            const dStr = d.toISOString().slice(0, 10);
-            promises.push(
-                this[loaderMethod](dStr, ...args)
-                    .then(data => { if (data) results[dStr] = data; })
-                    .catch(() => {})
-            );
-        }
-
-        await Promise.all(promises);
-        return {
-            dates: Object.keys(results).sort(),
-            data: results,
-            found: Object.keys(results).length,
-            requested: lookbackDays
-        };
-    }
-}
-
-module.exports = { CachedDataLoader };