bulltrackers-module 1.0.732 → 1.0.733

package/functions/computation-system/legacy/CachedDataLoaderOld.js

@@ -1,357 +0,0 @@
-/**
- * {
- * type: uploaded file
- * fileName: computation-system/data/CachedDataLoader.js
- * }
- */
-const { 
-    loadDailyInsights, 
-    loadDailySocialPostInsights, 
-    getRelevantShardRefs, 
-    getPriceShardRefs,
-    loadVerificationProfiles,
-    loadPopularInvestorRankings,
-    loadPIRatings,
-    loadPIPageViews,
-    loadWatchlistMembership: loadWatchlistMembershipData,
-    loadPIAlertHistory,
-    loadPIWatchlistData,
-    loadPopularInvestorMasterList
-} = require('../utils/data_loader');
-const { getAvailabilityWindow } = require('../data/AvailabilityChecker');
-const zlib = require('zlib'); 
-
-// [NEW] Mapping of Loader Methods to Availability Flags
-const LOADER_DEPENDENCY_MAP = {
-    'loadRankings': 'piRankings',
-    'loadRatings': 'piRatings',
-    'loadPageViews': 'piPageViews',
-    'loadWatchlistMembership': 'watchlistMembership',
-    'loadAlertHistory': 'piAlertHistory',
-    'loadInsights': 'hasInsights',
-    'loadSocial': 'hasSocial'
-};
-
-// [NEW] Mapping of Loader Methods to Collection Config Keys / Defaults
-// This allows us to construct references for Batch Reading without executing the opaque loader functions.
-const LOADER_COLLECTION_MAP = {
-    'loadRatings': { configKey: 'piRatingsCollection', default: 'PIRatingsData' },
-    'loadPageViews': { configKey: 'piPageViewsCollection', default: 'PIPageViewsData' },
-    'loadWatchlistMembership': { configKey: 'watchlistMembershipCollection', default: 'WatchlistMembershipData' },
-    'loadAlertHistory': { configKey: 'piAlertHistoryCollection', default: 'PIAlertHistoryData' },
-    'loadInsights': { configKey: 'insightsCollectionName', default: 'daily_instrument_insights' },
-    'loadRankings': { configKey: 'popularInvestorRankingsCollection', default: 'popular_investor_rankings' }
-};
-
-class CachedDataLoader {
-    constructor(config, dependencies) { 
-        this.config = config; 
-        this.deps = dependencies; 
-        this.cache = { 
-            mappings: null, 
-            insights: new Map(), 
-            social:   new Map(),
-            verifications: null,
-            rankings: new Map(),
-            ratings: new Map(),
-            pageViews: new Map(),
-            watchlistMembership: new Map(),
-            alertHistory: new Map(),
-            piWatchlistData: new Map(),
-            piMasterList: null
-        }; 
-    }
-
-    _tryDecompress(data) {
-        if (data && data._compressed === true && data.payload) {
-            try {
-                return JSON.parse(zlib.gunzipSync(data.payload).toString());
-            } catch (e) {
-                console.error('[CachedDataLoader] Decompression failed', e);
-                return {};
-            }
-        }
-        return data;
-    }
-
-    // ... [Existing single-day load methods remain unchanged] ...
-    async loadMappings() { 
-        if (this.cache.mappings) return this.cache.mappings; 
-        const { calculationUtils } = this.deps; 
-        this.cache.mappings = await calculationUtils.loadInstrumentMappings(); 
-        return this.cache.mappings; 
-    }
-
-    async loadInsights(dateStr) { 
-        if (this.cache.insights.has(dateStr)) return this.cache.insights.get(dateStr); 
-        const collectionName = this.config.insightsCollectionName || 'daily_instrument_insights';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'insights' from: ${path}`);
-        const promise = loadDailyInsights(this.config, this.deps, dateStr); 
-        this.cache.insights.set(dateStr, promise); 
-        return promise; 
-    }
-
-    async loadSocial(dateStr) { 
-        if (this.cache.social.has(dateStr)) return this.cache.social.get(dateStr); 
-        const collectionName = this.config.socialInsightsCollection || 'daily_social_insights';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'social' from: ${path}`);
-        const promise = loadDailySocialPostInsights(this.config, this.deps, dateStr); 
-        this.cache.social.set(dateStr, promise); 
-        return promise; 
-    }
-
-    async loadVerifications() {
-        if (this.cache.verifications) return this.cache.verifications;
-        const collectionName = this.config.verificationsCollection || 'verification_profiles';
-        const path = `${collectionName}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'verifications' from: ${path}`);
-        const verifications = await loadVerificationProfiles(this.config, this.deps);
-        this.cache.verifications = verifications;
-        return verifications;
-    }
-
-    async loadRankings(dateStr) {
-        if (this.cache.rankings.has(dateStr)) return this.cache.rankings.get(dateStr);
-        const collectionName = this.config.popularInvestorRankingsCollection || 'popular_investor_rankings';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'rankings' from: ${path}`);
-        const promise = loadPopularInvestorRankings(this.config, this.deps, dateStr);
-        this.cache.rankings.set(dateStr, promise);
-        return promise;
-    }
-
-    async loadRatings(dateStr) {
-        if (this.cache.ratings.has(dateStr)) return this.cache.ratings.get(dateStr);
-        const collectionName = this.config.piRatingsCollection || 'PIRatingsData';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'ratings' from: ${path}`);
-        const promise = loadPIRatings(this.config, this.deps, dateStr);
-        this.cache.ratings.set(dateStr, promise);
-        return promise;
-    }
-
-    async loadPageViews(dateStr) {
-        if (this.cache.pageViews.has(dateStr)) return this.cache.pageViews.get(dateStr);
-        const collectionName = this.config.piPageViewsCollection || 'PIPageViewsData';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'pageViews' from: ${path}`);
-        const promise = loadPIPageViews(this.config, this.deps, dateStr);
-        this.cache.pageViews.set(dateStr, promise);
-        return promise;
-    }
-
-    async loadWatchlistMembership(dateStr) {
-        if (this.cache.watchlistMembership.has(dateStr)) return this.cache.watchlistMembership.get(dateStr);
-        const collectionName = this.config.watchlistMembershipCollection || 'WatchlistMembershipData';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'watchlistMembership' from: ${path}`);
-        const promise = loadWatchlistMembershipData(this.config, this.deps, dateStr);
-        this.cache.watchlistMembership.set(dateStr, promise);
-        return promise;
-    }
-
-    async loadAlertHistory(dateStr) {
-        if (this.cache.alertHistory.has(dateStr)) return this.cache.alertHistory.get(dateStr);
-        const collectionName = this.config.piAlertHistoryCollection || 'PIAlertHistoryData';
-        const path = `${collectionName}/${dateStr}`;
-        this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Loading Root Data 'alertHistory' from: ${path}`);
-        const promise = loadPIAlertHistory(this.config, this.deps, dateStr);
-        this.cache.alertHistory.set(dateStr, promise);
-        return promise;
-    }
-
-    async loadPIWatchlistData(piCid) {
-        const piCidStr = String(piCid);
-        if (this.cache.piWatchlistData.has(piCidStr)) return this.cache.piWatchlistData.get(piCidStr);
-        const promise = loadPIWatchlistData(this.config, this.deps, piCidStr);
-        this.cache.piWatchlistData.set(piCidStr, promise);
-        return promise;
-    }
-
-    async getPriceShardReferences() { 
-        return getPriceShardRefs(this.config, this.deps); 
-    }
-
-    async getSpecificPriceShardReferences(targetInstrumentIds) { 
-        return getRelevantShardRefs(this.config, this.deps, targetInstrumentIds); 
-    }
-
-    async loadPriceShard(docRef) { 
-        try { 
-            const snap = await docRef.get(); 
-            if (!snap.exists) return {}; 
-            return this._tryDecompress(snap.data()); 
-        } catch (e) { 
-            console.error(`Error loading shard ${docRef.path}:`, e); 
-            return {}; 
-        } 
-    }
-
-    async loadPIMasterList() {
-        if (this.cache.piMasterList) return this.cache.piMasterList;
-        const data = await loadPopularInvestorMasterList(this.config, this.deps);
-        this.cache.piMasterList = data;
-        return data;
-    }
-
-    // --- [UPDATED] Batched Series Loading Logic ---
-    /**
-     * Optimistically loads a series of root data over a lookback period using Batch Reads.
-     * 1. Checks Availability Index (Range Query).
-     * 2. Constructs Refs for all existing dates.
-     * 3. Fetches all in ONE db.getAll() request.
-     */
-    async loadSeries(loaderMethod, dateStr, lookbackDays) {
-        if (!this[loaderMethod]) throw new Error(`[CachedDataLoader] Unknown method ${loaderMethod}`);
-        
-        // 1. Calculate Date Range
-        const endDate = new Date(dateStr);
-        const startDate = new Date(endDate);
-        startDate.setUTCDate(startDate.getUTCDate() - (lookbackDays - 1));
-        
-        const startStr = startDate.toISOString().slice(0, 10);
-        const endStr = endDate.toISOString().slice(0, 10);
-
-        // 2. Pre-flight: Fetch Availability Window
-        let availabilityMap = new Map();
-        try {
-            availabilityMap = await getAvailabilityWindow(this.deps, startStr, endStr);
-        } catch (e) {
-            console.warn(`[CachedDataLoader] Availability check failed for series. Falling back to optimistic batch fetch. Error: ${e.message}`);
-        }
-
-        // 3. Identify Collection & Required Flag
-        const collectionInfo = LOADER_COLLECTION_MAP[loaderMethod];
-        const requiredFlag = LOADER_DEPENDENCY_MAP[loaderMethod];
-        
-        if (!collectionInfo) {
-            // Fallback for methods not in the batch map (use legacy parallel loop)
-            return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays);
-        }
-
-        const collectionName = this.config[collectionInfo.configKey] || collectionInfo.default;
-        const batchRefs = [];
-        const dateKeyMap = []; // Keep track of which date corresponds to which ref index
-
-        // 4. Build Batch References
-        for (let i = 0; i < lookbackDays; i++) {
-            const d = new Date(endDate);
-            d.setUTCDate(d.getUTCDate() - i);
-            const dString = d.toISOString().slice(0, 10);
-            
-            // Check Availability
-            const dayStatus = availabilityMap.get(dString);
-            let shouldFetch = false;
-
-            if (availabilityMap.size > 0) {
-                // If index exists, trust it
-                if (dayStatus && (!requiredFlag || dayStatus[requiredFlag])) {
-                    shouldFetch = true;
-                }
-            } else {
-                // If index check failed/empty, try optimistically
-                shouldFetch = true;
-            }
-
-            if (shouldFetch) {
-                const ref = this.deps.db.collection(collectionName).doc(dString);
-                batchRefs.push(ref);
-                dateKeyMap.push(dString);
-            }
-        }
-
-        // 5. Execute Batch Read
-        const results = {};
-        let foundCount = 0;
-
-        if (batchRefs.length > 0) {
-            // Log summary of all paths being loaded
-            const paths = batchRefs.map(ref => ref.path).join(', ');
-            this.deps.logger?.log('INFO', `[CachedDataLoader] 📂 Batch loading ${batchRefs.length} documents for '${loaderMethod}': ${paths}`);
-            
-            try {
-                const snapshots = await this.deps.db.getAll(...batchRefs);
-                
-                snapshots.forEach((snap, index) => {
-                    if (snap.exists) {
-                        const dString = dateKeyMap[index];
-                        const rawData = snap.data();
-                        
-                        // Decompress and clean data
-                        const decompressed = this._tryDecompress(rawData);
-                        
-                        // Handle standard data shapes (removing metadata fields if necessary)
-                        // Most root data loaders return the full object, so we do too.
-                        // Specific logic from data_loader.js (like stripping 'date' key) is handled here generically
-                        // or by the consumer. For series data, returning the whole object is usually safer.
-                        
-                        // Special handling for cleaner output (mimicking data_loader.js logic)
-                        if (loaderMethod === 'loadRatings' || loaderMethod === 'loadPageViews' || 
-                            loaderMethod === 'loadWatchlistMembership' || loaderMethod === 'loadAlertHistory') {
-                            const { date, lastUpdated, ...cleanData } = decompressed;
-                            results[dString] = cleanData;
-                        } else if (loaderMethod === 'loadRankings') {
-                            results[dString] = decompressed.Items || [];
-                        } else {
-                            results[dString] = decompressed;
-                        }
-                        
-                        foundCount++;
-                    }
-                });
-            } catch (err) {
-                console.warn(`[CachedDataLoader] Batch fetch failed for ${loaderMethod}: ${err.message}. Falling back to individual fetches.`);
-                return this._loadSeriesLegacy(loaderMethod, dateStr, lookbackDays);
-            }
-        }
-
-        const summary = {
-            dates: Object.keys(results).sort(),
-            data: results,
-            found: foundCount,
-            requested: lookbackDays
-        };
-        
-        this.deps.logger?.log('INFO', `[CachedDataLoader] ✅ Loaded ${foundCount}/${lookbackDays} dates for '${loaderMethod}' (found: ${summary.dates.join(', ') || 'none'})`);
-        
-        return summary;
-    }
-
-    /**
-     * Legacy Fallback: Loads series using parallel promises (for custom/unmapped loaders)
-     */
-    async _loadSeriesLegacy(loaderMethod, dateStr, lookbackDays) {
-        const results = {};
-        const promises = [];
-        const endDate = new Date(dateStr);
-
-        for (let i = 0; i < lookbackDays; i++) {
-            const d = new Date(endDate);
-            d.setUTCDate(d.getUTCDate() - i);
-            const dString = d.toISOString().slice(0, 10);
-            
-            // Log path for legacy loader (will be logged by individual load methods)
-            promises.push(
-                this[loaderMethod](dString)
-                    .then(data => ({ date: dString, data }))
-                    .catch(() => ({ date: dString, data: null }))
-            );
-        }
-
-        const loaded = await Promise.all(promises);
-        loaded.forEach(({ date, data }) => {
-            if (data) results[date] = data;
-        });
-
-        return {
-            dates: Object.keys(results).sort(),
-            data: results,
-            found: Object.keys(results).length,
-            requested: lookbackDays
-        };
-    }
-}
-
-module.exports = { CachedDataLoader };