bulltrackers-module 1.0.731 → 1.0.733

package/functions/computation-system/data/DependencyFetcher.js

@@ -1,455 +0,0 @@
-/**
- * FILENAME: computation-system/data/DependencyFetcher.js
- * @fileoverview Fetches dependencies for computations.
- * REFACTORED: Unified fetch logic, streamlined decompression/sharding/GCS.
- * UPDATED: Properly checks isPage/isAlert flags to determine BigQuery vs Firestore routing.
- */
-const { normalizeName } = require('../utils/utils');
-const zlib = require('zlib');
-const { Storage } = require('@google-cloud/storage');
-
-const storage = new Storage(); // Singleton Client
-
-/**
- * Helper to check if a computation is isPage or isAlert by looking up manifest
- * @param {string} normalizedName - Normalized computation name
- * @param {object} config - Config object that may contain manifest info
- * @param {string} category - Computation category (fallback for alerts)
- * @returns {Promise<{isPage: boolean, isAlert: boolean}>}
- */
-async function checkComputationFlags(normalizedName, config, category) {
-    let isPage = false;
-    let isAlert = false;
-    
-    // Try to get manifest from config if available
-    if (config.getCalculations && typeof config.getCalculations === 'function') {
-        try {
-            const calculations = config.getCalculations(config);
-            const manifest = calculations.find(c => normalizeName(c.name) === normalizedName);
-            if (manifest) {
-                isPage = manifest.isPage === true;
-                isAlert = manifest.isAlertComputation === true;
-                return { isPage, isAlert };
-            }
-        } catch (e) {
-            // Fall through to category-based detection
-        }
-    }
-    
-    // Fallback: infer from category (alerts category = alert computation)
-    // Note: isPage cannot be inferred from category alone, so defaults to false
-    isAlert = category === 'alerts';
-    
-    return { isPage, isAlert };
-}
-
-// =============================================================================
-// HELPERS
-// =============================================================================
-
-/** Checks if data is effectively empty (null or only metadata keys) */
-function isDataEmpty(data) {
-    if (data == null) return true;
-    if (Array.isArray(data)) return data.length === 0;
-    if (typeof data === 'object') {
-        // Return true if NO keys exist that DON'T start with '_'
-        return !Object.keys(data).some(k => !k.startsWith('_'));
-    }
-    return false;
-}
-
-/** Robust decompression helper (Buffer, Base64, or Firestore Binary) */
-function tryDecompress(payload) {
-    if (!payload) return null;
-    try {
-        const buffer = (payload instanceof Buffer) ? payload : 
-                      (payload._byteString ? Buffer.from(payload._byteString, 'base64') : 
-                      (payload.toDate ? payload.toDate() : Buffer.from(payload)));
-        
-        return JSON.parse(zlib.gunzipSync(buffer).toString('utf8'));
-    } catch (e) {
-        throw new Error(`Decompression failed: ${e.message}`);
-    }
-}
-
-// =============================================================================
-// CORE FETCH LOGIC
-// =============================================================================
-
-/**
- * Fetches, decompresses, and reassembles (if sharded or on GCS) a single result document.
- * NEW: For non-alert, non-page computations, tries BigQuery first (cheaper, no sharding/compression).
- * @param {object} db - Firestore database instance
- * @param {object} config - Configuration object (may contain logger, manifestLookup, etc.)
- * @param {string} dateStr - Date string in YYYY-MM-DD format
- * @param {string} name - Computation name (normalized)
- * @param {string} category - Computation category
- * @returns {Promise<object|null>} Result data or null if not found
- */
-async function fetchSingleResult(db, config, dateStr, name, category) {
-    const { resultsCollection = 'computation_results', resultsSubcollection = 'results', computationsSubcollection = 'computations' } = config;
-    const log = config.logger || console;
-
-    // Check if this is an alert or page computation by looking up manifest
-    const { isPage: isPageComputation, isAlert: isAlertComputation } = await checkComputationFlags(name, config, category);
-    
-    // Try BigQuery first for non-alert, non-page computations (reduces Firestore reads)
-    // isPage and isAlert computations are always written to Firestore (in addition to BigQuery),
-    // so we should check Firestore for them, but can also try BigQuery as a fallback
-    if (!isAlertComputation && !isPageComputation && process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const { queryComputationResult } = require('../../core/utils/bigquery_utils');
-            const bigqueryResult = await queryComputationResult(name, category, dateStr, log);
-            
-            if (bigqueryResult && !isDataEmpty(bigqueryResult)) {
-                log.log('INFO', `[DependencyFetcher] ✅ Using BigQuery for ${name} (${dateStr}, ${category})`);
-                return bigqueryResult;
-            }
-        } catch (bqError) {
-            log.log('WARN', `[DependencyFetcher] BigQuery fetch failed for ${name}, falling back to Firestore: ${bqError.message}`);
-            // Fall through to Firestore
-        }
-    } else if (isAlertComputation || isPageComputation) {
-        log.log('INFO', `[DependencyFetcher] 📄 Using Firestore for ${isAlertComputation ? 'alert' : 'page'} computation ${name} (${dateStr})`);
-    }
-
-    // Fallback to Firestore (for alerts, pages, or if BigQuery fails)
-    const docRef = db.collection(resultsCollection).doc(dateStr)
-                     .collection(resultsSubcollection).doc(category)
-                     .collection(computationsSubcollection).doc(name);
-
-    const snap = await docRef.get();
-    if (!snap.exists) return null;
-
-    let data = snap.data();
-
-    // -------------------------------------------------------------------------
-    // 1. GCS POINTER HANDLER (New)
-    // -------------------------------------------------------------------------
-    if (data.gcsUri || (data._gcs && data.gcsBucket && data.gcsPath)) {
-        try {
-            const bucketName = data.gcsBucket || data.gcsUri.split('/')[2];
-            const fileName   = data.gcsPath || data.gcsUri.split('/').slice(3).join('/');
-            
-            // Stream download is memory efficient for large files
-            const [fileContent] = await storage.bucket(bucketName).file(fileName).download();
-            
-            // Assume Gzip (as writer does it), if fails try plain
-            try {
-                return JSON.parse(zlib.gunzipSync(fileContent).toString('utf8'));
-            } catch (gzipErr) {
-                // Fallback for uncompressed GCS files
-                return JSON.parse(fileContent.toString('utf8'));
-            }
-        } catch (e) {
-            log.log('ERROR', `[DependencyFetcher] ❌ GCS Fetch Failed for ${name}: ${e.message}`);
-            // Depending on strictness, we might return null here or allow it to fail hard.
-            // Returning null allows 'isDataEmpty' to catch it as "MISSING"
-            return null; 
-        }
-    }
-
-    // -------------------------------------------------------------------------
-    // 2. FIRESTORE COMPRESSED HANDLER
-    // -------------------------------------------------------------------------
-    if (data._compressed && data.payload) {
-        try {
-            const realData = tryDecompress(data.payload);
-            data = { ...data, ...realData }; // Merge payload into base
-            delete data.payload;
-        } catch (e) {
-            log.log('ERROR', `[DependencyFetcher] ❌ ${e.message} at ${docRef.path}`);
-            return null;
-        }
-    }
-
-    // -------------------------------------------------------------------------
-    // 3. FIRESTORE SHARDED HANDLER
-    // -------------------------------------------------------------------------
-    if (data._sharded) {
-        const shardSnaps = await docRef.collection('_shards').get();
-        if (shardSnaps.empty) {
-            log.log('ERROR', `[DependencyFetcher] ❌ Sharded doc has no shards: ${docRef.path}`);
-            return null;
-        }
-
-        // Initialize merged data with any non-metadata fields from the pointer doc
-        const merged = {};
-        Object.entries(data).forEach(([k, v]) => { if (!k.startsWith('_')) merged[k] = v; });
-
-        for (const shard of shardSnaps.docs) {
-            let sData = shard.data();
-            
-            // Decompress Shard if needed
-            if (sData._compressed && sData.payload) {
-                try {
-                    const decomp = tryDecompress(sData.payload);
-                    sData = (typeof decomp === 'string') ? JSON.parse(decomp) : decomp;
-                } catch (e) {
-                    log.log('ERROR', `[DependencyFetcher] ❌ Shard decompression failed (${shard.id}): ${e.message}`);
-                    continue; 
-                }
-            }
-
-            // Merge Shard Content
-            Object.entries(sData).forEach(([k, v]) => { 
-                if (!k.startsWith('_')) merged[k] = v; 
-            });
-        }
-        
-        // Return null if result is empty after merging
-        if (Object.keys(merged).length === 0) return null;
-        return merged;
-    }
-
-    // -------------------------------------------------------------------------
-    // 4. STANDARD DOCUMENT HANDLER
-    // -------------------------------------------------------------------------
-    const clean = {};
-    let hasContent = false;
-    Object.entries(data).forEach(([k, v]) => {
-        if (!k.startsWith('_')) { clean[k] = v; hasContent = true; }
-    });
-
-    return hasContent ? clean : null;
-}
-
-// =============================================================================
-// PUBLIC METHODS
-// =============================================================================
-
-/**
- * Fetches dependencies for a specific date (Standard pass).
- */
-async function fetchDependencies(date, calcs, config, deps, manifestLookup = {}, allowMissing = false) {
-    const { db, logger } = deps;
-    const dStr = date.toISOString().slice(0, 10);
-    
-    // 1. Resolve Dependencies (Normalize Name -> Original Name)
-    const needed = new Map(); 
-    calcs.forEach(c => {
-        // Priority: class.getDependencies() > instance.getDependencies() > manifest.dependencies
-        const getDeps = c.class?.getDependencies || c.getDependencies;
-        const reqs = (typeof getDeps === 'function') ? getDeps.call(c.class || c) : c.dependencies;
-        
-        if (Array.isArray(reqs)) {
-            reqs.forEach(r => needed.set(normalizeName(r), r));
-        }
-    });
-
-    if (needed.size === 0) return {};
-
-    logger.log('INFO', `[DependencyFetcher] 🔍 Fetching ${needed.size} dependencies for ${dStr}`);
-
-    const results = {};
-    const errors = []; 
-
-    // 2. Fetch All Dependencies in Parallel
-    await Promise.all(Array.from(needed.entries()).map(async ([norm, orig]) => {
-        const category = manifestLookup[norm] || 'analytics';
-        const path = `${config.resultsCollection || 'computation_results'}/${dStr}/.../${category}/${norm}`;
-        
-        try {
-            const data = await fetchSingleResult(db, { ...config, logger }, dStr, norm, category);
-            
-            if (!data || isDataEmpty(data)) {
-                // Determine severity based on context
-                const status = data ? 'EMPTY' : 'MISSING';
-                errors.push({ name: orig, path, reason: status });
-                
-                // Log immediately for visibility
-                const level = allowMissing ? 'INFO' : 'ERROR';
-                logger.log(level, `[DependencyFetcher] ⚠️ Dependency '${orig}' ${status} at ${path}`);
-            } else {
-                results[orig] = data; // Store using Original Name
-            }
-        } catch (e) {
-            errors.push({ name: orig, path, reason: e.message });
-            logger.log('ERROR', `[DependencyFetcher] ❌ Error loading '${orig}': ${e.message}`);
-        }
-    }));
-
-    // 3. Final Validation
-    if (errors.length > 0 && !allowMissing) {
-        throw new Error(`[DependencyFetcher] CRITICAL: Missing required dependencies: ${errors.map(e => e.name).join(', ')}`);
-    } else if (errors.length > 0) {
-        logger.log('INFO', `[DependencyFetcher] ⚠️ Allowed missing/empty dependencies in Historical context: ${errors.map(e => e.name).join(', ')}`);
-    }
-
-    return results;
-}
-
-/**
- * Fetches result series (Historical data) for lookbacks.
- * @param {string} endDateStr - The end date for the series
- * @param {string[]} calcNames - Original (case-sensitive) computation names
- * @param {Object} manifestLookup - Map of normalizedName -> category
- * @param {Object} config - Configuration object
- * @param {Object} deps - Dependencies (db, logger)
- * @param {number} lookbackDays - Number of days to look back
- */
-async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config, deps, lookbackDays) {
-    const { db, logger } = deps;
-    const results = {}; // normalizedName -> { date -> data }
-    const { resultsCollection = 'computation_results', resultsSubcollection = 'results', computationsSubcollection = 'computations' } = config;
-    
-    // Initialize results structure
-    calcNames.forEach(n => results[normalizeName(n)] = {});
-
-    // Generate Date List (going backwards from endDate)
-    const dates = [];
-    const d = new Date(endDateStr);
-    for (let i = 0; i < lookbackDays; i++) {
-        d.setUTCDate(d.getUTCDate() - 1);
-        dates.push(d.toISOString().slice(0, 10));
-    }
-    
-    const startDateStr = dates[dates.length - 1]; // Oldest date
-    const queryEndDateStr = dates[0]; // Newest date (for BigQuery query)
-
-    // [DEBUG] Log the manifest lookup and resolved categories
-    logger.log('INFO', `[DependencyFetcher] 🔍 ManifestLookup has ${Object.keys(manifestLookup).length} entries`);
-    for (const rawName of calcNames) {
-        const norm = normalizeName(rawName);
-        const category = manifestLookup[norm] || 'analytics';
-        const samplePath = `${resultsCollection}/${dates[0]}/${resultsSubcollection}/${category}/${computationsSubcollection}/${rawName}`;
-        logger.log('INFO', `[DependencyFetcher] 📍 '${rawName}' -> category='${category}' -> Path: ${samplePath}`);
-    }
-
-    // =========================================================================
-    // BIGQUERY FIRST: Try batch query for all dates at once
-    // =========================================================================
-    if (process.env.BIGQUERY_ENABLED !== 'false') {
-        try {
-            const { queryComputationResultsRange } = require('../../core/utils/bigquery_utils');
-            
-            // Query each computation in parallel
-            const bigqueryPromises = calcNames.map(async (rawName) => {
-                const norm = normalizeName(rawName);
-                const category = manifestLookup[norm] || 'analytics';
-                
-                const bigqueryRows = await queryComputationResultsRange(
-                    rawName,
-                    category,
-                    startDateStr,
-                    queryEndDateStr,
-                    logger
-                );
-                
-                if (bigqueryRows && bigqueryRows.length > 0) {
-                    logger.log('INFO', `[DependencyFetcher] ✅ Using BigQuery for ${rawName} series: ${bigqueryRows.length} dates`);
-                    
-                    // Map BigQuery results to results structure
-                    for (const row of bigqueryRows) {
-                        if (row.data && !isDataEmpty(row.data)) {
-                            results[norm][row.date] = row.data;
-                        }
-                    }
-                    
-                    return { name: rawName, found: bigqueryRows.length };
-                }
-                
-                return { name: rawName, found: 0 };
-            });
-            
-            const bigqueryResults = await Promise.all(bigqueryPromises);
-            const totalFound = bigqueryResults.reduce((sum, r) => sum + r.found, 0);
-            
-            if (totalFound > 0) {
-                logger.log('INFO', `[DependencyFetcher] ✅ BigQuery retrieved ${totalFound} computation result records across ${calcNames.length} computations`);
-                
-                // Fill in any missing dates from Firestore (fallback)
-                const missingOps = [];
-                for (const dateStr of dates) {
-                    for (const rawName of calcNames) {
-                        const norm = normalizeName(rawName);
-                        // Only fetch if we don't have this date already
-                        if (!results[norm] || !results[norm][dateStr]) {
-                            const category = manifestLookup[norm] || 'analytics';
-                            missingOps.push(async () => {
-                                const val = await fetchSingleResult(db, { ...config, logger }, dateStr, rawName, category);
-                                if (val && !isDataEmpty(val)) {
-                                    results[norm][dateStr] = val;
-                                }
-                            });
-                        }
-                    }
-                }
-                
-                // Fetch missing dates from Firestore
-                if (missingOps.length > 0) {
-                    logger.log('INFO', `[DependencyFetcher] 📂 Fetching ${missingOps.length} missing dates from Firestore (fallback)`);
-                    const BATCH_SIZE = 20;
-                    for (let i = 0; i < missingOps.length; i += BATCH_SIZE) {
-                        await Promise.all(missingOps.slice(i, i + BATCH_SIZE).map(fn => fn()));
-                    }
-                }
-                
-                // Log final summary
-                for (const rawName of calcNames) {
-                    const norm = normalizeName(rawName);
-                    const foundDates = Object.keys(results[norm] || {});
-                    logger.log('INFO', `[DependencyFetcher] ✅ '${rawName}' found data for ${foundDates.length}/${lookbackDays} days (BigQuery + Firestore)`);
-                }
-                
-                return results;
-            } else {
-                logger.log('INFO', `[DependencyFetcher] ⚠️ BigQuery returned no results, falling back to Firestore`);
-            }
-        } catch (bqError) {
-            logger.log('WARN', `[DependencyFetcher] BigQuery series query failed, falling back to Firestore: ${bqError.message}`);
-        }
-    }
-    
-    // =========================================================================
-    // FIRESTORE FALLBACK: Original logic (backwards compatibility)
-    // =========================================================================
-    logger.log('INFO', `[DependencyFetcher] 📂 Using Firestore for computation result series: ${calcNames.length} calcs x ${lookbackDays} days`);
-
-    // Build Fetch Operations
-    const ops = [];
-    for (const dateStr of dates) {
-        for (const rawName of calcNames) {
-            const norm = normalizeName(rawName);
-            const category = manifestLookup[norm] || 'analytics';
-            
-            ops.push(async () => {
-                const val = await fetchSingleResult(db, { ...config, logger }, dateStr, rawName, category);
-                if (val && !isDataEmpty(val)) {
-                    results[norm][dateStr] = val;
-                }
-            });
-        }
-    }
-
-    // Execute in Batches (Limit Concurrency)
-    logger.log('INFO', `[DependencyFetcher] 📚 Loading series: ${calcNames.length} calcs x ${lookbackDays} days (${ops.length} ops)`);
-    const BATCH_SIZE = 20;
-    for (let i = 0; i < ops.length; i += BATCH_SIZE) {
-        await Promise.all(ops.slice(i, i + BATCH_SIZE).map(fn => fn()));
-    }
-    
-    // [DEBUG] Log results summary
-    for (const rawName of calcNames) {
-        const norm = normalizeName(rawName);
-        const foundDates = Object.keys(results[norm] || {});
-        logger.log('INFO', `[DependencyFetcher] ✅ '${rawName}' found data for ${foundDates.length}/${lookbackDays} days`);
-    }
-
-    return results;
-}
-
-/**
- * Bridge function for WorkflowOrchestrator compatibility.
- */
-async function fetchExistingResults(dateStr, calcs, fullManifest, config, deps, isHistoricalContext) {
-    const lookup = {};
-    if (Array.isArray(fullManifest)) {
-        fullManifest.forEach(c => lookup[normalizeName(c.name)] = c.category || 'analytics');
-    }
-    // Ensure Date Object
-    const dateObj = new Date(dateStr.includes('T') ? dateStr : dateStr + 'T00:00:00Z');
-    return fetchDependencies(dateObj, calcs, config, deps, lookup, isHistoricalContext);
-}
-
-module.exports = { fetchDependencies, fetchResultSeries, fetchExistingResults };