bulltrackers-module 1.0.732 → 1.0.733

package/functions/computation-system/legacy/DependencyFetcherOld.js

@@ -1,478 +0,0 @@
-/**
- * FILENAME: computation-system/data/DependencyFetcher.js
- * @fileoverview Fetches dependencies for computations.
- * UPDATED: Added 'fetchExistingResults' bridge for WorkflowOrchestrator compatibility.
- * UPDATED: Uses 'manifestLookup' to resolve the correct category (Core vs Non-Core).
- * UPDATED: Supports automatic reassembly of sharded results (_shards subcollection).
- * UPDATED: Supports decompression of zipped results.
- */
-const { normalizeName } = require('../utils/utils');
-const zlib = require('zlib');
-
-/**
- * Checks if data is effectively empty (no usable content).
- * @param {any} data - The data to check
- * @returns {boolean} True if data is empty/null/undefined or contains no meaningful content
- */
-function isDataEmpty(data) {
-    if (!data || data === null || data === undefined) return true;
-    
-    // Check if it's an object with only metadata fields
-    if (typeof data === 'object' && !Array.isArray(data)) {
-        const keys = Object.keys(data);
-        // If only metadata/internal fields, consider it empty
-        const metadataFields = ['_completed', '_compressed', '_sharded', '_shardCount', '_isPageMode', '_pageCount', '_lastUpdated', '_expireAt'];
-        const hasOnlyMetadata = keys.length > 0 && keys.every(k => metadataFields.includes(k) || k.startsWith('_'));
-        
-        if (hasOnlyMetadata) return true;
-        
-        // If object has no keys (after filtering metadata), it's empty
-        const dataKeys = keys.filter(k => !k.startsWith('_'));
-        if (dataKeys.length === 0) return true;
-    }
-    
-    // Check if it's an empty array
-    if (Array.isArray(data) && data.length === 0) return true;
-    
-    return false;
-}
-
-/**
- * BRIDGE FUNCTION: Matches WorkflowOrchestrator signature.
- * Adapts (dateStr, calcs, manifest, ...) -> (dateObj, calcs, ..., manifestLookup).
- * This fixes the 'fetchExistingResults is not a function' TypeError.
- */
-async function fetchExistingResults(dateStr, calcs, fullManifest, config, deps, isHistoricalContext) {
-    const { logger } = deps;
-    
-    // DEBUG: Log entry
-    logger.log('INFO', `[DependencyFetcher] 📥 fetchExistingResults called for date: ${dateStr}, calcs: ${calcs.length}, isHistorical: ${isHistoricalContext}`);
-    logger.log('INFO', `[DependencyFetcher] 📥 Calcs being processed: ${calcs.map(c => {
-        const name = c.name || c.constructor?.name || 'unknown';
-        const hasClass = !!c.class;
-        const classType = c.class ? (typeof c.class === 'function' ? 'function' : typeof c.class) : 'none';
-        return `${name} (has class: ${hasClass}, class type: ${classType})`;
-    }).join(', ')}`);
-    
-    // 1. Build Manifest Lookup (Name -> Category)
-    const manifestLookup = {};
-    if (Array.isArray(fullManifest)) {
-        fullManifest.forEach(c => {
-            manifestLookup[normalizeName(c.name)] = c.category || 'analytics';
-        });
-        logger.log('INFO', `[DependencyFetcher] 📥 Built manifest lookup with ${Object.keys(manifestLookup).length} entries`);
-    } else {
-        logger.log('WARN', `[DependencyFetcher] ⚠️ fullManifest is not an array: ${typeof fullManifest}`);
-    }
-
-    // 2. Convert Date String to Date Object
-    // We append T00:00:00Z to ensure it parses as UTC date if only YYYY-MM-DD is provided.
-    const dateObj = new Date(dateStr + (dateStr.includes('T') ? '' : 'T00:00:00Z'));
-
-    // 3. Delegate to fetchDependencies
-    // CRITICAL: For historical context (yesterday's data), allow missing dependencies
-    // Historical lookbacks are optional - gaps in historical data are permissible
-    const result = await fetchDependencies(dateObj, calcs, config, deps, manifestLookup, isHistoricalContext);
-    
-    // DEBUG: Log result
-    const resultKeys = Object.keys(result);
-    logger.log('INFO', `[DependencyFetcher] 📤 fetchExistingResults returning ${resultKeys.length} dependencies: ${resultKeys.length > 0 ? resultKeys.join(', ') : 'NONE'}`);
-    
-    return result;
-}
-
-/**
- * Fetches dependencies for a specific date (Standard pass).
- * @param {Date} date - The target date.
- * @param {Array} calcs - The computations requiring dependencies.
- * @param {Object} config - System config.
- * @param {Object} deps - System dependencies (db, logger).
- * @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
- * @param {boolean} allowMissing - If true, missing/empty dependencies are allowed (for historical/lookback scenarios).
- */
-async function fetchDependencies(date, calcs, config, deps, manifestLookup = {}, allowMissing = false) {
-    const { db, logger } = deps;
-    const dStr = date.toISOString().slice(0, 10);
-    
-    // DEBUG: Log entry
-    logger.log('INFO', `[DependencyFetcher] 🔍 fetchDependencies called with ${calcs.length} calc(s): ${calcs.map(c => c.name || c.constructor?.name || 'unknown').join(', ')}`);
-    
-    // 1. Identify unique dependencies needed
-    // CHANGED: Use a Map to track { normalizedName: originalName }
-    const needed = new Map();
-
-    calcs.forEach(c => {
-        const calcName = c.name || c.constructor?.name || 'unknown';
-        
-        // DEBUG: Log what we're checking
-        logger.log('INFO', `[DependencyFetcher] 🔍 Processing calc: ${calcName}`);
-        logger.log('INFO', `[DependencyFetcher]   - has class: ${!!c.class}`);
-        logger.log('INFO', `[DependencyFetcher]   - class type: ${c.class ? (typeof c.class === 'function' ? 'function' : typeof c.class) : 'none'}`);
-        logger.log('INFO', `[DependencyFetcher]   - class.getDependencies: ${c.class && typeof c.class.getDependencies === 'function' ? 'YES' : 'NO'}`);
-        logger.log('INFO', `[DependencyFetcher]   - has getDependencies: ${typeof c.getDependencies === 'function' ? 'YES' : 'NO'}`);
-        logger.log('INFO', `[DependencyFetcher]   - has dependencies array: ${Array.isArray(c.dependencies) ? `YES (${c.dependencies.length} items)` : 'NO'}`);
-        if (Array.isArray(c.dependencies)) {
-            logger.log('INFO', `[DependencyFetcher]   - dependencies array: ${c.dependencies.join(', ')}`);
-        }
-        
-        // [FIX] Support both .getDependencies() method and .dependencies array
-        // CRITICAL: Prefer class.getDependencies() over manifest.dependencies
-        // because the class method returns original case-sensitive names,
-        // while manifest.dependencies contains normalized names
-        let reqs = [];
-        if (c.class && typeof c.class.getDependencies === 'function') {
-            // Use the class method - returns original case-sensitive names
-            reqs = c.class.getDependencies();
-            logger.log('INFO', `[DependencyFetcher] ✅ Using c.class.getDependencies() - returned: ${JSON.stringify(reqs)}`);
-        } else if (typeof c.getDependencies === 'function') {
-            // Fallback: direct method call (if c is the class itself)
-            reqs = c.getDependencies();
-            logger.log('INFO', `[DependencyFetcher] ✅ Using c.getDependencies() - returned: ${JSON.stringify(reqs)}`);
-        } else if (c.dependencies && Array.isArray(c.dependencies)) {
-            // Last resort: use manifest's dependencies array (normalized)
-            // This is less ideal because names are normalized, but we'll use them as-is
-            reqs = c.dependencies;
-            logger.log('INFO', `[DependencyFetcher] ⚠️ Using c.dependencies array (normalized) - returned: ${JSON.stringify(reqs)}`);
-        } else {
-            logger.log('WARN', `[DependencyFetcher] ❌ No way to get dependencies for ${calcName} - all methods failed`);
-        }
-            
-        if (Array.isArray(reqs)) {
-            logger.log('INFO', `[DependencyFetcher] ✅ Found ${reqs.length} dependencies for ${calcName}: ${reqs.join(', ')}`);
-            reqs.forEach(r => {
-                // We map the normalized version to the original requested version
-                // This ensures we fetch the right file (normalized) but return it 
-                // with the casing the user code expects (original).
-                needed.set(normalizeName(r), r);
-            });
-        } else {
-            logger.log('WARN', `[DependencyFetcher] ⚠️ reqs is not an array for ${calcName}: ${typeof reqs}`);
-        }
-    });
-    
-    if (needed.size === 0) {
-        logger.log('WARN', `[DependencyFetcher] ⚠️ No dependencies needed - returning empty object`);
-        return {};
-    }
-    
-    const calcNames = calcs.map(c => c.name || c.constructor?.name || 'unknown').join(', ');
-    logger.log('INFO', `[DependencyFetcher] Fetching ${needed.size} dependencies for computation(s): ${calcNames} (date: ${dStr})`);
-    
-    // DEBUG: Log what dependencies we're looking for
-    const depList = Array.from(needed.entries()).map(([norm, orig]) => `${orig} (normalized: ${norm})`).join(', ');
-    logger.log('INFO', `[DependencyFetcher] Dependencies requested: ${depList}`);
-    
-    const results = {};
-    const missingDeps = [];
-    const emptyDeps = [];
-    
-    // Helper to build path string
-    const buildPath = (category, normName) => {
-        return `${config.resultsCollection || 'computation_results'}/${dStr}/${config.resultsSubcollection || 'results'}/${category}/${config.computationsSubcollection || 'computations'}/${normName}`;
-    };
-    
-    // CHANGED: Iterate over the entries to access both normalized and original names
-    const promises = Array.from(needed.entries()).map(async ([normName, originalName]) => {
-        // Resolve Category from Lookup, default to 'analytics' if unknown
-        // Note: manifestLookup keys are expected to be normalized
-        const category = manifestLookup[normName] || 'analytics';
-        const path = buildPath(category, normName);
-        
-        try {
-            // Pass logger in config for fetchSingleResult
-            const fetchConfig = { ...config, logger };
-            
-            // Fetch using the normalized name (system standard)
-            const data = await fetchSingleResult(db, fetchConfig, dStr, normName, category);
-            
-            // CRITICAL: Validate that dependency exists and has data
-            if (!data) {
-                missingDeps.push({ name: originalName, normalizedName: normName, path });
-                // Log level depends on context - ERROR for current date, INFO for historical
-                if (allowMissing) {
-                    logger.log('INFO', `[DependencyFetcher] ⚠️ Missing dependency '${originalName}' (${normName}) from: ${path} (Historical context - allowed)`);
-                } else {
-                    logger.log('ERROR', `[DependencyFetcher] ❌ Missing required dependency '${originalName}' (${normName}) from: ${path}`);
-                }
-            } else if (isDataEmpty(data)) {
-                emptyDeps.push({ name: originalName, normalizedName: normName, path });
-                // Log level depends on context - ERROR for current date, INFO for historical
-                if (allowMissing) {
-                    logger.log('INFO', `[DependencyFetcher] ⚠️ Empty dependency '${originalName}' (${normName}) from: ${path} (Historical context - allowed)`);
-                } else {
-                    logger.log('ERROR', `[DependencyFetcher] ❌ Empty dependency '${originalName}' (${normName}) from: ${path} - Document exists but contains no usable data`);
-                }
-            } else {
-                // CHANGED: Store result using the ORIGINAL name so context.computed['CaseSensitive'] works
-                results[originalName] = data;
-                // DEBUG: Log successful dependency load
-                const dataKeys = Object.keys(data);
-                logger.log('INFO', `[DependencyFetcher] ✅ Stored dependency '${originalName}' in results. Keys: ${dataKeys.length} (sample: ${dataKeys.slice(0, 5).join(', ')})`);
-            }
-        } catch (e) {
-            missingDeps.push({ name: originalName, normalizedName: normName, path, error: e.message });
-            // Log level depends on context - ERROR for current date, INFO for historical
-            if (allowMissing) {
-                logger.log('INFO', `[DependencyFetcher] ⚠️ Failed to load dependency '${originalName}' (${normName}) from: ${path} - Error: ${e.message} (Historical context - allowed)`);
-            } else {
-                logger.log('ERROR', `[DependencyFetcher] ❌ Failed to load dependency '${originalName}' (${normName}) from: ${path} - Error: ${e.message}`);
-            }
-        }
-    });
-    
-    await Promise.all(promises);
-    
-    // DEBUG: Log what we're returning
-    const resultKeys = Object.keys(results);
-    logger.log('INFO', `[DependencyFetcher] ✅ Returning ${resultKeys.length} dependencies: ${resultKeys.join(', ')}`);
-    
-    // CRITICAL: Fail if any required dependencies are missing or empty
-    // EXCEPTION: For historical/lookback scenarios, missing dependencies are permissible
-    if ((missingDeps.length > 0 || emptyDeps.length > 0) && !allowMissing) {
-        const missingList = missingDeps.map(d => `'${d.name}' (path: ${d.path}${d.error ? `, error: ${d.error}` : ''})`).join(', ');
-        const emptyList = emptyDeps.map(d => `'${d.name}' (path: ${d.path})`).join(', ');
-        
-        const errorMsg = `[DependencyFetcher] ❌ CRITICAL: Cannot proceed - Required dependencies missing or empty for computation(s): ${calcNames}\n` +
-                        `Missing dependencies (${missingDeps.length}): ${missingList}\n` +
-                        (emptyDeps.length > 0 ? `Empty dependencies (${emptyDeps.length}): ${emptyList}\n` : '') +
-                        `Date: ${dStr}\n` +
-                        `This computation will FAIL and no results will be saved.`;
-        
-        logger.log('ERROR', errorMsg);
-        throw new Error(errorMsg);
-    } else if (missingDeps.length > 0 || emptyDeps.length > 0) {
-        // Historical/lookback context - log but allow missing dependencies
-        const missingList = missingDeps.map(d => `'${d.name}' (path: ${d.path})`).join(', ');
-        const emptyList = emptyDeps.map(d => `'${d.name}' (path: ${d.path})`).join(', ');
-        logger.log('INFO', `[DependencyFetcher] ⚠️ Historical/Lookback context: Missing/empty dependencies allowed for ${calcNames} on ${dStr}. Missing: ${missingList}${emptyDeps.length > 0 ? `, Empty: ${emptyList}` : ''}`);
-    }
-    
-    return results;
-}
-
-/**
- * Fetches result series (Historical data) for lookbacks.
- * @param {string} endDateStr - The most recent date.
- * @param {Array} calcNames - Names of computations to fetch.
- * @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
- */
-async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config, deps, lookbackDays) {
-    const { db, logger } = deps;
-    const results = {};
-    const dates = [];
-    
-    // Generate date list (starting from yesterday relative to endDateStr)
-    const d = new Date(endDateStr);
-    for (let i = 0; i < lookbackDays; i++) {
-        d.setUTCDate(d.getUTCDate() - 1); 
-        dates.push(d.toISOString().slice(0, 10));
-    }
-    
-    // Initialize structure
-    calcNames.forEach(name => { results[normalizeName(name)] = {}; });
-    
-    logger.log('INFO', `[DependencyFetcher] Loading series for ${calcNames.length} computation dependencies over ${lookbackDays} days: ${calcNames.join(', ')}`);
-
-    const fetchOps = [];
-    
-    for (const dateStr of dates) {
-        for (const rawName of calcNames) {
-            const normName = normalizeName(rawName);
-            const category = manifestLookup[normName] || 'analytics';
-            
-            fetchOps.push(async () => {
-                const fetchConfig = { ...config, logger };
-                const val = await fetchSingleResult(db, fetchConfig, dateStr, rawName, category);
-                // CRITICAL: For series/lookback, we allow missing dates (historical lookback may have gaps)
-                // This is expected behavior - not all historical dates will have data
-                // But we still validate that the data isn't empty if it exists
-                if (val && !isDataEmpty(val)) {
-                    if (!results[normName]) results[normName] = {};
-                    results[normName][dateStr] = val;
-                } else if (val && isDataEmpty(val)) {
-                    // Log but don't fail - series can have gaps, empty data is treated as missing
-                    logger.log('INFO', `[DependencyFetcher] ⚠️ Empty dependency '${rawName}' found at ${dateStr} in series (allowing gap - historical lookback)`);
-                }
-                // If val is null, that's fine - missing dates in historical series are permissible
-            });
-        }
-    }
-    
-    // Limited concurrency batch execution (Batch size 20)
-    const BATCH_SIZE = 20;
-    for (let i = 0; i < fetchOps.length; i += BATCH_SIZE) {
-        await Promise.all(fetchOps.slice(i, i + BATCH_SIZE).map(fn => fn()));
-    }
-    
-    return results;
-}
-
-/**
- * Core Helper: Fetches a single result, handles Sharding & Compression.
- */
-async function fetchSingleResult(db, config, dateStr, name, category) {
-    const resultsCollection = config.resultsCollection || 'computation_results';
-    const resultsSubcollection = config.resultsSubcollection || 'results';
-    const computationsSubcollection = config.computationsSubcollection || 'computations';
-    
-    const path = `${resultsCollection}/${dateStr}/${resultsSubcollection}/${category}/${computationsSubcollection}/${name}`;
-    
-    // Log path - use console.log if logger not available (for backward compatibility)
-    if (config.logger) {
-        config.logger.log('INFO', `[DependencyFetcher] 📂 Loading Dependency '${name}' from: ${path}`);
-    } else {
-        console.log(`[DependencyFetcher] 📂 Loading Dependency '${name}' from: ${path}`);
-    }
-    
-    const docRef = db.collection(resultsCollection)
-                     .doc(dateStr)
-                     .collection(resultsSubcollection)
-                     .doc(category)
-                     .collection(computationsSubcollection)
-                     .doc(name);
-                     
-    const snap = await docRef.get();
-    if (!snap.exists) {
-        // Log the missing document path clearly
-        if (config.logger) {
-            config.logger.log('ERROR', `[DependencyFetcher] ❌ Document does not exist at: ${path}`);
-        } else {
-            console.error(`[DependencyFetcher] ❌ Document does not exist at: ${path}`);
-        }
-        return null;
-    }
-    
-    let data = snap.data();
-    
-    // CRITICAL: Don't check if empty yet - we need to load shards/compressed data first
-    // A sharded document will only have metadata in the pointer doc, but the actual data is in shards
-    // A compressed document will only have metadata + payload, but the actual data is in the payload
-    
-    // 1. Handle Compression
-    if (data._compressed && data.payload) {
-        try {
-            const buffer = (data.payload instanceof Buffer) ? data.payload : data.payload.toDate();
-            const decompressed = zlib.gunzipSync(buffer);
-            const jsonStr = decompressed.toString('utf8');
-            const realData = JSON.parse(jsonStr);
-            // Merge decompressed data
-            data = { ...data, ...realData }; 
-            delete data.payload; 
-        } catch (e) {
-            const errorMsg = `Decompression failed for ${name}: ${e.message}`;
-            if (config.logger) {
-                config.logger.log('ERROR', `[DependencyFetcher] ❌ ${errorMsg} at: ${path}`);
-            } else {
-                console.error(`[DependencyFetcher] ❌ ${errorMsg} at: ${path}`);
-            }
-            return null;
-        }
-    }
-    
-    // 2. Handle Sharding (MUST happen before empty check)
-    if (data._sharded) {
-        const shardPath = `${path}/_shards`;
-        if (config.logger) {
-            config.logger.log('INFO', `[DependencyFetcher] 📂 Loading Shards for '${name}' from: ${shardPath}`);
-        } else {
-            console.log(`[DependencyFetcher] 📂 Loading Shards for '${name}' from: ${shardPath}`);
-        }
-        
-        const shardCol = docRef.collection('_shards');
-        const shardSnaps = await shardCol.get();
-        
-        if (shardSnaps.empty) {
-            // No shards found - this is a problem
-            if (config.logger) {
-                config.logger.log('ERROR', `[DependencyFetcher] ❌ Document marked as sharded but no shards found at: ${shardPath}`);
-            } else {
-                console.error(`[DependencyFetcher] ❌ Document marked as sharded but no shards found at: ${shardPath}`);
-            }
-            return null; // Return null so it gets caught as missing
-        }
-        
-        // Merge shard contents
-        let hasData = false;
-        shardSnaps.forEach(shard => {
-            let shardData = shard.data();
-            const shardId = shard.id;
-            if (config.logger) {
-                config.logger.log('TRACE', `[DependencyFetcher] 📂 Loading Shard '${shardId}' for '${name}' from: ${shardPath}/${shardId}`);
-            }
-            
-            // CRITICAL: Shards themselves can be compressed (common in big data)
-            // Decompress the shard if needed before merging
-            if (shardData._compressed && shardData.payload) {
-                try {
-                    const buffer = (shardData.payload instanceof Buffer) ? shardData.payload : 
-                                  (shardData.payload._byteString ? Buffer.from(shardData.payload._byteString, 'base64') :
-                                  Buffer.from(shardData.payload));
-                    const decompressed = zlib.gunzipSync(buffer);
-                    const jsonStr = decompressed.toString('utf8');
-                    const realData = JSON.parse(jsonStr);
-                    // If it's double-encoded, parse again
-                    const parsedData = (typeof realData === 'string') ? JSON.parse(realData) : realData;
-                    shardData = { ...shardData, ...parsedData };
-                    delete shardData.payload;
-                } catch (e) {
-                    if (config.logger) {
-                        config.logger.log('ERROR', `[DependencyFetcher] ❌ Failed to decompress shard '${shardId}' for '${name}': ${e.message}`);
-                    } else {
-                        console.error(`[DependencyFetcher] ❌ Failed to decompress shard '${shardId}' for '${name}': ${e.message}`);
-                    }
-                    // Continue with uncompressed data if decompression fails
-                }
-            }
-            
-            // Merge shard contents, ignoring internal metadata if it clashes
-            Object.entries(shardData).forEach(([k, v]) => {
-                if (!k.startsWith('_')) {
-                    data[k] = v;
-                    hasData = true;
-                }
-            });
-        });
-        
-        // If shards contained no actual data, treat as empty
-        if (!hasData) {
-            if (config.logger) {
-                config.logger.log('ERROR', `[DependencyFetcher] ❌ Shards found but contain no data at: ${shardPath}`);
-            } else {
-                console.error(`[DependencyFetcher] ❌ Shards found but contain no data at: ${shardPath}`);
-            }
-            return null;
-        }
-        
-        // After loading shards, remove shard metadata from data object for cleaner output
-        // Keep only the actual data fields
-        const cleanedData = {};
-        const dataKeys = [];
-        Object.entries(data).forEach(([k, v]) => {
-            if (!k.startsWith('_')) {
-                cleanedData[k] = v;
-                dataKeys.push(k);
-            }
-        });
-        data = cleanedData;
-        
-        // Log what we loaded for debugging
-        if (config.logger) {
-            config.logger.log('INFO', `[DependencyFetcher] ✅ Loaded ${shardSnaps.size} shard(s) for '${name}'. Data fields: ${dataKeys.length > 0 ? dataKeys.slice(0, 10).join(', ') + (dataKeys.length > 10 ? `... (+${dataKeys.length - 10} more)` : '') : 'none'}`);
-        }
-    }
-    
-    // Final validation: ensure we have usable data after all processing (decompression + sharding)
-    // Only check if we haven't already determined it's empty
-    if (isDataEmpty(data)) {
-        if (config.logger) {
-            config.logger.log('ERROR', `[DependencyFetcher] ❌ Dependency '${name}' loaded but is empty (no usable data) at: ${path}`);
-        } else {
-            console.error(`[DependencyFetcher] ❌ Dependency '${name}' loaded but is empty (no usable data) at: ${path}`);
-        }
-        return null;
-    }
-    
-    return data;
-}
-
-module.exports = { fetchDependencies, fetchResultSeries, fetchExistingResults };