bulltrackers-module 1.0.658 → 1.0.660

package/functions/computation-system/data/DependencyFetcher.js

@@ -1,478 +1,273 @@
 /**
  * FILENAME: computation-system/data/DependencyFetcher.js
  * @fileoverview Fetches dependencies for computations.
- * UPDATED: Added 'fetchExistingResults' bridge for WorkflowOrchestrator compatibility.
- * UPDATED: Uses 'manifestLookup' to resolve the correct category (Core vs Non-Core).
- * UPDATED: Supports automatic reassembly of sharded results (_shards subcollection).
- * UPDATED: Supports decompression of zipped results.
+ * REFACTORED: Unified fetch logic, streamlined decompression/sharding/GCS.
  */
 const { normalizeName } = require('../utils/utils');
 const zlib = require('zlib');
+const { Storage } = require('@google-cloud/storage');
 
-/**
- * Checks if data is effectively empty (no usable content).
- * @param {any} data - The data to check
- * @returns {boolean} True if data is empty/null/undefined or contains no meaningful content
- */
+const storage = new Storage(); // Singleton Client
+
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+/** Checks if data is effectively empty (null or only metadata keys) */
 function isDataEmpty(data) {
-    if (!data || data === null || data === undefined) return true;
-    
-    // Check if it's an object with only metadata fields
-    if (typeof data === 'object' && !Array.isArray(data)) {
-        const keys = Object.keys(data);
-        // If only metadata/internal fields, consider it empty
-        const metadataFields = ['_completed', '_compressed', '_sharded', '_shardCount', '_isPageMode', '_pageCount', '_lastUpdated', '_expireAt'];
-        const hasOnlyMetadata = keys.length > 0 && keys.every(k => metadataFields.includes(k) || k.startsWith('_'));
-        
-        if (hasOnlyMetadata) return true;
-        
-        // If object has no keys (after filtering metadata), it's empty
-        const dataKeys = keys.filter(k => !k.startsWith('_'));
-        if (dataKeys.length === 0) return true;
+    if (data == null) return true;
+    if (Array.isArray(data)) return data.length === 0;
+    if (typeof data === 'object') {
+        // Return true if NO keys exist that DON'T start with '_'
+        return !Object.keys(data).some(k => !k.startsWith('_'));
     }
-    
-    // Check if it's an empty array
-    if (Array.isArray(data) && data.length === 0) return true;
-    
     return false;
 }
 
+/** Robust decompression helper (Buffer, Base64, or Firestore Binary) */
+function tryDecompress(payload) {
+    if (!payload) return null;
+    try {
+        const buffer = (payload instanceof Buffer) ? payload : 
+                      (payload._byteString ? Buffer.from(payload._byteString, 'base64') : 
+                      (payload.toDate ? payload.toDate() : Buffer.from(payload)));
+        
+        return JSON.parse(zlib.gunzipSync(buffer).toString('utf8'));
+    } catch (e) {
+        throw new Error(`Decompression failed: ${e.message}`);
+    }
+}
+
+// =============================================================================
+// CORE FETCH LOGIC
+// =============================================================================
+
 /**
- * BRIDGE FUNCTION: Matches WorkflowOrchestrator signature.
- * Adapts (dateStr, calcs, manifest, ...) -> (dateObj, calcs, ..., manifestLookup).
- * This fixes the 'fetchExistingResults is not a function' TypeError.
+ * Fetches, decompresses, and reassembles (if sharded or on GCS) a single result document.
  */
-async function fetchExistingResults(dateStr, calcs, fullManifest, config, deps, isHistoricalContext) {
-    const { logger } = deps;
-    
-    // DEBUG: Log entry
-    logger.log('INFO', `[DependencyFetcher] 📥 fetchExistingResults called for date: ${dateStr}, calcs: ${calcs.length}, isHistorical: ${isHistoricalContext}`);
-    logger.log('INFO', `[DependencyFetcher] 📥 Calcs being processed: ${calcs.map(c => {
-        const name = c.name || c.constructor?.name || 'unknown';
-        const hasClass = !!c.class;
-        const classType = c.class ? (typeof c.class === 'function' ? 'function' : typeof c.class) : 'none';
-        return `${name} (has class: ${hasClass}, class type: ${classType})`;
-    }).join(', ')}`);
-    
-    // 1. Build Manifest Lookup (Name -> Category)
-    const manifestLookup = {};
-    if (Array.isArray(fullManifest)) {
-        fullManifest.forEach(c => {
-            manifestLookup[normalizeName(c.name)] = c.category || 'analytics';
-        });
-        logger.log('INFO', `[DependencyFetcher] 📥 Built manifest lookup with ${Object.keys(manifestLookup).length} entries`);
-    } else {
-        logger.log('WARN', `[DependencyFetcher] ⚠️ fullManifest is not an array: ${typeof fullManifest}`);
+async function fetchSingleResult(db, config, dateStr, name, category) {
+    const { resultsCollection = 'computation_results', resultsSubcollection = 'results', computationsSubcollection = 'computations' } = config;
+    const log = config.logger || console;
+
+    const docRef = db.collection(resultsCollection).doc(dateStr)
+                     .collection(resultsSubcollection).doc(category)
+                     .collection(computationsSubcollection).doc(name);
+
+    const snap = await docRef.get();
+    if (!snap.exists) return null;
+
+    let data = snap.data();
+
+    // -------------------------------------------------------------------------
+    // 1. GCS POINTER HANDLER (New)
+    // -------------------------------------------------------------------------
+    if (data.gcsUri || (data._gcs && data.gcsBucket && data.gcsPath)) {
+        try {
+            const bucketName = data.gcsBucket || data.gcsUri.split('/')[2];
+            const fileName   = data.gcsPath || data.gcsUri.split('/').slice(3).join('/');
+            
+            // Stream download is memory efficient for large files
+            const [fileContent] = await storage.bucket(bucketName).file(fileName).download();
+            
+            // Assume Gzip (as writer does it), if fails try plain
+            try {
+                return JSON.parse(zlib.gunzipSync(fileContent).toString('utf8'));
+            } catch (gzipErr) {
+                // Fallback for uncompressed GCS files
+                return JSON.parse(fileContent.toString('utf8'));
+            }
+        } catch (e) {
+            log.error(`[DependencyFetcher] ❌ GCS Fetch Failed for ${name}: ${e.message}`);
+            // Depending on strictness, we might return null here or allow it to fail hard.
+            // Returning null allows 'isDataEmpty' to catch it as "MISSING"
+            return null; 
+        }
     }
 
-    // 2. Convert Date String to Date Object
-    // We append T00:00:00Z to ensure it parses as UTC date if only YYYY-MM-DD is provided.
-    const dateObj = new Date(dateStr + (dateStr.includes('T') ? '' : 'T00:00:00Z'));
+    // -------------------------------------------------------------------------
+    // 2. FIRESTORE COMPRESSED HANDLER
+    // -------------------------------------------------------------------------
+    if (data._compressed && data.payload) {
+        try {
+            const realData = tryDecompress(data.payload);
+            data = { ...data, ...realData }; // Merge payload into base
+            delete data.payload;
+        } catch (e) {
+            log.error(`[DependencyFetcher] ❌ ${e.message} at ${docRef.path}`);
+            return null;
+        }
+    }
 
-    // 3. Delegate to fetchDependencies
-    // CRITICAL: For historical context (yesterday's data), allow missing dependencies
-    // Historical lookbacks are optional - gaps in historical data are permissible
-    const result = await fetchDependencies(dateObj, calcs, config, deps, manifestLookup, isHistoricalContext);
-    
-    // DEBUG: Log result
-    const resultKeys = Object.keys(result);
-    logger.log('INFO', `[DependencyFetcher] 📤 fetchExistingResults returning ${resultKeys.length} dependencies: ${resultKeys.length > 0 ? resultKeys.join(', ') : 'NONE'}`);
-    
-    return result;
+    // -------------------------------------------------------------------------
+    // 3. FIRESTORE SHARDED HANDLER
+    // -------------------------------------------------------------------------
+    if (data._sharded) {
+        const shardSnaps = await docRef.collection('_shards').get();
+        if (shardSnaps.empty) {
+            log.error(`[DependencyFetcher] ❌ Sharded doc has no shards: ${docRef.path}`);
+            return null;
+        }
+
+        // Initialize merged data with any non-metadata fields from the pointer doc
+        const merged = {};
+        Object.entries(data).forEach(([k, v]) => { if (!k.startsWith('_')) merged[k] = v; });
+
+        for (const shard of shardSnaps.docs) {
+            let sData = shard.data();
+            
+            // Decompress Shard if needed
+            if (sData._compressed && sData.payload) {
+                try {
+                    const decomp = tryDecompress(sData.payload);
+                    sData = (typeof decomp === 'string') ? JSON.parse(decomp) : decomp;
+                } catch (e) {
+                    log.error(`[DependencyFetcher] ❌ Shard decompression failed (${shard.id}): ${e.message}`);
+                    continue; 
+                }
+            }
+
+            // Merge Shard Content
+            Object.entries(sData).forEach(([k, v]) => { 
+                if (!k.startsWith('_')) merged[k] = v; 
+            });
+        }
+        
+        // Return null if result is empty after merging
+        if (Object.keys(merged).length === 0) return null;
+        return merged;
+    }
+
+    // -------------------------------------------------------------------------
+    // 4. STANDARD DOCUMENT HANDLER
+    // -------------------------------------------------------------------------
+    const clean = {};
+    let hasContent = false;
+    Object.entries(data).forEach(([k, v]) => {
+        if (!k.startsWith('_')) { clean[k] = v; hasContent = true; }
+    });
+
+    return hasContent ? clean : null;
 }
 
+// =============================================================================
+// PUBLIC METHODS
+// =============================================================================
+
 /**
  * Fetches dependencies for a specific date (Standard pass).
- * @param {Date} date - The target date.
- * @param {Array} calcs - The computations requiring dependencies.
- * @param {Object} config - System config.
- * @param {Object} deps - System dependencies (db, logger).
- * @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
- * @param {boolean} allowMissing - If true, missing/empty dependencies are allowed (for historical/lookback scenarios).
  */
 async function fetchDependencies(date, calcs, config, deps, manifestLookup = {}, allowMissing = false) {
     const { db, logger } = deps;
     const dStr = date.toISOString().slice(0, 10);
     
-    // DEBUG: Log entry
-    logger.log('INFO', `[DependencyFetcher] 🔍 fetchDependencies called with ${calcs.length} calc(s): ${calcs.map(c => c.name || c.constructor?.name || 'unknown').join(', ')}`);
-    
-    // 1. Identify unique dependencies needed
-    // CHANGED: Use a Map to track { normalizedName: originalName }
-    const needed = new Map();
-
+    // 1. Resolve Dependencies (Normalize Name -> Original Name)
+    const needed = new Map(); 
     calcs.forEach(c => {
-        const calcName = c.name || c.constructor?.name || 'unknown';
-        
-        // DEBUG: Log what we're checking
-        logger.log('INFO', `[DependencyFetcher] 🔍 Processing calc: ${calcName}`);
-        logger.log('INFO', `[DependencyFetcher]   - has class: ${!!c.class}`);
-        logger.log('INFO', `[DependencyFetcher]   - class type: ${c.class ? (typeof c.class === 'function' ? 'function' : typeof c.class) : 'none'}`);
-        logger.log('INFO', `[DependencyFetcher]   - class.getDependencies: ${c.class && typeof c.class.getDependencies === 'function' ? 'YES' : 'NO'}`);
-        logger.log('INFO', `[DependencyFetcher]   - has getDependencies: ${typeof c.getDependencies === 'function' ? 'YES' : 'NO'}`);
-        logger.log('INFO', `[DependencyFetcher]   - has dependencies array: ${Array.isArray(c.dependencies) ? `YES (${c.dependencies.length} items)` : 'NO'}`);
-        if (Array.isArray(c.dependencies)) {
-            logger.log('INFO', `[DependencyFetcher]   - dependencies array: ${c.dependencies.join(', ')}`);
-        }
+        // Priority: class.getDependencies() > instance.getDependencies() > manifest.dependencies
+        const getDeps = c.class?.getDependencies || c.getDependencies;
+        const reqs = (typeof getDeps === 'function') ? getDeps.call(c.class || c) : c.dependencies;
         
-        // [FIX] Support both .getDependencies() method and .dependencies array
-        // CRITICAL: Prefer class.getDependencies() over manifest.dependencies
-        // because the class method returns original case-sensitive names,
-        // while manifest.dependencies contains normalized names
-        let reqs = [];
-        if (c.class && typeof c.class.getDependencies === 'function') {
-            // Use the class method - returns original case-sensitive names
-            reqs = c.class.getDependencies();
-            logger.log('INFO', `[DependencyFetcher] ✅ Using c.class.getDependencies() - returned: ${JSON.stringify(reqs)}`);
-        } else if (typeof c.getDependencies === 'function') {
-            // Fallback: direct method call (if c is the class itself)
-            reqs = c.getDependencies();
-            logger.log('INFO', `[DependencyFetcher] ✅ Using c.getDependencies() - returned: ${JSON.stringify(reqs)}`);
-        } else if (c.dependencies && Array.isArray(c.dependencies)) {
-            // Last resort: use manifest's dependencies array (normalized)
-            // This is less ideal because names are normalized, but we'll use them as-is
-            reqs = c.dependencies;
-            logger.log('INFO', `[DependencyFetcher] ⚠️ Using c.dependencies array (normalized) - returned: ${JSON.stringify(reqs)}`);
-        } else {
-            logger.log('WARN', `[DependencyFetcher] ❌ No way to get dependencies for ${calcName} - all methods failed`);
-        }
-            
         if (Array.isArray(reqs)) {
-            logger.log('INFO', `[DependencyFetcher] ✅ Found ${reqs.length} dependencies for ${calcName}: ${reqs.join(', ')}`);
-            reqs.forEach(r => {
-                // We map the normalized version to the original requested version
-                // This ensures we fetch the right file (normalized) but return it 
-                // with the casing the user code expects (original).
-                needed.set(normalizeName(r), r);
-            });
-        } else {
-            logger.log('WARN', `[DependencyFetcher] ⚠️ reqs is not an array for ${calcName}: ${typeof reqs}`);
+            reqs.forEach(r => needed.set(normalizeName(r), r));
         }
     });
-    
-    if (needed.size === 0) {
-        logger.log('WARN', `[DependencyFetcher] ⚠️ No dependencies needed - returning empty object`);
-        return {};
-    }
-    
-    const calcNames = calcs.map(c => c.name || c.constructor?.name || 'unknown').join(', ');
-    logger.log('INFO', `[DependencyFetcher] Fetching ${needed.size} dependencies for computation(s): ${calcNames} (date: ${dStr})`);
-    
-    // DEBUG: Log what dependencies we're looking for
-    const depList = Array.from(needed.entries()).map(([norm, orig]) => `${orig} (normalized: ${norm})`).join(', ');
-    logger.log('INFO', `[DependencyFetcher] Dependencies requested: ${depList}`);
-    
+
+    if (needed.size === 0) return {};
+
+    logger.log('INFO', `[DependencyFetcher] 🔍 Fetching ${needed.size} dependencies for ${dStr}`);
+
     const results = {};
-    const missingDeps = [];
-    const emptyDeps = [];
-    
-    // Helper to build path string
-    const buildPath = (category, normName) => {
-        return `${config.resultsCollection || 'computation_results'}/${dStr}/${config.resultsSubcollection || 'results'}/${category}/${config.computationsSubcollection || 'computations'}/${normName}`;
-    };
-    
-    // CHANGED: Iterate over the entries to access both normalized and original names
-    const promises = Array.from(needed.entries()).map(async ([normName, originalName]) => {
-        // Resolve Category from Lookup, default to 'analytics' if unknown
-        // Note: manifestLookup keys are expected to be normalized
-        const category = manifestLookup[normName] || 'analytics';
-        const path = buildPath(category, normName);
+    const errors = []; 
+
+    // 2. Fetch All Dependencies in Parallel
+    await Promise.all(Array.from(needed.entries()).map(async ([norm, orig]) => {
+        const category = manifestLookup[norm] || 'analytics';
+        const path = `${config.resultsCollection || 'computation_results'}/${dStr}/.../${category}/${norm}`;
         
         try {
-            // Pass logger in config for fetchSingleResult
-            const fetchConfig = { ...config, logger };
+            const data = await fetchSingleResult(db, { ...config, logger }, dStr, norm, category);
             
-            // Fetch using the normalized name (system standard)
-            const data = await fetchSingleResult(db, fetchConfig, dStr, normName, category);
-            
-            // CRITICAL: Validate that dependency exists and has data
-            if (!data) {
-                missingDeps.push({ name: originalName, normalizedName: normName, path });
-                // Log level depends on context - ERROR for current date, INFO for historical
-                if (allowMissing) {
-                    logger.log('INFO', `[DependencyFetcher] ⚠️ Missing dependency '${originalName}' (${normName}) from: ${path} (Historical context - allowed)`);
-                } else {
-                    logger.log('ERROR', `[DependencyFetcher] ❌ Missing required dependency '${originalName}' (${normName}) from: ${path}`);
-                }
-            } else if (isDataEmpty(data)) {
-                emptyDeps.push({ name: originalName, normalizedName: normName, path });
-                // Log level depends on context - ERROR for current date, INFO for historical
-                if (allowMissing) {
-                    logger.log('INFO', `[DependencyFetcher] ⚠️ Empty dependency '${originalName}' (${normName}) from: ${path} (Historical context - allowed)`);
-                } else {
-                    logger.log('ERROR', `[DependencyFetcher] ❌ Empty dependency '${originalName}' (${normName}) from: ${path} - Document exists but contains no usable data`);
-                }
+            if (!data || isDataEmpty(data)) {
+                // Determine severity based on context
+                const status = data ? 'EMPTY' : 'MISSING';
+                errors.push({ name: orig, path, reason: status });
+                
+                // Log immediately for visibility
+                const level = allowMissing ? 'INFO' : 'ERROR';
+                logger.log(level, `[DependencyFetcher] ⚠️ Dependency '${orig}' ${status} at ${path}`);
             } else {
-                // CHANGED: Store result using the ORIGINAL name so context.computed['CaseSensitive'] works
-                results[originalName] = data;
-                // DEBUG: Log successful dependency load
-                const dataKeys = Object.keys(data);
-                logger.log('INFO', `[DependencyFetcher] ✅ Stored dependency '${originalName}' in results. Keys: ${dataKeys.length} (sample: ${dataKeys.slice(0, 5).join(', ')})`);
+                results[orig] = data; // Store using Original Name
             }
         } catch (e) {
-            missingDeps.push({ name: originalName, normalizedName: normName, path, error: e.message });
-            // Log level depends on context - ERROR for current date, INFO for historical
-            if (allowMissing) {
-                logger.log('INFO', `[DependencyFetcher] ⚠️ Failed to load dependency '${originalName}' (${normName}) from: ${path} - Error: ${e.message} (Historical context - allowed)`);
-            } else {
-                logger.log('ERROR', `[DependencyFetcher] ❌ Failed to load dependency '${originalName}' (${normName}) from: ${path} - Error: ${e.message}`);
-            }
+            errors.push({ name: orig, path, reason: e.message });
+            logger.log('ERROR', `[DependencyFetcher] ❌ Error loading '${orig}': ${e.message}`);
         }
-    });
-    
-    await Promise.all(promises);
-    
-    // DEBUG: Log what we're returning
-    const resultKeys = Object.keys(results);
-    logger.log('INFO', `[DependencyFetcher] ✅ Returning ${resultKeys.length} dependencies: ${resultKeys.join(', ')}`);
-    
-    // CRITICAL: Fail if any required dependencies are missing or empty
-    // EXCEPTION: For historical/lookback scenarios, missing dependencies are permissible
-    if ((missingDeps.length > 0 || emptyDeps.length > 0) && !allowMissing) {
-        const missingList = missingDeps.map(d => `'${d.name}' (path: ${d.path}${d.error ? `, error: ${d.error}` : ''})`).join(', ');
-        const emptyList = emptyDeps.map(d => `'${d.name}' (path: ${d.path})`).join(', ');
-        
-        const errorMsg = `[DependencyFetcher] ❌ CRITICAL: Cannot proceed - Required dependencies missing or empty for computation(s): ${calcNames}\n` +
-                        `Missing dependencies (${missingDeps.length}): ${missingList}\n` +
-                        (emptyDeps.length > 0 ? `Empty dependencies (${emptyDeps.length}): ${emptyList}\n` : '') +
-                        `Date: ${dStr}\n` +
-                        `This computation will FAIL and no results will be saved.`;
-        
-        logger.log('ERROR', errorMsg);
-        throw new Error(errorMsg);
-    } else if (missingDeps.length > 0 || emptyDeps.length > 0) {
-        // Historical/lookback context - log but allow missing dependencies
-        const missingList = missingDeps.map(d => `'${d.name}' (path: ${d.path})`).join(', ');
-        const emptyList = emptyDeps.map(d => `'${d.name}' (path: ${d.path})`).join(', ');
-        logger.log('INFO', `[DependencyFetcher] ⚠️ Historical/Lookback context: Missing/empty dependencies allowed for ${calcNames} on ${dStr}. Missing: ${missingList}${emptyDeps.length > 0 ? `, Empty: ${emptyList}` : ''}`);
+    }));
+
+    // 3. Final Validation
+    if (errors.length > 0 && !allowMissing) {
+        throw new Error(`[DependencyFetcher] CRITICAL: Missing required dependencies: ${errors.map(e => e.name).join(', ')}`);
+    } else if (errors.length > 0) {
+        logger.log('INFO', `[DependencyFetcher] ⚠️ Allowed missing/empty dependencies in Historical context: ${errors.map(e => e.name).join(', ')}`);
     }
-    
+
     return results;
 }
 
 /**
  * Fetches result series (Historical data) for lookbacks.
- * @param {string} endDateStr - The most recent date.
- * @param {Array} calcNames - Names of computations to fetch.
- * @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
  */
 async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config, deps, lookbackDays) {
     const { db, logger } = deps;
-    const results = {};
-    const dates = [];
+    const results = {}; // normalizedName -> { date -> data }
     
-    // Generate date list (starting from yesterday relative to endDateStr)
+    // Initialize results structure
+    calcNames.forEach(n => results[normalizeName(n)] = {});
+
+    // Generate Date List
+    const dates = [];
     const d = new Date(endDateStr);
     for (let i = 0; i < lookbackDays; i++) {
-        d.setUTCDate(d.getUTCDate() - 1); 
+        d.setUTCDate(d.getUTCDate() - 1);
         dates.push(d.toISOString().slice(0, 10));
     }
-    
-    // Initialize structure
-    calcNames.forEach(name => { results[normalizeName(name)] = {}; });
-    
-    logger.log('INFO', `[DependencyFetcher] Loading series for ${calcNames.length} computation dependencies over ${lookbackDays} days: ${calcNames.join(', ')}`);
 
-    const fetchOps = [];
-    
+    // Build Fetch Operations
+    const ops = [];
     for (const dateStr of dates) {
         for (const rawName of calcNames) {
-            const normName = normalizeName(rawName);
-            const category = manifestLookup[normName] || 'analytics';
-            
-            fetchOps.push(async () => {
-                const fetchConfig = { ...config, logger };
-                const val = await fetchSingleResult(db, fetchConfig, dateStr, rawName, category);
-                // CRITICAL: For series/lookback, we allow missing dates (historical lookback may have gaps)
-                // This is expected behavior - not all historical dates will have data
-                // But we still validate that the data isn't empty if it exists
+            const norm = normalizeName(rawName);
+            const category = manifestLookup[norm] || 'analytics';
+            ops.push(async () => {
+                const val = await fetchSingleResult(db, { ...config, logger }, dateStr, rawName, category);
                 if (val && !isDataEmpty(val)) {
-                    if (!results[normName]) results[normName] = {};
-                    results[normName][dateStr] = val;
-                } else if (val && isDataEmpty(val)) {
-                    // Log but don't fail - series can have gaps, empty data is treated as missing
-                    logger.log('INFO', `[DependencyFetcher] ⚠️ Empty dependency '${rawName}' found at ${dateStr} in series (allowing gap - historical lookback)`);
+                    results[norm][dateStr] = val;
                 }
-                // If val is null, that's fine - missing dates in historical series are permissible
             });
         }
     }
-    
-    // Limited concurrency batch execution (Batch size 20)
+
+    // Execute in Batches (Limit Concurrency)
+    logger.log('INFO', `[DependencyFetcher] 📚 Loading series: ${calcNames.length} calcs x ${lookbackDays} days (${ops.length} ops)`);
     const BATCH_SIZE = 20;
-    for (let i = 0; i < fetchOps.length; i += BATCH_SIZE) {
-        await Promise.all(fetchOps.slice(i, i + BATCH_SIZE).map(fn => fn()));
+    for (let i = 0; i < ops.length; i += BATCH_SIZE) {
+        await Promise.all(ops.slice(i, i + BATCH_SIZE).map(fn => fn()));
     }
-    
+
     return results;
 }
 
 /**
- * Core Helper: Fetches a single result, handles Sharding & Compression.
+ * Bridge function for WorkflowOrchestrator compatibility.
  */
-async function fetchSingleResult(db, config, dateStr, name, category) {
-    const resultsCollection = config.resultsCollection || 'computation_results';
-    const resultsSubcollection = config.resultsSubcollection || 'results';
-    const computationsSubcollection = config.computationsSubcollection || 'computations';
-    
-    const path = `${resultsCollection}/${dateStr}/${resultsSubcollection}/${category}/${computationsSubcollection}/${name}`;
-    
-    // Log path - use console.log if logger not available (for backward compatibility)
-    if (config.logger) {
-        config.logger.log('INFO', `[DependencyFetcher] 📂 Loading Dependency '${name}' from: ${path}`);
-    } else {
-        console.log(`[DependencyFetcher] 📂 Loading Dependency '${name}' from: ${path}`);
-    }
-    
-    const docRef = db.collection(resultsCollection)
-                     .doc(dateStr)
-                     .collection(resultsSubcollection)
-                     .doc(category)
-                     .collection(computationsSubcollection)
-                     .doc(name);
-                     
-    const snap = await docRef.get();
-    if (!snap.exists) {
-        // Log the missing document path clearly
-        if (config.logger) {
-            config.logger.log('ERROR', `[DependencyFetcher] ❌ Document does not exist at: ${path}`);
-        } else {
-            console.error(`[DependencyFetcher] ❌ Document does not exist at: ${path}`);
-        }
-        return null;
-    }
-    
-    let data = snap.data();
-    
-    // CRITICAL: Don't check if empty yet - we need to load shards/compressed data first
-    // A sharded document will only have metadata in the pointer doc, but the actual data is in shards
-    // A compressed document will only have metadata + payload, but the actual data is in the payload
-    
-    // 1. Handle Compression
-    if (data._compressed && data.payload) {
-        try {
-            const buffer = (data.payload instanceof Buffer) ? data.payload : data.payload.toDate();
-            const decompressed = zlib.gunzipSync(buffer);
-            const jsonStr = decompressed.toString('utf8');
-            const realData = JSON.parse(jsonStr);
-            // Merge decompressed data
-            data = { ...data, ...realData }; 
-            delete data.payload; 
-        } catch (e) {
-            const errorMsg = `Decompression failed for ${name}: ${e.message}`;
-            if (config.logger) {
-                config.logger.log('ERROR', `[DependencyFetcher] ❌ ${errorMsg} at: ${path}`);
-            } else {
-                console.error(`[DependencyFetcher] ❌ ${errorMsg} at: ${path}`);
-            }
-            return null;
-        }
-    }
-    
-    // 2. Handle Sharding (MUST happen before empty check)
-    if (data._sharded) {
-        const shardPath = `${path}/_shards`;
-        if (config.logger) {
-            config.logger.log('INFO', `[DependencyFetcher] 📂 Loading Shards for '${name}' from: ${shardPath}`);
-        } else {
-            console.log(`[DependencyFetcher] 📂 Loading Shards for '${name}' from: ${shardPath}`);
-        }
-        
-        const shardCol = docRef.collection('_shards');
-        const shardSnaps = await shardCol.get();
-        
-        if (shardSnaps.empty) {
-            // No shards found - this is a problem
-            if (config.logger) {
-                config.logger.log('ERROR', `[DependencyFetcher] ❌ Document marked as sharded but no shards found at: ${shardPath}`);
-            } else {
-                console.error(`[DependencyFetcher] ❌ Document marked as sharded but no shards found at: ${shardPath}`);
-            }
-            return null; // Return null so it gets caught as missing
-        }
-        
-        // Merge shard contents
-        let hasData = false;
-        shardSnaps.forEach(shard => {
-            let shardData = shard.data();
-            const shardId = shard.id;
-            if (config.logger) {
-                config.logger.log('TRACE', `[DependencyFetcher] 📂 Loading Shard '${shardId}' for '${name}' from: ${shardPath}/${shardId}`);
-            }
-            
-            // CRITICAL: Shards themselves can be compressed (common in big data)
-            // Decompress the shard if needed before merging
-            if (shardData._compressed && shardData.payload) {
-                try {
-                    const buffer = (shardData.payload instanceof Buffer) ? shardData.payload : 
-                                  (shardData.payload._byteString ? Buffer.from(shardData.payload._byteString, 'base64') :
-                                  Buffer.from(shardData.payload));
-                    const decompressed = zlib.gunzipSync(buffer);
-                    const jsonStr = decompressed.toString('utf8');
-                    const realData = JSON.parse(jsonStr);
-                    // If it's double-encoded, parse again
-                    const parsedData = (typeof realData === 'string') ? JSON.parse(realData) : realData;
-                    shardData = { ...shardData, ...parsedData };
-                    delete shardData.payload;
-                } catch (e) {
-                    if (config.logger) {
-                        config.logger.log('ERROR', `[DependencyFetcher] ❌ Failed to decompress shard '${shardId}' for '${name}': ${e.message}`);
-                    } else {
-                        console.error(`[DependencyFetcher] ❌ Failed to decompress shard '${shardId}' for '${name}': ${e.message}`);
-                    }
-                    // Continue with uncompressed data if decompression fails
-                }
-            }
-            
-            // Merge shard contents, ignoring internal metadata if it clashes
-            Object.entries(shardData).forEach(([k, v]) => {
-                if (!k.startsWith('_')) {
-                    data[k] = v;
-                    hasData = true;
-                }
-            });
-        });
-        
-        // If shards contained no actual data, treat as empty
-        if (!hasData) {
-            if (config.logger) {
-                config.logger.log('ERROR', `[DependencyFetcher] ❌ Shards found but contain no data at: ${shardPath}`);
-            } else {
-                console.error(`[DependencyFetcher] ❌ Shards found but contain no data at: ${shardPath}`);
-            }
-            return null;
-        }
-        
-        // After loading shards, remove shard metadata from data object for cleaner output
-        // Keep only the actual data fields
-        const cleanedData = {};
-        const dataKeys = [];
-        Object.entries(data).forEach(([k, v]) => {
-            if (!k.startsWith('_')) {
-                cleanedData[k] = v;
-                dataKeys.push(k);
-            }
-        });
-        data = cleanedData;
-        
-        // Log what we loaded for debugging
-        if (config.logger) {
-            config.logger.log('INFO', `[DependencyFetcher] ✅ Loaded ${shardSnaps.size} shard(s) for '${name}'. Data fields: ${dataKeys.length > 0 ? dataKeys.slice(0, 10).join(', ') + (dataKeys.length > 10 ? `... (+${dataKeys.length - 10} more)` : '') : 'none'}`);
-        }
-    }
-    
-    // Final validation: ensure we have usable data after all processing (decompression + sharding)
-    // Only check if we haven't already determined it's empty
-    if (isDataEmpty(data)) {
-        if (config.logger) {
-            config.logger.log('ERROR', `[DependencyFetcher] ❌ Dependency '${name}' loaded but is empty (no usable data) at: ${path}`);
-        } else {
-            console.error(`[DependencyFetcher] ❌ Dependency '${name}' loaded but is empty (no usable data) at: ${path}`);
-        }
-        return null;
+async function fetchExistingResults(dateStr, calcs, fullManifest, config, deps, isHistoricalContext) {
+    const lookup = {};
+    if (Array.isArray(fullManifest)) {
+        fullManifest.forEach(c => lookup[normalizeName(c.name)] = c.category || 'analytics');
     }
-    
-    return data;
+    // Ensure Date Object
+    const dateObj = new Date(dateStr.includes('T') ? dateStr : dateStr + 'T00:00:00Z');
+    return fetchDependencies(dateObj, calcs, config, deps, lookup, isHistoricalContext);
 }
 
 module.exports = { fetchDependencies, fetchResultSeries, fetchExistingResults };