bulltrackers-module 1.0.646 → 1.0.648

package/functions/computation-system/data/DependencyFetcher.js

@@ -9,6 +9,34 @@
 const { normalizeName } = require('../utils/utils');
 const zlib = require('zlib');
 
+/**
+ * Checks if data is effectively empty (no usable content).
+ * @param {any} data - The data to check
+ * @returns {boolean} True if data is empty/null/undefined or contains no meaningful content
+ */
+function isDataEmpty(data) {
+    if (!data || data === null || data === undefined) return true;
+    
+    // Check if it's an object with only metadata fields
+    if (typeof data === 'object' && !Array.isArray(data)) {
+        const keys = Object.keys(data);
+        // If only metadata/internal fields, consider it empty
+        const metadataFields = ['_completed', '_compressed', '_sharded', '_shardCount', '_isPageMode', '_pageCount', '_lastUpdated', '_expireAt'];
+        const hasOnlyMetadata = keys.length > 0 && keys.every(k => metadataFields.includes(k) || k.startsWith('_'));
+        
+        if (hasOnlyMetadata) return true;
+        
+        // If object has no keys (after filtering metadata), it's empty
+        const dataKeys = keys.filter(k => !k.startsWith('_'));
+        if (dataKeys.length === 0) return true;
+    }
+    
+    // Check if it's an empty array
+    if (Array.isArray(data) && data.length === 0) return true;
+    
+    return false;
+}
+
 /**
  * BRIDGE FUNCTION: Matches WorkflowOrchestrator signature.
  * Adapts (dateStr, calcs, manifest, ...) -> (dateObj, calcs, ..., manifestLookup).
@@ -28,7 +56,9 @@ async function fetchExistingResults(dateStr, calcs, fullManifest, config, deps,
     const dateObj = new Date(dateStr + (dateStr.includes('T') ? '' : 'T00:00:00Z'));
 
     // 3. Delegate to fetchDependencies
-    return fetchDependencies(dateObj, calcs, config, deps, manifestLookup);
+    // CRITICAL: For historical context (yesterday's data), allow missing dependencies
+    // Historical lookbacks are optional - gaps in historical data are permissible
+    return fetchDependencies(dateObj, calcs, config, deps, manifestLookup, isHistoricalContext);
 }
 
 /**
@@ -38,8 +68,9 @@ async function fetchExistingResults(dateStr, calcs, fullManifest, config, deps,
  * @param {Object} config - System config.
  * @param {Object} deps - System dependencies (db, logger).
  * @param {Object} manifestLookup - Map of { [calcName]: categoryString }.
+ * @param {boolean} allowMissing - If true, missing/empty dependencies are allowed (for historical/lookback scenarios).
  */
-async function fetchDependencies(date, calcs, config, deps, manifestLookup = {}) {
+async function fetchDependencies(date, calcs, config, deps, manifestLookup = {}, allowMissing = false) {
     const { db, logger } = deps;
     const dStr = date.toISOString().slice(0, 10);
     
@@ -69,27 +100,83 @@ async function fetchDependencies(date, calcs, config, deps, manifestLookup = {})
     logger.log('INFO', `[DependencyFetcher] Fetching ${needed.size} dependencies for computation(s): ${calcNames} (date: ${dStr})`);
     
     const results = {};
+    const missingDeps = [];
+    const emptyDeps = [];
+    
+    // Helper to build path string
+    const buildPath = (category, normName) => {
+        return `${config.resultsCollection || 'computation_results'}/${dStr}/${config.resultsSubcollection || 'results'}/${category}/${config.computationsSubcollection || 'computations'}/${normName}`;
+    };
+    
     // CHANGED: Iterate over the entries to access both normalized and original names
     const promises = Array.from(needed.entries()).map(async ([normName, originalName]) => {
+        // Resolve Category from Lookup, default to 'analytics' if unknown
+        // Note: manifestLookup keys are expected to be normalized
+        const category = manifestLookup[normName] || 'analytics';
+        const path = buildPath(category, normName);
+        
         try {
-            // Resolve Category from Lookup, default to 'analytics' if unknown
-            // Note: manifestLookup keys are expected to be normalized
-            const category = manifestLookup[normName] || 'analytics'; 
-            
             // Pass logger in config for fetchSingleResult
             const fetchConfig = { ...config, logger };
             
             // Fetch using the normalized name (system standard)
             const data = await fetchSingleResult(db, fetchConfig, dStr, normName, category);
             
-            // CHANGED: Store result using the ORIGINAL name so context.computed['CaseSensitive'] works
-            if (data) results[originalName] = data;
+            // CRITICAL: Validate that dependency exists and has data
+            if (!data) {
+                missingDeps.push({ name: originalName, normalizedName: normName, path });
+                // Log level depends on context - ERROR for current date, INFO for historical
+                if (allowMissing) {
+                    logger.log('INFO', `[DependencyFetcher] ⚠️ Missing dependency '${originalName}' (${normName}) from: ${path} (Historical context - allowed)`);
+                } else {
+                    logger.log('ERROR', `[DependencyFetcher] ❌ Missing required dependency '${originalName}' (${normName}) from: ${path}`);
+                }
+            } else if (isDataEmpty(data)) {
+                emptyDeps.push({ name: originalName, normalizedName: normName, path });
+                // Log level depends on context - ERROR for current date, INFO for historical
+                if (allowMissing) {
+                    logger.log('INFO', `[DependencyFetcher] ⚠️ Empty dependency '${originalName}' (${normName}) from: ${path} (Historical context - allowed)`);
+                } else {
+                    logger.log('ERROR', `[DependencyFetcher] ❌ Empty dependency '${originalName}' (${normName}) from: ${path} - Document exists but contains no usable data`);
+                }
+            } else {
+                // CHANGED: Store result using the ORIGINAL name so context.computed['CaseSensitive'] works
+                results[originalName] = data;
+            }
         } catch (e) {
-            logger.log('WARN', `[DependencyFetcher] Failed to load dependency ${originalName}: ${e.message}`);
+            missingDeps.push({ name: originalName, normalizedName: normName, path, error: e.message });
+            // Log level depends on context - ERROR for current date, INFO for historical
+            if (allowMissing) {
+                logger.log('INFO', `[DependencyFetcher] ⚠️ Failed to load dependency '${originalName}' (${normName}) from: ${path} - Error: ${e.message} (Historical context - allowed)`);
+            } else {
+                logger.log('ERROR', `[DependencyFetcher] ❌ Failed to load dependency '${originalName}' (${normName}) from: ${path} - Error: ${e.message}`);
+            }
         }
     });
     
     await Promise.all(promises);
+    
+    // CRITICAL: Fail if any required dependencies are missing or empty
+    // EXCEPTION: For historical/lookback scenarios, missing dependencies are permissible
+    if ((missingDeps.length > 0 || emptyDeps.length > 0) && !allowMissing) {
+        const missingList = missingDeps.map(d => `'${d.name}' (path: ${d.path}${d.error ? `, error: ${d.error}` : ''})`).join(', ');
+        const emptyList = emptyDeps.map(d => `'${d.name}' (path: ${d.path})`).join(', ');
+        
+        const errorMsg = `[DependencyFetcher] ❌ CRITICAL: Cannot proceed - Required dependencies missing or empty for computation(s): ${calcNames}\n` +
+                        `Missing dependencies (${missingDeps.length}): ${missingList}\n` +
+                        (emptyDeps.length > 0 ? `Empty dependencies (${emptyDeps.length}): ${emptyList}\n` : '') +
+                        `Date: ${dStr}\n` +
+                        `This computation will FAIL and no results will be saved.`;
+        
+        logger.log('ERROR', errorMsg);
+        throw new Error(errorMsg);
+    } else if (missingDeps.length > 0 || emptyDeps.length > 0) {
+        // Historical/lookback context - log but allow missing dependencies
+        const missingList = missingDeps.map(d => `'${d.name}' (path: ${d.path})`).join(', ');
+        const emptyList = emptyDeps.map(d => `'${d.name}' (path: ${d.path})`).join(', ');
+        logger.log('INFO', `[DependencyFetcher] ⚠️ Historical/Lookback context: Missing/empty dependencies allowed for ${calcNames} on ${dStr}. Missing: ${missingList}${emptyDeps.length > 0 ? `, Empty: ${emptyList}` : ''}`);
+    }
+    
     return results;
 }
 
@@ -126,10 +213,17 @@ async function fetchResultSeries(endDateStr, calcNames, manifestLookup, config,
             fetchOps.push(async () => {
                 const fetchConfig = { ...config, logger };
                 const val = await fetchSingleResult(db, fetchConfig, dateStr, rawName, category);
-                if (val) {
+                // CRITICAL: For series/lookback, we allow missing dates (historical lookback may have gaps)
+                // This is expected behavior - not all historical dates will have data
+                // But we still validate that the data isn't empty if it exists
+                if (val && !isDataEmpty(val)) {
                     if (!results[normName]) results[normName] = {};
                     results[normName][dateStr] = val;
+                } else if (val && isDataEmpty(val)) {
+                    // Log but don't fail - series can have gaps, empty data is treated as missing
+                    logger.log('INFO', `[DependencyFetcher] ⚠️ Empty dependency '${rawName}' found at ${dateStr} in series (allowing gap - historical lookback)`);
                 }
+                // If val is null, that's fine - missing dates in historical series are permissible
             });
         }
     }
@@ -168,10 +262,22 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
                      .doc(name);
                      
     const snap = await docRef.get();
-    if (!snap.exists) return null;
+    if (!snap.exists) {
+        // Log the missing document path clearly
+        if (config.logger) {
+            config.logger.log('ERROR', `[DependencyFetcher] ❌ Document does not exist at: ${path}`);
+        } else {
+            console.error(`[DependencyFetcher] ❌ Document does not exist at: ${path}`);
+        }
+        return null;
+    }
     
     let data = snap.data();
     
+    // CRITICAL: Don't check if empty yet - we need to load shards/compressed data first
+    // A sharded document will only have metadata in the pointer doc, but the actual data is in shards
+    // A compressed document will only have metadata + payload, but the actual data is in the payload
+    
     // 1. Handle Compression
     if (data._compressed && data.payload) {
         try {
@@ -183,12 +289,17 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
             data = { ...data, ...realData }; 
             delete data.payload; 
         } catch (e) {
-            console.warn(`[DependencyFetcher] Decompression failed for ${name}: ${e.message}`);
+            const errorMsg = `Decompression failed for ${name}: ${e.message}`;
+            if (config.logger) {
+                config.logger.log('ERROR', `[DependencyFetcher] ❌ ${errorMsg} at: ${path}`);
+            } else {
+                console.error(`[DependencyFetcher] ❌ ${errorMsg} at: ${path}`);
+            }
             return null;
         }
     }
     
-    // 2. Handle Sharding
+    // 2. Handle Sharding (MUST happen before empty check)
     if (data._sharded) {
         const shardPath = `${path}/_shards`;
         if (config.logger) {
@@ -200,21 +311,95 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
         const shardCol = docRef.collection('_shards');
         const shardSnaps = await shardCol.get();
         
-        if (!shardSnaps.empty) {
-            shardSnaps.forEach(shard => {
-                const shardData = shard.data();
-                const shardId = shard.id;
-                if (config.logger) {
-                    config.logger.log('TRACE', `[DependencyFetcher] 📂 Loading Shard '${shardId}' for '${name}' from: ${shardPath}/${shardId}`);
-                }
-                // Merge shard contents, ignoring internal metadata if it clashes
-                Object.entries(shardData).forEach(([k, v]) => {
-                    if (!k.startsWith('_')) {
-                        data[k] = v;
+        if (shardSnaps.empty) {
+            // No shards found - this is a problem
+            if (config.logger) {
+                config.logger.log('ERROR', `[DependencyFetcher] ❌ Document marked as sharded but no shards found at: ${shardPath}`);
+            } else {
+                console.error(`[DependencyFetcher] ❌ Document marked as sharded but no shards found at: ${shardPath}`);
+            }
+            return null; // Return null so it gets caught as missing
+        }
+        
+        // Merge shard contents
+        let hasData = false;
+        shardSnaps.forEach(shard => {
+            let shardData = shard.data();
+            const shardId = shard.id;
+            if (config.logger) {
+                config.logger.log('TRACE', `[DependencyFetcher] 📂 Loading Shard '${shardId}' for '${name}' from: ${shardPath}/${shardId}`);
+            }
+            
+            // CRITICAL: Shards themselves can be compressed (common in big data)
+            // Decompress the shard if needed before merging
+            if (shardData._compressed && shardData.payload) {
+                try {
+                    const buffer = (shardData.payload instanceof Buffer) ? shardData.payload : 
+                                  (shardData.payload._byteString ? Buffer.from(shardData.payload._byteString, 'base64') :
+                                  Buffer.from(shardData.payload));
+                    const decompressed = zlib.gunzipSync(buffer);
+                    const jsonStr = decompressed.toString('utf8');
+                    const realData = JSON.parse(jsonStr);
+                    // If it's double-encoded, parse again
+                    const parsedData = (typeof realData === 'string') ? JSON.parse(realData) : realData;
+                    shardData = { ...shardData, ...parsedData };
+                    delete shardData.payload;
+                } catch (e) {
+                    if (config.logger) {
+                        config.logger.log('ERROR', `[DependencyFetcher] ❌ Failed to decompress shard '${shardId}' for '${name}': ${e.message}`);
+                    } else {
+                        console.error(`[DependencyFetcher] ❌ Failed to decompress shard '${shardId}' for '${name}': ${e.message}`);
                     }
-                });
+                    // Continue with uncompressed data if decompression fails
+                }
+            }
+            
+            // Merge shard contents, ignoring internal metadata if it clashes
+            Object.entries(shardData).forEach(([k, v]) => {
+                if (!k.startsWith('_')) {
+                    data[k] = v;
+                    hasData = true;
+                }
             });
+        });
+        
+        // If shards contained no actual data, treat as empty
+        if (!hasData) {
+            if (config.logger) {
+                config.logger.log('ERROR', `[DependencyFetcher] ❌ Shards found but contain no data at: ${shardPath}`);
+            } else {
+                console.error(`[DependencyFetcher] ❌ Shards found but contain no data at: ${shardPath}`);
+            }
+            return null;
+        }
+        
+        // After loading shards, remove shard metadata from data object for cleaner output
+        // Keep only the actual data fields
+        const cleanedData = {};
+        const dataKeys = [];
+        Object.entries(data).forEach(([k, v]) => {
+            if (!k.startsWith('_')) {
+                cleanedData[k] = v;
+                dataKeys.push(k);
+            }
+        });
+        data = cleanedData;
+        
+        // Log what we loaded for debugging
+        if (config.logger) {
+            config.logger.log('INFO', `[DependencyFetcher] ✅ Loaded ${shardSnaps.size} shard(s) for '${name}'. Data fields: ${dataKeys.length > 0 ? dataKeys.slice(0, 10).join(', ') + (dataKeys.length > 10 ? `... (+${dataKeys.length - 10} more)` : '') : 'none'}`);
+        }
+    }
+    
+    // Final validation: ensure we have usable data after all processing (decompression + sharding)
+    // Only check if we haven't already determined it's empty
+    if (isDataEmpty(data)) {
+        if (config.logger) {
+            config.logger.log('ERROR', `[DependencyFetcher] ❌ Dependency '${name}' loaded but is empty (no usable data) at: ${path}`);
+        } else {
+            console.error(`[DependencyFetcher] ❌ Dependency '${name}' loaded but is empty (no usable data) at: ${path}`);
         }
+        return null;
     }
     
     return data;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.646",
+  "version": "1.0.648",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [