bulltrackers-module 1.0.647 → 1.0.649

package/functions/computation-system/data/DependencyFetcher.js

@@ -274,15 +274,9 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
     
     let data = snap.data();
     
-    // Check if document exists but is effectively empty (only metadata)
-    if (isDataEmpty(data)) {
-        if (config.logger) {
-            config.logger.log('ERROR', `[DependencyFetcher] ❌ Document exists but is empty at: ${path} (only contains metadata fields)`);
-        } else {
-            console.error(`[DependencyFetcher] ❌ Document exists but is empty at: ${path} (only contains metadata fields)`);
-        }
-        return null; // Return null so it gets caught as missing
-    }
+    // CRITICAL: Don't check if empty yet - we need to load shards/compressed data first
+    // A sharded document will only have metadata in the pointer doc, but the actual data is in shards
+    // A compressed document will only have metadata + payload, but the actual data is in the payload
     
     // 1. Handle Compression
     if (data._compressed && data.payload) {
@@ -305,7 +299,7 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
         }
     }
     
-    // 2. Handle Sharding
+    // 2. Handle Sharding (MUST happen before empty check)
     if (data._sharded) {
         const shardPath = `${path}/_shards`;
         if (config.logger) {
@@ -330,11 +324,36 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
         // Merge shard contents
         let hasData = false;
         shardSnaps.forEach(shard => {
-            const shardData = shard.data();
+            let shardData = shard.data();
             const shardId = shard.id;
             if (config.logger) {
                 config.logger.log('TRACE', `[DependencyFetcher] 📂 Loading Shard '${shardId}' for '${name}' from: ${shardPath}/${shardId}`);
             }
+            
+            // CRITICAL: Shards themselves can be compressed (common in big data)
+            // Decompress the shard if needed before merging
+            if (shardData._compressed && shardData.payload) {
+                try {
+                    const buffer = (shardData.payload instanceof Buffer) ? shardData.payload : 
+                                  (shardData.payload._byteString ? Buffer.from(shardData.payload._byteString, 'base64') :
+                                  Buffer.from(shardData.payload));
+                    const decompressed = zlib.gunzipSync(buffer);
+                    const jsonStr = decompressed.toString('utf8');
+                    const realData = JSON.parse(jsonStr);
+                    // If it's double-encoded, parse again
+                    const parsedData = (typeof realData === 'string') ? JSON.parse(realData) : realData;
+                    shardData = { ...shardData, ...parsedData };
+                    delete shardData.payload;
+                } catch (e) {
+                    if (config.logger) {
+                        config.logger.log('ERROR', `[DependencyFetcher] ❌ Failed to decompress shard '${shardId}' for '${name}': ${e.message}`);
+                    } else {
+                        console.error(`[DependencyFetcher] ❌ Failed to decompress shard '${shardId}' for '${name}': ${e.message}`);
+                    }
+                    // Continue with uncompressed data if decompression fails
+                }
+            }
+            
             // Merge shard contents, ignoring internal metadata if it clashes
             Object.entries(shardData).forEach(([k, v]) => {
                 if (!k.startsWith('_')) {
@@ -353,9 +372,27 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
             }
             return null;
         }
+        
+        // After loading shards, remove shard metadata from data object for cleaner output
+        // Keep only the actual data fields
+        const cleanedData = {};
+        const dataKeys = [];
+        Object.entries(data).forEach(([k, v]) => {
+            if (!k.startsWith('_')) {
+                cleanedData[k] = v;
+                dataKeys.push(k);
+            }
+        });
+        data = cleanedData;
+        
+        // Log what we loaded for debugging
+        if (config.logger) {
+            config.logger.log('INFO', `[DependencyFetcher] ✅ Loaded ${shardSnaps.size} shard(s) for '${name}'. Data fields: ${dataKeys.length > 0 ? dataKeys.slice(0, 10).join(', ') + (dataKeys.length > 10 ? `... (+${dataKeys.length - 10} more)` : '') : 'none'}`);
+        }
     }
     
-    // Final validation: ensure we have usable data after all processing
+    // Final validation: ensure we have usable data after all processing (decompression + sharding)
+    // Only check if we haven't already determined it's empty
     if (isDataEmpty(data)) {
         if (config.logger) {
             config.logger.log('ERROR', `[DependencyFetcher] ❌ Dependency '${name}' loaded but is empty (no usable data) at: ${path}`);

package/functions/computation-system/executors/MetaExecutor.js

@@ -196,9 +196,50 @@ class MetaExecutor {
                 seriesData
             });
 
+            // DEBUG: Log dependency availability
+            const depNames = c.getDependencies ? c.getDependencies() : (c.dependencies || []);
+            depNames.forEach(depName => {
+                const depData = context.computed[depName];
+                if (depData) {
+                    const keys = Object.keys(depData);
+                    logger.log('INFO', `[MetaExecutor] ✅ Dependency '${depName}' available for ${c.name}. Keys: ${keys.length} (sample: ${keys.slice(0, 5).join(', ')})`);
+                } else {
+                    logger.log('ERROR', `[MetaExecutor] ❌ Dependency '${depName}' MISSING for ${c.name} in context.computed`);
+                }
+            });
+
             try {
                 const result = await inst.process(context);
-                inst.results = { [dStr]: { global: result } };
+                
+                // DEBUG: Log result before saving
+                if (result && typeof result === 'object') {
+                    const resultKeys = Object.keys(result);
+                    logger.log('INFO', `[MetaExecutor] ✅ ${c.name} computed result. Keys: ${resultKeys.length} (sample: ${resultKeys.slice(0, 10).join(', ')})`);
+                    if (resultKeys.length === 0) {
+                        logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} returned EMPTY result object!`);
+                    }
+                } else {
+                    logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} returned non-object result: ${typeof result} - ${result}`);
+                }
+                
+                // CRITICAL FIX: Do NOT overwrite this.results - the computation already sets it correctly
+                // The computation's process() method sets this.results, and getResult() returns it
+                // We only use the return value for logging/debugging
+                // inst.results should already be set by the computation's process() method
+                if (!inst.results) {
+                    logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} did not set this.results - using return value as fallback`);
+                    inst.results = result;
+                }
+                
+                // DEBUG: Verify what getResult() will return
+                const finalResult = await inst.getResult();
+                if (finalResult && typeof finalResult === 'object') {
+                    const finalKeys = Object.keys(finalResult);
+                    logger.log('INFO', `[MetaExecutor] ✅ ${c.name} getResult() will return ${finalKeys.length} keys`);
+                } else {
+                    logger.log('WARN', `[MetaExecutor] ⚠️ ${c.name} getResult() returns: ${typeof finalResult}`);
+                }
+                
                 state[c.name] = inst;
             } catch (e) {
                 logger.log('ERROR', `Meta calc ${c.name} failed: ${e.message}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.647",
+  "version": "1.0.649",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [