bulltrackers-module 1.0.647 → 1.0.648

package/functions/computation-system/data/DependencyFetcher.js

@@ -274,15 +274,9 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
     
     let data = snap.data();
     
-    // Check if document exists but is effectively empty (only metadata)
-    if (isDataEmpty(data)) {
-        if (config.logger) {
-            config.logger.log('ERROR', `[DependencyFetcher] ❌ Document exists but is empty at: ${path} (only contains metadata fields)`);
-        } else {
-            console.error(`[DependencyFetcher] ❌ Document exists but is empty at: ${path} (only contains metadata fields)`);
-        }
-        return null; // Return null so it gets caught as missing
-    }
+    // CRITICAL: Don't check if empty yet - we need to load shards/compressed data first
+    // A sharded document will only have metadata in the pointer doc, but the actual data is in shards
+    // A compressed document will only have metadata + payload, but the actual data is in the payload
     
     // 1. Handle Compression
     if (data._compressed && data.payload) {
@@ -305,7 +299,7 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
         }
     }
     
-    // 2. Handle Sharding
+    // 2. Handle Sharding (MUST happen before empty check)
     if (data._sharded) {
         const shardPath = `${path}/_shards`;
         if (config.logger) {
@@ -330,11 +324,36 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
         // Merge shard contents
         let hasData = false;
         shardSnaps.forEach(shard => {
-            const shardData = shard.data();
+            let shardData = shard.data();
             const shardId = shard.id;
             if (config.logger) {
                 config.logger.log('TRACE', `[DependencyFetcher] 📂 Loading Shard '${shardId}' for '${name}' from: ${shardPath}/${shardId}`);
             }
+            
+            // CRITICAL: Shards themselves can be compressed (common in big data)
+            // Decompress the shard if needed before merging
+            if (shardData._compressed && shardData.payload) {
+                try {
+                    const buffer = (shardData.payload instanceof Buffer) ? shardData.payload : 
+                                  (shardData.payload._byteString ? Buffer.from(shardData.payload._byteString, 'base64') :
+                                  Buffer.from(shardData.payload));
+                    const decompressed = zlib.gunzipSync(buffer);
+                    const jsonStr = decompressed.toString('utf8');
+                    const realData = JSON.parse(jsonStr);
+                    // If it's double-encoded, parse again
+                    const parsedData = (typeof realData === 'string') ? JSON.parse(realData) : realData;
+                    shardData = { ...shardData, ...parsedData };
+                    delete shardData.payload;
+                } catch (e) {
+                    if (config.logger) {
+                        config.logger.log('ERROR', `[DependencyFetcher] ❌ Failed to decompress shard '${shardId}' for '${name}': ${e.message}`);
+                    } else {
+                        console.error(`[DependencyFetcher] ❌ Failed to decompress shard '${shardId}' for '${name}': ${e.message}`);
+                    }
+                    // Continue with uncompressed data if decompression fails
+                }
+            }
+            
             // Merge shard contents, ignoring internal metadata if it clashes
             Object.entries(shardData).forEach(([k, v]) => {
                 if (!k.startsWith('_')) {
@@ -353,9 +372,27 @@ async function fetchSingleResult(db, config, dateStr, name, category) {
             }
             return null;
         }
+        
+        // After loading shards, remove shard metadata from data object for cleaner output
+        // Keep only the actual data fields
+        const cleanedData = {};
+        const dataKeys = [];
+        Object.entries(data).forEach(([k, v]) => {
+            if (!k.startsWith('_')) {
+                cleanedData[k] = v;
+                dataKeys.push(k);
+            }
+        });
+        data = cleanedData;
+        
+        // Log what we loaded for debugging
+        if (config.logger) {
+            config.logger.log('INFO', `[DependencyFetcher] ✅ Loaded ${shardSnaps.size} shard(s) for '${name}'. Data fields: ${dataKeys.length > 0 ? dataKeys.slice(0, 10).join(', ') + (dataKeys.length > 10 ? `... (+${dataKeys.length - 10} more)` : '') : 'none'}`);
+        }
     }
     
-    // Final validation: ensure we have usable data after all processing
+    // Final validation: ensure we have usable data after all processing (decompression + sharding)
+    // Only check if we haven't already determined it's empty
     if (isDataEmpty(data)) {
         if (config.logger) {
             config.logger.log('ERROR', `[DependencyFetcher] ❌ Dependency '${name}' loaded but is empty (no usable data) at: ${path}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.647",
+  "version": "1.0.648",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [