bulltrackers-module 1.0.441 → 1.0.443

package/functions/generic-api/user-api/helpers/data_helpers.js

@@ -1668,25 +1668,57 @@ async function checkPiInComputationDate(db, insightsCollection, resultsSub, comp
             return { found: false, profileData: null, computationData: null };
         }
         
-        // Decompress if needed
         const rawData = computationDoc.data();
-        let computationData = tryDecompress(rawData);
-        
-        // Handle string decompression result
-        if (typeof computationData === 'string') {
-            try {
-                computationData = JSON.parse(computationData);
-            } catch (e) {
-                logger.log('WARN', `[checkPiInComputationDate] Failed to parse decompressed string for date ${dateStr}:`, e.message);
-                return { found: false, profileData: null, computationData: null };
+        let computationData = null;
+        
+        // Check if data is sharded
+        if (rawData._sharded === true && rawData._shardCount) {
+            // Data is stored in shards - read all shards and merge
+            const shardsCol = computationRef.collection('_shards');
+            const shardCount = rawData._shardCount;
+            
+            logger.log('INFO', `[checkPiInComputationDate] Reading ${shardCount} shards for date ${dateStr}`);
+            
+            computationData = {};
+            
+            // Read all shards (shard_0, shard_1, ..., shard_N-1)
+            for (let i = 0; i < shardCount; i++) {
+                const shardDoc = await shardsCol.doc(`shard_${i}`).get();
+                if (shardDoc.exists) {
+                    const shardData = shardDoc.data();
+                    // Merge shard data into computationData
+                    Object.assign(computationData, shardData);
+                } else {
+                    logger.log('WARN', `[checkPiInComputationDate] Shard shard_${i} missing for date ${dateStr}`);
+                }
+            }
+        } else {
+            // Data is in the main document (compressed or raw)
+            computationData = tryDecompress(rawData);
+            
+            // Handle string decompression result
+            if (typeof computationData === 'string') {
+                try {
+                    computationData = JSON.parse(computationData);
+                } catch (e) {
+                    logger.log('WARN', `[checkPiInComputationDate] Failed to parse decompressed string for date ${dateStr}:`, e.message);
+                    return { found: false, profileData: null, computationData: null };
+                }
             }
         }
         
         // Check if CID exists in the computation data
         if (computationData && typeof computationData === 'object' && !Array.isArray(computationData)) {
+            // Filter out metadata keys that start with underscore
+            const cids = Object.keys(computationData).filter(key => !key.startsWith('_'));
+            
+            // Check if the requested CID exists
             const profileData = computationData[cidStr];
             if (profileData) {
+                logger.log('INFO', `[checkPiInComputationDate] Found CID ${cidStr} in date ${dateStr} (total CIDs: ${cids.length})`);
                 return { found: true, profileData, computationData };
+            } else {
+                logger.log('INFO', `[checkPiInComputationDate] CID ${cidStr} not found in date ${dateStr}. Available CIDs: ${cids.slice(0, 10).join(', ')}${cids.length > 10 ? '...' : ''}`);
             }
         }
         
@@ -1795,8 +1827,11 @@ async function getPiProfile(req, res, dependencies, config) {
                 logger
             );
             
+            // Filter out metadata keys (those starting with _) when listing available CIDs
             const allAvailableCids = latestResult.computationData && typeof latestResult.computationData === 'object' && !Array.isArray(latestResult.computationData)
-                ? Object.keys(latestResult.computationData).sort()
+                ? Object.keys(latestResult.computationData)
+                    .filter(key => !key.startsWith('_'))
+                    .sort()
                 : [];
             
             return res.status(404).json({ 

package/functions/generic-api/user-api/helpers/on_demand_fetch_helpers.js

@@ -295,10 +295,39 @@ async function getPiFetchStatus(req, res, dependencies, config) {
                     
                     const doc = await docRef.get();
                     if (doc.exists) {
-                        const { tryDecompress } = require('./data_helpers');
-                        const data = tryDecompress(doc.data());
+                        const docData = doc.data();
+                        let mergedData = null;
                         
-                        if (data && data[String(piCidNum)]) {
+                        // Check if data is sharded
+                        if (docData._sharded === true && docData._shardCount) {
+                            // Data is stored in shards - read all shards and merge
+                            const shardsCol = docRef.collection('_shards');
+                            const shardsSnapshot = await shardsCol.get();
+                            
+                            if (!shardsSnapshot.empty) {
+                                mergedData = {};
+                                for (const shardDoc of shardsSnapshot.docs) {
+                                    const shardData = shardDoc.data();
+                                    Object.assign(mergedData, shardData);
+                                }
+                            }
+                        } else {
+                            // Data is in the main document (compressed or not)
+                            const { tryDecompress } = require('./data_helpers');
+                            mergedData = tryDecompress(docData);
+                            
+                            // Handle string decompression result
+                            if (typeof mergedData === 'string') {
+                                try {
+                                    mergedData = JSON.parse(mergedData);
+                                } catch (e) {
+                                    logger.log('WARN', `[getPiFetchStatus] Failed to parse decompressed string for date ${dateStrFormatted}:`, e.message);
+                                    mergedData = null;
+                                }
+                            }
+                        }
+                        
+                        if (mergedData && typeof mergedData === 'object' && mergedData[String(piCidNum)]) {
                             // Computation completed! Update status
                             status = 'completed';
                             await requestsSnapshot.docs[0].ref.update({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.441",
+  "version": "1.0.443",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [