bulltrackers-module 1.0.545 → 1.0.547

package/functions/computation-system/persistence/ResultCommitter.js

@@ -235,19 +235,18 @@ async function fetchContracts(db, calcNames) {
 
 async function writeSingleResult(result, docRef, name, dateContext, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD', isInitialWrite = false) {
     const opCounts = { writes: 0, deletes: 0 };
+    
+    // [FIX 1] Always check for shards if we might compress, not just on 'isInitialWrite'. 
+    // This prevents zombie shards from persisting if a previous run failed.
     let wasSharded = false;
-    let shouldWipeShards = false;
-    let rootMergeOption = !isInitialWrite; 
-
-    if (isInitialWrite) {
-        try {
-            const currentSnap = await docRef.get();
-            if (currentSnap.exists) {
-                const d = currentSnap.data();
-                wasSharded = (d._sharded === true);
-                if (wasSharded) shouldWipeShards = true;
-            }
-        } catch (e) {}
+    try {
+        const currentSnap = await docRef.get();
+        if (currentSnap.exists) {
+            const d = currentSnap.data();
+            wasSharded = (d._sharded === true);
+        }
+    } catch (e) {
+        // Ignore read errors, proceed with write attempts
     }
 
     // --- COMPRESSION STRATEGY ---
@@ -259,13 +258,25 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
             const compressedBuffer = zlib.gzipSync(rawBuffer);
             if (compressedBuffer.length < 900 * 1024) {
                 logger.log('INFO', `[Compression] ${name}: Compressed ${(rawBuffer.length/1024).toFixed(0)}KB -> ${(compressedBuffer.length/1024).toFixed(0)}KB.`);
-                const compressedPayload = { _compressed: true, _completed: true, _lastUpdated: new Date().toISOString(), payload: compressedBuffer };
                 
-                if (shouldWipeShards) {
+                const compressedPayload = { 
+                    _compressed: true, 
+                    _completed: true, 
+                    _lastUpdated: new Date().toISOString(), 
+                    payload: compressedBuffer 
+                };
+                
+                // [FIX 2] Self-Healing: If we are writing compressed, we MUST ensure shards are gone.
+                // We use a batch to atomically write the blob and delete the shards.
+                if (wasSharded) {
                     const updates = [];
                     const shardCol = docRef.collection('_shards');
                     const shardDocs = await shardCol.listDocuments();
+                    
+                    // Add delete ops for all shards
                     shardDocs.forEach(d => updates.push({ type: 'DELETE', ref: d }));
+                    
+                    // [FIX 3] Use merge: false (overwrite) to clear old indexes and ensure we don't inherit "too many indexes" error.
                     updates.push({ ref: docRef, data: compressedPayload, options: { merge: false } });
                     
                     opCounts.deletes += shardDocs.length;
@@ -273,30 +284,47 @@ async function writeSingleResult(result, docRef, name, dateContext, logger, conf
 
                     await commitBatchInChunks(config, deps, updates, `${name}::Cleanup+Compress`);
                 } else {
-                    await docRef.set(compressedPayload, { merge: rootMergeOption });
+                    // [FIX 3] Use merge: false here as well for consistency and safety.
+                    await docRef.set(compressedPayload, { merge: false });
                     opCounts.writes += 1;
                 }
                 
                 return { totalSize: compressedBuffer.length, isSharded: false, shardCount: 1, nextShardIndex: startShardIndex, opCounts };
             }
         }
-    } catch (compErr) {}
+    } catch (compErr) {
+        // [FIX 4] Log the error but allow fallthrough to sharding (Self-Healing).
+        // This ensures that if compression fails (e.g., unexpected encoding issue), we still try to save the data via sharding.
+        logger.log('WARN', `[SelfHealing] Compression failed for ${name}, reverting to sharding. Error: ${compErr.message}`);
+    }
 
+    // --- SHARDING STRATEGY (Fallback) ---
     const strategies = [ { bytes: 900 * 1024, keys: null }, { bytes: 450 * 1024, keys: 10000 }, { bytes: 200 * 1024, keys: 2000 }, { bytes: 100 * 1024, keys: 50 } ];
     let committed = false; let lastError = null; 
     let finalStats = { totalSize: 0, isSharded: false, shardCount: 1, nextShardIndex: startShardIndex };
+    let rootMergeOption = !isInitialWrite; 
+
+    // If we detected shards earlier but failed compression, we need to make sure we wipe them if we are about to re-shard 
+    // (though prepareAutoShardedWrites handles the _shards logic, we might need to be careful about the root doc).
+    // In this specific flow, prepareAutoShardedWrites handles new shard creation. The existing logic below handles wiping shards if needed.
+    // However, since we might have skipped the compression block, let's ensure 'wasSharded' logic is respected here if we need to reset.
+    let shouldWipeShards = wasSharded; 
 
     for (let attempt = 0; attempt < strategies.length; attempt++) {
         if (committed) break;
         const constraints = strategies[attempt];
         try {
             const updates = await prepareAutoShardedWrites(result, docRef, logger, constraints.bytes, constraints.keys, startShardIndex, flushMode);
+            
             if (shouldWipeShards) {
                  const shardCol = docRef.collection('_shards');
                  const shardDocs = await shardCol.listDocuments();
+                 
+                 // Prepend delete operations for existing shards to ensure clean slate
                  shardDocs.forEach(d => updates.unshift({ type: 'DELETE', ref: d }));
-                 shouldWipeShards = false;
+                 shouldWipeShards = false; // Only do this once
             }
+            
             const rootUpdate = updates.find(u => u.ref.path === docRef.path && u.type !== 'DELETE');
             if (rootUpdate) { rootUpdate.options = { merge: rootMergeOption }; }
 

package/functions/task-engine/helpers/popular_investor_helpers.js

@@ -108,22 +108,22 @@ async function processPortfolio(context, config, taskData, isPI) {
     data.username = username;
 
     if (isPI) {
-        // Deep Dive Logic for PIs
-        if (config.enablePiDeepPortfolio !== false) {
-            const topPositions = (data.AggregatedPositions || []).sort((a, b) => b.Invested - a.Invested).slice(0, 10);
-            const deepPositions = [];
-            for (const pos of topPositions) {
-                try {
-                    const posUrl = `${config.ETORO_API_POSITIONS_URL}?cid=${cid}&InstrumentID=${pos.InstrumentID}&client_request_id=${uuid}`;
-                    const deepRes = await fetchWithRetry(posUrl, requestOptions, proxyManager, logger, 'DeepPos', headerManager);
-                    deepPositions.push({ instrumentId: pos.InstrumentID, ...(await deepRes.json()) });
-                } catch (e) {} // Skip failed deep positions
-            }
-            if (deepPositions.length) await storePopularInvestorPortfolio({ db, logger, collectionRegistry, cid, date: today, portfolioData: data, deepPortfolioData: { positions: deepPositions } });
-            else await storePopularInvestorPortfolio({ db, logger, collectionRegistry, cid, date: today, portfolioData: data });
-        } else {
-            await storePopularInvestorPortfolio({ db, logger, collectionRegistry, cid, date: today, portfolioData: data });
-        }
+        // Deep Dive Logic for PIs - DISABLED
+        // if (config.enablePiDeepPortfolio === true) {
+        //     const topPositions = (data.AggregatedPositions || []).sort((a, b) => b.Invested - a.Invested).slice(0, 10);
+        //     const deepPositions = [];
+        //     for (const pos of topPositions) {
+        //         try {
+        //             const posUrl = `${config.ETORO_API_POSITIONS_URL}?cid=${cid}&InstrumentID=${pos.InstrumentID}&client_request_id=${uuid}`;
+        //             const deepRes = await fetchWithRetry(posUrl, requestOptions, proxyManager, logger, 'DeepPos', headerManager);
+        //             deepPositions.push({ instrumentId: pos.InstrumentID, ...(await deepRes.json()) });
+        //         } catch (e) {} // Skip failed deep positions
+        //     }
+        //     if (deepPositions.length) await storePopularInvestorPortfolio({ db, logger, collectionRegistry, cid, date: today, portfolioData: data, deepPortfolioData: { positions: deepPositions } });
+        //     else await storePopularInvestorPortfolio({ db, logger, collectionRegistry, cid, date: today, portfolioData: data });
+        // } else {
+        await storePopularInvestorPortfolio({ db, logger, collectionRegistry, cid, date: today, portfolioData: data });
+        // }
     } else {
         await storeSignedInUserPortfolio({ db, logger, collectionRegistry, cid, date: today, portfolioData: data });
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.545",
+  "version": "1.0.547",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [