bulltrackers-module 1.0.276 → 1.0.277

package/functions/computation-system/data/CachedDataLoader.js

@@ -1,5 +1,6 @@
 /**
  * @fileoverview Execution-scoped data loader with caching.
+ * UPDATED: Handles Decompression of Shards.
  */
 const { 
     loadDailyInsights, 
@@ -7,6 +8,7 @@ const {
     getRelevantShardRefs, 
     getPriceShardRefs 
 } = require('../utils/data_loader');
+const zlib = require('zlib'); // [NEW]
 
 class CachedDataLoader {
     constructor(config, dependencies) { 
@@ -19,6 +21,19 @@ class CachedDataLoader {
         }; 
     }
 
+    // [NEW] Decompression Helper
+    _tryDecompress(data) {
+        if (data && data._compressed === true && data.payload) {
+            try {
+                return JSON.parse(zlib.gunzipSync(data.payload).toString());
+            } catch (e) {
+                console.error('[CachedDataLoader] Decompression failed', e);
+                return {};
+            }
+        }
+        return data;
+    }
+
     async loadMappings() { 
         if (this.cache.mappings) return this.cache.mappings; 
         const { calculationUtils } = this.deps; 
@@ -52,7 +67,8 @@ class CachedDataLoader {
         try { 
             const snap = await docRef.get(); 
             if (!snap.exists) return {}; 
-            return snap.data(); 
+            // [UPDATED] Use decompression helper
+            return this._tryDecompress(snap.data()); 
         } catch (e) { 
             console.error(`Error loading shard ${docRef.path}:`, e); 
             return {}; 

package/functions/computation-system/data/DependencyFetcher.js

@@ -1,7 +1,8 @@
 /**
- * @fileoverview Fetches results from previous computations, handling auto-sharding hydration.
+ * @fileoverview Fetches results from previous computations, handling auto-sharding and decompression.
  */
 const { normalizeName } = require('../utils/utils');
+const zlib = require('zlib'); // [NEW]
 
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
@@ -39,7 +40,20 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
             const name = names[i]; 
             if (!doc.exists) return; 
             const data = doc.data(); 
-            if (data._sharded === true) { 
+
+            // --- [NEW] DECOMPRESSION LOGIC ---
+            if (data._compressed === true && data.payload) {
+                try {
+                    // Firestore returns Buffers automatically
+                    const unzipped = zlib.gunzipSync(data.payload);
+                    fetched[name] = JSON.parse(unzipped.toString());
+                } catch (e) {
+                    console.error(`[Hydration] Failed to decompress ${name}:`, e);
+                    fetched[name] = {};
+                }
+            }
+            // --- END NEW LOGIC ---
+            else if (data._sharded === true) { 
                 hydrationPromises.push(hydrateAutoShardedResult(doc.ref, name)); 
             } else if (data._completed) { 
                 fetched[name] = data; 

package/functions/computation-system/persistence/ResultCommitter.js

@@ -1,14 +1,16 @@
 /**
  * @fileoverview Handles saving computation results with observability and Smart Cleanup.
+ * UPDATED: Implements GZIP Compression for efficient storage.
  * UPDATED: Implements Content-Based Hashing (ResultHash) for dependency short-circuiting.
  */
-const { commitBatchInChunks, generateDataHash } = require('../utils/utils'); // [UPDATED] Import generateDataHash
+const { commitBatchInChunks, generateDataHash } = require('../utils/utils');
 const { updateComputationStatus }          = require('./StatusRepository');
 const { batchStoreSchemas }                = require('../utils/schema_capture');
 const { generateProcessId, PROCESS_TYPES } = require('../logger/logger');
 const { HeuristicValidator }               = require('./ResultsValidator');
 const validationOverrides                  = require('../config/validation_overrides'); 
 const pLimit                               = require('p-limit'); 
+const zlib                                 = require('zlib'); // [NEW] Compression Lib
 
 const NON_RETRYABLE_ERRORS = [
     'PERMISSION_DENIED',     'DATA_LOSS',             'FAILED_PRECONDITION'
@@ -61,7 +63,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             
             const isEmpty = !result || (typeof result === 'object' && Object.keys(result).length === 0);
             
-            // [NEW] Calculate Result Hash (Content-Based)
+            // Calculate Result Hash (Content-Based)
             const resultHash = isEmpty ? 'empty' : generateDataHash(result);
 
             // Handle Empty Results
@@ -73,8 +75,8 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 if (calc.manifest.hash) { 
                     successUpdates[name] = { 
                         hash:        calc.manifest.hash,
-                        resultHash:  resultHash, // [NEW] Store result hash
-                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {}, // [NEW] Capture dep context
+                        resultHash:  resultHash, 
+                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {},
                         category:    calc.manifest.category, 
                         composition: calc.manifest.composition,
                         metrics: runMetrics 
@@ -107,8 +109,8 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 if (calc.manifest.hash) { 
                     successUpdates[name] = {  
                         hash:        calc.manifest.hash, 
-                        resultHash:  resultHash, // [NEW]
-                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {}, // [NEW]
+                        resultHash:  resultHash, 
+                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {},
                         category:    calc.manifest.category, 
                         composition: calc.manifest.composition,
                         metrics: runMetrics 
@@ -134,8 +136,8 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 if (calc.manifest.hash) { 
                     successUpdates[name] = {  
                         hash:        calc.manifest.hash, 
-                        resultHash:  resultHash, // [NEW]
-                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {}, // [NEW]
+                        resultHash:  resultHash,
+                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {},
                         category:    calc.manifest.category,
                         composition: calc.manifest.composition, 
                         metrics: runMetrics 
@@ -170,6 +172,44 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 }
 
 async function writeSingleResult(result, docRef, name, dateContext, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD') {
+    
+    // --- [NEW] COMPRESSION STRATEGY ---
+    // Try to compress before falling back to complex sharding
+    try {
+        const jsonString = JSON.stringify(result);
+        const rawBuffer = Buffer.from(jsonString);
+
+        // Only attempt if meaningful size (> 50KB)
+        if (rawBuffer.length > 50 * 1024) { 
+            const compressedBuffer = zlib.gzipSync(rawBuffer);
+            
+            // If compressed fits in one document (< 900KB safety limit)
+            if (compressedBuffer.length < 900 * 1024) {
+                logger.log('INFO', `[Compression] ${name}: Compressed ${(rawBuffer.length/1024).toFixed(0)}KB -> ${(compressedBuffer.length/1024).toFixed(0)}KB. Saved as Blob.`);
+                
+                const compressedPayload = {
+                    _compressed: true,
+                    _completed: true,
+                    _lastUpdated: new Date().toISOString(),
+                    payload: compressedBuffer
+                };
+                
+                // Write immediately
+                await docRef.set(compressedPayload, { merge: true });
+                
+                return { 
+                    totalSize: compressedBuffer.length, 
+                    isSharded: false, 
+                    shardCount: 1, 
+                    nextShardIndex: startShardIndex 
+                };
+            }
+        }
+    } catch (compErr) {
+        logger.log('WARN', `[Compression] Failed to compress ${name}. Falling back to standard sharding.`, compErr);
+    }
+    // --- END COMPRESSION STRATEGY ---
+
     const strategies = [ 
         { bytes: 900 * 1024, keys: null }, 
         { bytes: 450 * 1024, keys: 10000 }, 

package/functions/computation-system/utils/data_loader.js

@@ -4,7 +4,22 @@
  * --- NEW: Added streamPortfolioData async generator ---
  * --- FIXED: streamPortfolioData and streamHistoryData now accept optional 'providedRefs' ---
  * --- UPDATE: Added Smart Shard Indexing for specific ticker lookups ---
+ * --- UPDATE: Added GZIP Decompression Support for robust data loading ---
  */
+const zlib = require('zlib'); // [NEW]
+
+// [NEW] Helper for decompressing any doc if needed
+function tryDecompress(data) {
+    if (data && data._compressed === true && data.payload) {
+        try {
+            return JSON.parse(zlib.gunzipSync(data.payload).toString());
+        } catch (e) {
+            console.error('[DataLoader] Decompression failed', e);
+            return {};
+        }
+    }
+    return data;
+}
 
 /** --- Data Loader Sub-Pipes (Stateless, Dependency-Injection) --- */
 
@@ -39,7 +54,10 @@ async function loadDataByRefs(config, deps, refs) {
         const snapshots = await withRetry(() => db.getAll(...batchRefs), `getAll(batch ${Math.floor(i / batchSize)})`);
         for (const doc of snapshots) { 
             if (!doc.exists) continue; 
-            const data = doc.data(); 
+            const rawData = doc.data(); 
+            // [UPDATED] Decompress if needed
+            const data = tryDecompress(rawData);
+            
             if (data && typeof data === 'object') Object.assign(mergedPortfolios, data); 
             else logger.log('WARN', `Doc ${doc.id} exists but data is not an object`, data); 
         } 
@@ -68,7 +86,8 @@ async function loadDailyInsights(config, deps, dateString) {
         const docSnap = await withRetry(() => docRef.get(), `getInsights(${dateString})`);
         if (!docSnap.exists) { logger.log('WARN', `Insights not found for ${dateString}`); return null; }
         logger.log('TRACE', `Successfully loaded insights for ${dateString}`);
-        return docSnap.data();
+        // [UPDATED] Decompress
+        return tryDecompress(docSnap.data());
     } catch (error) { 
         logger.log('ERROR', `Failed to load daily insights for ${dateString}`, { errorMessage: error.message });
         return null;
@@ -86,7 +105,10 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
         const querySnapshot = await withRetry(() => postsCollectionRef.get(), `getSocialPosts(${dateString})`);
         if (querySnapshot.empty) { logger.log('WARN', `No social post insights for ${dateString}`); return null; }
         const postsMap = {};
-        querySnapshot.forEach(doc => { postsMap[doc.id] = doc.data(); });
+        querySnapshot.forEach(doc => { 
+            // [UPDATED] Decompress individual posts if needed
+            postsMap[doc.id] = tryDecompress(doc.data()); 
+        });
         logger.log('TRACE', `Loaded ${Object.keys(postsMap).length} social post insights`);
         return postsMap;
     } catch (error) { 
@@ -168,12 +190,6 @@ async function getPriceShardRefs(config, deps) {
  * when only specific tickers are needed.
  */
 
-/**
- * Ensures the Price Shard Index exists. If not, builds it by scanning all shards.
- * @param {object} config 
- * @param {object} deps 
- * @returns {Promise<Object>} The lookup map { "instrumentId": "shardDocId" }
- */
 /**
  * Ensures the Price Shard Index exists. If not, builds it by scanning all shards.
  * [FIX] Added TTL check to ensure new instruments are discovered.
@@ -205,7 +221,10 @@ async function ensurePriceShardIndex(config, deps) {
     
     snapshot.forEach(doc => {
         shardCount++;
-        const data = doc.data(); 
+        // [UPDATED] Robustly handle compressed shards during indexing
+        const rawData = doc.data(); 
+        const data = tryDecompress(rawData);
+
         if (data.history) {
             Object.keys(data.history).forEach(instId => {
                 index[instId] = doc.id;
@@ -273,4 +292,4 @@ module.exports = {
     getPriceShardRefs,
     ensurePriceShardIndex,
     getRelevantShardRefs
-};
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.276",
+  "version": "1.0.277",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [