bulltrackers-module 1.0.276 → 1.0.278

package/functions/computation-system/data/CachedDataLoader.js

@@ -1,5 +1,6 @@
 /**
  * @fileoverview Execution-scoped data loader with caching.
+ * UPDATED: Handles Decompression of Shards.
  */
 const { 
     loadDailyInsights, 
@@ -7,6 +8,7 @@ const {
     getRelevantShardRefs, 
     getPriceShardRefs 
 } = require('../utils/data_loader');
+const zlib = require('zlib'); // [NEW]
 
 class CachedDataLoader {
     constructor(config, dependencies) { 
@@ -19,6 +21,19 @@ class CachedDataLoader {
         }; 
     }
 
+    // [NEW] Decompression Helper
+    _tryDecompress(data) {
+        if (data && data._compressed === true && data.payload) {
+            try {
+                return JSON.parse(zlib.gunzipSync(data.payload).toString());
+            } catch (e) {
+                console.error('[CachedDataLoader] Decompression failed', e);
+                return {};
+            }
+        }
+        return data;
+    }
+
     async loadMappings() { 
         if (this.cache.mappings) return this.cache.mappings; 
         const { calculationUtils } = this.deps; 
@@ -52,7 +67,8 @@ class CachedDataLoader {
         try { 
             const snap = await docRef.get(); 
             if (!snap.exists) return {}; 
-            return snap.data(); 
+            // [UPDATED] Use decompression helper
+            return this._tryDecompress(snap.data()); 
         } catch (e) { 
             console.error(`Error loading shard ${docRef.path}:`, e); 
             return {}; 

package/functions/computation-system/data/DependencyFetcher.js

@@ -1,7 +1,8 @@
 /**
- * @fileoverview Fetches results from previous computations, handling auto-sharding hydration.
+ * @fileoverview Fetches results from previous computations, handling auto-sharding and decompression.
  */
 const { normalizeName } = require('../utils/utils');
+const zlib = require('zlib'); // [NEW]
 
 async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config, { db }, includeSelf = false) {
     const manifestMap = new Map(fullManifest.map(c => [normalizeName(c.name), c]));
@@ -39,7 +40,20 @@ async function fetchExistingResults(dateStr, calcsInPass, fullManifest, config,
             const name = names[i]; 
             if (!doc.exists) return; 
             const data = doc.data(); 
-            if (data._sharded === true) { 
+
+            // --- [NEW] DECOMPRESSION LOGIC ---
+            if (data._compressed === true && data.payload) {
+                try {
+                    // Firestore returns Buffers automatically
+                    const unzipped = zlib.gunzipSync(data.payload);
+                    fetched[name] = JSON.parse(unzipped.toString());
+                } catch (e) {
+                    console.error(`[Hydration] Failed to decompress ${name}:`, e);
+                    fetched[name] = {};
+                }
+            }
+            // --- END NEW LOGIC ---
+            else if (data._sharded === true) { 
                 hydrationPromises.push(hydrateAutoShardedResult(doc.ref, name)); 
             } else if (data._completed) { 
                 fetched[name] = data; 

package/functions/computation-system/persistence/ResultCommitter.js

@@ -1,14 +1,17 @@
 /**
  * @fileoverview Handles saving computation results with observability and Smart Cleanup.
+ * UPDATED: Implements GZIP Compression for efficient storage.
  * UPDATED: Implements Content-Based Hashing (ResultHash) for dependency short-circuiting.
+ * UPDATED: Auto-enforces Weekend Mode validation for 'Price-Only' computations.
  */
-const { commitBatchInChunks, generateDataHash } = require('../utils/utils'); // [UPDATED] Import generateDataHash
+const { commitBatchInChunks, generateDataHash } = require('../utils/utils');
 const { updateComputationStatus }          = require('./StatusRepository');
 const { batchStoreSchemas }                = require('../utils/schema_capture');
 const { generateProcessId, PROCESS_TYPES } = require('../logger/logger');
 const { HeuristicValidator }               = require('./ResultsValidator');
 const validationOverrides                  = require('../config/validation_overrides'); 
 const pLimit                               = require('p-limit'); 
+const zlib                                 = require('zlib'); // [NEW] Compression Lib
 
 const NON_RETRYABLE_ERRORS = [
     'PERMISSION_DENIED',     'DATA_LOSS',             'FAILED_PRECONDITION'
@@ -26,30 +29,49 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
     const pid            = generateProcessId(PROCESS_TYPES.STORAGE, passName, dStr);
     
     // Options defaults
-    const flushMode    = options.flushMode || 'STANDARD';
-    const shardIndexes = options.shardIndexes || {};
+    const flushMode        = options.flushMode || 'STANDARD';
+    const shardIndexes     = options.shardIndexes || {};
     const nextShardIndexes = {};
 
     const fanOutLimit = pLimit(10); 
 
     for (const name in stateObj) {
-        const calc = stateObj[name];
+        const calc      = stateObj[name];
         const execStats = calc._executionStats || { processedUsers: 0, skippedUsers: 0 };
         const currentShardIndex = shardIndexes[name] || 0;
 
         const runMetrics = {
-            storage: { sizeBytes: 0, isSharded: false, shardCount: 1, keys: 0 },
+            storage:    { sizeBytes: 0, isSharded: false, shardCount: 1, keys: 0 },
             validation: { isValid: true, anomalies: [] },
             execution: execStats
         };
 
         try {
             const result = await calc.getResult();
-            const overrides   = validationOverrides[calc.manifest.name] || {};
+            const configOverrides = validationOverrides[calc.manifest.name] || {};
             
+            // --- [NEW] AUTO-ENFORCE WEEKEND MODE FOR PRICE-ONLY CALCS ---
+            // If a calculation depends SOLELY on 'price', we assume market closures
+            // will cause 0s/Flatlines on weekends, so we enforce lenient validation.
+            const dataDeps = calc.manifest.rootDataDependencies || [];
+            const isPriceOnly = (dataDeps.length === 1 && dataDeps[0] === 'price');
+            
+            let effectiveOverrides = { ...configOverrides };
+            
+            if (isPriceOnly && !effectiveOverrides.weekend) {
+                effectiveOverrides.weekend = {
+                    maxZeroPct: 100,
+                    maxFlatlinePct: 100,
+                    maxNullPct: 100 // Allow full nulls (e.g. holidays)
+                };
+            }
+            // -----------------------------------------------------------
+
             // Validation
             if (result && Object.keys(result).length > 0) {
-                const healthCheck = HeuristicValidator.analyze(calc.manifest.name, result, overrides);
+                // [FIX] Added 'dStr' as 3rd argument to match HeuristicValidator signature
+                const healthCheck = HeuristicValidator.analyze(calc.manifest.name, result, dStr, effectiveOverrides);
+                
                 if (!healthCheck.valid) {
                     runMetrics.validation.isValid = false;
                     runMetrics.validation.anomalies.push(healthCheck.reason);
@@ -61,7 +83,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             
             const isEmpty = !result || (typeof result === 'object' && Object.keys(result).length === 0);
             
-            // [NEW] Calculate Result Hash (Content-Based)
+            // Calculate Result Hash (Content-Based)
             const resultHash = isEmpty ? 'empty' : generateDataHash(result);
 
             // Handle Empty Results
@@ -73,8 +95,8 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 if (calc.manifest.hash) { 
                     successUpdates[name] = { 
                         hash:        calc.manifest.hash,
-                        resultHash:  resultHash, // [NEW] Store result hash
-                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {}, // [NEW] Capture dep context
+                        resultHash:  resultHash, 
+                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {},
                         category:    calc.manifest.category, 
                         composition: calc.manifest.composition,
                         metrics: runMetrics 
@@ -89,7 +111,7 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
             const isMultiDate = resultKeys.length > 0 && resultKeys.every(k => /^\d{4}-\d{2}-\d{2}$/.test(k));
 
             if (isMultiDate) {
-                const datePromises = resultKeys.map((historicalDate) => fanOutLimit(async () => {
+                const datePromises  = resultKeys.map((historicalDate) => fanOutLimit(async () => {
                     const dailyData = result[historicalDate];
                     if (!dailyData || Object.keys(dailyData).length === 0) return;
 
@@ -107,8 +129,8 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 if (calc.manifest.hash) { 
                     successUpdates[name] = {  
                         hash:        calc.manifest.hash, 
-                        resultHash:  resultHash, // [NEW]
-                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {}, // [NEW]
+                        resultHash:  resultHash, 
+                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {},
                         category:    calc.manifest.category, 
                         composition: calc.manifest.composition,
                         metrics: runMetrics 
@@ -134,8 +156,8 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
                 if (calc.manifest.hash) { 
                     successUpdates[name] = {  
                         hash:        calc.manifest.hash, 
-                        resultHash:  resultHash, // [NEW]
-                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {}, // [NEW]
+                        resultHash:  resultHash,
+                        dependencyResultHashes: calc.manifest.dependencyResultHashes || {},
                         category:    calc.manifest.category,
                         composition: calc.manifest.composition, 
                         metrics: runMetrics 
@@ -170,6 +192,44 @@ async function commitResults(stateObj, dStr, passName, config, deps, skipStatusW
 }
 
 async function writeSingleResult(result, docRef, name, dateContext, logger, config, deps, startShardIndex = 0, flushMode = 'STANDARD') {
+    
+    // --- [NEW] COMPRESSION STRATEGY ---
+    // Try to compress before falling back to complex sharding
+    try {
+        const jsonString = JSON.stringify(result);
+        const rawBuffer = Buffer.from(jsonString);
+
+        // Only attempt if meaningful size (> 50KB)
+        if (rawBuffer.length > 50 * 1024) { 
+            const compressedBuffer = zlib.gzipSync(rawBuffer);
+            
+            // If compressed fits in one document (< 900KB safety limit)
+            if (compressedBuffer.length < 900 * 1024) {
+                logger.log('INFO', `[Compression] ${name}: Compressed ${(rawBuffer.length/1024).toFixed(0)}KB -> ${(compressedBuffer.length/1024).toFixed(0)}KB. Saved as Blob.`);
+                
+                const compressedPayload = {
+                    _compressed: true,
+                    _completed: true,
+                    _lastUpdated: new Date().toISOString(),
+                    payload: compressedBuffer
+                };
+                
+                // Write immediately
+                await docRef.set(compressedPayload, { merge: true });
+                
+                return { 
+                    totalSize: compressedBuffer.length, 
+                    isSharded: false, 
+                    shardCount: 1, 
+                    nextShardIndex: startShardIndex 
+                };
+            }
+        }
+    } catch (compErr) {
+        logger.log('WARN', `[Compression] Failed to compress ${name}. Falling back to standard sharding.`, compErr);
+    }
+    // --- END COMPRESSION STRATEGY ---
+
     const strategies = [ 
         { bytes: 900 * 1024, keys: null }, 
         { bytes: 450 * 1024, keys: 10000 }, 

package/functions/computation-system/persistence/ResultsValidator.js

@@ -24,11 +24,11 @@ class HeuristicValidator {
         const sampleSize = Math.min(totalItems, 100);
         const step = Math.floor(totalItems / sampleSize);
         
-        let zeroCount = 0;
-        let nullCount = 0;
-        let nanCount = 0; 
+        let zeroCount        = 0;
+        let nullCount        = 0;
+        let nanCount         = 0; 
         let emptyVectorCount = 0; 
-        let analyzedCount = 0;
+        let analyzedCount    = 0;
         
         const numericValues = []; 
 
@@ -82,9 +82,9 @@ class HeuristicValidator {
 
         // Default Thresholds
         let thresholds = {
-            maxZeroPct: overrides.maxZeroPct ?? 99, 
-            maxNullPct: overrides.maxNullPct ?? 90,
-            maxNanPct:  overrides.maxNanPct  ?? 0,   
+            maxZeroPct:     overrides.maxZeroPct     ?? 99, 
+            maxNullPct:     overrides.maxNullPct     ?? 90,
+            maxNanPct:      overrides.maxNanPct      ?? 0,   
             maxFlatlinePct: overrides.maxFlatlinePct ?? 95 
         };
 

package/functions/computation-system/tools/BuildReporter.js

@@ -4,6 +4,7 @@
  * REFACTORED: Strict 5-category reporting with date-based exclusion logic.
  * UPDATED: Added transactional locking to prevent duplicate reports on concurrent cold starts.
  * UPDATED: Adds 'pass' number to detail records for better waterfall visibility.
+ * FIXED: Ensures 'latest' pointer updates even if detail writes fail.
  */
 
 const { analyzeDateExecution }                                             = require('../WorkflowOrchestrator');
@@ -242,15 +243,34 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
         scanRange: `${datesToCheck[0]} to ${datesToCheck[datesToCheck.length-1]}` 
     };
 
+    // 1. Write the main report header (Specific Version)
     const reportRef = db.collection('computation_build_records').doc(buildId);
     await reportRef.set(reportHeader);
 
+    // 2. Write Details (Protected)
+    // [FIX] We wrap this in try-catch so that if the massive detail write fails,
+    // we still update the 'latest' pointer to the new version.
+    let detailsSuccess = true;
     if (detailWrites.length > 0) {
         logger.log('INFO', `[BuildReporter] Writing ${detailWrites.length} detail records...`);
-        await commitBatchInChunks(config, dependencies, detailWrites, 'BuildReportDetails');
+        try {
+            await commitBatchInChunks(config, dependencies, detailWrites, 'BuildReportDetails');
+        } catch (detailErr) {
+            detailsSuccess = false;
+            logger.log('ERROR', `[BuildReporter] ⚠️ Failed to write all details, but Report Header is saved.`, detailErr);
+        }
     }
 
-    await db.collection('computation_build_records').doc('latest').set({ ...reportHeader, note: "Latest build report pointer (See subcollection for details)." });
+    // 3. Update 'latest' pointer
+    // This now runs even if details failed, preventing the version mismatch bug.
+    const latestMetadata = { 
+        ...reportHeader, 
+        note: detailsSuccess 
+            ? "Latest build report pointer (See subcollection for details)." 
+            : "Latest build report pointer (WARNING: Partial detail records due to write error)."
+    };
+    
+    await db.collection('computation_build_records').doc('latest').set(latestMetadata);
 
     logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, New: ${totalRun}.`);
     

package/functions/computation-system/utils/data_loader.js

@@ -4,7 +4,22 @@
  * --- NEW: Added streamPortfolioData async generator ---
  * --- FIXED: streamPortfolioData and streamHistoryData now accept optional 'providedRefs' ---
  * --- UPDATE: Added Smart Shard Indexing for specific ticker lookups ---
+ * --- UPDATE: Added GZIP Decompression Support for robust data loading ---
  */
+const zlib = require('zlib'); // [NEW]
+
+// [NEW] Helper for decompressing any doc if needed
+function tryDecompress(data) {
+    if (data && data._compressed === true && data.payload) {
+        try {
+            return JSON.parse(zlib.gunzipSync(data.payload).toString());
+        } catch (e) {
+            console.error('[DataLoader] Decompression failed', e);
+            return {};
+        }
+    }
+    return data;
+}
 
 /** --- Data Loader Sub-Pipes (Stateless, Dependency-Injection) --- */
 
@@ -39,7 +54,10 @@ async function loadDataByRefs(config, deps, refs) {
         const snapshots = await withRetry(() => db.getAll(...batchRefs), `getAll(batch ${Math.floor(i / batchSize)})`);
         for (const doc of snapshots) { 
             if (!doc.exists) continue; 
-            const data = doc.data(); 
+            const rawData = doc.data(); 
+            // [UPDATED] Decompress if needed
+            const data = tryDecompress(rawData);
+            
             if (data && typeof data === 'object') Object.assign(mergedPortfolios, data); 
             else logger.log('WARN', `Doc ${doc.id} exists but data is not an object`, data); 
         } 
@@ -68,7 +86,8 @@ async function loadDailyInsights(config, deps, dateString) {
         const docSnap = await withRetry(() => docRef.get(), `getInsights(${dateString})`);
         if (!docSnap.exists) { logger.log('WARN', `Insights not found for ${dateString}`); return null; }
         logger.log('TRACE', `Successfully loaded insights for ${dateString}`);
-        return docSnap.data();
+        // [UPDATED] Decompress
+        return tryDecompress(docSnap.data());
     } catch (error) { 
         logger.log('ERROR', `Failed to load daily insights for ${dateString}`, { errorMessage: error.message });
         return null;
@@ -86,7 +105,10 @@ async function loadDailySocialPostInsights(config, deps, dateString) {
         const querySnapshot = await withRetry(() => postsCollectionRef.get(), `getSocialPosts(${dateString})`);
         if (querySnapshot.empty) { logger.log('WARN', `No social post insights for ${dateString}`); return null; }
         const postsMap = {};
-        querySnapshot.forEach(doc => { postsMap[doc.id] = doc.data(); });
+        querySnapshot.forEach(doc => { 
+            // [UPDATED] Decompress individual posts if needed
+            postsMap[doc.id] = tryDecompress(doc.data()); 
+        });
         logger.log('TRACE', `Loaded ${Object.keys(postsMap).length} social post insights`);
         return postsMap;
     } catch (error) { 
@@ -168,12 +190,6 @@ async function getPriceShardRefs(config, deps) {
  * when only specific tickers are needed.
  */
 
-/**
- * Ensures the Price Shard Index exists. If not, builds it by scanning all shards.
- * @param {object} config 
- * @param {object} deps 
- * @returns {Promise<Object>} The lookup map { "instrumentId": "shardDocId" }
- */
 /**
  * Ensures the Price Shard Index exists. If not, builds it by scanning all shards.
  * [FIX] Added TTL check to ensure new instruments are discovered.
@@ -205,7 +221,10 @@ async function ensurePriceShardIndex(config, deps) {
     
     snapshot.forEach(doc => {
         shardCount++;
-        const data = doc.data(); 
+        // [UPDATED] Robustly handle compressed shards during indexing
+        const rawData = doc.data(); 
+        const data = tryDecompress(rawData);
+
         if (data.history) {
             Object.keys(data.history).forEach(instId => {
                 index[instId] = doc.id;
@@ -273,4 +292,4 @@ module.exports = {
     getPriceShardRefs,
     ensurePriceShardIndex,
     getRelevantShardRefs
-};
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.276",
+  "version": "1.0.278",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [