bulltrackers-module 1.0.659 → 1.0.661

package/functions/alert-system/helpers/alert_helpers.js

@@ -5,9 +5,12 @@
 
 const { FieldValue } = require('@google-cloud/firestore');
 const zlib = require('zlib');
+const { Storage } = require('@google-cloud/storage');
 const { getAlertTypeByComputation, generateAlertMessage } = require('./alert_type_registry');
 // Migration helpers removed - write directly to new path
 
+const storage = new Storage(); // Singleton GCS Client
+
 /**
  * Process alerts for a specific PI from computation results
  */
@@ -474,10 +477,48 @@ function readComputationResults(docData) {
 }
 
 /**
- * Read computation results, handling sharded data
+ * Read computation results, handling GCS pointers, sharded data, and compressed data
+ * UPDATED: Added GCS pointer support to read from GCS when data is offloaded
  */
-async function readComputationResultsWithShards(db, docData, docRef) {
+async function readComputationResultsWithShards(db, docData, docRef, logger = null) {
   try {
+    // -------------------------------------------------------------------------
+    // 1. GCS POINTER HANDLER (Check first - highest priority)
+    // -------------------------------------------------------------------------
+    if (docData.gcsUri || (docData._gcs && docData.gcsBucket && docData.gcsPath)) {
+      try {
+        const bucketName = docData.gcsBucket || docData.gcsUri.split('/')[2];
+        const fileName = docData.gcsPath || docData.gcsUri.split('/').slice(3).join('/');
+        
+        if (logger) {
+          logger.log('INFO', `[AlertSystem] Reading computation results from GCS: ${fileName}`);
+        }
+        
+        // Stream download is memory efficient for large files
+        const [fileContent] = await storage.bucket(bucketName).file(fileName).download();
+        
+        // Assume Gzip (as writer does it), if fails try plain
+        let decompressedData;
+        try {
+          decompressedData = JSON.parse(zlib.gunzipSync(fileContent).toString('utf8'));
+        } catch (gzipErr) {
+          // Fallback for uncompressed GCS files
+          decompressedData = JSON.parse(fileContent.toString('utf8'));
+        }
+        
+        // Process the decompressed data through readComputationResults
+        return readComputationResults(decompressedData);
+      } catch (gcsErr) {
+        if (logger) {
+          logger.log('ERROR', `[AlertSystem] GCS fetch failed, falling back to Firestore: ${gcsErr.message}`);
+        }
+        // Fall through to Firestore logic below
+      }
+    }
+
+    // -------------------------------------------------------------------------
+    // 2. FIRESTORE SHARDED HANDLER
+    // -------------------------------------------------------------------------
     if (docData._sharded === true && docData._shardCount) {
       const shardsCol = docRef.collection('_shards');
       const shardsSnapshot = await shardsCol.get();
@@ -492,9 +533,17 @@ async function readComputationResultsWithShards(db, docData, docRef) {
         return readComputationResults(mergedData);
       }
     }
+
+    // -------------------------------------------------------------------------
+    // 3. FIRESTORE COMPRESSED OR DIRECT DATA HANDLER
+    // -------------------------------------------------------------------------
     return readComputationResults(docData);
   } catch (error) {
-    console.error('[readComputationResultsWithShards] Error reading sharded results', error);
+    if (logger) {
+      logger.log('ERROR', `[AlertSystem] Error reading computation results: ${error.message}`);
+    } else {
+      console.error('[readComputationResultsWithShards] Error reading sharded results', error);
+    }
     return { cids: [], metadata: {}, perUserData: {} };
   }
 }

package/functions/alert-system/index.js

@@ -72,9 +72,9 @@ async function handleAlertTrigger(message, context, config, dependencies) {
       return;
     }
     
-    // 3. Read and decompress computation results
+    // 3. Read and decompress computation results (handling GCS, shards, and compression)
     const docData = docSnapshot.data();
-    const results = readComputationResults(docData);
+    const results = await readComputationResultsWithShards(db, docData, docRef, logger);
     
     if (!results.cids || results.cids.length === 0) {
       logger.log('INFO', `[AlertTrigger] No PIs found in computation results for ${computationName}`);
@@ -188,7 +188,7 @@ async function handleComputationResultWrite(change, context, config, dependencie
     // If it's PopularInvestorProfileMetrics, check for all-clear notifications only
     if (isProfileMetrics) {
       const docData = change.after.data();
-      const results = await readComputationResultsWithShards(db, docData, change.after.ref);
+      const results = await readComputationResultsWithShards(db, docData, change.after.ref, logger);
       if (results.cids && results.cids.length > 0) {
         await checkAndSendAllClearNotifications(db, logger, results.cids, date, config, dependencies);
       }
@@ -203,9 +203,9 @@ async function handleComputationResultWrite(change, context, config, dependencie
     
     logger.log('INFO', `[AlertTrigger] Processing alert computation: ${computationName} for date ${date}`);
     
-    // 2. Read and decompress computation results (handling shards)
+    // 2. Read and decompress computation results (handling GCS, shards, and compression)
     const docData = change.after.data();
-    const results = await readComputationResultsWithShards(db, docData, change.after.ref);
+    const results = await readComputationResultsWithShards(db, docData, change.after.ref, logger);
     
     if (!results.cids || results.cids.length === 0) {
       logger.log('INFO', `[AlertTrigger] No PIs found in computation results for ${computationName}`);

package/functions/api-v2/helpers/data-fetchers/firestore.js

@@ -1,10 +1,13 @@
 // Firestore helper functions for fetching data from collections
 const { FieldValue, Timestamp } = require('@google-cloud/firestore');
+const { Storage } = require('@google-cloud/storage');
 const { dispatchSyncRequest } = require('../task_engine_helper.js');
 const { sanitizeCid, sanitizeDocId } = require('../security_utils.js');
 const crypto = require('crypto');
 const zlib = require('zlib');
 
+const storage = new Storage(); // Singleton GCS Client
+
 // 1. Fetch latest stored snapshots of user data from a user-centric collection
 
 // Examples
@@ -1210,14 +1213,40 @@ const getComputationResults = async (db, computationName, dateStr, userId = null
 
         const pointerData = pointerSnap.data();
 
-        // 2. Strategy: Compressed Data
+        // 2. Strategy: GCS Pointer (Check first - highest priority)
+        // If _gcs is true or gcsUri exists, the data is stored in GCS
+        // Note: Page mode is exempt from GCS logic (handled separately below)
+        if (pointerData._isPageMode !== true && (pointerData.gcsUri || (pointerData._gcs && pointerData.gcsBucket && pointerData.gcsPath))) {
+            try {
+                const bucketName = pointerData.gcsBucket || pointerData.gcsUri.split('/')[2];
+                const fileName = pointerData.gcsPath || pointerData.gcsUri.split('/').slice(3).join('/');
+                
+                console.log(`[Computation] Reading from GCS: ${fileName} for ${computationName}`);
+                
+                // Stream download is memory efficient for large files
+                const [fileContent] = await storage.bucket(bucketName).file(fileName).download();
+                
+                // Assume Gzip (as writer does it), if fails try plain
+                try {
+                    return JSON.parse(zlib.gunzipSync(fileContent).toString('utf8'));
+                } catch (gzipErr) {
+                    // Fallback for uncompressed GCS files
+                    return JSON.parse(fileContent.toString('utf8'));
+                }
+            } catch (gcsErr) {
+                console.error(`[Computation] GCS fetch failed for ${computationName}, falling back to Firestore: ${gcsErr.message}`);
+                // Fall through to Firestore strategies below
+            }
+        }
+
+        // 3. Strategy: Compressed Data
         // If _compressed is true, the data is inside the payload field, just zipped.
         if (pointerData._compressed === true) {
             console.log(`[Computation] Reading compressed data for ${computationName}`);
             return tryDecompress(pointerData);
         }
 
-        // 3. Strategy: Sharded Data
+        // 4. Strategy: Sharded Data
         // If _sharded is true, we must fetch N documents from the _shards subcollection.
         if (pointerData._sharded === true) {
             const shardCount = pointerData._shardCount || 0;
@@ -1260,14 +1289,15 @@ const getComputationResults = async (db, computationName, dateStr, userId = null
             return reassembledData;
         }
 
-        // 4. Strategy: Page Mode (User Centric)
+        // 5. Strategy: Page Mode (User Centric)
         // If _isPageMode is true, we delegate to the pageCollection helper.
+        // Note: Page mode is exempt from GCS logic (uses individual user documents)
         if (pointerData._isPageMode === true) {
             console.log(`[Computation] Fetching page mode data for ${computationName} / User: ${userId}`);
             return await pageCollection(db, dateStr, computationName, userId);
         }
 
-        // 5. Strategy: Standard (Direct Read)
+        // 6. Strategy: Standard (Direct Read)
         // If no flags are set, the data is in the pointer document itself.
         console.log(`[Computation] Returning direct pointer data for ${computationName}`);
         return pointerData;

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -190,15 +190,25 @@ async function handleForceRun(config, dependencies, computationManifest, reqBody
         const hasFailedDep = report.failedDependency.some(t => normalizeName(t.name) === targetComputationNormalized);
         const isImpossible = report.impossible.some(t => normalizeName(t.name) === targetComputationNormalized);
         const isBlocked = report.blocked.some(t => normalizeName(t.name) === targetComputationNormalized);
+        const isSkipped = report.skipped.some(t => normalizeName(t.name) === targetComputationNormalized);
         
-        if (isRunnable || needsReRun || hasFailedDep) {
-            runnableDates.push(date);
-        } else if (isImpossible) {
+        // For force runs: treat skipped computations (already stored with valid hash) as runnable
+        // They will overwrite with the same result, which is fine for testing
+        // Only mark as impossible if root data or dependencies don't exist at all
+        if (isImpossible) {
             skippedDates.push({ date, reason: report.impossible.find(t => normalizeName(t.name) === targetComputationNormalized)?.reason || 'Impossible' });
+        } else if (isRunnable || needsReRun || hasFailedDep || isSkipped) {
+            // Runnable, needs re-run, has failed deps (but not impossible), or skipped (already stored)
+            // All of these are runnable for force runs - will overwrite existing results if needed
+            runnableDates.push(date);
         } else if (isBlocked) {
-            skippedDates.push({ date, reason: report.blocked.find(t => normalizeName(t.name) === targetComputationNormalized)?.reason || 'Blocked' });
+            // Blocked usually means waiting for data - for force runs, if root data exists, still runnable
+            // Only skip if truly impossible (handled above)
+            runnableDates.push(date);
         } else {
-            skippedDates.push({ date, reason: 'Not runnable (unknown reason)' });
+            // Unknown state - for force runs, if root data exists (which it does, since result is not null), treat as runnable
+            logger.log('WARN', `[ForceRun] Computation ${computationName} in unknown state for ${date}, treating as runnable`);
+            runnableDates.push(date);
         }
     }
     

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.659",
+  "version": "1.0.661",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [