bulltrackers-module 1.0.667 → 1.0.669

package/functions/computation-system/helpers/computation_dispatcher.js

@@ -14,12 +14,69 @@ const { runFinalSweepCheck } = require('../tools/FinalSweepReporter');
 // 1. IMPORT SNAPSHOT SERVICE
 const { generateDailySnapshots } = require('../services/SnapshotService');
 const crypto = require('crypto');
+// Import Google Auth Library for OAuth 2.0 authentication
+let GoogleAuth = null;
+try {
+    GoogleAuth = require('google-auth-library').GoogleAuth;
+} catch (e) {
+    // google-auth-library might not be installed, will handle gracefully
+}
 
 const BASE_SECONDS_PER_WEIGHT_UNIT = 3;
 const SESSION_CACHE_DURATION_MS = 1000 * 60 * 30; // 30 Minutes
 const STALE_LOCK_THRESHOLD_MS = 1000 * 60 * 15;
 
-// ... [SHARED UTILS and SHARED ASYNC HELPERS remain exactly the same] ...
+/**
+ * Helper function to send an authenticated callback to Cloud Workflows
+ * Uses OAuth 2.0 access token from the default service account
+ */
+async function sendAuthenticatedCallback(callbackUrl, payload, logger, directFetch) {
+    if (!GoogleAuth) {
+        throw new Error('google-auth-library is required for authenticated callbacks. Please install: npm install google-auth-library');
+    }
+    
+    try {
+        // Get OAuth 2.0 access token using default service account credentials
+        const auth = new GoogleAuth({
+            scopes: ['https://www.googleapis.com/auth/cloud-platform']
+        });
+        const accessToken = await auth.getAccessToken();
+        
+        if (!accessToken) {
+            throw new Error('Failed to obtain OAuth 2.0 access token');
+        }
+        
+        logger.log('INFO', '[Dispatcher] ✅ Obtained OAuth 2.0 access token for callback');
+        
+        // Add timeout to prevent hanging (30 seconds should be plenty for a callback)
+        const timeoutPromise = new Promise((_, reject) => 
+            setTimeout(() => reject(new Error('Callback timeout after 30s')), 30000)
+        );
+        
+        const fetchPromise = directFetch(callbackUrl, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'Authorization': `Bearer ${accessToken}`
+            },
+            body: JSON.stringify(payload)
+        });
+        
+        const response = await Promise.race([fetchPromise, timeoutPromise]);
+        
+        if (!response.ok) {
+            const errorText = await response.text().catch(() => 'Unable to read response');
+            throw new Error(`Callback returned non-OK status: ${response.status} ${response.statusText}. Response: ${errorText.substring(0, 200)}`);
+        }
+        
+        logger.log('INFO', `[Dispatcher] ✅ Callback sent successfully (status: ${response.status})`);
+        return response;
+    } catch (err) {
+        logger.log('ERROR', `[Dispatcher] Authenticated callback failed: ${err.message}. Stack: ${err.stack?.substring(0, 300)}`);
+        throw err;
+    }
+}
+
 
 function getMillis(field) {
     if (!field) return 0;
@@ -168,27 +225,9 @@ async function handleSnapshot(config, dependencies, reqBody) {
             if (callbackUrl) {
                 logger.log('INFO', `[Dispatcher] 📞 Calling back Workflow at: ${callbackUrl}`);
                 try {
-                    // Add timeout to prevent hanging (30 seconds should be plenty for a callback)
-                    const timeoutPromise = new Promise((_, reject) => 
-                        setTimeout(() => reject(new Error('Callback timeout after 30s')), 30000)
-                    );
-                    
-                    const fetchPromise = directFetch(callbackUrl, {
-                        method: 'POST',
-                        headers: { 'Content-Type': 'application/json' },
-                        body: JSON.stringify(finalResult)
-                    });
-                    
-                    const response = await Promise.race([fetchPromise, timeoutPromise]);
-                    
-                    if (!response.ok) {
-                        const errorText = await response.text().catch(() => 'Unable to read response');
-                        logger.log('ERROR', `[Dispatcher] Callback returned non-OK status: ${response.status} ${response.statusText}. Response: ${errorText.substring(0, 200)}`);
-                    } else {
-                        logger.log('INFO', `[Dispatcher] ✅ Callback sent successfully (status: ${response.status})`);
-                    }
+                    await sendAuthenticatedCallback(callbackUrl, finalResult, logger, directFetch);
                 } catch (err) {
-                    logger.log('ERROR', `[Dispatcher] Callback failed: ${err.message}. Stack: ${err.stack?.substring(0, 300)}`);
+                    logger.log('ERROR', `[Dispatcher] Callback failed: ${err.message}`);
                 }
             } else {
                 logger.log('WARN', '[Dispatcher] No callback URL provided, workflow will not be notified');
@@ -240,27 +279,9 @@ async function handleSnapshot(config, dependencies, reqBody) {
         if (callbackUrl) {
             logger.log('INFO', `[Dispatcher] 📞 Calling back Workflow at: ${callbackUrl}`);
             try {
-                // Add timeout to prevent hanging (30 seconds should be plenty for a callback)
-                const timeoutPromise = new Promise((_, reject) => 
-                    setTimeout(() => reject(new Error('Callback timeout after 30s')), 30000)
-                );
-                
-                const fetchPromise = directFetch(callbackUrl, {
-                    method: 'POST',
-                    headers: { 'Content-Type': 'application/json' },
-                    body: JSON.stringify(finalResult)
-                });
-                
-                const response = await Promise.race([fetchPromise, timeoutPromise]);
-                
-                if (!response.ok) {
-                    const errorText = await response.text().catch(() => 'Unable to read response');
-                    logger.log('ERROR', `[Dispatcher] Callback returned non-OK status: ${response.status} ${response.statusText}. Response: ${errorText.substring(0, 200)}`);
-                } else {
-                    logger.log('INFO', `[Dispatcher] ✅ Callback sent successfully (status: ${response.status})`);
-                }
+                await sendAuthenticatedCallback(callbackUrl, finalResult, logger, directFetch);
             } catch (err) {
-                logger.log('ERROR', `[Dispatcher] Callback failed: ${err.message}. Stack: ${err.stack?.substring(0, 300)}`);
+                logger.log('ERROR', `[Dispatcher] Callback failed: ${err.message}`);
             }
         } else {
             logger.log('WARN', '[Dispatcher] No callback URL provided, workflow will not be notified');
@@ -275,27 +296,9 @@ async function handleSnapshot(config, dependencies, reqBody) {
         if (callbackUrl) {
             logger.log('INFO', `[Dispatcher] 📞 Calling back Workflow with error at: ${callbackUrl}`);
             try {
-                // Add timeout to prevent hanging (30 seconds should be plenty for a callback)
-                const timeoutPromise = new Promise((_, reject) => 
-                    setTimeout(() => reject(new Error('Callback timeout after 30s')), 30000)
-                );
-                
-                const fetchPromise = directFetch(callbackUrl, {
-                    method: 'POST',
-                    headers: { 'Content-Type': 'application/json' },
-                    body: JSON.stringify(errorResult)
-                });
-                
-                const response = await Promise.race([fetchPromise, timeoutPromise]);
-                
-                if (!response.ok) {
-                    const errorText = await response.text().catch(() => 'Unable to read response');
-                    logger.log('ERROR', `[Dispatcher] Error callback returned non-OK status: ${response.status} ${response.statusText}. Response: ${errorText.substring(0, 200)}`);
-                } else {
-                    logger.log('INFO', `[Dispatcher] ✅ Error callback sent successfully (status: ${response.status})`);
-                }
+                await sendAuthenticatedCallback(callbackUrl, errorResult, logger, directFetch);
             } catch (err) {
-                logger.log('ERROR', `[Dispatcher] Error callback failed: ${err.message}. Stack: ${err.stack?.substring(0, 300)}`);
+                logger.log('ERROR', `[Dispatcher] Error callback failed: ${err.message}`);
             }
         } else {
             logger.log('WARN', '[Dispatcher] No callback URL provided, workflow will not be notified of error');

package/functions/computation-system/utils/data_loader.js

@@ -38,7 +38,18 @@ async function tryLoadFromGCS(config, dateString, snapshotName, logger) {
         if (exists) {
             logger.log('INFO', `[DataLoader] ⚡️ GCS HIT: ${snapshotName} for ${dateString}`);
             const [content] = await file.download();
-            return JSON.parse(zlib.gunzipSync(content).toString());
+            
+            // FIX: GCS client auto-decompresses if Content-Encoding is gzip.
+            // We try gunzip first; if it fails with header check, it's likely already JSON.
+            try {
+                return JSON.parse(zlib.gunzipSync(content).toString());
+            } catch (zipError) {
+                if (zipError.message && zipError.message.includes('incorrect header check')) {
+                    // Content was already decompressed by the client
+                    return JSON.parse(content.toString());
+                }
+                throw zipError;
+            }
         }
     } catch (e) {
         logger.log('WARN', `[DataLoader] GCS Check Failed (${snapshotName}): ${e.message}`);
@@ -434,7 +445,19 @@ async function* streamPortfolioData(config, deps, dateString, providedRefs = nul
             if (exists) {
                 logger.log('INFO', `[DataLoader] ⚡️ STREAMING: Hydrating Portfolios from GCS Snapshot`);
                 const [content] = await file.download();
-                const fullData = JSON.parse(zlib.gunzipSync(content).toString());
+                
+                // FIX: Handle Double Decompression
+                let fullData;
+                try {
+                    fullData = JSON.parse(zlib.gunzipSync(content).toString());
+                } catch (zipError) {
+                     if (zipError.message && zipError.message.includes('incorrect header check')) {
+                        fullData = JSON.parse(content.toString());
+                    } else {
+                        throw zipError;
+                    }
+                }
+
                 yield fullData; // Yield all in one chunk as it fits in memory
                 return;
             }
@@ -473,7 +496,7 @@ async function* streamHistoryData(config, deps, dateString, providedRefs = null,
             if (exists) {
                 logger.log('INFO', `[DataLoader] ⚡️ STREAMING: Hydrating History from GCS (JSONL)`);
                 
-                const fileStream = file.createReadStream().pipe(zlib.createGunzip());
+                const fileStream = file.createReadStream();
                 const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
 
                 let currentBatch = {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.667",
+  "version": "1.0.669",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [