bulltrackers-module 1.0.290 → 1.0.291

package/functions/computation-system/helpers/computation_worker.js

@@ -1,14 +1,15 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes tasks, executes logic, and signals Workflow upon Batch Completion.
- * UPDATED: Implements "Last Worker" Callback Pattern.
+ * UPDATED: Implements IAM Auth for Workflow Callbacks.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
 const { getManifest }         = require('../topology/ManifestLoader');
 const { StructuredLogger }    = require('../logger/logger'); 
 const { recordRunAttempt }    = require('../persistence/RunRecorder'); 
-const https                   = require('https'); // [NEW] Required for callback
+const https                   = require('https'); 
+const { GoogleAuth }          = require('google-auth-library'); // [NEW] Required for Auth
 
 let calculationPackage;
 try { calculationPackage = require('aiden-shared-calculations-unified');
@@ -19,40 +20,56 @@ const MAX_RETRIES = 3;
 
 /**
  * [NEW] Helper: Fires the webhook back to Google Cloud Workflows.
+ * UPDATED: Now generates an IAM Bearer Token to authenticate the request.
  */
-function triggerWorkflowCallback(url, status, logger) {
-    if (!url) return Promise.resolve();
+async function triggerWorkflowCallback(url, status, logger) {
+    if (!url) return;
     logger.log('INFO', `[Worker] 🔔 BATCH COMPLETE! Triggering Workflow Callback: ${status}`);
     
-    return new Promise((resolve, reject) => {
-        const body = JSON.stringify({ 
-            status: status, 
-            timestamp: new Date().toISOString() 
-        });
-        
-        const req = https.request(url, {
-            method: 'POST',
-            headers: {
-                'Content-Type': 'application/json',
-                'Content-Length': Buffer.byteLength(body)
-            }
-        }, (res) => {
-            if (res.statusCode >= 200 && res.statusCode < 300) {
-                resolve();
-            } else {
-                logger.log('WARN', `Callback responded with ${res.statusCode}`);
-                resolve(); // Don't crash the worker if callback fails, logic is done.
-            }
+    try {
+        // 1. Get OAuth2 Access Token (Required for Workflows Callbacks)
+        const auth = new GoogleAuth({
+            scopes: ['https://www.googleapis.com/auth/cloud-platform']
         });
+        const client = await auth.getClient();
+        const accessToken = await client.getAccessToken();
+        const token = accessToken.token;
 
-        req.on('error', (e) => {
-            logger.log('ERROR', `Failed to trigger callback: ${e.message}`);
-            resolve(); 
+        // 2. Send Authenticated Request
+        return new Promise((resolve, reject) => {
+            const body = JSON.stringify({ 
+                status: status, 
+                timestamp: new Date().toISOString() 
+            });
+            
+            const req = https.request(url, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Content-Length': Buffer.byteLength(body),
+                    'Authorization': `Bearer ${token}` // <--- CRITICAL FIX
+                }
+            }, (res) => {
+                if (res.statusCode >= 200 && res.statusCode < 300) {
+                    resolve();
+                } else {
+                    logger.log('WARN', `Callback responded with ${res.statusCode}`);
+                    // We resolve anyway to avoid crashing the worker logic
+                    resolve(); 
+                }
+            });
+
+            req.on('error', (e) => {
+                logger.log('ERROR', `Failed to trigger callback: ${e.message}`);
+                resolve(); 
+            });
+            
+            req.write(body);
+            req.end();
         });
-        
-        req.write(body);
-        req.end();
-    });
+    } catch (e) {
+        logger.log('ERROR', `Failed to generate auth token for callback: ${e.message}`);
+    }
 }
 
 /**
@@ -133,12 +150,8 @@ async function handleComputationTask(message, config, dependencies) {
                     failedAt: new Date()
                 }, { merge: true });
 
-                // [CRITICAL] Even if it failed, we MUST decrement the counter.
-                // Otherwise the workflow waits 24h for a task that will never finish.
                 const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
                 if (callbackUrl) {
-                    // We signal SUCCESS to the workflow because the *Batch* is finished processing (even if this task failed).
-                    // The "monitor" or next pass can handle data gaps.
                     await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger);
                 }
 
@@ -162,7 +175,6 @@ async function handleComputationTask(message, config, dependencies) {
     try { computationManifest = getManifest(config.activeProductLines || [], calculations, runDependencies);
     } catch (manifestError) { 
         logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
-        // Do NOT decrement here, let PubSub retry the task.
         return; 
     }
 
@@ -178,7 +190,6 @@ async function handleComputationTask(message, config, dependencies) {
         const successUpdates = result?.updates?.successUpdates || {};
 
         if (failureReport.length > 0) {
-            // Logic/Storage failure (handled internally by executor)
             const failReason = failureReport[0]; 
             throw new Error(failReason.error.message || 'Computation Logic Failed');
         } 
@@ -189,7 +200,6 @@ async function handleComputationTask(message, config, dependencies) {
                 logger.log('WARN', `[Worker] ⚠️ Empty Result: ${computation}`);
             }
 
-            // 1. Mark Ledger as COMPLETED
             await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).doc(computation).update({
                 status: 'COMPLETED',
                 completedAt: new Date()
@@ -197,18 +207,14 @@ async function handleComputationTask(message, config, dependencies) {
 
             await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, { durationMs: duration }, triggerReason);
 
-            // 2. [NEW] Decrement Batch Counter & Check for Callback
+            // Decrement & Callback
             const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
-            
-            // 3. [NEW] If last one, fire callback
             if (callbackUrl) {
                 await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger);
             }
         }
     } catch (err) { 
         // --- ERROR HANDLING ---
-        
-        // Check for Permanent/Deterministic Errors
         const isDeterministicError = err.stage === 'SHARDING_LIMIT_EXCEEDED' || 
                                      err.stage === 'QUALITY_CIRCUIT_BREAKER' ||
                                      err.stage === 'SEMANTIC_GATE' ||
@@ -233,28 +239,23 @@ async function handleComputationTask(message, config, dependencies) {
 
                  await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'PERMANENT_FAIL' }, { durationMs: 0 }, triggerReason);
                  
-                 // [CRITICAL] Permanent failure -> Must decrement so workflow doesn't hang
                  const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
                  if (callbackUrl) {
                      await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger);
                  }
 
-                 return; // Do NOT throw, consume the message
+                 return; 
              } catch (dlqErr) { logger.log('FATAL', `[Worker] Failed to write to DLQ`, dlqErr); }
         }
 
-        // Standard Retryable Error (Transient)
         if (retryCount >= MAX_RETRIES) {
-             // Let the top-level poison check handle the decrement on the *next* delivery (or handle here if you prefer).
-             // Standard practice: throw so PubSub handles the backoff and redelivery.
-             // The poison logic at the top of this function will catch it on attempt N+1.
              throw err; 
         }
 
         logger.log('ERROR', `[Worker] ❌ Crash: ${computation}: ${err.message}`); 
         await recordRunAttempt(db, { date, computation, pass }, 'CRASH', {  message: err.message, stack: err.stack, stage: 'SYSTEM_CRASH' }, { durationMs: 0 }, triggerReason);
         
-        throw err; // Trigger Pub/Sub retry
+        throw err; 
     }
 }
 

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,6 +1,7 @@
 # Cloud Workflows Definition for BullTrackers Computation Pipeline
 # Orchestrates 5 sequential passes using Event-Driven Callbacks (Zero Polling).
-# UPDATED: Implements "Callback Pattern" to eliminate arbitrary sleeps.
+# FIXED: Replaced invalid 'sys' callback functions with 'events' library functions.
+# FIXED: Proper extraction of 'callback_details.url' for the dispatcher.
 
 main:
   params: [input]
@@ -40,11 +41,17 @@ main:
                           assign:
                             - attempt_count: ${attempt_count + 1}
                       
-                      # 1. GENERATE CALLBACK URL
-                      # This creates a unique HTTP endpoint that points specifically to this step execution.
+                      # 1. GENERATE CALLBACK ENDPOINT
+                      # We use the 'events' library. This returns an object containing the URL.
                       - create_callback:
-                          call: sys.create_callback_url
-                          result: callback_url
+                          call: events.create_callback_endpoint
+                          args:
+                              http_callback_method: "POST"
+                          result: callback_details
+
+                      - extract_callback_url:
+                          assign:
+                              - callback_url: ${callback_details.url}
 
                       - log_start:
                           call: sys.log
@@ -53,21 +60,20 @@ main:
                             severity: "INFO"
 
                       # 2. TRIGGER DISPATCHER
-                      # We pass the 'callback_url' to the dispatcher so it can hand it to the workers.
+                      # We pass the extracted 'callback_url' string to the dispatcher.
                       - trigger_dispatcher:
                           call: http.post
                           args:
                             url: ${dispatcher_url}
                             body:
                               date: ${date_to_run}
-                              callbackUrl: ${callback_url} # <--- CRITICAL: Passing the token
+                              callbackUrl: ${callback_url} 
                             auth:
                               type: OIDC
                             timeout: 1800 # 30 mins max for dispatch analysis
                           result: dispatch_response
 
                       # 3. CHECK FOR "NOTHING TO DO"
-                      # If the dispatcher found 0 tasks, it returns immediate success. We skip waiting.
                       - check_immediate_completion:
                           switch:
                             - condition: ${dispatch_response.body.dispatched == 0}
@@ -83,19 +89,20 @@ main:
                                     next: pass_retry_loop
 
                       # 4. WAIT FOR WORKER SIGNAL
-                      # The workflow freezes here (sleeps) until a worker hits the callback_url.
-                      # This eliminates the need for polling logic.
+                      # We must pass the original 'callback_details' object here, not the URL string.
                       - wait_for_completion:
-                          call: sys.await_callback
+                          call: events.await_callback
                           args:
-                            callback_url: ${callback_url}
-                            timeout: 86400 # Wait up to 24 hours for the batch to finish
+                            callback: ${callback_details}
+                            timeout: 86400 # Wait up to 24 hours
                           result: callback_request
 
                       # 5. PROCESS SIGNAL
-                      - evaluate_signal:
+                      - parse_signal:
                           assign:
                             - signal_data: ${callback_request.http_request.body}
+
+                      - evaluate_signal:
                           switch:
                             - condition: ${signal_data.status == "SUCCESS"}
                               steps:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.290",
+  "version": "1.0.291",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -35,11 +35,12 @@
     "cors": "^2.8.5",
     "dotenv": "latest",
     "express": "^4.19.2",
+    "google-auth-library": "^10.5.0",
+    "graphviz": "latest",
     "node-graphviz": "^0.1.1",
     "p-limit": "^3.1.0",
     "require-all": "^3.0.0",
-    "sharedsetup": "latest",
-    "graphviz": "latest"
+    "sharedsetup": "latest"
   },
   "devDependencies": {
     "bulltracker-deployer": "file:../bulltracker-deployer"