bulltrackers-module 1.0.290 → 1.0.292

package/functions/computation-system/helpers/computation_worker.js

@@ -1,14 +1,15 @@
 /**
  * FILENAME: computation-system/helpers/computation_worker.js
  * PURPOSE: Consumes tasks, executes logic, and signals Workflow upon Batch Completion.
- * UPDATED: Implements "Last Worker" Callback Pattern.
+ * UPDATED: Implements IAM Auth for Workflow Callbacks.
  */
 
 const { executeDispatchTask } = require('../WorkflowOrchestrator.js');
 const { getManifest }         = require('../topology/ManifestLoader');
 const { StructuredLogger }    = require('../logger/logger'); 
 const { recordRunAttempt }    = require('../persistence/RunRecorder'); 
-const https                   = require('https'); // [NEW] Required for callback
+const https                   = require('https'); 
+const { GoogleAuth }          = require('google-auth-library'); // [NEW] Required for Auth
 
 let calculationPackage;
 try { calculationPackage = require('aiden-shared-calculations-unified');
@@ -19,40 +20,56 @@ const MAX_RETRIES = 3;
 
 /**
  * [NEW] Helper: Fires the webhook back to Google Cloud Workflows.
+ * UPDATED: Now generates an IAM Bearer Token to authenticate the request.
  */
-function triggerWorkflowCallback(url, status, logger) {
-    if (!url) return Promise.resolve();
+async function triggerWorkflowCallback(url, status, logger) {
+    if (!url) return;
     logger.log('INFO', `[Worker] 🔔 BATCH COMPLETE! Triggering Workflow Callback: ${status}`);
     
-    return new Promise((resolve, reject) => {
-        const body = JSON.stringify({ 
-            status: status, 
-            timestamp: new Date().toISOString() 
-        });
-        
-        const req = https.request(url, {
-            method: 'POST',
-            headers: {
-                'Content-Type': 'application/json',
-                'Content-Length': Buffer.byteLength(body)
-            }
-        }, (res) => {
-            if (res.statusCode >= 200 && res.statusCode < 300) {
-                resolve();
-            } else {
-                logger.log('WARN', `Callback responded with ${res.statusCode}`);
-                resolve(); // Don't crash the worker if callback fails, logic is done.
-            }
+    try {
+        // 1. Get OAuth2 Access Token (Required for Workflows Callbacks)
+        const auth = new GoogleAuth({
+            scopes: ['https://www.googleapis.com/auth/cloud-platform']
         });
+        const client = await auth.getClient();
+        const accessToken = await client.getAccessToken();
+        const token = accessToken.token;
 
-        req.on('error', (e) => {
-            logger.log('ERROR', `Failed to trigger callback: ${e.message}`);
-            resolve(); 
+        // 2. Send Authenticated Request
+        return new Promise((resolve, reject) => {
+            const body = JSON.stringify({ 
+                status: status, 
+                timestamp: new Date().toISOString() 
+            });
+            
+            const req = https.request(url, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Content-Length': Buffer.byteLength(body),
+                    'Authorization': `Bearer ${token}` // <--- CRITICAL FIX
+                }
+            }, (res) => {
+                if (res.statusCode >= 200 && res.statusCode < 300) {
+                    resolve();
+                } else {
+                    logger.log('WARN', `Callback responded with ${res.statusCode}`);
+                    // We resolve anyway to avoid crashing the worker logic
+                    resolve(); 
+                }
+            });
+
+            req.on('error', (e) => {
+                logger.log('ERROR', `Failed to trigger callback: ${e.message}`);
+                resolve(); 
+            });
+            
+            req.write(body);
+            req.end();
         });
-        
-        req.write(body);
-        req.end();
-    });
+    } catch (e) {
+        logger.log('ERROR', `Failed to generate auth token for callback: ${e.message}`);
+    }
 }
 
 /**
@@ -133,12 +150,8 @@ async function handleComputationTask(message, config, dependencies) {
                     failedAt: new Date()
                 }, { merge: true });
 
-                // [CRITICAL] Even if it failed, we MUST decrement the counter.
-                // Otherwise the workflow waits 24h for a task that will never finish.
                 const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
                 if (callbackUrl) {
-                    // We signal SUCCESS to the workflow because the *Batch* is finished processing (even if this task failed).
-                    // The "monitor" or next pass can handle data gaps.
                     await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger);
                 }
 
@@ -162,7 +175,6 @@ async function handleComputationTask(message, config, dependencies) {
     try { computationManifest = getManifest(config.activeProductLines || [], calculations, runDependencies);
     } catch (manifestError) { 
         logger.log('FATAL', `[Worker] Failed to load Manifest: ${manifestError.message}`);
-        // Do NOT decrement here, let PubSub retry the task.
         return; 
     }
 
@@ -178,7 +190,6 @@ async function handleComputationTask(message, config, dependencies) {
         const successUpdates = result?.updates?.successUpdates || {};
 
         if (failureReport.length > 0) {
-            // Logic/Storage failure (handled internally by executor)
             const failReason = failureReport[0]; 
             throw new Error(failReason.error.message || 'Computation Logic Failed');
         } 
@@ -189,7 +200,6 @@ async function handleComputationTask(message, config, dependencies) {
                 logger.log('WARN', `[Worker] ⚠️ Empty Result: ${computation}`);
             }
 
-            // 1. Mark Ledger as COMPLETED
             await db.collection(`computation_audit_ledger/${date}/passes/${pass}/tasks`).doc(computation).update({
                 status: 'COMPLETED',
                 completedAt: new Date()
@@ -197,18 +207,14 @@ async function handleComputationTask(message, config, dependencies) {
 
             await recordRunAttempt(db, { date, computation, pass }, 'SUCCESS', null, { durationMs: duration }, triggerReason);
 
-            // 2. [NEW] Decrement Batch Counter & Check for Callback
+            // Decrement & Callback
             const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
-            
-            // 3. [NEW] If last one, fire callback
             if (callbackUrl) {
                 await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger);
             }
         }
     } catch (err) { 
         // --- ERROR HANDLING ---
-        
-        // Check for Permanent/Deterministic Errors
         const isDeterministicError = err.stage === 'SHARDING_LIMIT_EXCEEDED' || 
                                      err.stage === 'QUALITY_CIRCUIT_BREAKER' ||
                                      err.stage === 'SEMANTIC_GATE' ||
@@ -233,28 +239,23 @@ async function handleComputationTask(message, config, dependencies) {
 
                  await recordRunAttempt(db, { date, computation, pass }, 'FAILURE', { message: err.message, stage: err.stage || 'PERMANENT_FAIL' }, { durationMs: 0 }, triggerReason);
                  
-                 // [CRITICAL] Permanent failure -> Must decrement so workflow doesn't hang
                  const callbackUrl = await decrementAndCheck(db, metaStatePath, logger);
                  if (callbackUrl) {
                      await triggerWorkflowCallback(callbackUrl, 'SUCCESS', logger);
                  }
 
-                 return; // Do NOT throw, consume the message
+                 return; 
              } catch (dlqErr) { logger.log('FATAL', `[Worker] Failed to write to DLQ`, dlqErr); }
         }
 
-        // Standard Retryable Error (Transient)
         if (retryCount >= MAX_RETRIES) {
-             // Let the top-level poison check handle the decrement on the *next* delivery (or handle here if you prefer).
-             // Standard practice: throw so PubSub handles the backoff and redelivery.
-             // The poison logic at the top of this function will catch it on attempt N+1.
              throw err; 
         }
 
         logger.log('ERROR', `[Worker] ❌ Crash: ${computation}: ${err.message}`); 
         await recordRunAttempt(db, { date, computation, pass }, 'CRASH', {  message: err.message, stack: err.stack, stage: 'SYSTEM_CRASH' }, { durationMs: 0 }, triggerReason);
         
-        throw err; // Trigger Pub/Sub retry
+        throw err; 
     }
 }
 

package/functions/computation-system/tools/BuildReporter.js

@@ -6,41 +6,29 @@ const SimRunner = require('../simulation/SimRunner');
 const pLimit = require('p-limit');
 const path = require('path');
 const crypto = require('crypto');
-const fs = require('fs'); // [NEW] Required for file reading
+const fs = require('fs'); 
 const packageJson = require(path.join(__dirname, '..', '..', '..', 'package.json'));
 const packageVersion = packageJson.version;
-const { generateCodeHash } = require('../utils/utils'); // Reuse your standard logic
+const { generateCodeHash } = require('../utils/utils'); 
 
-// Persistent Registry for SimHashes
 const SIMHASH_REGISTRY_COLLECTION = 'system_simhash_registry';
 
-// --- [NEW] RECURSIVE SYSTEM HASHING ---
-
-// 1. define the Root of the system (one level up from 'tools')
+// --- RECURSIVE SYSTEM HASHING ---
 const SYSTEM_ROOT = path.resolve(__dirname, '..');
-
-// 2. Define what to ignore to prevent noise or infinite loops
 const IGNORED_DIRS = new Set(['node_modules', '.git', '.idea', 'coverage', 'logs', 'tests']);
 const IGNORED_FILES = new Set(['package-lock.json', '.DS_Store', '.env']);
 
-/**
- * Recursively walks a directory and returns a list of file paths.
- */
 function walkSync(dir, fileList = []) {
     const files = fs.readdirSync(dir);
-    
     files.forEach(file => {
         if (IGNORED_FILES.has(file)) return;
-        
         const filePath = path.join(dir, file);
         const stat = fs.statSync(filePath);
-        
         if (stat.isDirectory()) {
             if (!IGNORED_DIRS.has(file)) {
                 walkSync(filePath, fileList);
             }
         } else {
-            // Only hash code files (add .yaml if you want workflows included)
             if (file.endsWith('.js') || file.endsWith('.json') || file.endsWith('.yaml')) {
                 fileList.push(filePath);
             }
@@ -49,69 +37,41 @@ function walkSync(dir, fileList = []) {
     return fileList;
 }
 
-/**
- * Generates a single hash representing the entire infrastructure code state.
- */
 function getInfrastructureHash() {
     try {
         const allFiles = walkSync(SYSTEM_ROOT);
-        allFiles.sort(); // Crucial for determinism
-
+        allFiles.sort(); 
         const bigHash = crypto.createHash('sha256');
-
         for (const filePath of allFiles) {
             const content = fs.readFileSync(filePath, 'utf8');
             const relativePath = path.relative(SYSTEM_ROOT, filePath);
-
-            // DECISION: How to clean?
             let cleanContent = content;
-
-            // 1. If it's JS, use your system standard for code hashing
             if (filePath.endsWith('.js')) {
-                // This strips comments and whitespace consistently with ManifestBuilder
-                // Note: generateCodeHash returns a hash, we can just use that hash
                 cleanContent = generateCodeHash(content); 
-            } 
-            // 2. If it's JSON/YAML, just strip basic whitespace to ignore indent changes
-            else {
+            } else {
                 cleanContent = content.replace(/\s+/g, '');
             }
-
-            // Feed the PATH and the CLEAN CONTENT into the master hash
             bigHash.update(`${relativePath}:${cleanContent}|`);
         }
-
         return bigHash.digest('hex');
     } catch (e) {
         console.warn(`[BuildReporter] ⚠️ Failed to generate infra hash: ${e.message}`);
-        return 'infra_hash_error'; // Fail safe so we run the report rather than crash
+        return 'infra_hash_error'; 
     }
 }
 
-/**
- * UPDATED: System Fingerprint = Manifest Hash + Infrastructure Hash
- */
 function getSystemFingerprint(manifest) {
-    // 1. Business Logic Hash (The Calculations)
     const sortedManifestHashes = manifest.map(c => c.hash).sort().join('|');
-    
-    // 2. Infrastructure Hash (The System Code)
     const infraHash = getInfrastructureHash();
-
-    // 3. Combine
     return crypto.createHash('sha256')
         .update(sortedManifestHashes + infraHash)
         .digest('hex');
 }
 
-/**
- * Helper: Determines if a calculation should be excluded from the report.
- */
 function isDateBeforeAvailability(dateStr, calcManifest) {
     const targetDate = new Date(dateStr + 'T00:00:00Z');
     const deps = calcManifest.rootDataDependencies || [];
     if (deps.length === 0) return false;
-
     for (const dep of deps) {
         let startDate = null;
         if (dep === 'portfolio')      startDate = DEFINITIVE_EARLIEST_DATES.portfolio;
@@ -119,19 +79,14 @@ function isDateBeforeAvailability(dateStr, calcManifest) {
         else if (dep === 'social')    startDate = DEFINITIVE_EARLIEST_DATES.social;
         else if (dep === 'insights')  startDate = DEFINITIVE_EARLIEST_DATES.insights;
         else if (dep === 'price')     startDate = DEFINITIVE_EARLIEST_DATES.price;
-
         if (startDate && targetDate < startDate) { return true; }
     }
     return false;
 }
 
-/**
- * Helper: Calculates the transitive closure of dependents (Blast Radius).
- */
 function calculateBlastRadius(targetCalcName, reverseGraph) {
     const impactSet = new Set();
     const queue = [targetCalcName];
-    
     while(queue.length > 0) {
         const current = queue.shift();
         const dependents = reverseGraph.get(current) || [];
@@ -142,7 +97,6 @@ function calculateBlastRadius(targetCalcName, reverseGraph) {
             }
         });
     }
-
     return {
         directDependents: (reverseGraph.get(targetCalcName) || []).length,
         totalCascadingDependents: impactSet.size,
@@ -150,14 +104,11 @@ function calculateBlastRadius(targetCalcName, reverseGraph) {
     };
 }
 
-/**
- * [NEW] Helper: Runs SimHash check with Caching and Registry Persistence.
- */
 async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, logger, simHashCache, db) {
     const trueReRuns = [];
     const stableUpdates = [];
     
-    // Limit concurrency for simulations
+    // Concurrency for simulations
     const limit = pLimit(10);
 
     const checks = candidates.map(item => limit(async () => {
@@ -170,16 +121,10 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
                 return;
             }
 
-            // 1. Check Cache first (Avoid re-simulating the same code for 100 different dates)
             let newSimHash = simHashCache.get(manifest.hash);
-
-            // 2. If Miss, Run Simulation & Persist to Registry
             if (!newSimHash) {
                 newSimHash = await SimRunner.run(manifest, manifestMap);
                 simHashCache.set(manifest.hash, newSimHash);
-                
-                // Write to Registry so Production Workers can find it without running SimRunner
-                // Fire-and-forget write to reduce latency
                 db.collection(SIMHASH_REGISTRY_COLLECTION).doc(manifest.hash).set({
                     simHash: newSimHash,
                     createdAt: new Date(),
@@ -187,17 +132,14 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
                 }).catch(err => logger.log('WARN', `Failed to write SimHash registry for ${manifest.name}: ${err.message}`));
             }
 
-            // 3. Compare
             if (newSimHash === stored.simHash) {
-                // BEHAVIORAL MATCH: Code changed, but output is identical.
                 stableUpdates.push({
                     ...item,
                     reason: "Code Updated (Logic Stable)",
-                    simHash: newSimHash, // New SimHash (same as old)
-                    newHash: manifest.hash // New Code Hash
+                    simHash: newSimHash, 
+                    newHash: manifest.hash 
                 });
             } else {
-                // BEHAVIORAL MISMATCH: Logic changed.
                 trueReRuns.push({
                     ...item,
                     reason: item.reason + ` [SimHash Mismatch]`,
@@ -215,9 +157,6 @@ async function verifyBehavioralStability(candidates, manifestMap, dailyStatus, l
     return { trueReRuns, stableUpdates };
 }
 
-/**
- * AUTO-RUN ENTRY POINT
- */
 async function ensureBuildReport(config, dependencies, manifest) {
     const { db, logger } = dependencies;
     const now = new Date();
@@ -234,40 +173,28 @@ async function ensureBuildReport(config, dependencies, manifest) {
 
         if (!shouldRun) { logger.log('INFO', `[BuildReporter] 🔒 Report for v${packageVersion} locked. Skipping.`); return; }
 
-        // [NEW] 1. Calculate Current System Fingerprint
         const currentSystemHash = getSystemFingerprint(manifest);
-
-        // [NEW] 2. Fetch Last Build's Fingerprint
         const latestBuildDoc = await db.collection('computation_build_records').doc('latest').get();
         
         if (latestBuildDoc.exists) {
             const latestData = latestBuildDoc.data();
-            
-            // [OPTIMIZATION] If signatures match, we can clone the report or just skip
             if (latestData.systemFingerprint === currentSystemHash) {
                 logger.log('INFO', `[BuildReporter] ⚡ System Fingerprint (${currentSystemHash.substring(0,8)}) matches latest build. Skipping Report.`);
-                
-                // Create a "Skipped" record for the new version so we know it deployed
                 await db.collection('computation_build_records').doc(buildId).set({
                     buildId,
                     packageVersion,
                     systemFingerprint: currentSystemHash,
                     status: 'SKIPPED_IDENTICAL',
-                    referenceBuild: latestData.buildId, // Pointer to the build that actually did the work
+                    referenceBuild: latestData.buildId,
                     generatedAt: new Date().toISOString()
                 });
-                
-                // Release lock and exit
                 lockRef.update({ status: 'SKIPPED', completedAt: new Date() }).catch(() => {});
                 return;
             }
         }
 
         logger.log('INFO', `[BuildReporter] 🚀 Change Detected. Running Pre-flight Report for v${packageVersion}...`);
-        
-        // Pass the fingerprint to generateBuildReport so it can save it
         await generateBuildReport(config, dependencies, manifest, 90, buildId, currentSystemHash);
-        
         lockRef.update({ status: 'COMPLETED', completedAt: new Date() }).catch(() => {});
 
     } catch (e) {
@@ -275,14 +202,9 @@ async function ensureBuildReport(config, dependencies, manifest) {
     }
 }
 
-/**
- * Generates the report, writes to Firestore, AND FIXES STABLE UPDATES.
- */
 async function generateBuildReport(config, dependencies, manifest, daysBack = 90, customBuildId = null, systemFingerprint = null) {
     const { db, logger } = dependencies;
     const buildId = customBuildId || `manual_${Date.now()}`;
-    
-    // Calculate fingerprint if not provided (for manual runs)
     const finalFingerprint = systemFingerprint || getSystemFingerprint(manifest);
 
     logger.log('INFO', `[BuildReporter] Generating Build Report: ${buildId} (Scope: ${daysBack} days, Fingerprint: ${finalFingerprint.substring(0,8)})...`);
@@ -293,8 +215,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     
     const datesToCheck = getExpectedDateStrings(startDate, today);
     const manifestMap  = new Map(manifest.map(c => [normalizeName(c.name), c]));
-    
-    // [OPTIMIZATION] Cache SimHashes across dates so we only calculate once per code version
     const simHashCache = new Map();
 
     const reverseGraph = new Map();
@@ -312,16 +232,16 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
     const reportHeader = { 
         buildId, 
         packageVersion, 
-        systemFingerprint: finalFingerprint, // Saved to Firestore
+        systemFingerprint: finalFingerprint, 
         generatedAt: new Date().toISOString(), 
         summary: {}, 
         _sharded: true 
     };
 
     let totalRun = 0, totalReRun = 0, totalStable = 0;
-    const detailWrites = []; 
 
-    const limit = pLimit(20);
+    // [FIX] Reduced concurrency from 20 to 5 to avoid Firestore DEADLINE_EXCEEDED
+    const limit = pLimit(5);
 
     const processingPromises = datesToCheck.map(dateStr => limit(async () => {
         try {
@@ -374,29 +294,22 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             // 1. RUN
             analysis.runnable.forEach(item => pushIfValid(dateSummary.run, item, "New Calculation"));
 
-            // 2. RE-RUN & STABLE Analysis (SimHash Integration)
+            // 2. RE-RUN & STABLE Analysis
             if (analysis.reRuns.length > 0) {
-                // Pass simHashCache and db for registry writes
                 const { trueReRuns, stableUpdates } = await verifyBehavioralStability(analysis.reRuns, manifestMap, dailyStatus, logger, simHashCache, db);
                 
                 trueReRuns.forEach(item => pushIfValid(dateSummary.rerun, item, "Logic Changed"));
                 stableUpdates.forEach(item => pushIfValid(dateSummary.stable, item, "Cosmetic Change"));
 
-                // [CRITICAL FIX] "Fix the Blast Radius"
-                // If updates are STABLE, we update the status NOW. 
-                // This implies: Code Hash changes, but Sim Hash stays same.
-                // The Dispatcher will see the new Code Hash in status matches the Manifest, so it won't dispatch.
                 if (stableUpdates.length > 0) {
                     const updatesPayload = {};
                     for (const stable of stableUpdates) {
                         const m = manifestMap.get(stable.name);
-                        // We preserve the *existing* resultHash because the logic is proven stable.
-                        // We update the 'hash' to the NEW code hash.
                         if (m && dailyStatus[stable.name]) {
                             updatesPayload[stable.name] = {
-                                hash: m.hash, // New Code Hash
-                                simHash: stable.simHash, // Same Sim Hash
-                                resultHash: dailyStatus[stable.name].resultHash, // Same Result Hash
+                                hash: m.hash, 
+                                simHash: stable.simHash, 
+                                resultHash: dailyStatus[stable.name].resultHash, 
                                 dependencyResultHashes: dailyStatus[stable.name].dependencyResultHashes || {},
                                 category: m.category,
                                 composition: m.composition,
@@ -404,7 +317,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
                             };
                         }
                     }
-                    // Perform the "Fix"
                     if (Object.keys(updatesPayload).length > 0) {
                         await updateComputationStatus(dateStr, updatesPayload, config, dependencies);
                         logger.log('INFO', `[BuildReporter] 🩹 Fixed ${Object.keys(updatesPayload).length} stable items for ${dateStr}. They will NOT re-run.`);
@@ -424,11 +336,8 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
             dateSummary.meta.totalIncluded = includedCount;
             dateSummary.meta.match = (includedCount === expectedCount);
 
-            // REMOVE THIS: Inefficient memory usage causes timeouts
-            // detailWrites.push({ ref: db.collection(...), data: dateSummary });
-
-            // ADD THIS (Write immediately):
-           await db.collection('computation_build_records')
+            // Write Immediately
+            await db.collection('computation_build_records')
             .doc(buildId)
             .collection('details')
             .doc(dateStr)
@@ -450,10 +359,6 @@ async function generateBuildReport(config, dependencies, manifest, daysBack = 90
 
     await db.collection('computation_build_records').doc(buildId).set(reportHeader);
     
-    // Parallel write details ---> Now redundant.
-    //const writeLimit = pLimit(15);
-    //await Promise.all(detailWrites.map(w => writeLimit(() => w.ref.set(w.data))));
-
     await db.collection('computation_build_records').doc('latest').set({ ...reportHeader, note: "Latest build report pointer." });
     
     logger.log('SUCCESS', `[BuildReporter] Report ${buildId} saved. Re-runs: ${totalReRun}, Stable (Fixed): ${totalStable}, New: ${totalRun}.`);

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -1,6 +1,7 @@
 # Cloud Workflows Definition for BullTrackers Computation Pipeline
 # Orchestrates 5 sequential passes using Event-Driven Callbacks (Zero Polling).
-# UPDATED: Implements "Callback Pattern" to eliminate arbitrary sleeps.
+# FIXED: Replaced invalid 'sys' callback functions with 'events' library functions.
+# FIXED: Proper extraction of 'callback_details.url' for the dispatcher.
 
 main:
   params: [input]
@@ -40,11 +41,17 @@ main:
                           assign:
                             - attempt_count: ${attempt_count + 1}
                       
-                      # 1. GENERATE CALLBACK URL
-                      # This creates a unique HTTP endpoint that points specifically to this step execution.
+                      # 1. GENERATE CALLBACK ENDPOINT
+                      # We use the 'events' library. This returns an object containing the URL.
                       - create_callback:
-                          call: sys.create_callback_url
-                          result: callback_url
+                          call: events.create_callback_endpoint
+                          args:
+                              http_callback_method: "POST"
+                          result: callback_details
+
+                      - extract_callback_url:
+                          assign:
+                              - callback_url: ${callback_details.url}
 
                       - log_start:
                           call: sys.log
@@ -53,21 +60,20 @@ main:
                             severity: "INFO"
 
                       # 2. TRIGGER DISPATCHER
-                      # We pass the 'callback_url' to the dispatcher so it can hand it to the workers.
+                      # We pass the extracted 'callback_url' string to the dispatcher.
                       - trigger_dispatcher:
                           call: http.post
                           args:
                             url: ${dispatcher_url}
                             body:
                               date: ${date_to_run}
-                              callbackUrl: ${callback_url} # <--- CRITICAL: Passing the token
+                              callbackUrl: ${callback_url} 
                             auth:
                               type: OIDC
                             timeout: 1800 # 30 mins max for dispatch analysis
                           result: dispatch_response
 
                       # 3. CHECK FOR "NOTHING TO DO"
-                      # If the dispatcher found 0 tasks, it returns immediate success. We skip waiting.
                       - check_immediate_completion:
                           switch:
                             - condition: ${dispatch_response.body.dispatched == 0}
@@ -83,19 +89,20 @@ main:
                                     next: pass_retry_loop
 
                       # 4. WAIT FOR WORKER SIGNAL
-                      # The workflow freezes here (sleeps) until a worker hits the callback_url.
-                      # This eliminates the need for polling logic.
+                      # We must pass the original 'callback_details' object here, not the URL string.
                       - wait_for_completion:
-                          call: sys.await_callback
+                          call: events.await_callback
                           args:
-                            callback_url: ${callback_url}
-                            timeout: 86400 # Wait up to 24 hours for the batch to finish
+                            callback: ${callback_details}
+                            timeout: 86400 # Wait up to 24 hours
                           result: callback_request
 
                       # 5. PROCESS SIGNAL
-                      - evaluate_signal:
+                      - parse_signal:
                           assign:
                             - signal_data: ${callback_request.http_request.body}
+
+                      - evaluate_signal:
                           switch:
                             - condition: ${signal_data.status == "SUCCESS"}
                               steps:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.290",
+  "version": "1.0.292",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [
@@ -35,11 +35,12 @@
     "cors": "^2.8.5",
     "dotenv": "latest",
     "express": "^4.19.2",
+    "google-auth-library": "^10.5.0",
+    "graphviz": "latest",
     "node-graphviz": "^0.1.1",
     "p-limit": "^3.1.0",
     "require-all": "^3.0.0",
-    "sharedsetup": "latest",
-    "graphviz": "latest"
+    "sharedsetup": "latest"
   },
   "devDependencies": {
     "bulltracker-deployer": "file:../bulltracker-deployer"