bulltrackers-module 1.0.284 → 1.0.285

package/functions/computation-system/helpers/monitor.js

@@ -0,0 +1,63 @@
+/**
+ * @fileoverview Monitor helper for Cloud Workflows.
+ * Checks the state of the Audit Ledger to determine if a pass is complete.
+ * This function is stateless and receives dependencies via injection.
+ */
+
+/**
+ * Checks the status of a specific computation pass.
+ * @param {object} req - Express request object (query: date, pass).
+ * @param {object} res - Express response object.
+ * @param {object} dependencies - Contains db (Firestore), logger.
+ */
+async function checkPassStatus(req, res, dependencies) {
+    const { db, logger } = dependencies;
+    const { date, pass } = req.query;
+
+    if (!date || !pass) {
+        return res.status(400).json({ error: "Missing 'date' or 'pass' query parameters." });
+    }
+
+    const ledgerPath = `computation_audit_ledger/${date}/passes/${pass}/tasks`;
+    logger.log('INFO', `[Monitor] Checking status for ${date} Pass ${pass} at ${ledgerPath}`);
+
+    try {
+        const tasksRef = db.collection(ledgerPath);
+
+        // 1. Check for Active Tasks (Blocking)
+        // If anything is PENDING or IN_PROGRESS, the system is still working.
+        const runningSnap = await tasksRef.where('status', 'in', ['PENDING', 'IN_PROGRESS']).get();
+        
+        if (!runningSnap.empty) {
+            logger.log('INFO', `[Monitor] Pass ${pass} is RUNNING. Active tasks: ${runningSnap.size}`);
+            return res.status(200).json({ 
+                state: 'RUNNING', 
+                activeCount: runningSnap.size 
+            });
+        }
+
+        // 2. Check for Failures (Retry Condition)
+        // If nothing is running, we check if anything ended in FAILED state.
+        // We consider these "retryable" by re-triggering the dispatcher.
+        const failedSnap = await tasksRef.where('status', '==', 'FAILED').get();
+        
+        if (!failedSnap.empty) {
+            logger.log('WARN', `[Monitor] Pass ${pass} finished with FAILURES. Count: ${failedSnap.size}`);
+            return res.status(200).json({ 
+                state: 'HAS_FAILURES', 
+                failureCount: failedSnap.size 
+            });
+        }
+
+        // 3. Clean Success
+        // No running tasks, no failed tasks.
+        logger.log('INFO', `[Monitor] Pass ${pass} COMPLETED successfully.`);
+        return res.status(200).json({ state: 'SUCCESS' });
+
+    } catch (error) {
+        logger.log('ERROR', `[Monitor] Failed to check status: ${error.message}`);
+        return res.status(500).json({ error: error.message });
+    }
+}
+
+module.exports = { checkPassStatus };

package/functions/computation-system/workflows/bulltrackers_pipeline.yaml

@@ -0,0 +1,143 @@
+# Cloud Workflows Definition for BullTrackers Computation Pipeline
+# Orchestrates 5 sequential passes with Self-Healing (Retry) logic.
+
+main:
+  params: [input]
+  steps:
+    - init:
+        assign:
+          - project: ${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}
+          - location: "europe-west1"
+          # If 'date' is provided in input, use it. Otherwise default to today (YYYY-MM-DD).
+          - date_to_run: ${default(map.get(input, "date"), text.substring(time.format(sys.now()), 0, 10))}
+          - passes: ["1", "2", "3", "4", "5"]
+          - max_retries: 3
+          - propagation_wait_seconds: 300 # 5 Minutes
+          # URL of the new Monitor Function
+          - monitor_url: ${"https://europe-west1-" + project + ".cloudfunctions.net/computation-monitor"}
+
+    # ======================================================
+    # MAIN LOOP: Iterate through Passes 1 to 5
+    # ======================================================
+    - run_passes:
+        for:
+          value: pass_id
+          in: ${passes}
+          steps:
+            - init_pass_vars:
+                assign:
+                  - attempt_count: 0
+                  - pass_success: false
+                  # Construct URL for the specific pass function (e.g. computation-pass-1)
+                  - dispatcher_url: ${"https://europe-west1-" + project + ".cloudfunctions.net/computation-pass-" + pass_id}
+
+            # -----------------------------------------------
+            # RETRY LOOP: Try to complete the pass up to 3 times
+            # -----------------------------------------------
+            - pass_retry_loop:
+                switch:
+                  - condition: ${attempt_count < max_retries and not pass_success}
+                    steps:
+                      - increment_attempt:
+                          assign:
+                            - attempt_count: ${attempt_count + 1}
+                      
+                      - log_start:
+                          call: sys.log
+                          args:
+                            text: ${"Starting Pass " + pass_id + " (Attempt " + attempt_count + ") for " + date_to_run}
+                            severity: "INFO"
+
+                      # 1. TRIGGER DISPATCHER (Fire and Forget mechanism via HTTP)
+                      # The dispatcher analyzes missing data and queues tasks.
+                      - trigger_dispatcher:
+                          call: http.get
+                          args:
+                            url: ${dispatcher_url}
+                            query:
+                              date: ${date_to_run}
+                            auth:
+                              type: OIDC
+                            timeout: 1800 # 30 mins max for dispatch analysis
+                          result: dispatch_response
+
+                      # 2. PROPAGATION WAIT
+                      # Wait for dispatcher to queue tasks and workers to start/finish
+                      - wait_for_propagation:
+                          call: sys.log
+                          args:
+                            text: ${"Pass " + pass_id + " dispatched. Waiting " + propagation_wait_seconds + "s for propagation..."}
+                          next: sleep_propagation
+                      
+                      - sleep_propagation:
+                          call: sys.sleep
+                          args:
+                            seconds: ${propagation_wait_seconds}
+
+                      # 3. MONITORING LOOP
+                      # Poll until RUNNING state clears
+                      - monitor_loop:
+                          call: http.get
+                          args:
+                            url: ${monitor_url}
+                            query:
+                              date: ${date_to_run}
+                              pass: ${pass_id}
+                            auth:
+                              type: OIDC
+                          result: status_resp
+
+                      - evaluate_status:
+                          switch:
+                            # CASE A: Still Running -> Sleep and Poll Again
+                            - condition: ${status_resp.body.state == "RUNNING"}
+                              steps:
+                                - log_running:
+                                    call: sys.log
+                                    args:
+                                      text: ${"Pass " + pass_id + " is RUNNING (" + status_resp.body.activeCount + " active). Waiting..."}
+                                - sleep_polling:
+                                    call: sys.sleep
+                                    args:
+                                      seconds: 60
+                                - next: monitor_loop
+
+                            # CASE B: Clean Success -> Mark done, Break Retry Loop
+                            - condition: ${status_resp.body.state == "SUCCESS"}
+                              steps:
+                                - log_success:
+                                    call: sys.log
+                                    args:
+                                      text: ${"Pass " + pass_id + " COMPLETED successfully."}
+                                      severity: "INFO"
+                                - mark_success:
+                                    assign:
+                                      - pass_success: true
+                                - next: pass_retry_loop # Will exit loop due to pass_success=true
+
+                            # CASE C: Failures Found -> Continue Retry Loop (will trigger dispatcher again)
+                            - condition: ${status_resp.body.state == "HAS_FAILURES"}
+                              steps:
+                                - log_failure:
+                                    call: sys.log
+                                    args:
+                                      text: ${"Pass " + pass_id + " has " + status_resp.body.failureCount + " FAILURES. Attempting Retry."}
+                                      severity: "WARNING"
+                                - next: pass_retry_loop
+
+            # -----------------------------------------------
+            # END RETRY LOOP
+            # -----------------------------------------------
+            
+            - check_final_status:
+                switch:
+                  - condition: ${not pass_success}
+                    steps:
+                      - log_giving_up:
+                          call: sys.log
+                          args:
+                            text: ${"Pass " + pass_id + " failed after " + max_retries + " attempts. Proceeding to next pass with potential gaps."}
+                            severity: "ERROR"
+
+    - finish:
+        return: "Pipeline Execution Complete"

package/index.js

@@ -29,8 +29,9 @@ const { handleUpdate }                                    = require('./functions
 const { build: buildManifest }                            = require('./functions/computation-system/context/ManifestBuilder');
 const { dispatchComputationPass }                         = require('./functions/computation-system/helpers/computation_dispatcher');
 const { handleComputationTask }                           = require('./functions/computation-system/helpers/computation_worker');
-// [NEW] Import Report Tools
 const { ensureBuildReport, generateBuildReport }          = require('./functions/computation-system/tools/BuildReporter');
+// [NEW] Import Monitor
+const { checkPassStatus }                                 = require('./functions/computation-system/helpers/monitor');
 
 const dataLoader                                          = require('./functions/computation-system/utils/data_loader');
 const computationUtils                                    = require('./functions/computation-system/utils/utils');
@@ -51,8 +52,7 @@ const { runBackfillAssetPrices }                          = require('./functions
 // Proxy
 const { handlePost }                                      = require('./functions/appscript-api/index');
 
-// NEW
-
+// Root Indexer
 const { runRootDataIndexer } = require('./functions/root-data-indexer/index'); 
 
 const core = {
@@ -92,9 +92,10 @@ const computationSystem = {
   dataLoader,
   computationUtils,
   buildManifest,
-  // [NEW] Export Tools
   ensureBuildReport,
   generateBuildReport,
+  // [NEW] Export Monitor Pipe
+  checkPassStatus
 };
 
 const api = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.284",
+  "version": "1.0.285",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [