@onlineapps/conn-orch-orchestrator 1.0.43 → 1.0.45

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@onlineapps/conn-orch-orchestrator",
-  "version": "1.0.43",
+  "version": "1.0.45",
   "description": "Workflow orchestration connector for OA Drive - handles message routing and workflow execution",
   "main": "src/index.js",
   "scripts": {

package/src/WorkflowOrchestrator.js

@@ -37,6 +37,14 @@ class WorkflowOrchestrator {
     this.logger = config.logger || console;
     this.defaultTimeout = config.defaultTimeout || 30000;
 
+    // Retry configuration for DLQ mechanism
+    this.retryConfig = {
+      maxAttempts: config.maxRetryAttempts || 3,
+      baseDelay: config.retryBaseDelay || 1000,
+      maxDelay: config.retryMaxDelay || 30000,
+      backoffMultiplier: config.retryBackoffMultiplier || 2
+    };
+
     // Create cookbook router - createRouter is an exported function, not a method
     const { createRouter } = this.cookbook;
     if (typeof createRouter === 'function') {
@@ -230,61 +238,159 @@ class WorkflowOrchestrator {
       };
 
     } catch (error) {
-      // Publish to workflow.failed queue with FULL context for monitoring
+      // DLQ/Retry mechanism: retry with exponential backoff before sending to DLQ
+      const dlqHistory = context._dlq_history || [];
+      const stepAttempts = dlqHistory.filter(h => h.step_id === current_step && h.attempt).length;
+      const attemptNumber = stepAttempts + 1;
+      
+      // Calculate step_index for failed step
+      let failedStepIndex = 0;
+      try {
+        const stepsArr = this._getStepsArray(cookbookDef);
+        const idx = stepsArr.findIndex(s => this._getStepId(s) === current_step);
+        if (idx >= 0) failedStepIndex = idx;
+      } catch (e) {
+        // Ignore - use default 0
+      }
+
+      // IMPORTANT: Publish progress event for FAILED step (so it appears in trace)
+      // This ensures every cookbook step is visible in monitoring, even failed ones
+      const { publishToMonitoringWorkflow } = require('@onlineapps/mq-client-core').monitoring;
+      try {
+        await publishToMonitoringWorkflow(this.mqClient, {
+          event_type: 'progress',
+          workflow_id: workflow_id,
+          service_name: serviceName,
+          step_index: failedStepIndex,
+          step_id: current_step,
+          cookbook: cookbookDef,
+          context: context,
+          output: null,
+          error: {
+            message: error.message,
+            attempt: attemptNumber
+          },
+          status: 'failed',
+          timestamp: new Date().toISOString()
+        }, this.logger, { workflow_id, step_id: current_step });
+      } catch (monitoringError) {
+        // Don't fail workflow if monitoring publish fails
+        this.logger.warn('Failed to publish failed step progress to monitoring', {
+          workflow_id,
+          error: monitoringError.message
+        });
+      }
+
+      // Add attempt to DLQ history
+      const attemptRecord = {
+        step_id: current_step,
+        step_index: failedStepIndex,
+        attempt: attemptNumber,
+        service: serviceName,
+        error: error.message,
+        at: new Date().toISOString()
+      };
+      
+      const updatedDlqHistory = [...dlqHistory, attemptRecord];
+      const updatedContext = {
+        ...context,
+        _dlq_history: updatedDlqHistory
+      };
+
+      this.logger.warn(`[WorkflowOrchestrator] Step failed, attempt ${attemptNumber}/${this.retryConfig.maxAttempts}`, {
+        workflow_id,
+        current_step,
+        service: serviceName,
+        error: error.message,
+        attempt: attemptNumber
+      });
+
+      // Check if we should retry
+      if (attemptNumber < this.retryConfig.maxAttempts) {
+        // Calculate delay with exponential backoff
+        const delay = Math.min(
+          this.retryConfig.baseDelay * Math.pow(this.retryConfig.backoffMultiplier, attemptNumber - 1),
+          this.retryConfig.maxDelay
+        );
+
+        this.logger.info(`[WorkflowOrchestrator] Scheduling retry ${attemptNumber + 1} in ${delay}ms`, {
+          workflow_id,
+          current_step,
+          delay
+        });
+
+        // Wait for delay
+        await new Promise(resolve => setTimeout(resolve, delay));
+
+        // Republish to same service queue for retry
+        const serviceQueue = `${serviceName}.workflow`;
+        await this.mqClient.publish(serviceQueue, {
+          workflow_id,
+          cookbook: cookbookDef,
+          current_step,
+          context: updatedContext
+        });
+
+        this.logger.info(`[WorkflowOrchestrator] Message republished for retry`, {
+          workflow_id,
+          current_step,
+          queue: serviceQueue,
+          nextAttempt: attemptNumber + 1
+        });
+
+        // Don't throw - message will be retried
+        return { retrying: true, attempt: attemptNumber, nextAttempt: attemptNumber + 1 };
+      }
+
+      // Max retries exhausted - send to workflow.failed (DLQ entry)
+      const dlqEntryRecord = {
+        dlq_entered: true,
+        step_id: current_step,
+        step_index: failedStepIndex,
+        service: serviceName,
+        reason: 'max_retries_exceeded',
+        total_attempts: attemptNumber,
+        at: new Date().toISOString()
+      };
+      
+      const finalDlqHistory = [...updatedDlqHistory, dlqEntryRecord];
+      const finalContext = {
+        ...context,
+        _dlq_history: finalDlqHistory
+      };
+
       try {
-        console.log(`[WorkflowOrchestrator] [PUBLISH] Preparing to publish workflow.failed for ${workflow_id}`);
-        this.logger.error(`[WorkflowOrchestrator] [PUBLISH] Workflow processing failed: ${error.message}`, {
+        console.log(`[WorkflowOrchestrator] [DLQ] Max retries exhausted, publishing to workflow.failed for ${workflow_id}`);
+        this.logger.error(`[WorkflowOrchestrator] [DLQ] Workflow entering DLQ after ${attemptNumber} attempts: ${error.message}`, {
           workflow_id,
           current_step,
+          attempts: attemptNumber,
           error: error.stack
         });
         
-        // Calculate step_index for failed step
-        let failedStepIndex = 0;
-        try {
-          const stepsArr = this._getStepsArray(cookbookDef);
-          const idx = stepsArr.findIndex(s => this._getStepId(s) === current_step);
-          if (idx >= 0) failedStepIndex = idx;
-        } catch (e) {
-          // Ignore - use default 0
-        }
-        
         await this.mqClient.publish('workflow.failed', {
           workflow_id,
           current_step,
-          step_id: current_step,           // Explicit step_id for monitoring
-          step_index: failedStepIndex,     // Step index for ordering
-          service: serviceName,            // Service that failed
-          cookbook: cookbookDef,           // Full cookbook for context
-          context: context,                // Full context
+          step_id: current_step,
+          step_index: failedStepIndex,
+          service: serviceName,
+          cookbook: cookbookDef,
+          context: finalContext,
           error: error.message,
           errorStack: error.stack,
+          _dlq_history: finalDlqHistory,
           failed_at: new Date().toISOString()
         });
         
-        console.log(`[WorkflowOrchestrator] [PUBLISH] ✓ Successfully published workflow.failed for ${workflow_id}`);
-        this.logger.info(`[WorkflowOrchestrator] [PUBLISH] ✓ Published workflow.failed: ${workflow_id}`);
+        console.log(`[WorkflowOrchestrator] [DLQ] ✓ Successfully published workflow.failed for ${workflow_id}`);
+        this.logger.info(`[WorkflowOrchestrator] [DLQ] ✓ Published workflow.failed: ${workflow_id}`);
       } catch (publishError) {
-        console.error(`[WorkflowOrchestrator] [PUBLISH] ✗ Failed to publish workflow.failed for ${workflow_id}:`, publishError.message);
-        this.logger.error(`[WorkflowOrchestrator] [PUBLISH] ✗ Failed to publish workflow.failed`, {
+        console.error(`[WorkflowOrchestrator] [DLQ] ✗ Failed to publish workflow.failed for ${workflow_id}:`, publishError.message);
+        this.logger.error(`[WorkflowOrchestrator] [DLQ] ✗ Failed to publish workflow.failed`, {
           workflow_id,
           originalError: error.message,
           publishError: publishError.message
         });
-        // Don't throw - original error is more important
-      }
-
-      // Handle error
-      if (this.errorHandler) {
-        await this.errorHandler.handleError(error, {
-          workflow_id,
-          current_step,
-          service: serviceName,
-          context
-        });
-      } else {
-        // Fallback to router DLQ
-        await this.router.routeToDLQ(cookbookDef, context, error, serviceName);
       }
 
       throw error;