@onlineapps/conn-orch-orchestrator 1.0.47 → 1.0.49

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@onlineapps/conn-orch-orchestrator",
-  "version": "1.0.47",
+  "version": "1.0.49",
   "description": "Workflow orchestration connector for OA Drive - handles message routing and workflow execution",
   "main": "src/index.js",
   "scripts": {

package/src/WorkflowOrchestrator.js

@@ -116,24 +116,19 @@ class WorkflowOrchestrator {
    * }, 'hello-service');
    */
   async processWorkflowMessage(message, serviceName) {
-    const { workflow_id, cookbook: cookbookDef, current_step, context } = message;
+    const { workflow_id, cookbook: cookbookDef, current_step } = message;
+    const startTime = Date.now();
 
     try {
       // Validate cookbook structure
       this.cookbook.validateCookbook(cookbookDef);
 
-      // Ensure delivery configuration is preserved in context (required by Delivery Dispatcher)
-      // Delivery comes from cookbook.delivery (set by Gateway) and must be passed through to workflow.completed
-      const enrichedContext = {
-        ...context,
-        // Preserve delivery from cookbook if not already in context
-        delivery: context?.delivery || cookbookDef?.delivery || { handler: 'none' }
-      };
-
-      // Find current step (supports both V1 'id' and V2 'step_id')
-      // V2 FORMAT: steps is an object keyed by step_id
+      // UNIFIED COOKBOOK: Everything is in cookbook, no separate context
+      // Find current step
       const stepsArray = this._getStepsArray(cookbookDef);
-      const step = stepsArray.find(s => this._getStepId(s) === current_step);
+      const currentIndex = stepsArray.findIndex(s => this._getStepId(s) === current_step);
+      const step = stepsArray[currentIndex];
+      
       if (!step) {
         throw new Error(`Step not found: ${current_step}`);
       }
@@ -141,93 +136,90 @@ class WorkflowOrchestrator {
       // Check if this step is for this service
       if (step.type === 'task' && step.service !== serviceName) {
         this.logger.warn(`Step ${current_step} is for ${step.service}, not ${serviceName}`);
-        if (this.router?.routeToService) {
-          await this.router.routeToService(step.service, message);
-        } else {
-          // Fallback: publish to service's workflow queue directly
-          const serviceQueue = `${step.service}.workflow`;
-          await this.mqClient.publish(serviceQueue, message);
-        }
+        const serviceQueue = `${step.service}.workflow`;
+        await this.mqClient.publish(serviceQueue, message);
         return { skipped: true, reason: 'wrong_service' };
       }
 
-      // Check cache if available
+      // Mark step as started
+      const updatedSteps = [...stepsArray];
+      updatedSteps[currentIndex] = {
+        ...step,
+        started_at: new Date().toISOString(),
+        status: 'in_progress'
+      };
+
+      // Execute the step (pass _input from cookbook as context for operations)
+      const stepContext = {
+        ...cookbookDef._input,
+        _system: cookbookDef._system,
+        delivery: cookbookDef.delivery
+      };
+      
       let result;
       if (this.cache && step.type === 'task') {
-        const cacheKey = this._getCacheKey(step, enrichedContext);
+        const cacheKey = this._getCacheKey(step, stepContext);
         result = await this.cache.get(cacheKey);
-
-        if (result) {
-          this.logger.info('Cache hit', { step: this._getStepId(step), cacheKey });
-        } else {
-          // Execute the step
-          result = await this._executeStep(step, enrichedContext, cookbookDef);
+        if (!result) {
+          result = await this._executeStep(step, stepContext, cookbookDef);
           await this.cache.set(cacheKey, result, { ttl: 300 });
         }
       } else {
-        // Execute without cache
-        result = await this._executeStep(step, enrichedContext, cookbookDef);
+        result = await this._executeStep(step, stepContext, cookbookDef);
       }
 
-      // Update context with result - steps as ARRAY (consistent with cookbook.steps)
-      // Each step preserves its definition (id/step_id, type, service, operation, input) and adds output
-      // stepsArray is already defined above via _getStepsArray()
-      const currentIndex = stepsArray.findIndex(s => this._getStepId(s) === current_step);
-      const stepDefinition = stepsArray[currentIndex];
-      
-      // Initialize steps array from cookbook if not present
-      const existingSteps = Array.isArray(enrichedContext.steps) 
-        ? [...enrichedContext.steps] 
-        : stepsArray.map(s => ({ ...s }));  // Deep copy of step definitions
-      
-      // Update the current step with output (preserve id, type, service, operation, input)
-      existingSteps[currentIndex] = {
-        ...stepDefinition,           // id, type, service, operation, input
-        output: result,              // Add output from operation
+      // Calculate duration
+      const duration_ms = Date.now() - startTime;
+
+      // Determine next step
+      const nextStep = stepsArray[currentIndex + 1];
+      const nextPointer = nextStep 
+        ? this._getStepId(nextStep) 
+        : 'api_delivery_dispatcher';  // Last step -> dispatcher
+
+      // Update step with output and completion info
+      updatedSteps[currentIndex] = {
+        ...step,
+        started_at: updatedSteps[currentIndex].started_at,
+        output: result,
         status: 'completed',
-        completed_at: new Date().toISOString()
+        completed_at: new Date().toISOString(),
+        duration_ms
       };
-      
-      const updatedContext = {
-        ...enrichedContext,
-        steps: existingSteps
+
+      // Build UPDATED UNIFIED COOKBOOK
+      const updatedCookbook = {
+        ...cookbookDef,
+        steps: updatedSteps,
+        _pointer: { next: nextStep ? this._getStepId(nextStep) : null }
       };
 
-      // Publish workflow.progress AFTER execution (so we have step output)
+      // Publish workflow.progress with UNIFIED COOKBOOK
       const { publishToMonitoringWorkflow } = require('@onlineapps/mq-client-core').monitoring;
       try {
-        console.log(`[WorkflowOrchestrator] [PROGRESS] Publishing progress for ${workflow_id}, step: ${current_step}, index: ${currentIndex}`);
         await publishToMonitoringWorkflow(this.mqClient, {
           event_type: 'progress',
           workflow_id: workflow_id,
           service_name: serviceName,
-          step_index: currentIndex >= 0 ? currentIndex : 0,
+          step_index: currentIndex,
           step_id: current_step,
-          cookbook: cookbookDef,
-          context: updatedContext,  // Context WITH step output
-          output: result,           // Step output directly
-          status: 'completed',      // Step completed (not in_progress)
+          cookbook: updatedCookbook,  // UNIFIED - contains everything
+          output: result,
+          status: 'completed',
           timestamp: new Date().toISOString()
         }, this.logger, { workflow_id, step_id: current_step });
-        console.log(`[WorkflowOrchestrator] [PROGRESS] ✓ Progress published for ${workflow_id}`);
       } catch (monitoringError) {
-        // Don't fail workflow if monitoring publish fails
-        console.error(`[WorkflowOrchestrator] [PROGRESS] ✗ Failed to publish progress:`, monitoringError);
         this.logger.warn('Failed to publish workflow.progress to monitoring', {
           workflow_id,
-          error: monitoringError.message || String(monitoringError)
+          error: monitoringError.message
         });
       }
 
-      // Find next step (currentIndex and stepsArray already defined above)
-      const nextStep = stepsArray[currentIndex + 1];
-
+      // Route to next step or complete workflow
       if (nextStep) {
-        // Route to next step
-        await this._routeToNextStep(nextStep, cookbookDef, updatedContext, workflow_id);
+        await this._routeToNextStep(nextStep, updatedCookbook, workflow_id);
       } else {
-        // Workflow completed - pass serviceName, cookbook and last step info for monitoring
-        await this._completeWorkflow(workflow_id, updatedContext, cookbookDef, serviceName, current_step, currentIndex);
+        await this._completeWorkflow(workflow_id, updatedCookbook, serviceName, current_step, currentIndex);
       }
 
       return {
@@ -238,39 +230,52 @@ class WorkflowOrchestrator {
       };
 
     } catch (error) {
-      // DLQ/Retry mechanism: retry with exponential backoff before sending to DLQ
-      const dlqHistory = context._dlq_history || [];
-      const stepAttempts = dlqHistory.filter(h => h.step_id === current_step && h.attempt).length;
-      const attemptNumber = stepAttempts + 1;
+      // UNIFIED COOKBOOK: _retry_history is stored INSIDE the step, not globally
+      const crypto = require('crypto');
       
-      // Calculate step_index for failed step
-      let failedStepIndex = 0;
-      try {
-        const stepsArr = this._getStepsArray(cookbookDef);
-        const idx = stepsArr.findIndex(s => this._getStepId(s) === current_step);
-        if (idx >= 0) failedStepIndex = idx;
-      } catch (e) {
-        // Ignore - use default 0
-      }
-
-      // NOTE: We do NOT publish workflow_progress on every retry attempt.
-      // Instead, we publish ONE progress event at the end (either success or final failure with DLQ entry).
-      // This keeps the trace clean - one entry per step, with complete _dlq_history in context.
+      // Find current step index
+      const stepsArray = this._getStepsArray(cookbookDef);
+      const currentIndex = stepsArray.findIndex(s => this._getStepId(s) === current_step);
+      const currentStepDef = stepsArray[currentIndex] || {};
+      
+      // Get existing retry history from the step itself
+      const stepRetryHistory = currentStepDef._retry_history || [];
+      const attemptNumber = stepRetryHistory.length + 1;
+      
+      // Generate error reference for log lookup (instead of full stack trace)
+      const errorRef = `err-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`;
+      
+      // Log full stack trace with error_ref for debugging
+      this.logger.error(`[${errorRef}] Step execution failed`, {
+        workflow_id,
+        step_id: current_step,
+        service: serviceName,
+        attempt: attemptNumber,
+        error: error.message,
+        stack: error.stack
+      });
 
-      // Add attempt to DLQ history
+      // Add attempt record to step's _retry_history
       const attemptRecord = {
-        step_id: current_step,
-        step_index: failedStepIndex,
         attempt: attemptNumber,
-        service: serviceName,
         error: error.message,
+        error_ref: errorRef,  // Reference to full stack in logs
         at: new Date().toISOString()
       };
       
-      const updatedDlqHistory = [...dlqHistory, attemptRecord];
-      const updatedContext = {
-        ...context,
-        _dlq_history: updatedDlqHistory
+      // Update steps array with retry history
+      const updatedSteps = [...stepsArray];
+      updatedSteps[currentIndex] = {
+        ...currentStepDef,
+        status: 'retrying',
+        _retry_history: [...stepRetryHistory, attemptRecord]
+      };
+
+      // Build updated UNIFIED COOKBOOK
+      const updatedCookbook = {
+        ...cookbookDef,
+        steps: updatedSteps,
+        _pointer: { next: current_step }  // Still pointing to this step (retrying)
       };
 
       this.logger.warn(`[WorkflowOrchestrator] Step failed, attempt ${attemptNumber}/${this.retryConfig.maxAttempts}`, {
@@ -278,12 +283,35 @@ class WorkflowOrchestrator {
         current_step,
         service: serviceName,
         error: error.message,
+        error_ref: errorRef,
         attempt: attemptNumber
       });
 
+      // Publish progress event for EACH retry attempt
+      const { publishToMonitoringWorkflow } = require('@onlineapps/mq-client-core').monitoring;
+      try {
+        await publishToMonitoringWorkflow(this.mqClient, {
+          event_type: 'progress',
+          workflow_id: workflow_id,
+          service_name: serviceName,
+          step_index: currentIndex,
+          step_id: current_step,
+          cookbook: updatedCookbook,  // UNIFIED cookbook with _retry_history in step
+          error: { message: error.message, error_ref: errorRef },
+          status: 'retry',
+          attempt: attemptNumber,
+          max_attempts: this.retryConfig.maxAttempts,
+          timestamp: new Date().toISOString()
+        }, this.logger, { workflow_id, step_id: current_step });
+      } catch (monitoringError) {
+        this.logger.warn('Failed to publish retry progress to monitoring', {
+          workflow_id,
+          error: monitoringError.message
+        });
+      }
+
       // Check if we should retry
       if (attemptNumber < this.retryConfig.maxAttempts) {
-        // Calculate delay with exponential backoff
         const delay = Math.min(
           this.retryConfig.baseDelay * Math.pow(this.retryConfig.backoffMultiplier, attemptNumber - 1),
           this.retryConfig.maxDelay
@@ -295,16 +323,14 @@ class WorkflowOrchestrator {
           delay
         });
 
-        // Wait for delay
         await new Promise(resolve => setTimeout(resolve, delay));
 
-        // Republish to same service queue for retry
+        // Republish with UNIFIED cookbook
         const serviceQueue = `${serviceName}.workflow`;
         await this.mqClient.publish(serviceQueue, {
           workflow_id,
-          cookbook: cookbookDef,
-          current_step,
-          context: updatedContext
+          cookbook: updatedCookbook,
+          current_step
         });
 
         this.logger.info(`[WorkflowOrchestrator] Message republished for retry`, {
@@ -314,25 +340,27 @@ class WorkflowOrchestrator {
           nextAttempt: attemptNumber + 1
         });
 
-        // Don't throw - message will be retried
         return { retrying: true, attempt: attemptNumber, nextAttempt: attemptNumber + 1 };
       }
 
-      // Max retries exhausted - send to workflow.failed (DLQ entry)
+      // Max retries exhausted - mark step as failed and send to DLQ
       const dlqEntryRecord = {
         dlq_entered: true,
-        step_id: current_step,
-        step_index: failedStepIndex,
-        service: serviceName,
         reason: 'max_retries_exceeded',
         total_attempts: attemptNumber,
         at: new Date().toISOString()
       };
       
-      const finalDlqHistory = [...updatedDlqHistory, dlqEntryRecord];
-      const finalContext = {
-        ...context,
-        _dlq_history: finalDlqHistory
+      updatedSteps[currentIndex] = {
+        ...updatedSteps[currentIndex],
+        status: 'failed',
+        _retry_history: [...updatedSteps[currentIndex]._retry_history, dlqEntryRecord]
+      };
+
+      const finalCookbook = {
+        ...cookbookDef,
+        steps: updatedSteps,
+        _pointer: { next: null }  // Workflow stuck - waiting for human intervention
       };
 
       try {
@@ -341,41 +369,32 @@ class WorkflowOrchestrator {
           workflow_id,
           current_step,
           attempts: attemptNumber,
-          error: error.stack
+          error_ref: errorRef  // Reference to full stack in logs
         });
         
         // Publish to workflow.failed (DLQ queue - message stays until manual requeue/discard)
+        // UNIFIED COOKBOOK contains everything including _retry_history in the step
         await this.mqClient.publish('workflow.failed', {
           workflow_id,
           current_step,
-          step_id: current_step,
-          step_index: failedStepIndex,
-          service: serviceName,
-          cookbook: cookbookDef,
-          context: finalContext,
-          error: error.message,
-          errorStack: error.stack,
-          _dlq_history: finalDlqHistory,
+          cookbook: finalCookbook,  // UNIFIED cookbook with full state
           failed_at: new Date().toISOString()
         });
         
         // ALSO publish to monitoring.workflow so dashboard shows the DLQ entry
-        // This is separate from the DLQ message - it's just for visibility
         try {
           await publishToMonitoringWorkflow(this.mqClient, {
             event_type: 'failed',
             workflow_id: workflow_id,
             service_name: serviceName,
-            step_index: failedStepIndex,
+            step_index: currentIndex,
             step_id: current_step,
-            cookbook: cookbookDef,
-            context: finalContext,
-            error: { message: error.message },
+            cookbook: finalCookbook,  // UNIFIED cookbook
+            error: { message: error.message, error_ref: errorRef },
             status: 'failed',
             timestamp: new Date().toISOString()
           }, this.logger, { workflow_id, step_id: current_step });
         } catch (monitoringError) {
-          // Don't fail if monitoring publish fails
           this.logger.warn('[WorkflowOrchestrator] Failed to publish DLQ entry to monitoring', {
             workflow_id,
             error: monitoringError.message
@@ -578,17 +597,17 @@ class WorkflowOrchestrator {
    * @private
    * @async
    * @param {Object} nextStep - Next step to execute
-   * @param {Object} cookbookDef - Full cookbook definition
-   * @param {Object} context - Updated context
+   * @param {Object} cookbook - UNIFIED cookbook (contains everything)
    * @param {string} workflow_id - Workflow ID
    */
-  async _routeToNextStep(nextStep, cookbookDef, context, workflow_id) {
+  async _routeToNextStep(nextStep, cookbook, workflow_id) {
     const nextStepId = this._getStepId(nextStep);
+    
+    // UNIFIED COOKBOOK: no separate context, everything is in cookbook
     const message = {
       workflow_id,
-      cookbook: cookbookDef,
-      current_step: nextStepId,
-      context
+      cookbook: cookbook,
+      current_step: nextStepId
     };
 
     if (nextStep.type === 'task') {
@@ -620,38 +639,37 @@ class WorkflowOrchestrator {
    * @private
    * @async
    * @param {string} workflow_id - Workflow ID
-   * @param {Object} finalContext - Final workflow context
+   * @param {Object} cookbook - UNIFIED cookbook (contains everything)
    * @param {string} serviceName - Service name that completed the workflow
    * @param {string} lastStepId - ID of the last completed step
    * @param {number} lastStepIndex - Index of the last completed step
    */
-  async _completeWorkflow(workflow_id, finalContext, cookbookDef = {}, serviceName = 'unknown', lastStepId = null, lastStepIndex = 0) {
+  async _completeWorkflow(workflow_id, cookbook, serviceName = 'unknown', lastStepId = null, lastStepIndex = 0) {
     try {
       console.log(`[WorkflowOrchestrator] [PUBLISH] Preparing to publish workflow.completed for ${workflow_id}`);
       this.logger.info(`[WorkflowOrchestrator] [PUBLISH] Preparing to publish workflow.completed`, {
         workflow_id,
         serviceName,
         lastStepId,
-        lastStepIndex,
-        hasMqClient: !!this.mqClient,
-        mqClientConnected: this.mqClient?._connected
+        lastStepIndex
       });
       
-      // Extract delivery configuration from context (passed from Gateway)
-      // Delivery Dispatcher requires: workflow_id, status, delivery (must be object, not null)
-      const delivery = finalContext?.delivery || cookbookDef?.delivery || { handler: 'none' };
+      // Update pointer to dispatcher (final step)
+      const finalCookbook = {
+        ...cookbook,
+        _pointer: { next: 'api_delivery_dispatcher' }
+      };
       
       // Build message in format expected by Delivery Dispatcher
+      // UNIFIED COOKBOOK contains everything including delivery config
       const workflowCompletedMessage = {
         workflow_id,
         status: 'completed',
-        service: serviceName, // Service that completed the workflow (for monitoring)
-        step_id: lastStepId, // Last step ID for monitoring
-        step_index: lastStepIndex, // Last step index for monitoring
-        cookbook: cookbookDef, // Full cookbook for monitoring trace
-        delivery: delivery, // Delivery configuration from Gateway context (must be object)
-        context: finalContext, // Full context for output resolution
-        steps: finalContext?.steps || [], // Steps results (array) for output resolution
+        service: serviceName,
+        step_id: lastStepId,
+        step_index: lastStepIndex,
+        cookbook: finalCookbook,  // UNIFIED cookbook with full state
+        delivery: cookbook.delivery || { handler: 'none' },
         completed_at: new Date().toISOString()
       };