@onlineapps/conn-orch-orchestrator 1.0.30 → 1.0.32

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@onlineapps/conn-orch-orchestrator",
-  "version": "1.0.30",
+  "version": "1.0.32",
   "description": "Workflow orchestration connector for OA Drive - handles message routing and workflow execution",
   "main": "src/index.js",
   "scripts": {
@@ -22,14 +22,13 @@
   "dependencies": {
     "@onlineapps/conn-base-monitoring": "^1.0.0",
     "@onlineapps/conn-infra-mq": "^1.1.0",
-    "@onlineapps/conn-orch-api-mapper": "^1.0.8",
-    "@onlineapps/conn-orch-cookbook": "^2.0.0",
     "@onlineapps/conn-orch-registry": "^1.1.4",
-    "@onlineapps/cookbook-core": "^2.1.2"
+    "@onlineapps/conn-orch-cookbook": "^2.0.0",
+    "@onlineapps/conn-orch-api-mapper": "^1.0.0"
   },
   "devDependencies": {
-    "eslint": "^8.30.0",
     "jest": "^29.5.0",
+    "eslint": "^8.30.0",
     "jsdoc-to-markdown": "^8.0.0"
   },
   "jest": {

package/src/WorkflowOrchestrator.js

@@ -51,6 +51,16 @@ class WorkflowOrchestrator {
     }
   }
 
+  /**
+   * Get step identifier (supports both V1 'id' and V2 'step_id')
+   * @private
+   * @param {Object} step - Step object
+   * @returns {string} Step identifier
+   */
+  _getStepId(step) {
+    return step?.step_id || step?.id;
+  }
+
   /**
    * Process a workflow message
    * @async
@@ -75,10 +85,7 @@ class WorkflowOrchestrator {
     const { workflow_id, cookbook: cookbookDef, current_step, context } = message;
 
     try {
-      // FAIL-FAST: Validate V2 format
-      this._validateV2Format(cookbookDef);
-      
-      // Validate against JSON Schema (V2 schema - steps as object)
+      // Validate cookbook structure
       this.cookbook.validateCookbook(cookbookDef);
 
       // Ensure delivery configuration is preserved in context (required by Delivery Dispatcher)
@@ -89,14 +96,11 @@ class WorkflowOrchestrator {
         delivery: context?.delivery || cookbookDef?.delivery || { handler: 'none' }
       };
 
-      // Find current step (V2: steps is an object keyed by step_id)
-      const step = cookbookDef.steps[current_step];
+      // Find current step (supports both V1 'id' and V2 'step_id')
+      const step = cookbookDef.steps.find(s => this._getStepId(s) === current_step);
       if (!step) {
-        throw new Error(`Step not found: ${current_step}. Available steps: ${Object.keys(cookbookDef.steps).join(', ')}`);
+        throw new Error(`Step not found: ${current_step}`);
       }
-      
-      // Get step_id (V2)
-      const stepId = step.step_id;
 
       // Check if this step is for this service
       if (step.type === 'task' && step.service !== serviceName) {
@@ -118,7 +122,7 @@ class WorkflowOrchestrator {
         result = await this.cache.get(cacheKey);
 
         if (result) {
-          this.logger.info('Cache hit', { step: stepId, cacheKey });
+          this.logger.info('Cache hit', { step: this._getStepId(step), cacheKey });
         } else {
           // Execute the step
           result = await this._executeStep(step, enrichedContext, cookbookDef);
@@ -129,55 +133,29 @@ class WorkflowOrchestrator {
         result = await this._executeStep(step, enrichedContext, cookbookDef);
       }
 
-      // Update context with result - steps as OBJECT (V2 format, keyed by step_id)
-      // Each step preserves its definition (step_id, type, service, operation, input) and adds output
-      const stepDefinition = cookbookDef.steps[current_step];
-      
-      // Initialize steps object from cookbook if not present
-      // V2: steps is an object keyed by step_id
-      let existingSteps = enrichedContext.steps || {};
+      // Update context with result - steps as ARRAY (consistent with cookbook.steps)
+      // Each step preserves its definition (id/step_id, type, service, operation, input) and adds output
+      const currentIndex = cookbookDef.steps.findIndex(s => this._getStepId(s) === current_step);
+      const stepDefinition = cookbookDef.steps[currentIndex];
       
-      // Initialize from cookbook if empty
-      if (Object.keys(existingSteps).length === 0) {
-        Object.entries(cookbookDef.steps).forEach(([key, s]) => {
-          existingSteps[key] = { ...s };
-        });
-      }
+      // Initialize steps array from cookbook if not present
+      const existingSteps = Array.isArray(enrichedContext.steps) 
+        ? [...enrichedContext.steps] 
+        : cookbookDef.steps.map(s => ({ ...s }));  // Deep copy of step definitions
       
-      // Update the current step with output (preserve step_id, type, service, operation, input)
-      existingSteps[stepId] = {
-        ...stepDefinition,           // step_id, type, service, operation, input
+      // Update the current step with output (preserve id, type, service, operation, input)
+      existingSteps[currentIndex] = {
+        ...stepDefinition,           // id, type, service, operation, input
         output: result,              // Add output from operation
         status: 'completed',
         completed_at: new Date().toISOString()
       };
       
-      // === STRUCTURED LOG: Context update ===
-      console.log(`[WorkflowOrchestrator:CONTEXT_UPDATE] ${JSON.stringify({
-        layer: 'orchestrator',
-        workflow_id: workflow_id,
-        step_id: stepId,
-        stored_output: {
-          _descriptor: result?._descriptor,
-          type: result?.type,
-          storage_ref: result?.storage_ref,
-          outputKeys: result && typeof result === 'object' ? Object.keys(result) : null
-        },
-        timestamp: new Date().toISOString()
-      })}`);
-
-      
       const updatedContext = {
         ...enrichedContext,
-        steps: existingSteps  // Object keyed by step_id
+        steps: existingSteps
       };
 
-      // V2: Get step order from object keys (JavaScript preserves insertion order for string keys)
-      const stepKeys = Object.keys(cookbookDef.steps);
-      const currentIndex = stepKeys.indexOf(current_step);
-      const nextStepKey = stepKeys[currentIndex + 1];
-      const nextStep = nextStepKey ? cookbookDef.steps[nextStepKey] : null;
-
       // Publish workflow.progress AFTER execution (so we have step output)
       const { publishToMonitoringWorkflow } = require('@onlineapps/mq-client-core').monitoring;
       try {
@@ -204,13 +182,15 @@ class WorkflowOrchestrator {
         });
       }
 
+      // Find next step (currentIndex already defined above)
+      const nextStep = cookbookDef.steps[currentIndex + 1];
+
       if (nextStep) {
         // Route to next step
-        const nextStepId = nextStep.step_id;
-        await this._routeToNextStep(nextStep, cookbookDef, updatedContext, workflow_id, nextStepId);
+        await this._routeToNextStep(nextStep, cookbookDef, updatedContext, workflow_id);
       } else {
         // Workflow completed - pass serviceName, cookbook and last step info for monitoring
-        await this._completeWorkflow(workflow_id, updatedContext, cookbookDef, serviceName, stepId, currentIndex);
+        await this._completeWorkflow(workflow_id, updatedContext, cookbookDef, serviceName, current_step, currentIndex);
       }
 
       return {
@@ -295,78 +275,25 @@ class WorkflowOrchestrator {
    * @returns {Promise<Object>} API call result
    */
   async _executeTaskStep(step, context) {
-    // FAIL-FAST: Verify context exists
-    if (!context) {
-      const error = new Error(`Context is missing for step ${step.step_id}`);
-      this.logger.error('[WorkflowOrchestrator] [FAIL-FAST] Missing context', { step_id: step.step_id });
-      throw error;
-    }
-    
-    // Debug: log context keys and api_input
-    console.log(`[WorkflowOrchestrator] [RESOLVE] Step ${step.step_id} - context keys:`, Object.keys(context || {}));
+    const stepId = this._getStepId(step);
+    // Resolve variable references in step.input (e.g. ${steps[0].output.message})
+    console.log(`[WorkflowOrchestrator] [RESOLVE] Step ${stepId} input BEFORE:`, JSON.stringify(step.input));
+    console.log(`[WorkflowOrchestrator] [RESOLVE] Context keys:`, Object.keys(context || {}));
+    console.log(`[WorkflowOrchestrator] [RESOLVE] Context.input_file:`, context?.input_file);
+    console.log(`[WorkflowOrchestrator] [RESOLVE] Context.steps:`, JSON.stringify(context.steps?.map(s => ({ id: this._getStepId(s), output: s.output }))));
     
-    // FAIL-FAST: Log api_input state (but don't fail if missing - it might not be needed)
-    if (context.api_input === undefined) {
-      console.warn(`[WorkflowOrchestrator] [WARNING] Step ${step.step_id} - context.api_input is undefined - template references like {{api_input.field}} will fail`);
-    } else if (context.api_input === null) {
-      console.warn(`[WorkflowOrchestrator] [WARNING] Step ${step.step_id} - context.api_input is null - template references like {{api_input.field}} will fail`);
-    } else if (typeof context.api_input !== 'object' || Array.isArray(context.api_input)) {
-      const error = new Error(`Invalid api_input type for step ${step.step_id}: expected object, got ${typeof context.api_input}`);
-      console.error(`[WorkflowOrchestrator] [FAIL-FAST] Invalid api_input type:`, {
-        step_id: step.step_id,
-        api_input_type: typeof context.api_input,
-        api_input_value: context.api_input,
-        is_array: Array.isArray(context.api_input)
-      });
-      throw error;
-    } else {
-      console.log(`[WorkflowOrchestrator] [RESOLVE] Step ${step.step_id} - context.api_input:`, JSON.stringify(context.api_input));
-    }
-    
-    console.log(`[WorkflowOrchestrator] [RESOLVE] Step ${step.step_id} - context.steps keys:`, 
-      context.steps ? Object.keys(context.steps) : 'NO STEPS');
-    
-    // Resolve variable references in step.input (e.g. {{steps.step_id.output.field}})
-    console.log(`[WorkflowOrchestrator] [RESOLVE] Step ${step.step_id} input BEFORE:`, JSON.stringify(step.input));
     const resolvedInput = this._resolveInputReferences(step.input, context);
-    console.log(`[WorkflowOrchestrator] [RESOLVE] Step ${step.step_id} input AFTER:`, JSON.stringify(resolvedInput));
-    
-    // FAIL-FAST: Check if template references were resolved
-    const inputStr = JSON.stringify(resolvedInput);
-    if (inputStr.includes('{{api_input.') || inputStr.includes('${api_input.')) {
-      console.error(`[WorkflowOrchestrator] [FAIL-FAST] Unresolved api_input template in step ${step.step_id}:`, {
-        step_id: step.step_id,
-        resolved_input: resolvedInput,
-        context_api_input: context.api_input,
-        context_keys: Object.keys(context)
-      });
-      throw new Error(`Template reference to api_input not resolved in step ${step.step_id} - check that api_input is in context`);
-    }
+    console.log(`[WorkflowOrchestrator] [RESOLVE] Step ${stepId} input AFTER:`, JSON.stringify(resolvedInput));
     
     // Use API mapper to call the service with resolved input
     const result = await this.apiMapper.callOperation(
-      step.operation,
+      step.operation || stepId,
       resolvedInput,
       context
     );
 
-    // === STRUCTURED LOG: Step result ===
-    console.log(`[WorkflowOrchestrator:STEP_RESULT] ${JSON.stringify({
-      layer: 'orchestrator',
-      step_id: step.step_id,
-      service: step.service,
-      operation: step.operation,
-      result: {
-        _descriptor: result?._descriptor,
-        type: result?.type,
-        storage_ref: result?.storage_ref,
-        resultKeys: result && typeof result === 'object' ? Object.keys(result) : null
-      },
-      timestamp: new Date().toISOString()
-    })}`);
-
     this.logger.info(`Task step executed`, {
-      step: step.step_id,
+      step: stepId,
       service: step.service,
       operation: step.operation,
       inputResolved: !!resolvedInput
@@ -377,9 +304,7 @@ class WorkflowOrchestrator {
   
   /**
    * Resolve variable references in input object
-   * Supports both syntaxes:
-   * - {{steps.step_id.output.field}}, {{api_input.field}} (preferred, V2)
-   * - ${steps[N].output.field}, ${api_input.field} (legacy)
+   * Supports ${steps[N].output.field}, ${api_input.field}, ${context.field}
    * @private
    * @param {Object} input - Input object with potential references
    * @param {Object} context - Current context with steps, api_input, etc
@@ -389,39 +314,12 @@ class WorkflowOrchestrator {
     if (!input) return input;
     
     const resolveValue = (value) => {
-      if (typeof value === 'string') {
-        let result = value;
-        
-        // Replace {{...}} references (V2 preferred syntax)
-        if (result.includes('{{')) {
-          result = result.replace(/\{\{([^}]+)\}\}/g, (match, path) => {
-            const resolved = this._getValueFromPath(path.trim(), context);
-            if (resolved !== undefined) {
-              // If resolved value is an object, stringify it for embedding in strings
-              if (typeof resolved === 'object' && result !== match) {
-                return JSON.stringify(resolved);
-              }
-              return resolved;
-            }
-            return match;
-          });
-        }
-        
-        // Replace ${...} references (legacy syntax)
-        if (result.includes('${')) {
-          result = result.replace(/\$\{([^}]+)\}/g, (match, path) => {
-            const resolved = this._getValueFromPath(path.trim(), context);
-            if (resolved !== undefined) {
-              if (typeof resolved === 'object' && result !== match) {
-                return JSON.stringify(resolved);
-              }
-              return resolved;
-            }
-            return match;
-          });
-        }
-        
-        return result;
+      if (typeof value === 'string' && value.includes('${')) {
+        // Replace ${...} references
+        return value.replace(/\$\{([^}]+)\}/g, (match, path) => {
+          const resolved = this._getValueFromPath(path.trim(), context);
+          return resolved !== undefined ? resolved : match;
+        });
       }
       if (Array.isArray(value)) {
         return value.map(resolveValue);
@@ -440,34 +338,16 @@ class WorkflowOrchestrator {
   /**
    * Get value from dot/bracket notation path
    * @private
-   * @param {string} path - Path like "steps.step_id.output.message" or "api_input.name"
+   * @param {string} path - Path like "steps[0].output.message" or "api_input.name"
    * @param {Object} context - Context object
    * @returns {*} Resolved value or undefined
    */
   _getValueFromPath(path, context) {
-    // FAIL-FAST: Verify context exists
-    if (!context) {
-      console.error('[WorkflowOrchestrator] [FAIL-FAST] _getValueFromPath called with null/undefined context', { path });
-      throw new Error(`Cannot resolve path '${path}': context is null or undefined`);
-    }
-    
-    // V2: steps is an object keyed by step_id, not an array
-    // Support both formats for backward compatibility during migration:
-    // - steps.step_id.output.field (V2, preferred)
-    // - steps[0].output.field (legacy, deprecated)
-    
-    // Normalize bracket notation to dot notation: steps[0] -> steps.0 (legacy)
-    // But prefer step_id access: steps.step_id -> steps.step_id (V2)
-    let normalized = path;
-    
-    // Handle legacy array access steps[N] - convert to numeric key
-    // But only if steps is actually an array (fallback for old data)
-    if (normalized.includes('[') && normalized.includes(']')) {
-      normalized = normalized.replace(/\[(\d+)\]/g, '.$1');
-    }
+    // Normalize bracket notation to dot notation: steps[0] -> steps.0
+    let normalized = path.replace(/\[(\d+)\]/g, '.$1');
     
     // Strip prefixes - we're already searching in context object
-    // Supports: ${context.X}, ${api_input.X}, ${steps.step_id.X} (V2)
+    // Supports: ${context.X}, ${api_input.X}, ${steps[N].X}
     if (normalized.startsWith('context.')) {
       normalized = normalized.substring(8); // Remove 'context.'
     }
@@ -475,46 +355,9 @@ class WorkflowOrchestrator {
     
     const parts = normalized.split('.');
     
-    // FAIL-FAST: Log api_input access attempts
-    if (normalized.startsWith('api_input.')) {
-      console.log(`[WorkflowOrchestrator] [GET_VALUE] Resolving api_input path: ${path} -> ${normalized}`);
-      console.log(`[WorkflowOrchestrator] [GET_VALUE] context.api_input exists:`, context.api_input !== undefined);
-      console.log(`[WorkflowOrchestrator] [GET_VALUE] context.api_input type:`, typeof context.api_input);
-      if (context.api_input) {
-        console.log(`[WorkflowOrchestrator] [GET_VALUE] context.api_input keys:`, Object.keys(context.api_input));
-      }
-    }
-    
     let current = context;
     for (const part of parts) {
-      if (current === undefined || current === null) {
-        // FAIL-FAST: Log failed resolution for api_input
-        if (normalized.startsWith('api_input.')) {
-          console.error(`[WorkflowOrchestrator] [FAIL-FAST] Failed to resolve api_input path: ${path}`, {
-            path,
-            normalized,
-            parts,
-            failed_at_part: part,
-            current_type: typeof current,
-            context_keys: Object.keys(context),
-            context_has_api_input: 'api_input' in context,
-            api_input_type: typeof context.api_input,
-            api_input_keys: context.api_input ? Object.keys(context.api_input) : null
-          });
-          throw new Error(`Failed to resolve api_input path '${path}': '${part}' is undefined in context`);
-        }
-        return undefined;
-      }
-      
-      // Special handling for steps: if accessing by numeric index and steps is object,
-      // try to find step by index in cookbook (fallback for legacy)
-      if (part === 'steps' && parts.length > 1) {
-        current = current.steps;
-        // If steps is object (V2), next part is step_id
-        // If steps is array (legacy), next part is numeric index
-        continue;
-      }
-      
+      if (current === undefined || current === null) return undefined;
       current = current[part];
     }
     return current;
@@ -549,7 +392,7 @@ class WorkflowOrchestrator {
     const result = await executor.executeStep(step);
 
     this.logger.info(`Control flow step executed`, {
-      step: step.step_id,
+      step: this._getStepId(step),
       type: step.type
     });
 
@@ -564,9 +407,9 @@ class WorkflowOrchestrator {
    * @param {Object} cookbookDef - Full cookbook definition
    * @param {Object} context - Updated context
    * @param {string} workflow_id - Workflow ID
-   * @param {string} nextStepId - Next step step_id (V2) or id (V1 fallback)
    */
-  async _routeToNextStep(nextStep, cookbookDef, context, workflow_id, nextStepId) {
+  async _routeToNextStep(nextStep, cookbookDef, context, workflow_id) {
+    const nextStepId = this._getStepId(nextStep);
     const message = {
       workflow_id,
       cookbook: cookbookDef,
@@ -624,21 +467,6 @@ class WorkflowOrchestrator {
       // Delivery Dispatcher requires: workflow_id, status, delivery (must be object, not null)
       const delivery = finalContext?.delivery || cookbookDef?.delivery || { handler: 'none' };
       
-      // Extract api_output from the last completed step
-      // steps is an object keyed by step_id in V2
-      let api_output = null;
-      if (lastStepId && finalContext?.steps?.[lastStepId]?.output) {
-        api_output = finalContext.steps[lastStepId].output;
-      } else if (finalContext?.steps) {
-        // Fallback: find any step with output
-        const stepKeys = Object.keys(finalContext.steps);
-        for (const key of stepKeys) {
-          if (finalContext.steps[key]?.output) {
-            api_output = finalContext.steps[key].output;
-          }
-        }
-      }
-      
       // Build message in format expected by Delivery Dispatcher
       const workflowCompletedMessage = {
         workflow_id,
@@ -646,11 +474,10 @@ class WorkflowOrchestrator {
         service: serviceName, // Service that completed the workflow (for monitoring)
         step_id: lastStepId, // Last step ID for monitoring
         step_index: lastStepIndex, // Last step index for monitoring
-        api_output: api_output, // Output from last step - for frontend/delivery
         cookbook: cookbookDef, // Full cookbook for monitoring trace
         delivery: delivery, // Delivery configuration from Gateway context (must be object)
         context: finalContext, // Full context for output resolution
-        steps: finalContext?.steps || {}, // Steps results (object) for output resolution
+        steps: finalContext?.steps || [], // Steps results (array) for output resolution
         completed_at: new Date().toISOString()
       };
       
@@ -678,14 +505,15 @@ class WorkflowOrchestrator {
    */
   _getCacheKey(step, context) {
     const crypto = require('crypto');
+    const stepId = this._getStepId(step);
     const data = JSON.stringify({
       service: step.service,
-      operation: step.operation,
+      operation: step.operation || stepId,
       input: step.input,
       contextInput: context.api_input
     });
     const hash = crypto.createHash('sha256').update(data).digest('hex');
-    return `workflow:step:${step.service}:${step.operation}:${hash}`;
+    return `workflow:step:${step.service}:${step.operation || stepId}:${hash}`;
   }
 
   /**
@@ -723,77 +551,6 @@ class WorkflowOrchestrator {
     const mapper = new ResponseMapper();
     return mapper.mapResponse(response, mapping);
   }
-
-  /**
-   * FAIL-FAST: Validate V2 format strictly
-   * Rejects V1 format immediately with clear error message
-   * @private
-   * @param {Object} cookbook - Cookbook definition
-   * @throws {Error} If cookbook is not V2 format
-   */
-  _validateV2Format(cookbook) {
-    if (!cookbook) {
-      throw new Error('FAIL-FAST: Cookbook is required');
-    }
-
-    // Check version format
-    const version = cookbook.version;
-    if (!version) {
-      throw new Error('FAIL-FAST: Cookbook version is required');
-    }
-
-    // V1 format detection - REJECT immediately
-    if (/^1\.\d+\.\d+$/.test(version)) {
-      throw new Error(`FAIL-FAST: V1 format (${version}) is DEPRECATED and no longer supported. Use V2 format (2.x.x) with step_id instead of id.`);
-    }
-
-    // Must be V2 format
-    if (!/^2\.\d+\.\d+$/.test(version)) {
-      throw new Error(`FAIL-FAST: Invalid version format '${version}'. Must be 2.x.x (e.g., 2.0.0)`);
-    }
-
-    // Check steps - V2 FORMAT: steps MUST be an OBJECT (keyed by step_id), NOT an array!
-    if (!cookbook.steps || typeof cookbook.steps !== 'object') {
-      throw new Error('FAIL-FAST: Cookbook must have steps object');
-    }
-    
-    // V1 format (array) is BANNED!
-    if (Array.isArray(cookbook.steps)) {
-      throw new Error('FAIL-FAST: V1 FORMAT BANNED! Cookbook steps must be an OBJECT (keyed by step_id), NOT an array!');
-    }
-    
-    const stepKeys = Object.keys(cookbook.steps);
-    if (stepKeys.length === 0) {
-      throw new Error('FAIL-FAST: Cookbook must have at least one step');
-    }
-
-    // Validate each step has step_id (V2) not id (V1)
-    const invalidSteps = [];
-    stepKeys.forEach((stepKey) => {
-      const step = cookbook.steps[stepKey];
-      if (step.id && !step.step_id) {
-        invalidSteps.push(`Step '${stepKey}': has 'id' (V1) but missing 'step_id' (V2)`);
-      }
-      if (!step.step_id) {
-        invalidSteps.push(`Step '${stepKey}': missing required 'step_id'`);
-      }
-      if (step.step_id !== stepKey) {
-        invalidSteps.push(`Step '${stepKey}': step_id '${step.step_id}' does not match key '${stepKey}'`);
-      }
-      if (!step.type) {
-        invalidSteps.push(`Step '${stepKey}': missing required 'type'`);
-      }
-    });
-
-    if (invalidSteps.length > 0) {
-      throw new Error(`FAIL-FAST: Invalid V2 cookbook format:\n${invalidSteps.join('\n')}`);
-    }
-
-    this.logger.info('[WorkflowOrchestrator] V2 format validation passed', {
-      version: cookbook.version,
-      steps: Object.keys(cookbook.steps)
-    });
-  }
 }
 
 module.exports = WorkflowOrchestrator;