@onlineapps/conn-orch-orchestrator 1.0.63 → 1.0.65

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@onlineapps/conn-orch-orchestrator",
-  "version": "1.0.63",
+  "version": "1.0.65",
   "description": "Workflow orchestration connector for OA Drive - handles message routing and workflow execution",
   "main": "src/index.js",
   "scripts": {
-    "test": "jest",
-    "test:watch": "jest --watch",
-    "test:coverage": "jest --coverage",
+    "test": "jest --config jest.config.js",
+    "test:watch": "jest --config jest.config.js --watch",
+    "test:coverage": "jest --config jest.config.js --coverage",
     "lint": "eslint src/",
     "docs": "jsdoc2md --files src/**/*.js > API.md"
   },
@@ -24,18 +24,11 @@
     "@onlineapps/conn-infra-mq": "^1.1.0",
     "@onlineapps/conn-orch-registry": "1.1.25",
     "@onlineapps/conn-orch-cookbook": "2.0.12",
-    "@onlineapps/conn-orch-api-mapper": "1.0.13"
+    "@onlineapps/conn-orch-api-mapper": "1.0.14"
   },
   "devDependencies": {
     "jest": "^29.5.0",
     "eslint": "^8.30.0",
     "jsdoc-to-markdown": "^8.0.0"
-  },
-  "jest": {
-    "testEnvironment": "node",
-    "coverageDirectory": "coverage",
-    "collectCoverageFrom": [
-      "src/**/*.js"
-    ]
   }
 }

package/src/WorkflowOrchestrator.js

@@ -141,6 +141,15 @@ class WorkflowOrchestrator {
         return { skipped: true, reason: 'wrong_service' };
       }
 
+      // OPTIONAL: pin control-flow execution to a specific service (explicit, no magic)
+      // If a control-flow step has "service", only that service is allowed to execute it.
+      if (step.type !== 'task' && step.service && step.service !== serviceName) {
+        this.logger.warn(`Control-flow step ${current_step} is pinned to ${step.service}, not ${serviceName}`);
+        const serviceQueue = `${step.service}.workflow`;
+        await this.mqClient.publish(serviceQueue, message);
+        return { skipped: true, reason: 'wrong_service_control_flow' };
+      }
+
       // Mark step as started (preserve existing retry_history if requeued)
       const updatedSteps = [...stepsArray];
       updatedSteps[currentIndex] = {
@@ -162,6 +171,8 @@ class WorkflowOrchestrator {
         }
       });
       const stepContext = {
+        workflow_id: workflow_id,
+        step_id: current_step,
         api_input: cookbookDef.api_input || {},
         steps: stepsArrayCtx,       // array (legacy consumers expect .map)
         steps_by_id: stepsMap,      // fast lookup by step_id for templating
@@ -174,11 +185,11 @@ class WorkflowOrchestrator {
         const cacheKey = this._getCacheKey(step, stepContext);
         result = await this.cache.get(cacheKey);
         if (!result) {
-          result = await this._executeStep(step, stepContext, cookbookDef);
+          result = await this._executeStep(step, stepContext, cookbookDef, serviceName);
           await this.cache.set(cacheKey, result, { ttl: 300 });
         }
       } else {
-        result = await this._executeStep(step, stepContext, cookbookDef);
+        result = await this._executeStep(step, stepContext, cookbookDef, serviceName);
       }
 
       // Calculate duration
@@ -456,13 +467,13 @@ class WorkflowOrchestrator {
    * @param {Object} cookbookDef - Full cookbook definition
    * @returns {Promise<Object>} Step result
    */
-  async _executeStep(step, context, cookbookDef) {
+  async _executeStep(step, context, cookbookDef, serviceName) {
     if (step.type === 'task') {
       // Task step - call service API
-      return await this._executeTaskStep(step, context);
+      return await this._executeTaskStep(step, context, serviceName);
     } else {
-      // Control flow step - use cookbook executor
-      return await this._executeControlFlowStep(step, context, cookbookDef);
+      // Control flow step
+      return await this._executeControlFlowStep(step, context, cookbookDef, serviceName);
     }
   }
 
@@ -474,7 +485,7 @@ class WorkflowOrchestrator {
    * @param {Object} context - Execution context
    * @returns {Promise<Object>} API call result
    */
-  async _executeTaskStep(step, context) {
+  async _executeTaskStep(step, context, serviceName) {
     const stepId = this._getStepId(step);
     // Resolve variable references in step.input (e.g. ${steps[0].output.message})
     console.log(`[WorkflowOrchestrator] [RESOLVE] Step ${stepId} input BEFORE:`, JSON.stringify(step.input));
@@ -482,14 +493,25 @@ class WorkflowOrchestrator {
     console.log(`[WorkflowOrchestrator] [RESOLVE] Context.api_input:`, JSON.stringify(context?.api_input));
     console.log(`[WorkflowOrchestrator] [RESOLVE] Context.steps:`, JSON.stringify(context.steps?.map(s => ({ id: this._getStepId(s), output: s.output }))));
     
-    const resolvedInput = this._resolveInputReferences(step.input, context);
+    const helperContext = {
+      data: context,
+      workflow_id: context?.workflow_id || null,
+      step_id: stepId,
+      service: serviceName || null
+    };
+
+    const resolvedInput = await this._resolveInputReferencesAsync(step.input, context, helperContext);
     console.log(`[WorkflowOrchestrator] [RESOLVE] Step ${stepId} input AFTER:`, JSON.stringify(resolvedInput));
     
     // Use API mapper to call the service with resolved input
     const result = await this.apiMapper.callOperation(
       step.operation || stepId,
       resolvedInput,
-      context
+      {
+        ...context,
+        workflow_id: context?.workflow_id,
+        step_id: stepId
+      }
     );
 
     this.logger.info(`Task step executed`, {
@@ -501,51 +523,166 @@ class WorkflowOrchestrator {
 
     return result;
   }
-  
+
   /**
-   * Resolve variable references in input object
-   * Supports {{steps.step_id.output.field}}, {{api_input.field}}, {{context.field}}
+   * Resolve template variables in input recursively (async)
+   *
+   * Supported:
+   * - {{steps.step_id.output.field}}, {{api_input.field}}, {{current.field}}
+   * - helper calls: {{normalizeString(api_input.text)}}, {{webalizeString(api_input.text)}}
+   * - async helper calls: {{string2file(api_input.text)}}, {{file2string(steps.x.output.file)}}
+   *
+   * Helper contract:
+   * - Helper is resolved from cookbook connector: this.cookbook.templateHelpers[helperName]
+   * - Helper is called with (...args, helperContext)
+   *
    * @private
-   * @param {Object} input - Input object with potential references
-   * @param {Object} context - Current context with steps, api_input, etc
-   * @returns {Object} Resolved input object
    */
-  _resolveInputReferences(input, context) {
+  async _resolveInputReferencesAsync(input, context, helperContext) {
     if (!input) return input;
-    
-    const resolveValue = (value) => {
+
+    const resolveValue = async (value) => {
       if (typeof value === 'string' && value.includes('{{')) {
-        // Check if the ENTIRE string is a single template reference
-        // e.g. "{{steps.greeting.output}}" should return the object directly
+        // Entire value is a single template expression -> preserve non-string types
         const singleRefMatch = value.match(/^\{\{([^}]+)\}\}$/);
         if (singleRefMatch) {
-          // Entire value is a single reference - return the resolved value directly (preserves objects)
-          const resolved = this._getValueFromPath(singleRefMatch[1].trim(), context);
+          const resolved = await this._resolveTemplateExpression(singleRefMatch[1].trim(), context, helperContext);
           return resolved !== undefined ? resolved : value;
         }
-        
-        // Multiple references or mixed with text - use string replacement
-        // e.g. "Hello {{name}}, your order {{orderId}}" becomes "Hello John, your order 123"
-        return value.replace(/\{\{([^}]+)\}\}/g, (match, path) => {
-          const resolved = this._getValueFromPath(path.trim(), context);
-          if (resolved === undefined) return match;
-          // For string concatenation, convert objects to JSON string
-          if (typeof resolved === 'object') return JSON.stringify(resolved);
-          return String(resolved);
-        });
+
+        // Mixed string with multiple expressions -> string replacement
+        const matches = [...value.matchAll(/\{\{([^}]+)\}\}/g)];
+        let out = value;
+        for (const match of matches) {
+          const full = match[0];
+          const expr = match[1].trim();
+          const resolved = await this._resolveTemplateExpression(expr, context, helperContext);
+          if (resolved === undefined) continue;
+          const replacement = typeof resolved === 'object' ? JSON.stringify(resolved) : String(resolved);
+          out = out.replace(full, replacement);
+        }
+        return out;
       }
+
       if (Array.isArray(value)) {
-        return value.map(resolveValue);
+        return Promise.all(value.map(resolveValue));
       }
+
       if (value && typeof value === 'object') {
-        return Object.fromEntries(
-          Object.entries(value).map(([k, v]) => [k, resolveValue(v)])
+        const entries = await Promise.all(
+          Object.entries(value).map(async ([k, v]) => [k, await resolveValue(v)])
         );
+        return Object.fromEntries(entries);
       }
+
       return value;
     };
-    
-    return resolveValue(input);
+
+    return await resolveValue(input);
+  }
+
+  /**
+   * Resolve a single template expression.
+   * @private
+   */
+  async _resolveTemplateExpression(expression, context, helperContext) {
+    // Detect helper call: helperName(...)
+    const helperCallMatch = expression.match(/^([a-zA-Z_][a-zA-Z0-9_]*)\(([\s\S]*)\)$/);
+    if (helperCallMatch) {
+      const helperName = helperCallMatch[1];
+      const argsRaw = helperCallMatch[2].trim();
+      return await this._executeTemplateHelper(helperName, argsRaw, context, helperContext);
+    }
+
+    // Plain path reference
+    return this._getValueFromPath(expression, context);
+  }
+
+  /**
+   * Execute helperName(args...) and return its result.
+   * @private
+   */
+  async _executeTemplateHelper(helperName, argsRaw, context, helperContext) {
+    const helpers = this.cookbook?.templateHelpers || {};
+    const helperFn = helpers[helperName];
+
+    if (typeof helperFn !== 'function') {
+      throw new Error(`[WorkflowOrchestrator] Unknown template helper '${helperName}' - Expected helper to exist in cookbook.templateHelpers`);
+    }
+
+    const argTokens = this._splitHelperArgs(argsRaw);
+    const args = [];
+    for (const token of argTokens) {
+      args.push(await this._evalHelperArg(token, context, helperContext));
+    }
+
+    // Convention: helper(...args, helperContext)
+    return await helperFn(...args, helperContext);
+  }
+
+  /**
+   * Split helper args by commas, respecting quotes.
+   * @private
+   */
+  _splitHelperArgs(argsRaw) {
+    if (!argsRaw) return [];
+    const args = [];
+    let cur = '';
+    let inSingle = false;
+    let inDouble = false;
+
+    for (let i = 0; i < argsRaw.length; i++) {
+      const ch = argsRaw[i];
+      const prev = argsRaw[i - 1];
+
+      if (ch === "'" && !inDouble && prev !== '\\') {
+        inSingle = !inSingle;
+        cur += ch;
+        continue;
+      }
+      if (ch === '"' && !inSingle && prev !== '\\') {
+        inDouble = !inDouble;
+        cur += ch;
+        continue;
+      }
+      if (ch === ',' && !inSingle && !inDouble) {
+        const trimmed = cur.trim();
+        if (trimmed.length > 0) args.push(trimmed);
+        cur = '';
+        continue;
+      }
+      cur += ch;
+    }
+
+    const last = cur.trim();
+    if (last.length > 0) args.push(last);
+    return args;
+  }
+
+  /**
+   * Evaluate a helper argument token.
+   * Supports: string literals, booleans/null/numbers, and path references.
+   * @private
+   */
+  async _evalHelperArg(token, context, helperContext) {
+    const t = token.trim();
+    if ((t.startsWith('"') && t.endsWith('"')) || (t.startsWith("'") && t.endsWith("'"))) {
+      return t.slice(1, -1);
+    }
+    if (t === 'true') return true;
+    if (t === 'false') return false;
+    if (t === 'null') return null;
+    if (t === 'undefined') return undefined;
+    if (/^-?\d+(\.\d+)?$/.test(t)) return Number(t);
+
+    // Default: treat as path in context
+    // Allow nested helper calls as args: helperName(...)
+    const nestedHelperMatch = t.match(/^([a-zA-Z_][a-zA-Z0-9_]*)\(([\s\S]*)\)$/);
+    if (nestedHelperMatch) {
+      return await this._executeTemplateHelper(nestedHelperMatch[1], nestedHelperMatch[2].trim(), context, helperContext);
+    }
+
+    return this._getValueFromPath(t, context);
   }
   
   /**
@@ -603,31 +740,273 @@ class WorkflowOrchestrator {
    * @param {Object} cookbookDef - Full cookbook definition
    * @returns {Promise<Object>} Control flow result
    */
-  async _executeControlFlowStep(step, context, cookbookDef) {
-    const { CookbookExecutor } = this.cookbook;
+  async _executeControlFlowStep(step, context, cookbookDef, serviceName) {
+    const stepId = this._getStepId(step);
 
-    // Create executor for control flow
-    const executor = new CookbookExecutor(
-      { steps: [step] },
-      {
-        logger: this.logger,
-        defaultTimeout: this.defaultTimeout
+    // Fail-fast: if pinned, it must already match (enforced earlier), but keep it explicit.
+    if (step.service && serviceName && step.service !== serviceName) {
+      throw new Error(`[WorkflowOrchestrator] Control-flow step '${stepId}' pinned to '${step.service}' - Current service is '${serviceName}'`);
+    }
+
+    switch (step.type) {
+      case 'steps':
+        return await this._executeStepsGroup(step, context, cookbookDef, serviceName);
+      case 'foreach':
+        return await this._executeForeach(step, context, cookbookDef, serviceName);
+      case 'switch':
+        return await this._executeSwitch(step, context, cookbookDef, serviceName);
+      case 'fork_join':
+        return await this._executeForkJoin(step, context, cookbookDef, serviceName);
+      default: {
+        // Fallback to legacy executor behavior (does NOT execute nested tasks)
+        const { CookbookExecutor } = this.cookbook;
+        if (!CookbookExecutor) {
+          throw new Error(`[WorkflowOrchestrator] CookbookExecutor missing - Cannot execute control-flow step '${stepId}'`);
+        }
+
+        const executor = new CookbookExecutor(
+          { steps: [step] },
+          { logger: this.logger, defaultTimeout: this.defaultTimeout }
+        );
+        executor.context.setInput(context.api_input || {});
+        executor.context.data.steps = context.steps || {};
+        return await executor.executeStep(step);
       }
-    );
+    }
+  }
 
-    // Set context
-    executor.context.setInput(context.api_input || {});
-    executor.context.data.steps = context.steps || {};
+  /**
+   * Execute nested steps group (V2.1: type=steps, steps: Step[])
+   * @private
+   */
+  async _executeStepsGroup(step, context, cookbookDef, serviceName) {
+    const stepId = this._getStepId(step);
+    const steps = Array.isArray(step.steps) ? step.steps : [];
 
-    // Execute the control flow step
-    const result = await executor.executeStep(step);
+    if (!Array.isArray(step.steps)) {
+      throw new Error(`[WorkflowOrchestrator] Steps step '${stepId}' invalid - Expected 'steps' to be an array`);
+    }
 
-    this.logger.info(`Control flow step executed`, {
-      step: this._getStepId(step),
-      type: step.type
-    });
+    const resultsByStepId = {};
+    const order = [];
 
-    return result;
+    for (const nested of steps) {
+      const nestedId = this._getStepId(nested);
+      if (!nestedId) {
+        throw new Error(`[WorkflowOrchestrator] Steps step '${stepId}' contains nested step without step_id`);
+      }
+      const output = await this._executeNestedStep(nested, context, cookbookDef, serviceName);
+      resultsByStepId[nestedId] = output;
+      order.push(nestedId);
+    }
+
+    return {
+      type: 'steps',
+      step_id: stepId,
+      order,
+      results: resultsByStepId
+    };
+  }
+
+  /**
+   * Execute foreach (V2.1: iterator + body[])
+   * @private
+   */
+  async _executeForeach(step, context, cookbookDef, serviceName) {
+    const stepId = this._getStepId(step);
+    const helperContext = {
+      data: context,
+      workflow_id: context?.workflow_id || null,
+      step_id: stepId,
+      service: serviceName || null
+    };
+
+    const iteratorResolved = await this._resolveInputReferencesAsync(step.iterator, context, helperContext);
+    const items = Array.isArray(iteratorResolved) ? iteratorResolved : iteratorResolved;
+
+    if (!Array.isArray(items)) {
+      throw new Error(`[WorkflowOrchestrator] Foreach '${stepId}' iterator must resolve to array - Got ${typeof items}`);
+    }
+
+    const body = Array.isArray(step.body) ? step.body : [];
+    if (!Array.isArray(step.body) || body.length === 0) {
+      throw new Error(`[WorkflowOrchestrator] Foreach '${stepId}' invalid - Expected non-empty 'body' array`);
+    }
+
+    const iterations = [];
+    for (let i = 0; i < items.length; i++) {
+      const item = items[i];
+      const iterationContext = {
+        ...context,
+        current: item,
+        index: i
+      };
+
+      const outputs = {};
+      for (const nested of body) {
+        const nestedId = this._getStepId(nested);
+        if (!nestedId) {
+          throw new Error(`[WorkflowOrchestrator] Foreach '${stepId}' body contains nested step without step_id`);
+        }
+        outputs[nestedId] = await this._executeNestedStep(nested, iterationContext, cookbookDef, serviceName);
+      }
+
+      iterations.push({ index: i, item, outputs });
+    }
+
+    return {
+      type: 'foreach',
+      step_id: stepId,
+      count: items.length,
+      iterations
+    };
+  }
+
+  /**
+   * Execute switch (V2.1: expression + cases{} + optional default)
+   * @private
+   */
+  async _executeSwitch(step, context, cookbookDef, serviceName) {
+    const stepId = this._getStepId(step);
+    const helperContext = {
+      data: context,
+      workflow_id: context?.workflow_id || null,
+      step_id: stepId,
+      service: serviceName || null
+    };
+
+    const value = await this._evaluateSwitchExpression(step.expression, context, helperContext);
+    const key = value === null ? 'null' : String(value);
+
+    const cases = step.cases && typeof step.cases === 'object' ? step.cases : null;
+    if (!cases) {
+      throw new Error(`[WorkflowOrchestrator] Switch '${stepId}' invalid - Expected 'cases' object`);
+    }
+
+    const selectedStep = cases[key] ? cases[key] : (step.default ? step.default : null);
+    const selectedCase = cases[key] ? key : (step.default ? 'default' : null);
+
+    if (!selectedStep) {
+      return {
+        type: 'switch',
+        step_id: stepId,
+        expression: step.expression,
+        value,
+        selectedCase: null,
+        result: null
+      };
+    }
+
+    const result = await this._executeNestedStep(selectedStep, context, cookbookDef, serviceName);
+
+    return {
+      type: 'switch',
+      step_id: stepId,
+      expression: step.expression,
+      value,
+      selectedCase,
+      result
+    };
+  }
+
+  /**
+   * Execute fork_join (V2.1: branches{} + join)
+   * @private
+   */
+  async _executeForkJoin(step, context, cookbookDef, serviceName) {
+    const stepId = this._getStepId(step);
+    const branches = step.branches && typeof step.branches === 'object' ? step.branches : null;
+    if (!branches) {
+      throw new Error(`[WorkflowOrchestrator] ForkJoin '${stepId}' invalid - Expected 'branches' object`);
+    }
+
+    const branchEntries = Object.entries(branches);
+    if (branchEntries.length === 0) {
+      throw new Error(`[WorkflowOrchestrator] ForkJoin '${stepId}' invalid - Expected at least 1 branch`);
+    }
+
+    const results = {};
+    await Promise.all(branchEntries.map(async ([branchName, branchStep]) => {
+      results[branchName] = await this._executeNestedStep(branchStep, context, cookbookDef, serviceName);
+    }));
+
+    const strategy = step.join?.strategy || 'all';
+
+    return {
+      type: 'fork_join',
+      step_id: stepId,
+      join: { strategy },
+      branches: results
+    };
+  }
+
+  /**
+   * Execute nested step within a control-flow step.
+   * @private
+   */
+  async _executeNestedStep(step, context, cookbookDef, serviceName) {
+    const stepId = this._getStepId(step);
+
+    if (!step || typeof step !== 'object') {
+      throw new Error('[WorkflowOrchestrator] Nested step invalid - Expected object');
+    }
+    if (!step.type) {
+      throw new Error(`[WorkflowOrchestrator] Nested step '${stepId || 'unknown'}' invalid - Missing 'type'`);
+    }
+
+    if (step.type === 'task') {
+      // Fail-fast: nested task must be for THIS service (ApiMapper is service-bound)
+      if (step.service && serviceName && step.service !== serviceName) {
+        throw new Error(`[WorkflowOrchestrator] Nested task '${stepId}' targets service '${step.service}' but is executing in '${serviceName}' - Expected same service`);
+      }
+      return await this._executeTaskStep(step, context, serviceName);
+    }
+
+    return await this._executeControlFlowStep(step, context, cookbookDef, serviceName);
+  }
+
+  /**
+   * Evaluate switch expression (explicit, predictable).
+   * Supports:
+   * - {{...}} templates (including helpers)
+   * - $api_input.* / $steps.* / $.api_input.* / $.steps.*
+   * - plain path like api_input.mode
+   * @private
+   */
+  async _evaluateSwitchExpression(expression, context, helperContext) {
+    if (expression === null || expression === undefined) {
+      return null;
+    }
+
+    if (typeof expression !== 'string') {
+      return expression;
+    }
+
+    if (expression.includes('{{')) {
+      return await this._resolveInputReferencesAsync(expression, context, helperContext);
+    }
+
+    // $api_input.x, $steps.x, $.api_input.x
+    if (expression.startsWith('$.')) {
+      const v = this._getValueFromPath(expression.substring(2), context);
+      return v !== undefined ? v : undefined;
+    }
+    if (expression.startsWith('$api_input.') || expression.startsWith('$steps.')) {
+      const v = this._getValueFromPath(expression.substring(1), context);
+      return v !== undefined ? v : undefined;
+    }
+
+    // plain dot path (api_input.mode)
+    const v = this._getValueFromPath(expression, context);
+    if (v !== undefined) return v;
+
+    // If it looks like a path (contains dot) but cannot be resolved -> fail-fast
+    if (expression.includes('.')) {
+      throw new Error(`[WorkflowOrchestrator] Switch expression could not be resolved: '${expression}'`);
+    }
+
+    // Otherwise treat as literal
+    return expression;
   }
 
   /**
@@ -664,11 +1043,17 @@ class WorkflowOrchestrator {
         });
       }
     } else {
-      // Control flow steps handled by workflow init queue
-      await this.mqClient.publish('workflow.init', message);
-      this.logger.info(`Routed control flow to workflow.init`, {
-        step: nextStepId
-      });
+      // Control flow steps:
+      // - if pinned (step.service), route to that service queue
+      // - otherwise fall back to workflow.init (any service may execute)
+      if (nextStep.service) {
+        const serviceQueue = `${nextStep.service}.workflow`;
+        await this.mqClient.publish(serviceQueue, message);
+        this.logger.info(`Routed control flow to pinned service queue: ${serviceQueue}`, { step: nextStepId });
+      } else {
+        await this.mqClient.publish('workflow.init', message);
+        this.logger.info(`Routed control flow to workflow.init`, { step: nextStepId });
+      }
     }
   }