@marktoflow/core 2.0.0-alpha.9 → 2.0.2

package/dist/engine.js

@@ -4,10 +4,72 @@
  * Executes workflow steps with retry logic, variable resolution,
  * and SDK invocation.
  */
-import { StepStatus, WorkflowStatus, createExecutionContext, createStepResult, isActionStep, isSubWorkflowStep, isIfStep, isSwitchStep, isForEachStep, isWhileStep, isMapStep, isFilterStep, isReduceStep, isParallelStep, isTryStep, } from './models.js';
+import { StepStatus, WorkflowStatus, createExecutionContext, createStepResult, isActionStep, isSubWorkflowStep, isIfStep, isSwitchStep, isForEachStep, isWhileStep, isMapStep, isFilterStep, isReduceStep, isParallelStep, isTryStep, isScriptStep, isWaitStep, isMergeStep, } from './models.js';
+import { mergePermissions, toSecurityPolicy, } from './permissions.js';
+import { loadPromptFile, resolvePromptTemplate, validatePromptInputs, } from './prompt-loader.js';
 import { DEFAULT_FAILOVER_CONFIG, AgentHealthTracker, FailoverReason, } from './failover.js';
 import { parseFile } from './parser.js';
 import { resolve, dirname } from 'node:path';
+import { executeBuiltInOperation, isBuiltInOperation } from './built-in-operations.js';
+import { renderTemplate } from './template-engine.js';
+import { executeScriptAsync } from './script-executor.js';
+// ============================================================================
+// Helper Functions
+// ============================================================================
+/**
+ * Parse a duration string like "2h", "30m", "5s", "100ms" into milliseconds.
+ */
+function parseDuration(duration) {
+    const match = duration.match(/^(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)$/i);
+    if (!match) {
+        const asNum = Number(duration);
+        if (!isNaN(asNum))
+            return asNum; // Treat bare numbers as milliseconds
+        throw new Error(`Invalid duration format: "${duration}". Use format like "2h", "30m", "5s", "100ms"`);
+    }
+    const value = parseFloat(match[1]);
+    const unit = match[2].toLowerCase();
+    switch (unit) {
+        case 'ms': return value;
+        case 's': return value * 1000;
+        case 'm': return value * 60 * 1000;
+        case 'h': return value * 60 * 60 * 1000;
+        case 'd': return value * 24 * 60 * 60 * 1000;
+        default: return value;
+    }
+}
+/**
+ * Convert error to string for display/logging
+ */
+function errorToString(error) {
+    if (!error)
+        return 'Unknown error';
+    if (typeof error === 'string')
+        return error;
+    if (error instanceof Error)
+        return error.message;
+    try {
+        return JSON.stringify(error);
+    }
+    catch {
+        return String(error);
+    }
+}
+/**
+ * Get a field value from an object by key path.
+ */
+function getField(item, field) {
+    if (!item || typeof item !== 'object')
+        return undefined;
+    const parts = field.split('.');
+    let current = item;
+    for (const part of parts) {
+        if (!current || typeof current !== 'object')
+            return undefined;
+        current = current[part];
+    }
+    return current;
+}
 // ============================================================================
 // Retry Policy
 // ============================================================================
@@ -92,26 +154,25 @@ export class CircuitBreaker {
 // ============================================================================
 /**
  * Resolve template variables in a value.
- * Supports {{variable}} and {{inputs.name}} syntax.
+ * Supports {{variable}}, {{inputs.name}}, and Nunjucks filters.
+ *
+ * Uses Nunjucks as the template engine with:
+ * - Legacy regex operator support (=~, !~, //) converted to filters
+ * - Custom filters for string, array, object, date operations
+ * - Jinja2-style control flow ({% for %}, {% if %}, etc.)
  */
 export function resolveTemplates(value, context) {
     if (typeof value === 'string') {
-        // Check if the entire string is a single template expression
-        const singleTemplateMatch = value.match(/^\{\{([^}]+)\}\}$/);
-        if (singleTemplateMatch) {
-            // Return the actual value without converting to string
-            const path = singleTemplateMatch[1].trim();
-            const resolved = resolveVariablePath(path, context);
-            // For single template expressions, return the actual value (could be object, array, etc.)
-            // If undefined, return empty string for backward compatibility
-            return resolved !== undefined ? resolved : '';
-        }
-        // Otherwise, do string interpolation
-        return value.replace(/\{\{([^}]+)\}\}/g, (_, varPath) => {
-            const path = varPath.trim();
-            const resolved = resolveVariablePath(path, context);
-            return resolved !== undefined ? String(resolved) : '';
-        });
+        // Build the template context with all available variables
+        // Spread inputs first, then variables (variables override inputs if same key)
+        // Also keep inputs accessible via inputs.* for explicit access
+        const templateContext = {
+            ...context.inputs, // Spread inputs at root level for direct access ({{ path }})
+            ...context.variables, // Variables override inputs if same key
+            inputs: context.inputs, // Also keep inputs accessible as inputs.*
+        };
+        // Use the new Nunjucks-based template engine with legacy syntax support
+        return renderTemplate(value, templateContext);
     }
     if (Array.isArray(value)) {
         return value.map((v) => resolveTemplates(v, context));
@@ -141,6 +202,11 @@ export function resolveVariablePath(path, context) {
     if (fromVars !== undefined) {
         return fromVars;
     }
+    // Check inputs (for bare variable names like "value" instead of "inputs.value")
+    const fromInputs = getNestedValue(context.inputs, path);
+    if (fromInputs !== undefined) {
+        return fromInputs;
+    }
     // Check step metadata (for status checks like: step_id.status)
     const fromStepMeta = getNestedValue(context.stepMetadata, path);
     if (fromStepMeta !== undefined) {
@@ -150,26 +216,63 @@ export function resolveVariablePath(path, context) {
     return getNestedValue(context, path);
 }
 /**
- * Get a nested value from an object using dot notation.
+ * Get a nested value from an object using dot notation and array indexing.
+ * Supports paths like: "user.name", "items[0].name", "data.users[1].email"
  */
 function getNestedValue(obj, path) {
     if (obj === null || obj === undefined) {
         return undefined;
     }
-    const parts = path.split('.');
-    let current = obj;
+    // Parse path into parts, handling both dot notation and array indexing
+    // Convert "a.b[0].c[1]" into ["a", "b", "0", "c", "1"]
+    const parts = [];
+    let current = '';
+    for (let i = 0; i < path.length; i++) {
+        const char = path[i];
+        if (char === '.') {
+            if (current) {
+                parts.push(current);
+                current = '';
+            }
+        }
+        else if (char === '[') {
+            if (current) {
+                parts.push(current);
+                current = '';
+            }
+        }
+        else if (char === ']') {
+            if (current) {
+                parts.push(current);
+                current = '';
+            }
+        }
+        else {
+            current += char;
+        }
+    }
+    if (current) {
+        parts.push(current);
+    }
+    // Traverse the object using the parsed parts
+    let result = obj;
     for (const part of parts) {
-        if (current === null || current === undefined) {
+        if (result === null || result === undefined) {
             return undefined;
         }
-        if (typeof current === 'object') {
-            current = current[part];
+        // Check if part is a number (array index)
+        const index = Number(part);
+        if (!isNaN(index) && Array.isArray(result)) {
+            result = result[index];
+        }
+        else if (typeof result === 'object') {
+            result = result[part];
         }
         else {
             return undefined;
         }
     }
-    return current;
+    return result;
 }
 export class WorkflowEngine {
     config;
@@ -181,13 +284,17 @@ export class WorkflowEngine {
     failoverConfig;
     healthTracker;
     failoverEvents = [];
-    workflowPath; // Base path for resolving sub-workflows
+    workflowPath; // Base path for resolving sub-workflows (public for CLI state tracking)
+    workflowPermissions; // Workflow-level permissions
+    promptCache = new Map(); // Cache for loaded prompts
     constructor(config = {}, events = {}, stateStore) {
         this.config = {
             defaultTimeout: config.defaultTimeout ?? 60000,
             maxRetries: config.maxRetries ?? 3,
             retryBaseDelay: config.retryBaseDelay ?? 1000,
             retryMaxDelay: config.retryMaxDelay ?? 30000,
+            defaultAgent: config.defaultAgent,
+            defaultModel: config.defaultModel,
         };
         this.retryPolicy = new RetryPolicy(this.config.maxRetries, this.config.retryBaseDelay, this.config.retryMaxDelay);
         this.events = events;
@@ -232,6 +339,15 @@ export class WorkflowEngine {
         if (isTryStep(step)) {
             return this.executeTryStep(step, context, sdkRegistry, stepExecutor);
         }
+        if (isScriptStep(step)) {
+            return this.executeScriptStep(step, context);
+        }
+        if (isWaitStep(step)) {
+            return this.executeWaitStep(step, context);
+        }
+        if (isMergeStep(step)) {
+            return this.executeMergeStep(step, context);
+        }
         // Default: action or workflow step
         return this.executeStepWithFailover(step, context, sdkRegistry, stepExecutor);
     }
@@ -242,13 +358,22 @@ export class WorkflowEngine {
         const context = createExecutionContext(workflow, inputs);
         const stepResults = [];
         const startedAt = new Date();
+        // Store workflow-level permissions and defaults
+        this.workflowPermissions = workflow.permissions;
+        // Use workflow defaults if not set in engine config
+        if (!this.config.defaultAgent && workflow.defaultAgent) {
+            this.config.defaultAgent = workflow.defaultAgent;
+        }
+        if (!this.config.defaultModel && workflow.defaultModel) {
+            this.config.defaultModel = workflow.defaultModel;
+        }
         context.status = WorkflowStatus.RUNNING;
         this.events.onWorkflowStart?.(workflow, context);
         if (this.stateStore) {
             this.stateStore.createExecution({
                 runId: context.runId,
                 workflowId: workflow.metadata.id,
-                workflowPath: 'unknown',
+                workflowPath: this.workflowPath ?? 'unknown',
                 status: WorkflowStatus.RUNNING,
                 startedAt: startedAt,
                 completedAt: null,
@@ -272,12 +397,21 @@ export class WorkflowEngine {
                 context.stepMetadata[step.id] = {
                     status: result.status.toLowerCase(),
                     retryCount: result.retryCount,
-                    ...(result.error ? { error: result.error } : {}),
+                    ...(result.error ? { error: errorToString(result.error) } : {}),
                 };
                 // Store output variable
                 if (step.outputVariable && result.status === StepStatus.COMPLETED) {
                     context.variables[step.outputVariable] = result.output;
                 }
+                // Check if this step set workflow outputs (from workflow.set_outputs action)
+                if (result.status === StepStatus.COMPLETED &&
+                    result.output &&
+                    typeof result.output === 'object' &&
+                    '__workflow_outputs__' in result.output) {
+                    const outputObj = result.output;
+                    const outputs = outputObj['__workflow_outputs__'];
+                    context.workflowOutputs = outputs;
+                }
                 // Handle failure
                 if (result.status === StepStatus.FAILED) {
                     // Get error action from step if it has error handling
@@ -287,7 +421,7 @@ export class WorkflowEngine {
                     }
                     if (errorAction === 'stop') {
                         context.status = WorkflowStatus.FAILED;
-                        const workflowError = result.error || `Step ${step.id} failed`;
+                        const workflowError = result.error ? errorToString(result.error) : `Step ${step.id} failed`;
                         const workflowResult = this.buildWorkflowResult(workflow, context, stepResults, startedAt, workflowError);
                         this.events.onWorkflowComplete?.(workflow, workflowResult);
                         return workflowResult;
@@ -302,7 +436,7 @@ export class WorkflowEngine {
                             });
                         }
                         context.status = WorkflowStatus.FAILED;
-                        const workflowError = result.error || `Step ${step.id} failed`;
+                        const workflowError = result.error ? errorToString(result.error) : `Step ${step.id} failed`;
                         const workflowResult = this.buildWorkflowResult(workflow, context, stepResults, startedAt, workflowError);
                         this.events.onWorkflowComplete?.(workflow, workflowResult);
                         return workflowResult;
@@ -336,6 +470,136 @@ export class WorkflowEngine {
         this.events.onWorkflowComplete?.(workflow, workflowResult);
         return workflowResult;
     }
+    /**
+     * Resume a paused execution (e.g., after form submission).
+     *
+     * @param runId - The execution run ID
+     * @param stepId - The step ID that was waiting
+     * @param resumeData - Data from the resume action (e.g., form submission)
+     * @param sdkRegistry - SDK registry for step execution
+     * @param stepExecutor - Step executor function
+     * @returns Workflow result from resumed execution
+     */
+    async resumeExecution(runId, stepId, resumeData, sdkRegistry, stepExecutor) {
+        if (!this.stateStore) {
+            throw new Error('Cannot resume execution: StateStore not configured');
+        }
+        // Load execution from state store
+        const execution = this.stateStore.getExecution(runId);
+        if (!execution) {
+            throw new Error(`Execution ${runId} not found`);
+        }
+        if (execution.status !== WorkflowStatus.RUNNING) {
+            throw new Error(`Cannot resume execution ${runId}: status is ${execution.status}`);
+        }
+        // Load workflow
+        const { workflow } = await parseFile(execution.workflowPath);
+        this.workflowPath = execution.workflowPath;
+        // Find the step that was waiting
+        const stepIndex = workflow.steps.findIndex(s => s.id === stepId);
+        if (stepIndex === -1) {
+            throw new Error(`Step ${stepId} not found in workflow`);
+        }
+        // Load all checkpoints to rebuild context
+        const checkpoints = this.stateStore.getCheckpoints(runId);
+        const stepResults = [];
+        const context = createExecutionContext(workflow, execution.inputs || {});
+        context.runId = runId;
+        context.status = WorkflowStatus.RUNNING;
+        // Rebuild context from checkpoints
+        for (let i = 0; i < stepIndex; i++) {
+            const checkpoint = checkpoints.find(cp => cp.stepIndex === i);
+            if (checkpoint) {
+                const step = workflow.steps[i];
+                // Recreate step result
+                const result = createStepResult(step.id, checkpoint.status, checkpoint.outputs, checkpoint.startedAt, checkpoint.retryCount, checkpoint.error || undefined);
+                stepResults.push(result);
+                // Restore step metadata
+                context.stepMetadata[step.id] = {
+                    status: checkpoint.status.toLowerCase(),
+                    retryCount: checkpoint.retryCount,
+                    ...(checkpoint.error ? { error: checkpoint.error } : {}),
+                };
+                // Restore output variable
+                if (step.outputVariable && checkpoint.status === StepStatus.COMPLETED) {
+                    context.variables[step.outputVariable] = checkpoint.outputs;
+                }
+            }
+        }
+        // Inject resume data into context
+        context.variables[`${stepId}_response`] = resumeData;
+        // Continue execution from next step
+        const startedAt = new Date(execution.startedAt);
+        this.workflowPermissions = workflow.permissions;
+        try {
+            for (let i = stepIndex + 1; i < workflow.steps.length; i++) {
+                const step = workflow.steps[i];
+                context.currentStepIndex = i;
+                const result = await this.executeStep(step, context, sdkRegistry, stepExecutor);
+                stepResults.push(result);
+                context.stepMetadata[step.id] = {
+                    status: result.status.toLowerCase(),
+                    retryCount: result.retryCount,
+                    ...(result.error ? { error: errorToString(result.error) } : {}),
+                };
+                if (step.outputVariable && result.status === StepStatus.COMPLETED) {
+                    context.variables[step.outputVariable] = result.output;
+                }
+                if (result.status === StepStatus.COMPLETED &&
+                    result.output &&
+                    typeof result.output === 'object' &&
+                    '__workflow_outputs__' in result.output) {
+                    const outputObj = result.output;
+                    const outputs = outputObj['__workflow_outputs__'];
+                    context.workflowOutputs = outputs;
+                }
+                if (result.status === StepStatus.FAILED) {
+                    let errorAction = 'stop';
+                    if ('errorHandling' in step && step.errorHandling?.action) {
+                        errorAction = step.errorHandling.action;
+                    }
+                    if (errorAction === 'stop' || errorAction === 'rollback') {
+                        if (errorAction === 'rollback' && this.rollbackRegistry) {
+                            await this.rollbackRegistry.rollbackAllAsync({
+                                context,
+                                inputs: context.inputs,
+                                variables: context.variables,
+                            });
+                        }
+                        context.status = WorkflowStatus.FAILED;
+                        const workflowError = result.error ? errorToString(result.error) : `Step ${step.id} failed`;
+                        const workflowResult = this.buildWorkflowResult(workflow, context, stepResults, startedAt, workflowError);
+                        this.events.onWorkflowComplete?.(workflow, workflowResult);
+                        return workflowResult;
+                    }
+                }
+            }
+            context.status = WorkflowStatus.COMPLETED;
+        }
+        catch (error) {
+            context.status = WorkflowStatus.FAILED;
+            if (this.stateStore) {
+                this.stateStore.updateExecution(runId, {
+                    status: WorkflowStatus.FAILED,
+                    completedAt: new Date(),
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+            const workflowResult = this.buildWorkflowResult(workflow, context, stepResults, startedAt, error instanceof Error ? error.message : String(error));
+            this.events.onWorkflowComplete?.(workflow, workflowResult);
+            return workflowResult;
+        }
+        const workflowResult = this.buildWorkflowResult(workflow, context, stepResults, startedAt);
+        if (this.stateStore) {
+            this.stateStore.updateExecution(runId, {
+                status: context.status,
+                completedAt: new Date(),
+                outputs: context.variables,
+            });
+        }
+        this.events.onWorkflowComplete?.(workflow, workflowResult);
+        return workflowResult;
+    }
     /**
      * Execute a workflow from a file.
      * This method automatically sets the workflow path for resolving sub-workflows.
@@ -390,14 +654,302 @@ export class WorkflowEngine {
         // Return the sub-workflow output
         return result.output;
     }
+    /**
+     * Execute a sub-workflow using an AI sub-agent.
+     * The agent interprets the workflow and executes it autonomously.
+     */
+    async executeSubWorkflowWithAgent(step, context, sdkRegistry, stepExecutor) {
+        // Resolve the sub-workflow path
+        const subWorkflowPath = this.workflowPath
+            ? resolve(dirname(this.workflowPath), step.workflow)
+            : resolve(step.workflow);
+        // Read the workflow file content
+        const { readFile } = await import('node:fs/promises');
+        const workflowContent = await readFile(subWorkflowPath, 'utf-8');
+        // Resolve inputs for the sub-workflow
+        const resolvedInputs = resolveTemplates(step.inputs, context);
+        // Get subagent configuration
+        const subagentConfig = step.subagentConfig || {};
+        const model = subagentConfig.model || step.model || this.config.defaultModel;
+        const maxTurns = subagentConfig.maxTurns || 10;
+        const systemPrompt = subagentConfig.systemPrompt || this.buildDefaultSubagentSystemPrompt();
+        const tools = subagentConfig.tools || ['Read', 'Write', 'Bash', 'Glob', 'Grep'];
+        // Build the prompt for the agent
+        const agentPrompt = this.buildSubagentPrompt(workflowContent, resolvedInputs, tools);
+        // Determine the agent action to use
+        const agentName = step.agent || this.config.defaultAgent || 'agent';
+        const agentAction = `${agentName}.chat.completions`;
+        // Build the messages array
+        const messages = [
+            { role: 'system', content: systemPrompt },
+            { role: 'user', content: agentPrompt },
+        ];
+        // Create a virtual action step to execute via the agent
+        const agentStep = {
+            id: `${step.id}-subagent`,
+            type: 'action',
+            action: agentAction,
+            inputs: {
+                model,
+                messages,
+                max_tokens: 8192,
+            },
+            model,
+            agent: agentName,
+        };
+        // Build executor context
+        const executorContext = this.buildStepExecutorContext(agentStep);
+        // Execute the agent call
+        let response;
+        let turns = 0;
+        let conversationMessages = [...messages];
+        let finalOutput = {};
+        while (turns < maxTurns) {
+            turns++;
+            try {
+                response = await stepExecutor({ ...agentStep, inputs: { ...agentStep.inputs, messages: conversationMessages } }, context, sdkRegistry, executorContext);
+                // Parse the response
+                const parsedResponse = this.parseSubagentResponse(response);
+                if (parsedResponse.completed) {
+                    finalOutput = parsedResponse.output || {};
+                    break;
+                }
+                // If agent needs to continue, add its response and continue
+                if (parsedResponse.message) {
+                    conversationMessages.push({ role: 'assistant', content: parsedResponse.message });
+                    // Agent might request a tool call - for now, we'll prompt it to continue
+                    conversationMessages.push({ role: 'user', content: 'Continue with the workflow execution.' });
+                }
+                else {
+                    // No clear continuation, assume completed
+                    finalOutput = parsedResponse.output || {};
+                    break;
+                }
+            }
+            catch (error) {
+                throw new Error(`Sub-agent execution failed at turn ${turns}: ${error instanceof Error ? error.message : String(error)}`);
+            }
+        }
+        if (turns >= maxTurns) {
+            throw new Error(`Sub-agent exceeded maximum turns (${maxTurns})`);
+        }
+        return finalOutput;
+    }
+    /**
+     * Build the default system prompt for sub-agent execution.
+     */
+    buildDefaultSubagentSystemPrompt() {
+        return `You are an AI agent executing a workflow. Your task is to interpret the workflow definition and execute each step in order.
+
+For each step:
+1. Understand what the step requires
+2. Execute the action described
+3. Store any outputs as specified
+
+When you complete all steps, respond with a JSON object containing the workflow outputs.
+
+Format your final response as:
+\`\`\`json
+{
+  "completed": true,
+  "output": { /* workflow outputs here */ }
+}
+\`\`\`
+
+If you encounter an error, respond with:
+\`\`\`json
+{
+  "completed": false,
+  "error": "description of the error"
+}
+\`\`\``;
+    }
+    /**
+     * Build the prompt for sub-agent workflow execution.
+     */
+    buildSubagentPrompt(workflowContent, inputs, tools) {
+        return `Execute the following workflow:
+
+## Workflow Definition
+\`\`\`markdown
+${workflowContent}
+\`\`\`
+
+## Inputs
+\`\`\`json
+${JSON.stringify(inputs, null, 2)}
+\`\`\`
+
+## Available Tools
+${tools.join(', ')}
+
+Execute the workflow steps in order and return the final outputs as JSON.`;
+    }
+    /**
+     * Parse the sub-agent's response to extract completion status and output.
+     */
+    parseSubagentResponse(response) {
+        // Try to extract content from various response formats
+        let content;
+        if (typeof response === 'string') {
+            content = response;
+        }
+        else if (response && typeof response === 'object') {
+            const resp = response;
+            // OpenAI-style response
+            if (resp.choices && Array.isArray(resp.choices)) {
+                const choice = resp.choices[0];
+                if (choice.message && typeof choice.message === 'object') {
+                    const message = choice.message;
+                    content = message.content;
+                }
+            }
+            // Anthropic-style response
+            else if (resp.content && Array.isArray(resp.content)) {
+                const textBlock = resp.content.find((c) => typeof c === 'object' && c !== null && c.type === 'text');
+                content = textBlock?.text;
+            }
+            // Direct content field
+            else if (typeof resp.content === 'string') {
+                content = resp.content;
+            }
+            // Direct message field
+            else if (typeof resp.message === 'string') {
+                content = resp.message;
+            }
+        }
+        if (!content) {
+            return { completed: false, message: 'No content in response' };
+        }
+        // Try to parse JSON from the response
+        const jsonMatch = content.match(/```json\n?([\s\S]*?)```/);
+        if (jsonMatch) {
+            try {
+                const parsed = JSON.parse(jsonMatch[1]);
+                const output = parsed.output;
+                const error = parsed.error;
+                return {
+                    completed: parsed.completed === true,
+                    ...(output !== undefined ? { output } : {}),
+                    ...(error !== undefined ? { error } : {}),
+                };
+            }
+            catch (e) {
+                console.warn('[marktoflow] Failed to parse JSON block in sub-agent response:', e.message);
+            }
+        }
+        // Try to parse raw JSON
+        try {
+            const parsed = JSON.parse(content);
+            if (typeof parsed.completed === 'boolean') {
+                const output = parsed.output;
+                const error = parsed.error;
+                return {
+                    completed: parsed.completed,
+                    ...(output !== undefined ? { output } : {}),
+                    ...(error !== undefined ? { error } : {}),
+                };
+            }
+        }
+        catch (e) {
+            console.warn('[marktoflow] Sub-agent response is not valid JSON:', e.message);
+        }
+        // Return the content as a message
+        return { completed: false, message: content };
+    }
+    /**
+     * Build the step executor context with effective model/agent/permissions.
+     */
+    buildStepExecutorContext(step) {
+        // Merge workflow and step permissions
+        const effectivePermissions = mergePermissions(this.workflowPermissions, step.permissions);
+        // Resolve effective model/agent (step overrides workflow defaults)
+        const effectiveModel = step.model || this.config.defaultModel;
+        const effectiveAgent = step.agent || this.config.defaultAgent;
+        return {
+            model: effectiveModel,
+            agent: effectiveAgent,
+            permissions: effectivePermissions,
+            securityPolicy: toSecurityPolicy(effectivePermissions),
+            basePath: this.workflowPath,
+        };
+    }
+    /**
+     * Load and resolve an external prompt file for a step.
+     */
+    async loadAndResolvePrompt(step, context) {
+        if (!step.prompt) {
+            return step.inputs;
+        }
+        // Check cache
+        let loadedPrompt = this.promptCache.get(step.prompt);
+        if (!loadedPrompt) {
+            loadedPrompt = await loadPromptFile(step.prompt, this.workflowPath);
+            this.promptCache.set(step.prompt, loadedPrompt);
+        }
+        // Resolve prompt inputs (from step.promptInputs, with template resolution)
+        const promptInputs = step.promptInputs
+            ? resolveTemplates(step.promptInputs, context)
+            : {};
+        // Validate prompt inputs
+        const validation = validatePromptInputs(loadedPrompt, promptInputs);
+        if (!validation.valid) {
+            throw new Error(`Invalid prompt inputs: ${validation.errors.join(', ')}`);
+        }
+        // Resolve the prompt template
+        const resolved = resolvePromptTemplate(loadedPrompt, promptInputs, context);
+        // Merge resolved prompt content into inputs
+        // The resolved content typically goes into a 'messages' or 'prompt' field
+        const resolvedInputs = { ...step.inputs };
+        // If inputs has a 'messages' array with a user message, inject prompt content
+        if (Array.isArray(resolvedInputs.messages)) {
+            resolvedInputs.messages = resolvedInputs.messages.map((msg) => {
+                if (typeof msg === 'object' && msg !== null) {
+                    const message = msg;
+                    if (message.role === 'user' && typeof message.content === 'string') {
+                        // Replace {{ prompt }} placeholder with resolved content
+                        return {
+                            ...message,
+                            content: message.content.replace(/\{\{\s*prompt\s*\}\}/g, resolved.content),
+                        };
+                    }
+                }
+                return msg;
+            });
+        }
+        else {
+            // Add resolved prompt as 'promptContent' for the executor to use
+            resolvedInputs.promptContent = resolved.content;
+        }
+        return resolvedInputs;
+    }
     /**
      * Execute a step with retry logic.
      */
     async executeStepWithRetry(step, context, sdkRegistry, stepExecutor) {
         const startedAt = new Date();
         let lastError;
+        // Build executor context with model/agent/permissions
+        const executorContext = this.buildStepExecutorContext(step);
         // Handle sub-workflow execution
         if (isSubWorkflowStep(step)) {
+            // Check if we should use subagent execution
+            if (step.useSubagent) {
+                try {
+                    this.events.onStepStart?.(step, context);
+                    const output = await this.executeWithTimeout(() => this.executeSubWorkflowWithAgent(step, context, sdkRegistry, stepExecutor), step.timeout ?? this.config.defaultTimeout);
+                    const result = createStepResult(step.id, StepStatus.COMPLETED, output, startedAt, 0);
+                    this.events.onStepComplete?.(step, result);
+                    return result;
+                }
+                catch (error) {
+                    lastError = error instanceof Error ? error : new Error(String(error));
+                    const result = createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, lastError);
+                    this.events.onStepComplete?.(step, result);
+                    return result;
+                }
+            }
+            // Standard sub-workflow execution
             try {
                 this.events.onStepStart?.(step, context);
                 const output = await this.executeWithTimeout(() => this.executeSubWorkflow(step, context, sdkRegistry, stepExecutor), step.timeout ?? this.config.defaultTimeout);
@@ -407,7 +959,8 @@ export class WorkflowEngine {
             }
             catch (error) {
                 lastError = error instanceof Error ? error : new Error(String(error));
-                const result = createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, lastError.message);
+                const result = createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, lastError // Pass full error object
+                );
                 this.events.onStepComplete?.(step, result);
                 return result;
             }
@@ -431,11 +984,30 @@ export class WorkflowEngine {
             }
             this.events.onStepStart?.(step, context);
             try {
-                // Resolve templates in inputs
-                const resolvedInputs = resolveTemplates(step.inputs, context);
+                // Load and resolve external prompt if specified
+                let resolvedInputs;
+                if (step.prompt) {
+                    resolvedInputs = await this.loadAndResolvePrompt(step, context);
+                    resolvedInputs = resolveTemplates(resolvedInputs, context);
+                }
+                else {
+                    // Resolve templates in inputs
+                    resolvedInputs = resolveTemplates(step.inputs, context);
+                }
                 const stepWithResolvedInputs = { ...step, inputs: resolvedInputs };
-                // Execute step
-                const output = await this.executeWithTimeout(() => stepExecutor(stepWithResolvedInputs, context, sdkRegistry), step.timeout ?? this.config.defaultTimeout);
+                // Check if this is a built-in operation
+                let output;
+                if (isBuiltInOperation(step.action)) {
+                    // Execute built-in operation directly (no timeout, no SDK executor needed)
+                    // For built-in operations, pass both resolved and unresolved inputs
+                    // to allow selective resolution of template expressions
+                    // Await in case operation is async (e.g., file operations)
+                    output = await executeBuiltInOperation(step.action, step.inputs, resolvedInputs, context);
+                }
+                else {
+                    // Execute step with executor context
+                    output = await this.executeWithTimeout(() => stepExecutor(stepWithResolvedInputs, context, sdkRegistry, executorContext), step.timeout ?? this.config.defaultTimeout);
+                }
                 circuitBreaker.recordSuccess();
                 const result = createStepResult(step.id, StepStatus.COMPLETED, output, startedAt, attempt);
                 this.events.onStepComplete?.(step, result);
@@ -453,7 +1025,8 @@ export class WorkflowEngine {
             }
         }
         // All retries exhausted
-        const result = createStepResult(step.id, StepStatus.FAILED, null, startedAt, maxRetries, lastError?.message);
+        const result = createStepResult(step.id, StepStatus.FAILED, null, startedAt, maxRetries, lastError // Pass full error object to preserve HTTP details, stack traces, etc.
+        );
         this.events.onStepComplete?.(step, result);
         return result;
     }
@@ -472,7 +1045,7 @@ export class WorkflowEngine {
             this.healthTracker.markHealthy(primaryTool);
             return primaryResult;
         }
-        const errorMessage = primaryResult.error ?? '';
+        const errorMessage = primaryResult.error ? errorToString(primaryResult.error) : '';
         const isTimeout = errorMessage.includes('timed out');
         if (isTimeout && !this.failoverConfig.failoverOnTimeout) {
             this.healthTracker.markUnhealthy(primaryTool, errorMessage);
@@ -576,8 +1149,18 @@ export class WorkflowEngine {
     /**
      * Resolve a condition value with support for nested properties.
      * Handles direct variable references and nested paths.
+     * Uses Nunjucks for template expressions with filters/regex.
      */
     resolveConditionValue(path, context) {
+        // If it looks like a template expression, resolve it
+        if (path.includes('|') || path.includes('=~') || path.includes('!~')) {
+            // Build template context
+            const templateContext = {
+                inputs: context.inputs,
+                ...context.variables,
+            };
+            return renderTemplate(`{{ ${path} }}`, templateContext);
+        }
         // First try to parse as a literal value (true, false, numbers, etc.)
         const parsedValue = this.parseValue(path);
         // If parseValue returned the same string, try to resolve as a variable
@@ -615,12 +1198,14 @@ export class WorkflowEngine {
      */
     buildWorkflowResult(workflow, context, stepResults, startedAt, error) {
         const completedAt = new Date();
+        // Use workflowOutputs if set by workflow.set_outputs, otherwise use all variables
+        const output = context.workflowOutputs || context.variables;
         return {
             workflowId: workflow.metadata.id,
             runId: context.runId,
             status: context.status,
             stepResults,
-            output: context.variables,
+            output,
             error,
             startedAt,
             completedAt,
@@ -704,9 +1289,17 @@ export class WorkflowEngine {
     }
     /**
      * Execute a for-each loop step.
+     * Supports optional batch_size and pause_between_batches for rate limiting.
      */
     async executeForEachStep(step, context, sdkRegistry, stepExecutor) {
         const startedAt = new Date();
+        const cleanupLoopVars = () => {
+            delete context.variables[step.itemVariable];
+            delete context.variables['loop'];
+            delete context.variables['batch'];
+            if (step.indexVariable)
+                delete context.variables[step.indexVariable];
+        };
         try {
             // Resolve items array
             const items = resolveTemplates(step.items, context);
@@ -716,10 +1309,15 @@ export class WorkflowEngine {
             if (items.length === 0) {
                 return createStepResult(step.id, StepStatus.SKIPPED, [], startedAt);
             }
-            // Execute steps for each item
+            const batchSize = step.batchSize;
+            const pauseBetweenBatches = step.pauseBetweenBatches;
+            // If batch mode, process items in batches
+            if (batchSize && batchSize > 0) {
+                return await this.executeForEachBatched(step, items, batchSize, pauseBetweenBatches ?? 0, context, sdkRegistry, stepExecutor, startedAt);
+            }
+            // Standard item-by-item execution
             const results = [];
             for (let i = 0; i < items.length; i++) {
-                // Inject loop variables
                 context.variables[step.itemVariable] = items[i];
                 context.variables['loop'] = {
                     index: i,
@@ -739,35 +1337,82 @@ export class WorkflowEngine {
                     if (result.status === StepStatus.FAILED) {
                         const errorAction = step.errorHandling?.action ?? 'stop';
                         if (errorAction === 'stop') {
-                            // Clean up loop variables
-                            delete context.variables[step.itemVariable];
-                            delete context.variables['loop'];
-                            if (step.indexVariable)
-                                delete context.variables[step.indexVariable];
+                            cleanupLoopVars();
                             return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, result.error);
                         }
-                        // 'continue' - skip to next iteration
                         break;
                     }
                 }
                 results.push(context.variables[step.itemVariable]);
             }
-            // Clean up loop variables
-            delete context.variables[step.itemVariable];
-            delete context.variables['loop'];
-            if (step.indexVariable)
-                delete context.variables[step.indexVariable];
+            cleanupLoopVars();
             return createStepResult(step.id, StepStatus.COMPLETED, results, startedAt);
         }
         catch (error) {
-            // Clean up loop variables on error
-            delete context.variables[step.itemVariable];
-            delete context.variables['loop'];
-            if (step.indexVariable)
-                delete context.variables[step.indexVariable];
+            cleanupLoopVars();
             return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
         }
     }
+    /**
+     * Execute for-each in batch mode.
+     * Items are split into batches; {{ batch }} contains the current batch array.
+     */
+    async executeForEachBatched(step, items, batchSize, pauseBetweenBatches, context, sdkRegistry, stepExecutor, startedAt) {
+        const results = [];
+        const totalBatches = Math.ceil(items.length / batchSize);
+        for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) {
+            const batchStart = batchIndex * batchSize;
+            const batchItems = items.slice(batchStart, batchStart + batchSize);
+            // Pause between batches (not before first batch)
+            if (batchIndex > 0 && pauseBetweenBatches > 0) {
+                await new Promise((resolve) => setTimeout(resolve, pauseBetweenBatches));
+            }
+            // Expose batch-level variables
+            context.variables['batch'] = batchItems;
+            context.variables['loop'] = {
+                index: batchIndex,
+                first: batchIndex === 0,
+                last: batchIndex === totalBatches - 1,
+                length: totalBatches,
+                batchSize,
+                batchStart,
+                totalItems: items.length,
+            };
+            // Process each item in the batch
+            for (let i = 0; i < batchItems.length; i++) {
+                const globalIndex = batchStart + i;
+                context.variables[step.itemVariable] = batchItems[i];
+                if (step.indexVariable) {
+                    context.variables[step.indexVariable] = globalIndex;
+                }
+                for (const iterStep of step.steps) {
+                    const result = await this.executeStep(iterStep, context, sdkRegistry, stepExecutor);
+                    if (result.status === StepStatus.COMPLETED && iterStep.outputVariable) {
+                        context.variables[iterStep.outputVariable] = result.output;
+                    }
+                    if (result.status === StepStatus.FAILED) {
+                        const errorAction = step.errorHandling?.action ?? 'stop';
+                        if (errorAction === 'stop') {
+                            delete context.variables[step.itemVariable];
+                            delete context.variables['loop'];
+                            delete context.variables['batch'];
+                            if (step.indexVariable)
+                                delete context.variables[step.indexVariable];
+                            return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, result.error);
+                        }
+                        break;
+                    }
+                }
+                results.push(context.variables[step.itemVariable]);
+            }
+        }
+        delete context.variables[step.itemVariable];
+        delete context.variables['loop'];
+        delete context.variables['batch'];
+        if (step.indexVariable)
+            delete context.variables[step.indexVariable];
+        return createStepResult(step.id, StepStatus.COMPLETED, results, startedAt);
+    }
     /**
      * Execute a while loop step.
      */
@@ -899,7 +1544,7 @@ export class WorkflowEngine {
                         branchResults.push(result.output);
                     }
                     if (result.status === StepStatus.FAILED) {
-                        throw new Error(`Branch ${branch.id} failed: ${result.error}`);
+                        throw new Error(`Branch ${branch.id} failed: ${errorToString(result.error)}`);
                     }
                 }
                 return { branchId: branch.id, context: branchContext, results: branchResults };
@@ -936,7 +1581,7 @@ export class WorkflowEngine {
                     context.variables[tryStep.outputVariable] = result.output;
                 }
                 if (result.status === StepStatus.FAILED) {
-                    tryError = new Error(result.error || 'Step failed');
+                    tryError = new Error(result.error ? errorToString(result.error) : 'Step failed');
                     break;
                 }
             }
@@ -954,7 +1599,7 @@ export class WorkflowEngine {
                         context.variables[catchStep.outputVariable] = result.output;
                     }
                     if (result.status === StepStatus.FAILED) {
-                        catchError = new Error(result.error || 'Catch block failed');
+                        catchError = new Error(result.error ? errorToString(result.error) : 'Catch block failed');
                         break;
                     }
                 }
@@ -998,6 +1643,239 @@ export class WorkflowEngine {
             return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
         }
     }
+    /**
+     * Execute a script step (inline JavaScript).
+     */
+    async executeScriptStep(step, context) {
+        const startedAt = new Date();
+        try {
+            // Resolve any templates in the code
+            const resolvedInputs = resolveTemplates(step.inputs, context);
+            // Execute the script with the workflow context
+            const result = await executeScriptAsync(resolvedInputs.code, {
+                variables: context.variables,
+                inputs: context.inputs,
+                steps: context.stepMetadata,
+            }, {
+                timeout: resolvedInputs.timeout ?? 5000,
+            });
+            if (!result.success) {
+                return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, result.error ?? 'Script execution failed');
+            }
+            return createStepResult(step.id, StepStatus.COMPLETED, result.value, startedAt);
+        }
+        catch (error) {
+            return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
+        }
+    }
+    // ============================================================================
+    // Wait Step Execution
+    // ============================================================================
+    /**
+     * Execute a wait/pause step.
+     *
+     * For mode=duration: In-process wait (suitable for short durations).
+     *   For persistent long-duration waits, a WaitManager should checkpoint
+     *   the execution and resume it later via the scheduler.
+     *
+     * For mode=webhook: Returns a resume URL. The execution will be
+     *   checkpointed and resumed when the URL is called.
+     *
+     * For mode=form: Returns form fields. The execution will be
+     *   checkpointed and resumed when the form is submitted.
+     */
+    async executeWaitStep(step, context) {
+        const startedAt = new Date();
+        try {
+            switch (step.mode) {
+                case 'duration': {
+                    if (!step.duration) {
+                        return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Wait step with mode=duration requires a duration');
+                    }
+                    const resolvedDuration = resolveTemplates(step.duration, context);
+                    const ms = parseDuration(resolvedDuration);
+                    // For short durations (under 5 minutes), do in-process wait
+                    if (ms <= 300000) {
+                        await new Promise((resolve) => setTimeout(resolve, ms));
+                        return createStepResult(step.id, StepStatus.COMPLETED, { waited: ms }, startedAt);
+                    }
+                    // For longer durations, checkpoint and schedule resume
+                    // The StateStore will persist the execution state
+                    if (this.stateStore) {
+                        this.stateStore.saveCheckpoint({
+                            runId: context.runId,
+                            stepIndex: context.currentStepIndex,
+                            stepName: step.id,
+                            status: StepStatus.COMPLETED,
+                            startedAt: startedAt,
+                            completedAt: new Date(),
+                            inputs: { mode: 'duration', resumeAt: new Date(Date.now() + ms).toISOString() },
+                            outputs: { waiting: true },
+                            error: null,
+                            retryCount: 0,
+                        });
+                    }
+                    // Set execution status to indicate waiting
+                    const resumeAt = new Date(Date.now() + ms).toISOString();
+                    return createStepResult(step.id, StepStatus.COMPLETED, {
+                        waiting: true,
+                        mode: 'duration',
+                        resumeAt,
+                        durationMs: ms,
+                    }, startedAt);
+                }
+                case 'webhook': {
+                    // Generate a unique resume URL
+                    const resumeToken = crypto.randomUUID();
+                    const webhookPath = step.webhookPath
+                        ? resolveTemplates(step.webhookPath, context)
+                        : `/resume/${context.runId}/${step.id}/${resumeToken}`;
+                    if (this.stateStore) {
+                        this.stateStore.saveCheckpoint({
+                            runId: context.runId,
+                            stepIndex: context.currentStepIndex,
+                            stepName: step.id,
+                            status: StepStatus.COMPLETED,
+                            startedAt: startedAt,
+                            completedAt: new Date(),
+                            inputs: { mode: 'webhook', resumeToken, webhookPath },
+                            outputs: { waiting: true },
+                            error: null,
+                            retryCount: 0,
+                        });
+                    }
+                    return createStepResult(step.id, StepStatus.COMPLETED, {
+                        waiting: true,
+                        mode: 'webhook',
+                        resumeToken,
+                        webhookPath,
+                    }, startedAt);
+                }
+                case 'form': {
+                    if (!step.fields || Object.keys(step.fields).length === 0) {
+                        return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Wait step with mode=form requires fields');
+                    }
+                    const resumeToken = crypto.randomUUID();
+                    if (this.stateStore) {
+                        this.stateStore.saveCheckpoint({
+                            runId: context.runId,
+                            stepIndex: context.currentStepIndex,
+                            stepName: step.id,
+                            status: StepStatus.COMPLETED,
+                            startedAt: startedAt,
+                            completedAt: new Date(),
+                            inputs: { mode: 'form', resumeToken, fields: step.fields },
+                            outputs: { waiting: true },
+                            error: null,
+                            retryCount: 0,
+                        });
+                    }
+                    return createStepResult(step.id, StepStatus.COMPLETED, {
+                        waiting: true,
+                        mode: 'form',
+                        resumeToken,
+                        fields: step.fields,
+                        formPath: `/form/${context.runId}/${step.id}/${resumeToken}`,
+                    }, startedAt);
+                }
+                default:
+                    return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, `Unknown wait mode: ${step.mode}`);
+            }
+        }
+        catch (error) {
+            return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
+        }
+    }
+    // ============================================================================
+    // Merge Step Execution
+    // ============================================================================
+    /**
+     * Execute a merge step - combines data from multiple sources.
+     */
+    async executeMergeStep(step, context) {
+        const startedAt = new Date();
+        try {
+            // Resolve all source expressions to arrays
+            const resolvedSources = [];
+            for (const source of step.sources) {
+                const resolved = resolveTemplates(source, context);
+                if (!Array.isArray(resolved)) {
+                    return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, `Merge source "${source}" did not resolve to an array`);
+                }
+                resolvedSources.push(resolved);
+            }
+            let result;
+            switch (step.mode) {
+                case 'append':
+                    result = resolvedSources.flat();
+                    break;
+                case 'match': {
+                    if (!step.matchField) {
+                        return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Merge mode "match" requires matchField');
+                    }
+                    // Return items that appear in ALL sources (by matchField)
+                    const fieldSets = resolvedSources.map((source) => new Set(source.map((item) => getField(item, step.matchField))));
+                    // Intersection of all field sets
+                    const commonKeys = fieldSets.reduce((acc, set) => new Set([...acc].filter((key) => set.has(key))));
+                    // Return items from first source that match
+                    result = resolvedSources[0].filter((item) => commonKeys.has(getField(item, step.matchField)));
+                    break;
+                }
+                case 'diff': {
+                    if (!step.matchField) {
+                        return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Merge mode "diff" requires matchField');
+                    }
+                    // Return items from first source NOT found in other sources
+                    const otherKeys = new Set(resolvedSources.slice(1).flat().map((item) => getField(item, step.matchField)));
+                    result = resolvedSources[0].filter((item) => !otherKeys.has(getField(item, step.matchField)));
+                    break;
+                }
+                case 'combine_by_field': {
+                    if (!step.matchField) {
+                        return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Merge mode "combine_by_field" requires matchField');
+                    }
+                    // Group items by matchField and merge their properties
+                    const grouped = new Map();
+                    const onConflict = step.onConflict ?? 'keep_last';
+                    for (const source of resolvedSources) {
+                        for (const item of source) {
+                            if (!item || typeof item !== 'object')
+                                continue;
+                            const key = getField(item, step.matchField);
+                            const existing = grouped.get(key);
+                            if (existing) {
+                                if (onConflict === 'keep_first') {
+                                    // Only add new fields
+                                    for (const [k, v] of Object.entries(item)) {
+                                        if (!(k in existing))
+                                            existing[k] = v;
+                                    }
+                                }
+                                else if (onConflict === 'keep_last') {
+                                    Object.assign(existing, item);
+                                }
+                                else {
+                                    // merge_fields: deep merge
+                                    Object.assign(existing, item);
+                                }
+                            }
+                            else {
+                                grouped.set(key, { ...item });
+                            }
+                        }
+                    }
+                    result = Array.from(grouped.values());
+                    break;
+                }
+                default:
+                    return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, `Unknown merge mode: ${step.mode}`);
+            }
+            return createStepResult(step.id, StepStatus.COMPLETED, result, startedAt);
+        }
+        catch (error) {
+            return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
+        }
+    }
     // ============================================================================
     // Helper Methods for Control Flow
     // ============================================================================