@xtr-dev/payload-automation 0.0.43 → 0.0.46

package/dist/core/workflow-executor.js

@@ -1,4 +1,4 @@
-import Handlebars from 'handlebars';
+import { evaluateCondition as evalCondition, resolveStepInput as resolveInput } from './expression-engine.js';
 export class WorkflowExecutor {
     payload;
     logger;
@@ -7,507 +7,294 @@ export class WorkflowExecutor {
         this.logger = logger;
     }
     /**
-   * Convert string values to appropriate types based on common patterns
-   */ convertValueType(value, key) {
-        if (typeof value !== 'string') {
-            return value;
+   * Resolve workflow steps by loading base step configurations and merging with overrides
+   */ async resolveWorkflowSteps(workflow) {
+        const resolvedSteps = [];
+        if (!workflow.steps || workflow.steps.length === 0) {
+            return resolvedSteps;
         }
-        // Type conversion patterns based on field names and values
-        const numericFields = [
-            'timeout',
-            'retries',
-            'delay',
-            'port',
-            'limit',
-            'offset',
-            'count',
-            'max',
-            'min'
-        ];
-        const booleanFields = [
-            'enabled',
-            'required',
-            'active',
-            'success',
-            'failed',
-            'complete'
-        ];
-        // Convert numeric fields
-        if (numericFields.some((field)=>key.toLowerCase().includes(field))) {
-            const numValue = Number(value);
-            if (!isNaN(numValue)) {
-                this.logger.debug({
-                    key,
-                    originalValue: value,
-                    convertedValue: numValue
-                }, 'Auto-converted field to number');
-                return numValue;
+        for(let i = 0; i < workflow.steps.length; i++){
+            const workflowStep = workflow.steps[i];
+            let baseStep;
+            if (typeof workflowStep.step === 'object' && workflowStep.step !== null) {
+                baseStep = workflowStep.step;
+            } else {
+                try {
+                    baseStep = await this.payload.findByID({
+                        collection: 'automation-steps',
+                        id: workflowStep.step,
+                        depth: 0
+                    });
+                } catch (error) {
+                    this.logger.error({
+                        stepId: workflowStep.step,
+                        stepIndex: i,
+                        error: error instanceof Error ? error.message : 'Unknown error'
+                    }, 'Failed to load step configuration');
+                    throw new Error(`Failed to load step ${workflowStep.step}: ${error instanceof Error ? error.message : 'Unknown error'}`);
+                }
             }
+            const baseConfig = baseStep.config || {};
+            const overrides = workflowStep.inputOverrides || {};
+            const mergedConfig = {
+                ...baseConfig,
+                ...overrides
+            };
+            const dependencies = (workflowStep.dependencies || []).map((d)=>d.stepIndex);
+            resolvedSteps.push({
+                stepIndex: i,
+                stepId: baseStep.id,
+                stepName: workflowStep.stepName || baseStep.name || `step-${i}`,
+                stepType: baseStep.type,
+                config: mergedConfig,
+                condition: workflowStep.condition,
+                dependencies,
+                retryOnFailure: baseStep.retryOnFailure,
+                maxRetries: baseStep.maxRetries || workflow.maxRetries || 3,
+                retryDelay: baseStep.retryDelay || workflow.retryDelay || 1000
+            });
         }
-        // Convert boolean fields
-        if (booleanFields.some((field)=>key.toLowerCase().includes(field))) {
-            if (value === 'true') return true;
-            if (value === 'false') return false;
-        }
-        // Try to parse as number if it looks numeric
-        if (/^\d+$/.test(value)) {
-            const numValue = parseInt(value, 10);
-            this.logger.debug({
-                key,
-                originalValue: value,
-                convertedValue: numValue
-            }, 'Auto-converted numeric string to number');
-            return numValue;
-        }
-        // Try to parse as float if it looks like a decimal
-        if (/^\d+\.\d+$/.test(value)) {
-            const floatValue = parseFloat(value);
-            this.logger.debug({
-                key,
-                originalValue: value,
-                convertedValue: floatValue
-            }, 'Auto-converted decimal string to number');
-            return floatValue;
-        }
-        // Return as string if no conversion applies
-        return value;
+        return resolvedSteps;
     }
     /**
-   * Classifies error types based on error messages
-   */ classifyErrorType(errorMessage) {
-        if (errorMessage.includes('timeout') || errorMessage.includes('ETIMEDOUT')) {
-            return 'timeout';
-        }
-        if (errorMessage.includes('ENOTFOUND') || errorMessage.includes('getaddrinfo')) {
-            return 'dns';
+   * Resolve step execution order based on dependencies
+   */ resolveExecutionOrder(steps) {
+        const indegree = new Map();
+        const dependents = new Map();
+        for (const step of steps){
+            indegree.set(step.stepIndex, step.dependencies.length);
+            dependents.set(step.stepIndex, []);
         }
-        if (errorMessage.includes('ECONNREFUSED') || errorMessage.includes('ECONNRESET')) {
-            return 'connection';
+        for (const step of steps){
+            for (const depIndex of step.dependencies){
+                const deps = dependents.get(depIndex) || [];
+                deps.push(step.stepIndex);
+                dependents.set(depIndex, deps);
+            }
         }
-        if (errorMessage.includes('network') || errorMessage.includes('fetch')) {
-            return 'network';
+        const executionBatches = [];
+        const processed = new Set();
+        while(processed.size < steps.length){
+            const currentBatch = [];
+            for (const step of steps){
+                if (!processed.has(step.stepIndex) && indegree.get(step.stepIndex) === 0) {
+                    currentBatch.push(step);
+                }
+            }
+            if (currentBatch.length === 0) {
+                throw new Error('Circular dependency detected in workflow steps');
+            }
+            executionBatches.push(currentBatch);
+            for (const step of currentBatch){
+                processed.add(step.stepIndex);
+                for (const depIndex of dependents.get(step.stepIndex) || []){
+                    indegree.set(depIndex, (indegree.get(depIndex) || 1) - 1);
+                }
+            }
         }
-        return 'unknown';
-    }
-    /**
-   * Evaluate a step condition using JSONPath
-   */ evaluateStepCondition(condition, context) {
-        return this.evaluateCondition(condition, context);
+        return executionBatches;
     }
     /**
    * Execute a single workflow step
-   */ async executeStep(step, stepIndex, context, req, workflowRunId) {
-        const stepName = step.name || 'step-' + stepIndex;
+   */ async executeStep(step, context, req, stepResults, jobMeta) {
+        const result = {
+            step: step.stepId,
+            stepName: step.stepName,
+            stepIndex: step.stepIndex,
+            status: 'running',
+            startedAt: new Date().toISOString(),
+            retryCount: 0
+        };
         this.logger.info({
-            hasStep: 'step' in step,
-            step: JSON.stringify(step),
-            stepName
+            stepName: step.stepName,
+            stepType: step.stepType,
+            stepIndex: step.stepIndex
         }, 'Executing step');
         // Check step condition if present
         if (step.condition) {
-            this.logger.debug({
-                condition: step.condition,
-                stepName,
-                availableSteps: Object.keys(context.steps),
-                completedSteps: Object.entries(context.steps).filter(([_, s])=>s.state === 'succeeded').map(([name])=>name),
-                triggerType: context.trigger?.type
-            }, 'Evaluating step condition');
-            const conditionMet = this.evaluateStepCondition(step.condition, context);
+            const conditionMet = await this.evaluateCondition(step.condition, context);
             if (!conditionMet) {
                 this.logger.info({
-                    condition: step.condition,
-                    stepName,
-                    contextSnapshot: JSON.stringify({
-                        stepOutputs: Object.entries(context.steps).reduce((acc, [name, step])=>{
-                            acc[name] = {
-                                state: step.state,
-                                hasOutput: !!step.output
-                            };
-                            return acc;
-                        }, {}),
-                        triggerData: context.trigger?.data ? 'present' : 'absent'
-                    })
+                    stepName: step.stepName,
+                    condition: step.condition
                 }, 'Step condition not met, skipping');
-                // Mark step as completed but skipped
-                context.steps[stepName] = {
-                    error: undefined,
-                    input: undefined,
-                    output: {
-                        reason: 'Condition not met',
-                        skipped: true
-                    },
-                    state: 'succeeded'
+                result.status = 'skipped';
+                result.completedAt = new Date().toISOString();
+                result.output = {
+                    reason: 'Condition not met',
+                    skipped: true
                 };
-                // Update workflow run context if needed
-                if (workflowRunId) {
-                    await this.updateWorkflowRunContext(workflowRunId, context, req);
-                }
-                return;
+                context.steps[step.stepName] = {
+                    state: 'skipped',
+                    output: result.output
+                };
+                return result;
             }
-            this.logger.info({
-                condition: step.condition,
-                stepName,
-                contextSnapshot: JSON.stringify({
-                    stepOutputs: Object.entries(context.steps).reduce((acc, [name, step])=>{
-                        acc[name] = {
-                            state: step.state,
-                            hasOutput: !!step.output
-                        };
-                        return acc;
-                    }, {}),
-                    triggerData: context.trigger?.data ? 'present' : 'absent'
-                })
-            }, 'Step condition met, proceeding with execution');
         }
-        // Initialize step context
-        context.steps[stepName] = {
-            error: undefined,
-            input: undefined,
-            output: undefined,
+        // Resolve input using JSONata expressions
+        const resolvedInput = await this.resolveStepInput(step.config, context);
+        result.input = resolvedInput;
+        context.steps[step.stepName] = {
             state: 'running',
-            _startTime: Date.now() // Track execution start time for independent duration tracking
+            input: resolvedInput
         };
-        // Move taskSlug declaration outside try block so it's accessible in catch
-        const taskSlug = step.type;
         try {
-            // Get input configuration from the step
-            const inputConfig = step.input || {};
-            // Resolve input data using Handlebars templates
-            const resolvedInput = this.resolveStepInput(inputConfig, context, taskSlug);
-            context.steps[stepName].input = resolvedInput;
-            if (!taskSlug) {
-                throw new Error(`Step ${stepName} is missing a task type`);
-            }
-            this.logger.info({
-                hasInput: !!resolvedInput,
-                hasReq: !!req,
-                stepName,
-                taskSlug
-            }, 'Queueing task');
             const job = await this.payload.jobs.queue({
                 input: resolvedInput,
                 req,
-                task: taskSlug
+                task: step.stepType
             });
-            // Run the specific job immediately and wait for completion
-            this.logger.info({
-                jobId: job.id
-            }, 'Running job immediately using runByID');
-            const runResults = await this.payload.jobs.runByID({
+            // Update the job with automation context fields
+            // This allows tracking which workflow run triggered this job
+            await this.payload.update({
+                collection: 'payload-jobs',
                 id: job.id,
+                data: {
+                    automationWorkflow: jobMeta.automationWorkflowId,
+                    automationWorkflowRun: jobMeta.automationWorkflowRunId,
+                    automationTrigger: jobMeta.automationTriggerId,
+                    automationStepName: step.stepName
+                },
                 req
             });
-            this.logger.info({
-                jobId: job.id,
-                runResult: runResults,
-                hasResult: !!runResults
-            }, 'Job run completed');
-            // Give a small delay to ensure job is fully processed
-            await new Promise((resolve)=>setTimeout(resolve, 100));
-            // Get the job result
-            const completedJob = await this.payload.findByID({
+            // Run the job and capture the result directly from runByID
+            // This is important because PayloadCMS may delete jobs on completion (deleteJobOnComplete: true by default)
+            const runResult = await this.payload.jobs.runByID({
                 id: job.id,
-                collection: 'payload-jobs',
                 req
             });
-            this.logger.info({
-                jobId: job.id,
-                totalTried: completedJob.totalTried,
-                hasError: completedJob.hasError,
-                taskStatus: completedJob.taskStatus ? Object.keys(completedJob.taskStatus) : 'null'
-            }, 'Retrieved job results');
-            const taskStatus = completedJob.taskStatus?.[completedJob.taskSlug]?.[completedJob.totalTried];
-            const isComplete = taskStatus?.complete === true;
-            const hasError = completedJob.hasError || !isComplete;
-            // Extract error information from job if available
-            let errorMessage;
-            if (hasError) {
-                // Try to get error from the latest log entry
-                if (completedJob.log && completedJob.log.length > 0) {
-                    const latestLog = completedJob.log[completedJob.log.length - 1];
-                    errorMessage = latestLog.error?.message || latestLog.error;
-                }
-                // Fallback to top-level error
-                if (!errorMessage && completedJob.error) {
-                    errorMessage = completedJob.error.message || completedJob.error;
-                }
-                // Try to get error from task output if available
-                if (!errorMessage && taskStatus?.output?.error) {
-                    errorMessage = taskStatus.output.error;
+            // Check the job status from the run result
+            // runByID returns { jobStatus: { [jobId]: { status: 'success' | 'error' | ... } }, ... }
+            const jobStatus = runResult?.jobStatus?.[job.id];
+            const jobSucceeded = jobStatus?.status === 'success';
+            if (jobSucceeded) {
+                // Job completed successfully - try to get output from the job if it still exists
+                // Note: Job may have been deleted if deleteJobOnComplete is true
+                let output = {};
+                try {
+                    const completedJob = await this.payload.findByID({
+                        id: job.id,
+                        collection: 'payload-jobs',
+                        req
+                    });
+                    const taskStatus = completedJob.taskStatus?.[completedJob.taskSlug]?.[completedJob.totalTried];
+                    output = taskStatus?.output || {};
+                } catch  {
+                    // Job was deleted after completion - this is expected behavior with deleteJobOnComplete: true
+                    // The job succeeded, so we proceed without the output
+                    this.logger.debug({
+                        stepName: step.stepName
+                    }, 'Job was deleted after successful completion (deleteJobOnComplete)');
                 }
-                // Check if task handler returned with state='failed'
-                if (!errorMessage && taskStatus?.state === 'failed') {
-                    errorMessage = 'Task handler returned a failed state';
-                    // Try to get more specific error from output
-                    if (taskStatus.output?.error) {
-                        errorMessage = taskStatus.output.error;
+                result.status = 'succeeded';
+                result.output = output;
+                result.completedAt = new Date().toISOString();
+                result.duration = new Date(result.completedAt).getTime() - new Date(result.startedAt).getTime();
+            } else {
+                // Job failed - try to get error details from the job
+                let errorMessage = 'Task failed';
+                try {
+                    const completedJob = await this.payload.findByID({
+                        id: job.id,
+                        collection: 'payload-jobs',
+                        req
+                    });
+                    const taskStatus = completedJob.taskStatus?.[completedJob.taskSlug]?.[completedJob.totalTried];
+                    if (completedJob.log && completedJob.log.length > 0) {
+                        const latestLog = completedJob.log[completedJob.log.length - 1];
+                        errorMessage = latestLog.error?.message || latestLog.error || errorMessage;
                     }
-                }
-                // Check for network errors in the job data
-                if (!errorMessage && completedJob.result) {
-                    const result = completedJob.result;
-                    if (result.error) {
-                        errorMessage = result.error;
+                    if (taskStatus?.output?.errorMessage) {
+                        errorMessage = taskStatus.output.errorMessage;
                     }
+                } catch  {
+                    // Job may have been deleted - use the job status from run result
+                    errorMessage = `Task failed with status: ${jobStatus?.status || 'unknown'}`;
                 }
-                // Final fallback to generic message with more detail
-                if (!errorMessage) {
-                    const jobDetails = {
-                        taskSlug,
-                        hasError: completedJob.hasError,
-                        taskStatus: taskStatus?.complete,
-                        totalTried: completedJob.totalTried
-                    };
-                    errorMessage = `Task ${taskSlug} failed without detailed error information. Job details: ${JSON.stringify(jobDetails)}`;
-                }
+                throw new Error(errorMessage);
             }
-            const result = {
-                error: errorMessage,
-                output: taskStatus?.output || {},
-                state: isComplete ? 'succeeded' : 'failed'
-            };
-            // Store the output and error
-            context.steps[stepName].output = result.output;
-            context.steps[stepName].state = result.state;
-            if (result.error) {
-                context.steps[stepName].error = result.error;
-            }
-            // Independent execution tracking (not dependent on PayloadCMS task status)
-            context.steps[stepName].executionInfo = {
-                completed: true,
-                success: result.state === 'succeeded',
-                executedAt: new Date().toISOString(),
-                duration: Date.now() - (context.steps[stepName]._startTime || Date.now())
+            context.steps[step.stepName] = {
+                state: 'succeeded',
+                input: resolvedInput,
+                output: result.output
             };
-            // For failed steps, try to extract detailed error information from the job logs
-            // This approach is more reliable than external storage and persists with the workflow
-            if (result.state === 'failed') {
-                const errorDetails = this.extractErrorDetailsFromJob(completedJob, context.steps[stepName], stepName);
-                if (errorDetails) {
-                    context.steps[stepName].errorDetails = errorDetails;
-                    this.logger.info({
-                        stepName,
-                        errorType: errorDetails.errorType,
-                        duration: errorDetails.duration,
-                        attempts: errorDetails.attempts
-                    }, 'Extracted detailed error information for failed step');
-                }
-            }
-            this.logger.debug({
-                context
-            }, 'Step execution context');
-            if (result.state !== 'succeeded') {
-                throw new Error(result.error || `Step ${stepName} failed`);
-            }
             this.logger.info({
-                output: result.output,
-                stepName
-            }, 'Step completed');
-            // Update workflow run with current step results if workflowRunId is provided
-            if (workflowRunId) {
-                await this.updateWorkflowRunContext(workflowRunId, context, req);
-            }
+                stepName: step.stepName,
+                duration: result.duration
+            }, 'Step completed successfully');
         } catch (error) {
             const errorMessage = error instanceof Error ? error.message : 'Unknown error';
-            context.steps[stepName].state = 'failed';
-            context.steps[stepName].error = errorMessage;
-            // Independent execution tracking for failed steps
-            context.steps[stepName].executionInfo = {
-                completed: true,
-                success: false,
-                executedAt: new Date().toISOString(),
-                duration: Date.now() - (context.steps[stepName]._startTime || Date.now()),
-                failureReason: errorMessage
+            result.status = 'failed';
+            result.error = errorMessage;
+            result.completedAt = new Date().toISOString();
+            result.duration = new Date(result.completedAt).getTime() - new Date(result.startedAt).getTime();
+            context.steps[step.stepName] = {
+                state: 'failed',
+                input: resolvedInput,
+                error: errorMessage
             };
             this.logger.error({
-                error: errorMessage,
-                input: context.steps[stepName].input,
-                stepName,
-                taskSlug
+                stepName: step.stepName,
+                error: errorMessage
             }, 'Step execution failed');
-            // Update workflow run with current step results if workflowRunId is provided
-            if (workflowRunId) {
-                try {
-                    await this.updateWorkflowRunContext(workflowRunId, context, req);
-                } catch (updateError) {
-                    this.logger.error({
-                        error: updateError instanceof Error ? updateError.message : 'Unknown error',
-                        stepName
-                    }, 'Failed to update workflow run context after step failure');
-                }
-            }
             throw error;
         }
+        return result;
     }
     /**
-   * Extracts detailed error information from job logs and input
-   */ extractErrorDetailsFromJob(job, stepContext, stepName) {
+   * Resolve step input using JSONata expressions
+   */ async resolveStepInput(config, context) {
         try {
-            // Get error information from multiple sources
-            const input = stepContext.input || {};
-            const logs = job.log || [];
-            const latestLog = logs[logs.length - 1];
-            // Extract error message from job error or log
-            const errorMessage = job.error?.message || latestLog?.error?.message || 'Unknown error';
-            // For timeout scenarios, check if it's a timeout based on duration and timeout setting
-            let errorType = this.classifyErrorType(errorMessage);
-            // Special handling for HTTP timeouts - if task failed and duration exceeds timeout, it's likely a timeout
-            if (errorType === 'unknown' && input.timeout && stepContext.executionInfo?.duration) {
-                const timeoutMs = parseInt(input.timeout) || 30000;
-                const actualDuration = stepContext.executionInfo.duration;
-                // If execution duration is close to or exceeds timeout, classify as timeout
-                if (actualDuration >= timeoutMs * 0.9) {
-                    errorType = 'timeout';
-                    this.logger.debug({
-                        timeoutMs,
-                        actualDuration,
-                        stepName
-                    }, 'Classified error as timeout based on duration analysis');
-                }
-            }
-            // Calculate duration from execution info if available
-            const duration = stepContext.executionInfo?.duration || 0;
-            // Extract attempt count from logs
-            const attempts = job.totalTried || 1;
-            return {
-                stepId: `${stepName}-${Date.now()}`,
-                errorType,
-                duration,
-                attempts,
-                finalError: errorMessage,
-                context: {
-                    url: input.url,
-                    method: input.method,
-                    timeout: input.timeout,
-                    statusCode: latestLog?.output?.status,
-                    headers: input.headers
-                },
-                timestamp: new Date().toISOString()
-            };
+            return await resolveInput(config, context, {
+                timeout: 5000
+            });
         } catch (error) {
             this.logger.warn({
-                error: error instanceof Error ? error.message : 'Unknown error',
-                stepName
-            }, 'Failed to extract error details from job');
-            return null;
+                error: error instanceof Error ? error.message : 'Unknown error'
+            }, 'Failed to resolve step input, using raw config');
+            return config;
         }
     }
     /**
-   * Resolve step execution order based on dependencies
-   */ resolveExecutionOrder(steps) {
-        const stepMap = new Map();
-        const dependencyGraph = new Map();
-        const indegree = new Map();
-        // Build the step map and dependency graph
-        for (const step of steps){
-            const stepName = step.name || `step-${steps.indexOf(step)}`;
-            const dependencies = step.dependencies || [];
-            stepMap.set(stepName, {
-                ...step,
-                name: stepName,
-                dependencies
+   * Evaluate a condition using JSONata
+   */ async evaluateCondition(condition, context) {
+        try {
+            return await evalCondition(condition, context, {
+                timeout: 5000
             });
-            dependencyGraph.set(stepName, dependencies);
-            indegree.set(stepName, dependencies.length);
-        }
-        // Topological sort to determine execution batches
-        const executionBatches = [];
-        const processed = new Set();
-        while(processed.size < steps.length){
-            const currentBatch = [];
-            // Find all steps with no remaining dependencies
-            for (const [stepName, inDegree] of indegree.entries()){
-                if (inDegree === 0 && !processed.has(stepName)) {
-                    const step = stepMap.get(stepName);
-                    if (step) {
-                        currentBatch.push(step);
-                    }
-                }
-            }
-            if (currentBatch.length === 0) {
-                throw new Error('Circular dependency detected in workflow steps');
-            }
-            executionBatches.push(currentBatch);
-            // Update indegrees for next iteration
-            for (const step of currentBatch){
-                processed.add(step.name);
-                // Reduce indegree for steps that depend on completed steps
-                for (const [otherStepName, dependencies] of dependencyGraph.entries()){
-                    if (dependencies.includes(step.name) && !processed.has(otherStepName)) {
-                        indegree.set(otherStepName, (indegree.get(otherStepName) || 0) - 1);
-                    }
-                }
-            }
-        }
-        return executionBatches;
-    }
-    /**
-   * Resolve step input using Handlebars templates with automatic type conversion
-   */ resolveStepInput(config, context, stepType) {
-        const resolved = {};
-        this.logger.debug({
-            configKeys: Object.keys(config),
-            contextSteps: Object.keys(context.steps),
-            triggerType: context.trigger?.type,
-            stepType
-        }, 'Starting step input resolution with Handlebars');
-        for (const [key, value] of Object.entries(config)){
-            if (typeof value === 'string') {
-                // Check if the string contains Handlebars templates
-                if (value.includes('{{') && value.includes('}}')) {
-                    this.logger.debug({
-                        key,
-                        template: value,
-                        availableSteps: Object.keys(context.steps),
-                        hasTriggerData: !!context.trigger?.data,
-                        hasTriggerDoc: !!context.trigger?.doc
-                    }, 'Processing Handlebars template');
-                    try {
-                        const template = Handlebars.compile(value);
-                        const result = template(context);
-                        this.logger.debug({
-                            key,
-                            template: value,
-                            result: JSON.stringify(result).substring(0, 200),
-                            resultType: typeof result
-                        }, 'Handlebars template resolved successfully');
-                        resolved[key] = this.convertValueType(result, key);
-                    } catch (error) {
-                        this.logger.warn({
-                            error: error instanceof Error ? error.message : 'Unknown error',
-                            key,
-                            template: value,
-                            contextSnapshot: JSON.stringify(context).substring(0, 500)
-                        }, 'Failed to resolve Handlebars template');
-                        resolved[key] = value; // Keep original value if resolution fails
-                    }
-                } else {
-                    // Regular string, apply type conversion
-                    resolved[key] = this.convertValueType(value, key);
-                }
-            } else if (typeof value === 'object' && value !== null) {
-                // Recursively resolve nested objects
-                this.logger.debug({
-                    key,
-                    nestedKeys: Object.keys(value)
-                }, 'Recursively resolving nested object');
-                resolved[key] = this.resolveStepInput(value, context, stepType);
-            } else {
-                // Keep literal values as-is
-                resolved[key] = value;
-            }
+        } catch (error) {
+            this.logger.warn({
+                condition,
+                error: error instanceof Error ? error.message : 'Unknown error'
+            }, 'Failed to evaluate condition');
+            return false;
         }
-        this.logger.debug({
-            resolvedKeys: Object.keys(resolved),
-            originalKeys: Object.keys(config)
-        }, 'Step input resolution completed');
-        return resolved;
     }
     /**
-   * Safely serialize an object, handling circular references and non-serializable values
+   * Safely serialize an object for storage
    */ safeSerialize(obj) {
         const seen = new WeakSet();
-        const serialize = (value)=>{
+        // Keys to completely exclude from serialization
+        const excludeKeys = new Set([
+            'table',
+            'schema',
+            '_',
+            '__',
+            'payload',
+            'res',
+            'transactionID',
+            'i18n',
+            'fallbackLocale'
+        ]);
+        // For req object, only keep these useful debugging properties
+        const reqAllowedKeys = new Set([
+            'payloadAPI',
+            'locale',
+            'user',
+            'method',
+            'url'
+        ]);
+        const serialize = (value, parentKey)=>{
             if (value === null || typeof value !== 'object') {
                 return value;
             }
@@ -516,18 +303,20 @@ export class WorkflowExecutor {
             }
             seen.add(value);
             if (Array.isArray(value)) {
-                return value.map(serialize);
+                return value.map((v)=>serialize(v));
             }
             const result = {};
             for (const [key, val] of Object.entries(value)){
                 try {
-                    // Skip non-serializable properties that are likely internal database objects
-                    if (key === 'table' || key === 'schema' || key === '_' || key === '__') {
+                    if (excludeKeys.has(key)) {
                         continue;
                     }
-                    result[key] = serialize(val);
+                    // Special handling for req object - only include allowed keys
+                    if (parentKey === 'req' && !reqAllowedKeys.has(key)) {
+                        continue;
+                    }
+                    result[key] = serialize(val, key);
                 } catch  {
-                    // Skip properties that can't be accessed or serialized
                     result[key] = '[Non-serializable]';
                 }
             }
@@ -536,277 +325,165 @@ export class WorkflowExecutor {
         return serialize(obj);
     }
     /**
-   * Update workflow run with current context
-   */ async updateWorkflowRunContext(workflowRunId, context, req) {
-        const serializeContext = ()=>({
-                steps: this.safeSerialize(context.steps),
-                trigger: {
-                    type: context.trigger.type,
-                    collection: context.trigger.collection,
-                    data: this.safeSerialize(context.trigger.data),
-                    doc: this.safeSerialize(context.trigger.doc),
-                    operation: context.trigger.operation,
-                    previousDoc: this.safeSerialize(context.trigger.previousDoc),
-                    triggeredAt: context.trigger.triggeredAt,
-                    user: context.trigger.req?.user
-                }
+   * Execute a workflow with the given context
+   */ async execute(workflow, context, req, firedTrigger) {
+        this.logger.info({
+            workflowId: workflow.id,
+            workflowName: workflow.name,
+            triggerId: firedTrigger?.id,
+            triggerName: firedTrigger?.name
+        }, 'Starting workflow execution');
+        const resolvedSteps = await this.resolveWorkflowSteps(workflow);
+        const stepResults = [];
+        for (const step of resolvedSteps){
+            stepResults.push({
+                step: step.stepId,
+                stepName: step.stepName,
+                stepIndex: step.stepIndex,
+                status: 'pending'
             });
-        await this.payload.update({
-            id: workflowRunId,
+        }
+        const workflowRun = await this.payload.create({
             collection: 'workflow-runs',
             data: {
-                context: serializeContext()
+                workflow: workflow.id,
+                workflowVersion: 1,
+                firedTrigger: firedTrigger?.id,
+                triggerData: this.safeSerialize(context.trigger),
+                status: 'running',
+                startedAt: new Date().toISOString(),
+                triggeredBy: context.trigger.req?.user?.email || 'system',
+                stepResults,
+                context: this.safeSerialize(context),
+                inputs: this.safeSerialize(context.trigger),
+                logs: [
+                    {
+                        timestamp: new Date().toISOString(),
+                        level: 'info',
+                        message: 'Workflow execution started'
+                    }
+                ]
             },
             req
         });
-    }
-    /**
-   * Evaluate a condition using Handlebars templates and comparison operators
-   */ evaluateCondition(condition, context) {
-        this.logger.debug({
-            condition,
-            contextKeys: Object.keys(context),
-            triggerType: context.trigger?.type,
-            triggerData: context.trigger?.data,
-            triggerDoc: context.trigger?.doc ? 'present' : 'absent'
-        }, 'Starting condition evaluation');
-        try {
-            // Check if this is a comparison expression
-            const comparisonMatch = condition.match(/^(.+?)\s*(==|!=|>|<|>=|<=)\s*(.+)$/);
-            if (comparisonMatch) {
-                const [, leftExpr, operator, rightExpr] = comparisonMatch;
-                // Evaluate left side (could be Handlebars template or JSONPath)
-                const leftValue = this.resolveConditionValue(leftExpr.trim(), context);
-                // Evaluate right side (could be Handlebars template, JSONPath, or literal)
-                const rightValue = this.resolveConditionValue(rightExpr.trim(), context);
-                this.logger.debug({
-                    condition,
-                    leftExpr: leftExpr.trim(),
-                    leftValue,
-                    operator,
-                    rightExpr: rightExpr.trim(),
-                    rightValue,
-                    leftType: typeof leftValue,
-                    rightType: typeof rightValue
-                }, 'Evaluating comparison condition');
-                // Perform comparison
-                let result;
-                switch(operator){
-                    case '!=':
-                        result = leftValue !== rightValue;
-                        break;
-                    case '<':
-                        result = Number(leftValue) < Number(rightValue);
-                        break;
-                    case '<=':
-                        result = Number(leftValue) <= Number(rightValue);
-                        break;
-                    case '==':
-                        result = leftValue === rightValue;
-                        break;
-                    case '>':
-                        result = Number(leftValue) > Number(rightValue);
-                        break;
-                    case '>=':
-                        result = Number(leftValue) >= Number(rightValue);
-                        break;
-                    default:
-                        throw new Error(`Unknown comparison operator: ${operator}`);
-                }
-                this.logger.debug({
-                    condition,
-                    result,
-                    leftValue,
-                    rightValue,
-                    operator
-                }, 'Comparison condition evaluation completed');
-                return result;
-            } else {
-                // Treat as template or JSONPath boolean evaluation
-                const result = this.resolveConditionValue(condition, context);
-                this.logger.debug({
-                    condition,
-                    result,
-                    resultType: Array.isArray(result) ? 'array' : typeof result,
-                    resultLength: Array.isArray(result) ? result.length : undefined
-                }, 'Boolean evaluation result');
-                // Handle different result types
-                let finalResult;
-                if (Array.isArray(result)) {
-                    finalResult = result.length > 0 && Boolean(result[0]);
-                } else {
-                    finalResult = Boolean(result);
-                }
-                this.logger.debug({
-                    condition,
-                    finalResult,
-                    originalResult: result
-                }, 'Boolean condition evaluation completed');
-                return finalResult;
-            }
-        } catch (error) {
-            this.logger.warn({
-                condition,
-                error: error instanceof Error ? error.message : 'Unknown error',
-                errorStack: error instanceof Error ? error.stack : undefined
-            }, 'Failed to evaluate condition');
-            // If condition evaluation fails, assume false
-            return false;
-        }
-    }
-    /**
-   * Resolve a condition value using Handlebars templates or JSONPath
-   */ resolveConditionValue(expr, context) {
-        // Handle string literals
-        if (expr.startsWith('"') && expr.endsWith('"') || expr.startsWith("'") && expr.endsWith("'")) {
-            return expr.slice(1, -1) // Remove quotes
-            ;
-        }
-        // Handle boolean literals
-        if (expr === 'true') {
-            return true;
-        }
-        if (expr === 'false') {
-            return false;
-        }
-        // Handle number literals
-        if (/^-?\d+(?:\.\d+)?$/.test(expr)) {
-            return Number(expr);
-        }
-        // Handle Handlebars templates
-        if (expr.includes('{{') && expr.includes('}}')) {
-            try {
-                const template = Handlebars.compile(expr);
-                return template(context);
-            } catch (error) {
-                this.logger.warn({
-                    error: error instanceof Error ? error.message : 'Unknown error',
-                    expr
-                }, 'Failed to resolve Handlebars condition');
-                return false;
-            }
-        }
-        // Return as string if nothing else matches
-        return expr;
-    }
-    /**
-   * Execute a workflow with the given context
-   */ async execute(workflow, context, req) {
-        this.logger.info({
-            workflowId: workflow.id,
-            workflowName: workflow.name
-        }, 'Starting workflow execution');
-        const serializeContext = ()=>({
-                steps: this.safeSerialize(context.steps),
-                trigger: {
-                    type: context.trigger.type,
-                    collection: context.trigger.collection,
-                    data: this.safeSerialize(context.trigger.data),
-                    doc: this.safeSerialize(context.trigger.doc),
-                    operation: context.trigger.operation,
-                    previousDoc: this.safeSerialize(context.trigger.previousDoc),
-                    triggeredAt: context.trigger.triggeredAt,
-                    user: context.trigger.req?.user
-                }
-            });
         this.logger.info({
-            workflowId: workflow.id,
-            workflowName: workflow.name,
-            contextSummary: {
-                triggerType: context.trigger.type,
-                triggerCollection: context.trigger.collection,
-                triggerOperation: context.trigger.operation,
-                hasDoc: !!context.trigger.doc,
-                userEmail: context.trigger.req?.user?.email
-            }
-        }, 'About to create workflow run record');
-        // Create a workflow run record
-        let workflowRun;
-        try {
-            workflowRun = await this.payload.create({
-                collection: 'workflow-runs',
-                data: {
-                    context: serializeContext(),
-                    startedAt: new Date().toISOString(),
-                    status: 'running',
-                    triggeredBy: context.trigger.req?.user?.email || 'system',
-                    workflow: workflow.id,
-                    workflowVersion: 1 // Default version since generated type doesn't have _version field
-                },
-                req
-            });
-            this.logger.info({
-                workflowRunId: workflowRun.id,
-                workflowId: workflow.id,
-                workflowName: workflow.name
-            }, 'Workflow run record created successfully');
-        } catch (error) {
-            this.logger.error({
-                error: error instanceof Error ? error.message : 'Unknown error',
-                errorStack: error instanceof Error ? error.stack : undefined,
-                workflowId: workflow.id,
-                workflowName: workflow.name
-            }, 'Failed to create workflow run record');
-            throw error;
-        }
+            workflowRunId: workflowRun.id,
+            workflowId: workflow.id
+        }, 'Workflow run record created');
+        // Create job metadata for tracking workflow context in payload-jobs
+        const jobMeta = {
+            automationWorkflowId: workflow.id,
+            automationWorkflowRunId: workflowRun.id,
+            automationTriggerId: firedTrigger?.id
+        };
         try {
-            // Resolve execution order based on dependencies
-            const executionBatches = this.resolveExecutionOrder(workflow.steps || []);
+            const executionBatches = this.resolveExecutionOrder(resolvedSteps);
             this.logger.info({
-                batchSizes: executionBatches.map((batch)=>batch.length),
-                totalBatches: executionBatches.length
+                batchCount: executionBatches.length,
+                batchSizes: executionBatches.map((b)=>b.length)
             }, 'Resolved step execution order');
-            // Execute each batch in sequence, but steps within each batch in parallel
             for(let batchIndex = 0; batchIndex < executionBatches.length; batchIndex++){
                 const batch = executionBatches[batchIndex];
                 this.logger.info({
                     batchIndex,
                     stepCount: batch.length,
-                    stepNames: batch.map((s)=>s.name)
+                    stepNames: batch.map((s)=>s.stepName)
                 }, 'Executing batch');
-                // Execute all steps in this batch in parallel
-                const batchPromises = batch.map((step, stepIndex)=>this.executeStep(step, stepIndex, context, req, workflowRun.id));
-                // Wait for all steps in the current batch to complete
+                const batchPromises = batch.map(async (step)=>{
+                    try {
+                        const result = await this.executeStep(step, context, req, stepResults, jobMeta);
+                        const idx = stepResults.findIndex((r)=>r.stepIndex === step.stepIndex);
+                        if (idx !== -1) {
+                            stepResults[idx] = result;
+                        }
+                        return result;
+                    } catch (error) {
+                        const errorMessage = error instanceof Error ? error.message : 'Unknown error';
+                        const idx = stepResults.findIndex((r)=>r.stepIndex === step.stepIndex);
+                        if (idx !== -1) {
+                            stepResults[idx] = {
+                                ...stepResults[idx],
+                                status: 'failed',
+                                error: errorMessage,
+                                completedAt: new Date().toISOString()
+                            };
+                        }
+                        if (workflow.errorHandling === 'stop') {
+                            throw error;
+                        }
+                        this.logger.warn({
+                            stepName: step.stepName,
+                            error: errorMessage
+                        }, 'Step failed but continuing due to error handling setting');
+                    }
+                });
                 await Promise.all(batchPromises);
-                this.logger.info({
-                    batchIndex,
-                    stepCount: batch.length
-                }, 'Batch completed');
+                await this.payload.update({
+                    id: workflowRun.id,
+                    collection: 'workflow-runs',
+                    data: {
+                        stepResults,
+                        context: this.safeSerialize(context)
+                    },
+                    req
+                });
+            }
+            const outputs = {};
+            for (const result of stepResults){
+                if (result.status === 'succeeded' && result.output) {
+                    outputs[result.stepName] = result.output;
+                }
             }
-            // Update workflow run as completed
             await this.payload.update({
                 id: workflowRun.id,
                 collection: 'workflow-runs',
                 data: {
+                    status: 'completed',
                     completedAt: new Date().toISOString(),
-                    context: serializeContext(),
-                    status: 'completed'
+                    stepResults,
+                    context: this.safeSerialize(context),
+                    outputs,
+                    logs: [
+                        ...workflowRun.logs || [],
+                        {
+                            timestamp: new Date().toISOString(),
+                            level: 'info',
+                            message: 'Workflow execution completed successfully'
+                        }
+                    ]
                 },
                 req
             });
             this.logger.info({
-                runId: workflowRun.id,
-                workflowId: workflow.id,
-                workflowName: workflow.name
+                workflowRunId: workflowRun.id,
+                workflowId: workflow.id
             }, 'Workflow execution completed');
         } catch (error) {
-            // Update workflow run as failed
+            const errorMessage = error instanceof Error ? error.message : 'Unknown error';
             await this.payload.update({
                 id: workflowRun.id,
                 collection: 'workflow-runs',
                 data: {
+                    status: 'failed',
                     completedAt: new Date().toISOString(),
-                    context: serializeContext(),
-                    error: error instanceof Error ? error.message : 'Unknown error',
-                    status: 'failed'
+                    stepResults,
+                    context: this.safeSerialize(context),
+                    error: errorMessage,
+                    logs: [
+                        ...workflowRun.logs || [],
+                        {
+                            timestamp: new Date().toISOString(),
+                            level: 'error',
+                            message: `Workflow execution failed: ${errorMessage}`
+                        }
+                    ]
                 },
                 req
             });
             this.logger.error({
-                error: error instanceof Error ? error.message : 'Unknown error',
-                runId: workflowRun.id,
+                workflowRunId: workflowRun.id,
                 workflowId: workflow.id,
-                workflowName: workflow.name
+                error: errorMessage
             }, 'Workflow execution failed');
             throw error;
         }