@marktoflow/core 2.0.0-alpha.15 → 2.0.0-alpha.16

package/dist/engine.js

@@ -4,7 +4,7 @@
  * Executes workflow steps with retry logic, variable resolution,
  * and SDK invocation.
  */
-import { StepStatus, WorkflowStatus, createExecutionContext, createStepResult, isActionStep, isSubWorkflowStep, isIfStep, isSwitchStep, isForEachStep, isWhileStep, isMapStep, isFilterStep, isReduceStep, isParallelStep, isTryStep, isScriptStep, } from './models.js';
+import { StepStatus, WorkflowStatus, createExecutionContext, createStepResult, isActionStep, isSubWorkflowStep, isIfStep, isSwitchStep, isForEachStep, isWhileStep, isMapStep, isFilterStep, isReduceStep, isParallelStep, isTryStep, isScriptStep, isWaitStep, isMergeStep, } from './models.js';
 import { mergePermissions, toSecurityPolicy, } from './permissions.js';
 import { loadPromptFile, resolvePromptTemplate, validatePromptInputs, } from './prompt-loader.js';
 import { DEFAULT_FAILOVER_CONFIG, AgentHealthTracker, FailoverReason, } from './failover.js';
@@ -16,6 +16,28 @@ import { executeScriptAsync } from './script-executor.js';
 // ============================================================================
 // Helper Functions
 // ============================================================================
+/**
+ * Parse a duration string like "2h", "30m", "5s", "100ms" into milliseconds.
+ */
+function parseDuration(duration) {
+    const match = duration.match(/^(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)$/i);
+    if (!match) {
+        const asNum = Number(duration);
+        if (!isNaN(asNum))
+            return asNum; // Treat bare numbers as milliseconds
+        throw new Error(`Invalid duration format: "${duration}". Use format like "2h", "30m", "5s", "100ms"`);
+    }
+    const value = parseFloat(match[1]);
+    const unit = match[2].toLowerCase();
+    switch (unit) {
+        case 'ms': return value;
+        case 's': return value * 1000;
+        case 'm': return value * 60 * 1000;
+        case 'h': return value * 60 * 60 * 1000;
+        case 'd': return value * 24 * 60 * 60 * 1000;
+        default: return value;
+    }
+}
 /**
  * Convert error to string for display/logging
  */
@@ -33,6 +55,21 @@ function errorToString(error) {
         return String(error);
     }
 }
+/**
+ * Get a field value from an object by key path.
+ */
+function getField(item, field) {
+    if (!item || typeof item !== 'object')
+        return undefined;
+    const parts = field.split('.');
+    let current = item;
+    for (const part of parts) {
+        if (!current || typeof current !== 'object')
+            return undefined;
+        current = current[part];
+    }
+    return current;
+}
 // ============================================================================
 // Retry Policy
 // ============================================================================
@@ -247,7 +284,7 @@ export class WorkflowEngine {
     failoverConfig;
     healthTracker;
     failoverEvents = [];
-    workflowPath; // Base path for resolving sub-workflows
+    workflowPath; // Base path for resolving sub-workflows (public for CLI state tracking)
     workflowPermissions; // Workflow-level permissions
     promptCache = new Map(); // Cache for loaded prompts
     constructor(config = {}, events = {}, stateStore) {
@@ -305,6 +342,12 @@ export class WorkflowEngine {
         if (isScriptStep(step)) {
             return this.executeScriptStep(step, context);
         }
+        if (isWaitStep(step)) {
+            return this.executeWaitStep(step, context);
+        }
+        if (isMergeStep(step)) {
+            return this.executeMergeStep(step, context);
+        }
         // Default: action or workflow step
         return this.executeStepWithFailover(step, context, sdkRegistry, stepExecutor);
     }
@@ -330,7 +373,7 @@ export class WorkflowEngine {
             this.stateStore.createExecution({
                 runId: context.runId,
                 workflowId: workflow.metadata.id,
-                workflowPath: 'unknown',
+                workflowPath: this.workflowPath ?? 'unknown',
                 status: WorkflowStatus.RUNNING,
                 startedAt: startedAt,
                 completedAt: null,
@@ -427,6 +470,136 @@ export class WorkflowEngine {
         this.events.onWorkflowComplete?.(workflow, workflowResult);
         return workflowResult;
     }
+    /**
+     * Resume a paused execution (e.g., after form submission).
+     *
+     * @param runId - The execution run ID
+     * @param stepId - The step ID that was waiting
+     * @param resumeData - Data from the resume action (e.g., form submission)
+     * @param sdkRegistry - SDK registry for step execution
+     * @param stepExecutor - Step executor function
+     * @returns Workflow result from resumed execution
+     */
+    async resumeExecution(runId, stepId, resumeData, sdkRegistry, stepExecutor) {
+        if (!this.stateStore) {
+            throw new Error('Cannot resume execution: StateStore not configured');
+        }
+        // Load execution from state store
+        const execution = this.stateStore.getExecution(runId);
+        if (!execution) {
+            throw new Error(`Execution ${runId} not found`);
+        }
+        if (execution.status !== WorkflowStatus.RUNNING) {
+            throw new Error(`Cannot resume execution ${runId}: status is ${execution.status}`);
+        }
+        // Load workflow
+        const { workflow } = await parseFile(execution.workflowPath);
+        this.workflowPath = execution.workflowPath;
+        // Find the step that was waiting
+        const stepIndex = workflow.steps.findIndex(s => s.id === stepId);
+        if (stepIndex === -1) {
+            throw new Error(`Step ${stepId} not found in workflow`);
+        }
+        // Load all checkpoints to rebuild context
+        const checkpoints = this.stateStore.getCheckpoints(runId);
+        const stepResults = [];
+        const context = createExecutionContext(workflow, execution.inputs || {});
+        context.runId = runId;
+        context.status = WorkflowStatus.RUNNING;
+        // Rebuild context from checkpoints
+        for (let i = 0; i < stepIndex; i++) {
+            const checkpoint = checkpoints.find(cp => cp.stepIndex === i);
+            if (checkpoint) {
+                const step = workflow.steps[i];
+                // Recreate step result
+                const result = createStepResult(step.id, checkpoint.status, checkpoint.outputs, checkpoint.startedAt, checkpoint.retryCount, checkpoint.error || undefined);
+                stepResults.push(result);
+                // Restore step metadata
+                context.stepMetadata[step.id] = {
+                    status: checkpoint.status.toLowerCase(),
+                    retryCount: checkpoint.retryCount,
+                    ...(checkpoint.error ? { error: checkpoint.error } : {}),
+                };
+                // Restore output variable
+                if (step.outputVariable && checkpoint.status === StepStatus.COMPLETED) {
+                    context.variables[step.outputVariable] = checkpoint.outputs;
+                }
+            }
+        }
+        // Inject resume data into context
+        context.variables[`${stepId}_response`] = resumeData;
+        // Continue execution from next step
+        const startedAt = new Date(execution.startedAt);
+        this.workflowPermissions = workflow.permissions;
+        try {
+            for (let i = stepIndex + 1; i < workflow.steps.length; i++) {
+                const step = workflow.steps[i];
+                context.currentStepIndex = i;
+                const result = await this.executeStep(step, context, sdkRegistry, stepExecutor);
+                stepResults.push(result);
+                context.stepMetadata[step.id] = {
+                    status: result.status.toLowerCase(),
+                    retryCount: result.retryCount,
+                    ...(result.error ? { error: errorToString(result.error) } : {}),
+                };
+                if (step.outputVariable && result.status === StepStatus.COMPLETED) {
+                    context.variables[step.outputVariable] = result.output;
+                }
+                if (result.status === StepStatus.COMPLETED &&
+                    result.output &&
+                    typeof result.output === 'object' &&
+                    '__workflow_outputs__' in result.output) {
+                    const outputObj = result.output;
+                    const outputs = outputObj['__workflow_outputs__'];
+                    context.workflowOutputs = outputs;
+                }
+                if (result.status === StepStatus.FAILED) {
+                    let errorAction = 'stop';
+                    if ('errorHandling' in step && step.errorHandling?.action) {
+                        errorAction = step.errorHandling.action;
+                    }
+                    if (errorAction === 'stop' || errorAction === 'rollback') {
+                        if (errorAction === 'rollback' && this.rollbackRegistry) {
+                            await this.rollbackRegistry.rollbackAllAsync({
+                                context,
+                                inputs: context.inputs,
+                                variables: context.variables,
+                            });
+                        }
+                        context.status = WorkflowStatus.FAILED;
+                        const workflowError = result.error ? errorToString(result.error) : `Step ${step.id} failed`;
+                        const workflowResult = this.buildWorkflowResult(workflow, context, stepResults, startedAt, workflowError);
+                        this.events.onWorkflowComplete?.(workflow, workflowResult);
+                        return workflowResult;
+                    }
+                }
+            }
+            context.status = WorkflowStatus.COMPLETED;
+        }
+        catch (error) {
+            context.status = WorkflowStatus.FAILED;
+            if (this.stateStore) {
+                this.stateStore.updateExecution(runId, {
+                    status: WorkflowStatus.FAILED,
+                    completedAt: new Date(),
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+            const workflowResult = this.buildWorkflowResult(workflow, context, stepResults, startedAt, error instanceof Error ? error.message : String(error));
+            this.events.onWorkflowComplete?.(workflow, workflowResult);
+            return workflowResult;
+        }
+        const workflowResult = this.buildWorkflowResult(workflow, context, stepResults, startedAt);
+        if (this.stateStore) {
+            this.stateStore.updateExecution(runId, {
+                status: context.status,
+                completedAt: new Date(),
+                outputs: context.variables,
+            });
+        }
+        this.events.onWorkflowComplete?.(workflow, workflowResult);
+        return workflowResult;
+    }
     /**
      * Execute a workflow from a file.
      * This method automatically sets the workflow path for resolving sub-workflows.
@@ -1116,9 +1289,17 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
     }
     /**
      * Execute a for-each loop step.
+     * Supports optional batch_size and pause_between_batches for rate limiting.
      */
     async executeForEachStep(step, context, sdkRegistry, stepExecutor) {
         const startedAt = new Date();
+        const cleanupLoopVars = () => {
+            delete context.variables[step.itemVariable];
+            delete context.variables['loop'];
+            delete context.variables['batch'];
+            if (step.indexVariable)
+                delete context.variables[step.indexVariable];
+        };
         try {
             // Resolve items array
             const items = resolveTemplates(step.items, context);
@@ -1128,10 +1309,15 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
             if (items.length === 0) {
                 return createStepResult(step.id, StepStatus.SKIPPED, [], startedAt);
             }
-            // Execute steps for each item
+            const batchSize = step.batchSize;
+            const pauseBetweenBatches = step.pauseBetweenBatches;
+            // If batch mode, process items in batches
+            if (batchSize && batchSize > 0) {
+                return await this.executeForEachBatched(step, items, batchSize, pauseBetweenBatches ?? 0, context, sdkRegistry, stepExecutor, startedAt);
+            }
+            // Standard item-by-item execution
             const results = [];
             for (let i = 0; i < items.length; i++) {
-                // Inject loop variables
                 context.variables[step.itemVariable] = items[i];
                 context.variables['loop'] = {
                     index: i,
@@ -1151,35 +1337,82 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
                     if (result.status === StepStatus.FAILED) {
                         const errorAction = step.errorHandling?.action ?? 'stop';
                         if (errorAction === 'stop') {
-                            // Clean up loop variables
-                            delete context.variables[step.itemVariable];
-                            delete context.variables['loop'];
-                            if (step.indexVariable)
-                                delete context.variables[step.indexVariable];
+                            cleanupLoopVars();
                             return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, result.error);
                         }
-                        // 'continue' - skip to next iteration
                         break;
                     }
                 }
                 results.push(context.variables[step.itemVariable]);
             }
-            // Clean up loop variables
-            delete context.variables[step.itemVariable];
-            delete context.variables['loop'];
-            if (step.indexVariable)
-                delete context.variables[step.indexVariable];
+            cleanupLoopVars();
             return createStepResult(step.id, StepStatus.COMPLETED, results, startedAt);
         }
         catch (error) {
-            // Clean up loop variables on error
-            delete context.variables[step.itemVariable];
-            delete context.variables['loop'];
-            if (step.indexVariable)
-                delete context.variables[step.indexVariable];
+            cleanupLoopVars();
             return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
         }
     }
+    /**
+     * Execute for-each in batch mode.
+     * Items are split into batches; {{ batch }} contains the current batch array.
+     */
+    async executeForEachBatched(step, items, batchSize, pauseBetweenBatches, context, sdkRegistry, stepExecutor, startedAt) {
+        const results = [];
+        const totalBatches = Math.ceil(items.length / batchSize);
+        for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) {
+            const batchStart = batchIndex * batchSize;
+            const batchItems = items.slice(batchStart, batchStart + batchSize);
+            // Pause between batches (not before first batch)
+            if (batchIndex > 0 && pauseBetweenBatches > 0) {
+                await new Promise((resolve) => setTimeout(resolve, pauseBetweenBatches));
+            }
+            // Expose batch-level variables
+            context.variables['batch'] = batchItems;
+            context.variables['loop'] = {
+                index: batchIndex,
+                first: batchIndex === 0,
+                last: batchIndex === totalBatches - 1,
+                length: totalBatches,
+                batchSize,
+                batchStart,
+                totalItems: items.length,
+            };
+            // Process each item in the batch
+            for (let i = 0; i < batchItems.length; i++) {
+                const globalIndex = batchStart + i;
+                context.variables[step.itemVariable] = batchItems[i];
+                if (step.indexVariable) {
+                    context.variables[step.indexVariable] = globalIndex;
+                }
+                for (const iterStep of step.steps) {
+                    const result = await this.executeStep(iterStep, context, sdkRegistry, stepExecutor);
+                    if (result.status === StepStatus.COMPLETED && iterStep.outputVariable) {
+                        context.variables[iterStep.outputVariable] = result.output;
+                    }
+                    if (result.status === StepStatus.FAILED) {
+                        const errorAction = step.errorHandling?.action ?? 'stop';
+                        if (errorAction === 'stop') {
+                            delete context.variables[step.itemVariable];
+                            delete context.variables['loop'];
+                            delete context.variables['batch'];
+                            if (step.indexVariable)
+                                delete context.variables[step.indexVariable];
+                            return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, result.error);
+                        }
+                        break;
+                    }
+                }
+                results.push(context.variables[step.itemVariable]);
+            }
+        }
+        delete context.variables[step.itemVariable];
+        delete context.variables['loop'];
+        delete context.variables['batch'];
+        if (step.indexVariable)
+            delete context.variables[step.indexVariable];
+        return createStepResult(step.id, StepStatus.COMPLETED, results, startedAt);
+    }
     /**
      * Execute a while loop step.
      */
@@ -1436,6 +1669,214 @@ Execute the workflow steps in order and return the final outputs as JSON.`;
         }
     }
     // ============================================================================
+    // Wait Step Execution
+    // ============================================================================
+    /**
+     * Execute a wait/pause step.
+     *
+     * For mode=duration: In-process wait (suitable for short durations).
+     *   For persistent long-duration waits, a WaitManager should checkpoint
+     *   the execution and resume it later via the scheduler.
+     *
+     * For mode=webhook: Returns a resume URL. The execution will be
+     *   checkpointed and resumed when the URL is called.
+     *
+     * For mode=form: Returns form fields. The execution will be
+     *   checkpointed and resumed when the form is submitted.
+     */
+    async executeWaitStep(step, context) {
+        const startedAt = new Date();
+        try {
+            switch (step.mode) {
+                case 'duration': {
+                    if (!step.duration) {
+                        return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Wait step with mode=duration requires a duration');
+                    }
+                    const resolvedDuration = resolveTemplates(step.duration, context);
+                    const ms = parseDuration(resolvedDuration);
+                    // For short durations (under 5 minutes), do in-process wait
+                    if (ms <= 300000) {
+                        await new Promise((resolve) => setTimeout(resolve, ms));
+                        return createStepResult(step.id, StepStatus.COMPLETED, { waited: ms }, startedAt);
+                    }
+                    // For longer durations, checkpoint and schedule resume
+                    // The StateStore will persist the execution state
+                    if (this.stateStore) {
+                        this.stateStore.saveCheckpoint({
+                            runId: context.runId,
+                            stepIndex: context.currentStepIndex,
+                            stepName: step.id,
+                            status: StepStatus.COMPLETED,
+                            startedAt: startedAt,
+                            completedAt: new Date(),
+                            inputs: { mode: 'duration', resumeAt: new Date(Date.now() + ms).toISOString() },
+                            outputs: { waiting: true },
+                            error: null,
+                            retryCount: 0,
+                        });
+                    }
+                    // Set execution status to indicate waiting
+                    const resumeAt = new Date(Date.now() + ms).toISOString();
+                    return createStepResult(step.id, StepStatus.COMPLETED, {
+                        waiting: true,
+                        mode: 'duration',
+                        resumeAt,
+                        durationMs: ms,
+                    }, startedAt);
+                }
+                case 'webhook': {
+                    // Generate a unique resume URL
+                    const resumeToken = crypto.randomUUID();
+                    const webhookPath = step.webhookPath
+                        ? resolveTemplates(step.webhookPath, context)
+                        : `/resume/${context.runId}/${step.id}/${resumeToken}`;
+                    if (this.stateStore) {
+                        this.stateStore.saveCheckpoint({
+                            runId: context.runId,
+                            stepIndex: context.currentStepIndex,
+                            stepName: step.id,
+                            status: StepStatus.COMPLETED,
+                            startedAt: startedAt,
+                            completedAt: new Date(),
+                            inputs: { mode: 'webhook', resumeToken, webhookPath },
+                            outputs: { waiting: true },
+                            error: null,
+                            retryCount: 0,
+                        });
+                    }
+                    return createStepResult(step.id, StepStatus.COMPLETED, {
+                        waiting: true,
+                        mode: 'webhook',
+                        resumeToken,
+                        webhookPath,
+                    }, startedAt);
+                }
+                case 'form': {
+                    if (!step.fields || Object.keys(step.fields).length === 0) {
+                        return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Wait step with mode=form requires fields');
+                    }
+                    const resumeToken = crypto.randomUUID();
+                    if (this.stateStore) {
+                        this.stateStore.saveCheckpoint({
+                            runId: context.runId,
+                            stepIndex: context.currentStepIndex,
+                            stepName: step.id,
+                            status: StepStatus.COMPLETED,
+                            startedAt: startedAt,
+                            completedAt: new Date(),
+                            inputs: { mode: 'form', resumeToken, fields: step.fields },
+                            outputs: { waiting: true },
+                            error: null,
+                            retryCount: 0,
+                        });
+                    }
+                    return createStepResult(step.id, StepStatus.COMPLETED, {
+                        waiting: true,
+                        mode: 'form',
+                        resumeToken,
+                        fields: step.fields,
+                        formPath: `/form/${context.runId}/${step.id}/${resumeToken}`,
+                    }, startedAt);
+                }
+                default:
+                    return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, `Unknown wait mode: ${step.mode}`);
+            }
+        }
+        catch (error) {
+            return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
+        }
+    }
+    // ============================================================================
+    // Merge Step Execution
+    // ============================================================================
+    /**
+     * Execute a merge step - combines data from multiple sources.
+     */
+    async executeMergeStep(step, context) {
+        const startedAt = new Date();
+        try {
+            // Resolve all source expressions to arrays
+            const resolvedSources = [];
+            for (const source of step.sources) {
+                const resolved = resolveTemplates(source, context);
+                if (!Array.isArray(resolved)) {
+                    return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, `Merge source "${source}" did not resolve to an array`);
+                }
+                resolvedSources.push(resolved);
+            }
+            let result;
+            switch (step.mode) {
+                case 'append':
+                    result = resolvedSources.flat();
+                    break;
+                case 'match': {
+                    if (!step.matchField) {
+                        return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Merge mode "match" requires matchField');
+                    }
+                    // Return items that appear in ALL sources (by matchField)
+                    const fieldSets = resolvedSources.map((source) => new Set(source.map((item) => getField(item, step.matchField))));
+                    // Intersection of all field sets
+                    const commonKeys = fieldSets.reduce((acc, set) => new Set([...acc].filter((key) => set.has(key))));
+                    // Return items from first source that match
+                    result = resolvedSources[0].filter((item) => commonKeys.has(getField(item, step.matchField)));
+                    break;
+                }
+                case 'diff': {
+                    if (!step.matchField) {
+                        return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Merge mode "diff" requires matchField');
+                    }
+                    // Return items from first source NOT found in other sources
+                    const otherKeys = new Set(resolvedSources.slice(1).flat().map((item) => getField(item, step.matchField)));
+                    result = resolvedSources[0].filter((item) => !otherKeys.has(getField(item, step.matchField)));
+                    break;
+                }
+                case 'combine_by_field': {
+                    if (!step.matchField) {
+                        return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, 'Merge mode "combine_by_field" requires matchField');
+                    }
+                    // Group items by matchField and merge their properties
+                    const grouped = new Map();
+                    const onConflict = step.onConflict ?? 'keep_last';
+                    for (const source of resolvedSources) {
+                        for (const item of source) {
+                            if (!item || typeof item !== 'object')
+                                continue;
+                            const key = getField(item, step.matchField);
+                            const existing = grouped.get(key);
+                            if (existing) {
+                                if (onConflict === 'keep_first') {
+                                    // Only add new fields
+                                    for (const [k, v] of Object.entries(item)) {
+                                        if (!(k in existing))
+                                            existing[k] = v;
+                                    }
+                                }
+                                else if (onConflict === 'keep_last') {
+                                    Object.assign(existing, item);
+                                }
+                                else {
+                                    // merge_fields: deep merge
+                                    Object.assign(existing, item);
+                                }
+                            }
+                            else {
+                                grouped.set(key, { ...item });
+                            }
+                        }
+                    }
+                    result = Array.from(grouped.values());
+                    break;
+                }
+                default:
+                    return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, `Unknown merge mode: ${step.mode}`);
+            }
+            return createStepResult(step.id, StepStatus.COMPLETED, result, startedAt);
+        }
+        catch (error) {
+            return createStepResult(step.id, StepStatus.FAILED, null, startedAt, 0, error instanceof Error ? error.message : String(error));
+        }
+    }
+    // ============================================================================
     // Helper Methods for Control Flow
     // ============================================================================
     /**