keystone-cli 0.8.0 → 1.0.1

package/src/runner/workflow-runner.ts

@@ -1,4 +1,6 @@
 import { randomUUID } from 'node:crypto';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
 import { dirname, join } from 'node:path';
 import { MemoryDb } from '../db/memory-db.ts';
 import { type RunStatus, WorkflowDb } from '../db/workflow-db.ts';
@@ -7,14 +9,22 @@ import { ExpressionEvaluator } from '../expression/evaluator.ts';
 import type { Step, Workflow, WorkflowStep } from '../parser/schema.ts';
 import { WorkflowParser } from '../parser/workflow-parser.ts';
 import { StepStatus, type StepStatusType, WorkflowStatus } from '../types/status.ts';
+import { ConfigLoader } from '../utils/config-loader.ts';
 import { extractJson } from '../utils/json-parser.ts';
 import { Redactor } from '../utils/redactor.ts';
+import { formatSchemaErrors, validateJsonSchema } from '../utils/schema-validator.ts';
 import { WorkflowRegistry } from '../utils/workflow-registry.ts';
 import { ForeachExecutor } from './foreach-executor.ts';
 import { type LLMMessage, getAdapter } from './llm-adapter.ts';
 import { MCPManager } from './mcp-manager.ts';
+import { ResourcePoolManager } from './resource-pool.ts';
 import { withRetry } from './retry.ts';
-import { type StepResult, WorkflowSuspendedError, executeStep } from './step-executor.ts';
+import {
+  type StepResult,
+  WorkflowSuspendedError,
+  WorkflowWaitingError,
+  executeStep,
+} from './step-executor.ts';
 import { withTimeout } from './timeout.ts';
 
 import { ConsoleLogger, type Logger } from '../utils/logger.ts';
@@ -51,8 +61,22 @@ class RedactingLogger implements Logger {
   }
 }
 
+class StepExecutionError extends Error {
+  constructor(public readonly result: StepResult) {
+    super(result.error || 'Step failed');
+    this.name = 'StepExecutionError';
+  }
+}
+
+function getWakeAt(output: unknown): string | undefined {
+  if (!output || typeof output !== 'object') return undefined;
+  const wakeAt = (output as { wakeAt?: unknown }).wakeAt;
+  return typeof wakeAt === 'string' ? wakeAt : undefined;
+}
+
 export interface RunOptions {
   inputs?: Record<string, unknown>;
+  secrets?: Record<string, string>;
   dbPath?: string;
   memoryDbPath?: string;
   resumeRunId?: string;
@@ -63,8 +87,14 @@ export interface RunOptions {
   resumeInputs?: Record<string, unknown>;
   dryRun?: boolean;
   debug?: boolean;
+  dedup?: boolean;
   getAdapter?: typeof getAdapter;
+  executeStep?: typeof executeStep;
   depth?: number;
+  allowSuccessResume?: boolean;
+  resourcePoolManager?: ResourcePoolManager;
+  allowInsecure?: boolean;
+  artifactRoot?: string;
 }
 
 export interface StepContext {
@@ -94,14 +124,17 @@ export class WorkflowRunner {
   private workflow: Workflow;
   private db: WorkflowDb;
   private memoryDb: MemoryDb;
-  private runId: string;
+  private _runId!: string;
   private stepContexts: Map<string, StepContext | ForeachStepContext> = new Map();
-  private inputs: Record<string, unknown>;
+  private inputs!: Record<string, unknown>;
   private secrets: Record<string, string>;
   private redactor: Redactor;
+  private rawLogger!: Logger;
+  private secretValues: string[] = [];
+  private redactAtRest = true;
   private resumeRunId?: string;
   private restored = false;
-  private logger: Logger;
+  private logger!: Logger;
   private mcpManager: MCPManager;
   private options: RunOptions;
   private signalHandler?: (signal: string) => void;
@@ -109,7 +142,25 @@ export class WorkflowRunner {
   private hasWarnedMemory = false;
   private static readonly MEMORY_WARNING_THRESHOLD = 1000;
   private static readonly MAX_RECURSION_DEPTH = 10;
+  private static readonly REDACTED_PLACEHOLDER = '***REDACTED***';
   private depth = 0;
+  private lastFailedStep?: { id: string; error: string };
+  private abortController = new AbortController();
+  private resourcePool!: ResourcePoolManager;
+
+  /**
+   * Get the abort signal for cancellation checks
+   */
+  public get abortSignal(): AbortSignal {
+    return this.abortController.signal;
+  }
+
+  /**
+   * Check if the workflow has been canceled
+   */
+  private get isCanceled(): boolean {
+    return this.abortController.signal.aborted;
+  }
 
   constructor(workflow: Workflow, options: RunOptions = {}) {
     this.workflow = workflow;
@@ -125,29 +176,69 @@ export class WorkflowRunner {
     this.db = new WorkflowDb(options.dbPath);
     this.memoryDb = new MemoryDb(options.memoryDbPath);
     this.secrets = this.loadSecrets();
-    this.redactor = new Redactor(this.secrets);
-    // Wrap the logger with a redactor to prevent secret leakage in logs
+    this.redactor = new Redactor(this.secrets, { forcedSecrets: this.secretValues });
+
+    this.initLogger(options);
+    this.mcpManager = options.mcpManager || new MCPManager();
+    this.initResourcePool(options);
+    this.initRun(options);
+
+    this.setupSignalHandlers();
+  }
+
+  private initLogger(options: RunOptions): void {
     const rawLogger = options.logger || new ConsoleLogger();
+    this.rawLogger = rawLogger;
     this.logger = new RedactingLogger(rawLogger, this.redactor);
-    this.mcpManager = options.mcpManager || new MCPManager();
+  }
+
+  private initResourcePool(options: RunOptions): void {
+    if (options.resourcePoolManager) {
+      this.resourcePool = options.resourcePoolManager;
+    } else {
+      const config = ConfigLoader.load();
+      const globalPools = config.concurrency?.pools || {};
+      const workflowPools: Record<string, number> = {};
+
+      if (this.workflow.pools) {
+        const baseContext = this.buildContext();
+        for (const [name, limit] of Object.entries(this.workflow.pools)) {
+          if (typeof limit === 'string') {
+            workflowPools[name] = Number(ExpressionEvaluator.evaluate(limit, baseContext));
+          } else {
+            workflowPools[name] = limit;
+          }
+        }
+      }
+
+      this.resourcePool = new ResourcePoolManager(this.logger, {
+        defaultLimit: config.concurrency?.default || 10,
+        pools: { ...globalPools, ...workflowPools },
+      });
+    }
+  }
 
+  private initRun(options: RunOptions): void {
     if (options.resumeRunId) {
-      // Resume existing run
-      this.runId = options.resumeRunId;
+      this._runId = options.resumeRunId;
       this.resumeRunId = options.resumeRunId;
-      this.inputs = options.resumeInputs || {}; // Start with resume inputs, will be merged with DB inputs in restoreState
+      this.inputs = options.resumeInputs || {};
     } else {
-      // Start new run
       this.inputs = options.inputs || {};
-      this.runId = randomUUID();
+      this._runId = randomUUID();
     }
-
-    this.setupSignalHandlers();
   }
 
   /**
    * Get the current run ID
    */
+  public get runId(): string {
+    return this._runId;
+  }
+
+  /**
+   * Get the current run ID (method for mocking compatibility)
+   */
   public getRunId(): string {
     return this.runId;
   }
@@ -161,14 +252,17 @@ export class WorkflowRunner {
       throw new Error(`Run ${this.runId} not found`);
     }
 
-    // Only allow resuming failed, paused, or running (crash recovery) runs
+    // Only allow resuming failed, paused, canceled, or running (crash recovery) runs
+    // Unless specifically allowed (e.g. for rollback/compensation)
     if (
       run.status !== WorkflowStatus.FAILED &&
       run.status !== WorkflowStatus.PAUSED &&
-      run.status !== WorkflowStatus.RUNNING
+      run.status !== WorkflowStatus.RUNNING &&
+      run.status !== WorkflowStatus.CANCELED &&
+      !(this.options.allowSuccessResume && run.status === WorkflowStatus.SUCCESS)
     ) {
       throw new Error(
-        `Cannot resume run with status '${run.status}'. Only 'failed', 'paused', or 'running' runs can be resumed.`
+        `Cannot resume run with status '${run.status}'. Only 'failed', 'paused', 'canceled', or 'running' runs can be resumed.`
       );
     }
 
@@ -178,6 +272,10 @@ export class WorkflowRunner {
       );
     }
 
+    if (run.status === WorkflowStatus.CANCELED) {
+      this.logger.log('📋 Resuming a previously canceled run. Completed steps will be skipped.');
+    }
+
     // Restore inputs from the previous run to ensure consistency
     // Merge with any resumeInputs provided (e.g. answers to human steps)
     try {
@@ -252,6 +350,7 @@ export class WorkflowRunner {
                   ? (output as Record<string, unknown>)
                   : {},
               status: exec.status as typeof StepStatus.SUCCESS | typeof StepStatus.SKIPPED,
+              error: exec.error || undefined,
             };
             outputs[exec.iteration_index] = output;
           } else {
@@ -261,6 +360,7 @@ export class WorkflowRunner {
               output: null,
               outputs: {},
               status: exec.status as StepStatusType,
+              error: exec.error || undefined,
             };
           }
         }
@@ -275,8 +375,8 @@ export class WorkflowRunner {
             if (parsed.__foreachItems && Array.isArray(parsed.__foreachItems)) {
               expectedCount = parsed.__foreachItems.length;
             }
-          } catch {
-            // Parse error, fall through to expression evaluation
+          } catch (_e) {
+            // ignore parse errors
           }
         }
 
@@ -332,7 +432,8 @@ export class WorkflowRunner {
         if (
           exec.status === StepStatus.SUCCESS ||
           exec.status === StepStatus.SKIPPED ||
-          exec.status === StepStatus.SUSPENDED
+          exec.status === StepStatus.SUSPENDED ||
+          exec.status === StepStatus.WAITING
         ) {
           let output: unknown = null;
           try {
@@ -341,15 +442,35 @@ export class WorkflowRunner {
             this.logger.warn(`Failed to parse output for step ${stepId}: ${error}`);
             output = { error: 'Failed to parse output' };
           }
+
+          // If step is WAITING, check if timer has elapsed
+          let effectiveStatus = exec.status as StepContext['status'];
+          if (exec.status === StepStatus.WAITING) {
+            const timer = await this.db.getTimerByStep(this.runId, stepId);
+            const timerId = timer?.id;
+            const wakeAt = timer?.wake_at;
+            if (timerId && wakeAt && new Date(wakeAt) <= new Date()) {
+              // Timer elapsed!
+              await this.db.completeTimer(timerId);
+              await this.db.completeStep(exec.id, StepStatus.SUCCESS, output);
+              effectiveStatus = StepStatus.SUCCESS;
+            }
+          }
+          let effectiveError = exec.error || undefined;
+          if (exec.status === StepStatus.WAITING && effectiveStatus === StepStatus.SUCCESS) {
+            effectiveError = undefined;
+          }
+
           this.stepContexts.set(stepId, {
             output,
             outputs:
               typeof output === 'object' && output !== null && !Array.isArray(output)
                 ? (output as Record<string, unknown>)
                 : {},
-            status: exec.status as StepContext['status'],
+            status: effectiveStatus,
+            error: effectiveError,
           });
-          if (exec.status !== StepStatus.SUSPENDED) {
+          if (effectiveStatus !== StepStatus.SUSPENDED && effectiveStatus !== StepStatus.WAITING) {
             completedStepIds.add(stepId);
           }
         }
@@ -366,8 +487,10 @@ export class WorkflowRunner {
   private setupSignalHandlers(): void {
     const handler = async (signal: string) => {
       if (this.isStopping) return;
-      this.logger.log(`\n\n🛑 Received ${signal}. Cleaning up...`);
-      await this.stop(WorkflowStatus.FAILED, `Cancelled by user (${signal})`);
+      this.logger.log(`\n\n🛑 Received ${signal}. Canceling workflow...`);
+      // Signal cancellation to all running steps
+      this.abortController.abort();
+      await this.stop(WorkflowStatus.CANCELED, `Canceled by user (${signal})`);
 
       // Only exit if not embedded
       if (!this.options.preventExit) {
@@ -381,6 +504,90 @@ export class WorkflowRunner {
     process.on('SIGTERM', handler);
   }
 
+  /**
+   * Process compensations (rollback)
+   */
+  private async processCompensations(errorReason: string): Promise<void> {
+    this.logger.log(`\n↩️  Initiating rollback due to: ${errorReason}`);
+
+    try {
+      // Get all pending compensations
+      const compensations = await this.db.getPendingCompensations(this.runId);
+
+      if (compensations.length === 0) {
+        this.logger.log('  No pending compensations found.');
+        return;
+      }
+
+      this.logger.log(`  Found ${compensations.length} compensation(s) to execute.`);
+
+      // Execute in reverse order (LIFO) - already sorted by query
+      for (const compRecord of compensations) {
+        const stepDef = JSON.parse(compRecord.definition) as Step;
+        this.logger.log(`  Running compensation: ${stepDef.id} (undoing ${compRecord.step_id})`);
+
+        await this.db.updateCompensationStatus(compRecord.id, 'running');
+
+        // Build context for compensation
+        // It has access to the original step's output via steps.<step_id>.output
+        const context = this.buildContext();
+
+        try {
+          // Execute the compensation step
+          const result = await executeStep(stepDef, context, this.logger, {
+            executeWorkflowFn: this.executeSubWorkflow.bind(this),
+            mcpManager: this.mcpManager,
+            memoryDb: this.memoryDb,
+            workflowDir: this.options.workflowDir,
+            dryRun: this.options.dryRun,
+            runId: this.runId,
+            artifactRoot: this.options.artifactRoot,
+            redactForStorage: this.redactForStorage.bind(this),
+          });
+
+          if (result.status === 'success') {
+            this.logger.log(`  ✓ Compensation ${stepDef.id} succeeded`);
+            await this.db.updateCompensationStatus(compRecord.id, 'success', result.output);
+          } else {
+            this.logger.error(`  ✗ Compensation ${stepDef.id} failed: ${result.error}`);
+            await this.db.updateCompensationStatus(
+              compRecord.id,
+              'failed',
+              result.output,
+              result.error
+            );
+          }
+        } catch (err) {
+          const errMsg = err instanceof Error ? err.message : String(err);
+          this.logger.error(`  ✗ Compensation ${stepDef.id} crashed: ${errMsg}`);
+          await this.db.updateCompensationStatus(compRecord.id, 'failed', null, errMsg);
+        }
+
+        // 2. Recursive rollback for sub-workflows
+        // Try to find if this step was a workflow step with a subRunId
+        const stepExec = await this.db.getMainStep(this.runId, compRecord.step_id);
+        const stepOutput = stepExec?.output;
+        if (stepOutput) {
+          try {
+            const output = JSON.parse(stepOutput);
+            const subRunId = output?.__subRunId;
+            if (subRunId) {
+              await this.cascadeRollback(subRunId, errorReason);
+            }
+          } catch (_e) {
+            // ignore parse errors
+          }
+        }
+      }
+
+      this.logger.log('  Rollback completed.\n');
+    } catch (error) {
+      this.logger.error(
+        `  ⚠️ Error during rollback processing: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+
   /**
    * Stop the runner and cleanup resources
    */
@@ -391,8 +598,18 @@ export class WorkflowRunner {
     try {
       this.removeSignalHandlers();
 
+      // Trigger rollback if failing or canceled
+      if (status === WorkflowStatus.FAILED || status === WorkflowStatus.CANCELED) {
+        await this.processCompensations(error || status);
+      }
+
       // Update run status in DB
-      await this.db.updateRunStatus(this.runId, status, undefined, error);
+      await this.db.updateRunStatus(
+        this.runId,
+        status,
+        undefined,
+        error ? this.redactForStorage(error) : undefined
+      );
 
       // Stop all MCP clients
       await this.mcpManager.stopAll();
@@ -418,7 +635,7 @@ export class WorkflowRunner {
    * Load secrets from environment
    */
   private loadSecrets(): Record<string, string> {
-    const secrets: Record<string, string> = {};
+    const secrets: Record<string, string> = { ...(this.options.secrets || {}) };
 
     // Common non-secret environment variables to exclude from redaction
     const blocklist = new Set([
@@ -469,18 +686,251 @@ export class WorkflowRunner {
     return secrets;
   }
 
+  private refreshRedactor(): void {
+    this.redactor = new Redactor(this.loadSecrets(), { forcedSecrets: this.secretValues });
+    this.logger = new RedactingLogger(this.rawLogger, this.redactor);
+  }
+
+  private redactForStorage<T>(value: T): T {
+    if (!this.redactAtRest) return value;
+    return this.redactor.redactValue(value) as T;
+  }
+
+  private validateSchema(
+    kind: 'input' | 'output',
+    schema: unknown,
+    data: unknown,
+    stepId: string
+  ): void {
+    try {
+      const result = validateJsonSchema(schema, data);
+      if (result.valid) return;
+      const details = result.errors.map((line: string) => `  - ${line}`).join('\n');
+      throw new Error(
+        `${kind === 'input' ? 'Input' : 'Output'} schema validation failed for step "${stepId}":\n${details}`
+      );
+    } catch (error) {
+      if (error instanceof Error) {
+        if (error.message.includes('schema validation failed for step')) {
+          throw error;
+        }
+        throw new Error(
+          `${kind === 'input' ? 'Input' : 'Output'} schema error for step "${stepId}": ${error.message}`
+        );
+      }
+      throw error;
+    }
+  }
+
+  private buildStepInputs(step: Step, context: ExpressionContext): Record<string, unknown> {
+    const stripUndefined = (value: Record<string, unknown>) => {
+      const result: Record<string, unknown> = {};
+      for (const [key, val] of Object.entries(value)) {
+        if (val !== undefined) {
+          result[key] = val;
+        }
+      }
+      return result;
+    };
+
+    switch (step.type) {
+      case 'shell': {
+        let env: Record<string, string> | undefined;
+        if (step.env) {
+          env = {};
+          for (const [key, value] of Object.entries(step.env)) {
+            env[key] = ExpressionEvaluator.evaluateString(value as string, context);
+          }
+        }
+        return stripUndefined({
+          run: ExpressionEvaluator.evaluateString(
+            (step as import('../parser/schema.ts').ShellStep).run,
+            context
+          ),
+          dir: step.dir ? ExpressionEvaluator.evaluateString(step.dir, context) : undefined,
+          env,
+          allowInsecure: step.allowInsecure,
+        });
+      }
+      case 'file':
+        return stripUndefined({
+          path: ExpressionEvaluator.evaluateString(
+            (step as import('../parser/schema.ts').FileStep).path,
+            context
+          ),
+          content:
+            (step as import('../parser/schema.ts').FileStep).content !== undefined
+              ? ExpressionEvaluator.evaluateString(
+                  (step as import('../parser/schema.ts').FileStep).content as string,
+                  context
+                )
+              : undefined,
+          op: step.op,
+          allowOutsideCwd: step.allowOutsideCwd,
+        });
+      case 'request': {
+        let headers: Record<string, string> | undefined;
+        if (step.headers) {
+          headers = {};
+          for (const [key, value] of Object.entries(step.headers)) {
+            headers[key] = ExpressionEvaluator.evaluateString(value as string, context);
+          }
+        }
+        return stripUndefined({
+          url: ExpressionEvaluator.evaluateString(
+            (step as import('../parser/schema.ts').RequestStep).url,
+            context
+          ),
+          method: step.method,
+          headers,
+          body:
+            step.body !== undefined
+              ? ExpressionEvaluator.evaluateObject(step.body, context)
+              : undefined,
+          allowInsecure: step.allowInsecure,
+        });
+      }
+      case 'human':
+        return stripUndefined({
+          message: ExpressionEvaluator.evaluateString(
+            (step as import('../parser/schema.ts').HumanStep).message,
+            context
+          ),
+          inputType: step.inputType,
+        });
+      case 'sleep': {
+        const evaluated = ExpressionEvaluator.evaluate(step.duration.toString(), context);
+        return { duration: Number(evaluated) };
+      }
+      case 'llm':
+        return stripUndefined({
+          agent: step.agent,
+          provider: step.provider,
+          model: step.model,
+          prompt: ExpressionEvaluator.evaluateString(step.prompt, context),
+          tools: step.tools,
+          maxIterations: step.maxIterations,
+          useGlobalMcp: step.useGlobalMcp,
+          allowClarification: step.allowClarification,
+          mcpServers: step.mcpServers,
+          useStandardTools: step.useStandardTools,
+          allowOutsideCwd: step.allowOutsideCwd,
+          allowInsecure: step.allowInsecure,
+        });
+      case 'workflow':
+        return stripUndefined({
+          path: (step as import('../parser/schema.ts').WorkflowStep).path,
+          inputs: step.inputs
+            ? ExpressionEvaluator.evaluateObject(step.inputs, context)
+            : undefined,
+        });
+      case 'script':
+        return stripUndefined({
+          run: step.run,
+          allowInsecure: step.allowInsecure,
+        });
+      case 'engine': {
+        const env: Record<string, string> = {};
+        for (const [key, value] of Object.entries(step.env || {})) {
+          env[key] = ExpressionEvaluator.evaluateString(value as string, context);
+        }
+        return stripUndefined({
+          command: ExpressionEvaluator.evaluateString(
+            (step as import('../parser/schema.ts').EngineStep).command,
+            context
+          ),
+          args: (step as import('../parser/schema.ts').EngineStep).args?.map((arg) =>
+            ExpressionEvaluator.evaluateString(arg, context)
+          ),
+          input:
+            (step as import('../parser/schema.ts').EngineStep).input !== undefined
+              ? ExpressionEvaluator.evaluateObject(
+                  (step as import('../parser/schema.ts').EngineStep).input,
+                  context
+                )
+              : undefined,
+          env,
+          cwd: ExpressionEvaluator.evaluateString(
+            (step as import('../parser/schema.ts').EngineStep).cwd,
+            context
+          ),
+        });
+      }
+      case 'memory':
+        return stripUndefined({
+          op: step.op,
+          query: step.query ? ExpressionEvaluator.evaluateString(step.query, context) : undefined,
+          text: step.text ? ExpressionEvaluator.evaluateString(step.text, context) : undefined,
+          model: step.model,
+          metadata: step.metadata
+            ? ExpressionEvaluator.evaluateObject(step.metadata, context)
+            : undefined,
+          limit: step.limit,
+        });
+      default:
+        return {};
+    }
+  }
+
+  /**
+   * Collect primitive secret values from structured inputs.
+   */
+  private static collectSecretValues(
+    value: unknown,
+    sink: Set<string>,
+    seen: WeakSet<object>
+  ): void {
+    if (value === null || value === undefined) return;
+
+    if (typeof value === 'string') {
+      sink.add(value);
+      return;
+    }
+
+    if (typeof value === 'number' || typeof value === 'boolean' || typeof value === 'bigint') {
+      sink.add(String(value));
+      return;
+    }
+
+    if (typeof value !== 'object') return;
+
+    if (seen.has(value)) return;
+    seen.add(value);
+
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        WorkflowRunner.collectSecretValues(item, sink, seen);
+      }
+      return;
+    }
+
+    for (const item of Object.values(value as Record<string, unknown>)) {
+      WorkflowRunner.collectSecretValues(item, sink, seen);
+    }
+  }
+
   /**
    * Apply workflow defaults to inputs and validate types
    */
   private applyDefaultsAndValidate(): void {
     if (!this.workflow.inputs) return;
 
+    const secretValues = new Set<string>();
+
     for (const [key, config] of Object.entries(this.workflow.inputs)) {
       // Apply default if missing
       if (this.inputs[key] === undefined && config.default !== undefined) {
         this.inputs[key] = config.default;
       }
 
+      if (config.secret) {
+        if (this.inputs[key] === WorkflowRunner.REDACTED_PLACEHOLDER) {
+          throw new Error(
+            `Secret input "${key}" was redacted at rest. Please provide it again to resume this run.`
+          );
+        }
+      }
+
       // Validate required inputs
       if (this.inputs[key] === undefined) {
         throw new Error(`Missing required input: ${key}`);
@@ -502,7 +952,42 @@ export class WorkflowRunner {
       if (type === 'array' && !Array.isArray(value)) {
         throw new Error(`Input "${key}" must be an array, got ${typeof value}`);
       }
+      if (
+        type === 'object' &&
+        (typeof value !== 'object' || value === null || Array.isArray(value))
+      ) {
+        throw new Error(`Input "${key}" must be an object, got ${typeof value}`);
+      }
+
+      if (config.values) {
+        if (type !== 'string' && type !== 'number' && type !== 'boolean') {
+          throw new Error(`Input "${key}" cannot use enum values with type "${type}"`);
+        }
+        for (const allowed of config.values) {
+          const matchesType =
+            (type === 'string' && typeof allowed === 'string') ||
+            (type === 'number' && typeof allowed === 'number') ||
+            (type === 'boolean' && typeof allowed === 'boolean');
+          if (!matchesType) {
+            throw new Error(
+              `Input "${key}" enum value ${JSON.stringify(allowed)} must be a ${type}`
+            );
+          }
+        }
+        if (!config.values.includes(value as string | number | boolean)) {
+          throw new Error(
+            `Input "${key}" must be one of: ${config.values.map((v) => JSON.stringify(v)).join(', ')}`
+          );
+        }
+      }
+
+      if (config.secret && value !== undefined && value !== WorkflowRunner.REDACTED_PLACEHOLDER) {
+        WorkflowRunner.collectSecretValues(value, secretValues, new WeakSet());
+      }
     }
+
+    this.secretValues = Array.from(secretValues);
+    this.refreshRedactor();
   }
 
   /**
@@ -515,6 +1000,7 @@ export class WorkflowRunner {
         output?: unknown;
         outputs?: Record<string, unknown>;
         status?: string;
+        error?: string;
         items?: StepContext[];
       }
     > = {};
@@ -526,6 +1012,7 @@ export class WorkflowRunner {
           output: ctx.output,
           outputs: ctx.outputs,
           status: ctx.status,
+          error: ctx.error,
           items: ctx.items,
         };
       } else {
@@ -533,21 +1020,49 @@ export class WorkflowRunner {
           output: ctx.output,
           outputs: ctx.outputs,
           status: ctx.status,
+          error: ctx.error,
         };
       }
     }
 
-    return {
+    const baseContext: ExpressionContext = {
       inputs: this.inputs,
-      secrets: this.secrets,
+      secrets: this.loadSecrets(), // Access secrets from options
+      secretValues: this.secretValues,
       steps: stepsContext,
       item,
       index,
-      env: this.workflow.env,
+      env: {},
       output: item
         ? undefined
         : this.stepContexts.get(this.workflow.steps.find((s) => !s.foreach)?.id || '')?.output,
+      last_failed_step: this.lastFailedStep,
     };
+
+    const resolvedEnv: Record<string, string> = {};
+    for (const [key, value] of Object.entries(process.env)) {
+      if (value !== undefined) {
+        resolvedEnv[key] = value;
+      }
+    }
+
+    if (this.workflow.env) {
+      for (const [key, value] of Object.entries(this.workflow.env)) {
+        try {
+          resolvedEnv[key] = ExpressionEvaluator.evaluateString(value, {
+            ...baseContext,
+            env: resolvedEnv,
+          });
+        } catch (error) {
+          this.logger.warn(
+            `Warning: Failed to evaluate workflow env "${key}": ${error instanceof Error ? error.message : String(error)}`
+          );
+        }
+      }
+    }
+
+    baseContext.env = resolvedEnv;
+    return baseContext;
   }
 
   /**
@@ -607,6 +1122,104 @@ export class WorkflowRunner {
     }
   }
 
+  private async claimIdempotencyRecord(
+    scopedKey: string,
+    stepId: string,
+    ttlSeconds?: number
+  ): Promise<
+    | { status: 'hit'; output: unknown; error?: string }
+    | { status: 'claimed' }
+    | { status: 'in-flight' }
+  > {
+    try {
+      await this.db.clearExpiredIdempotencyRecord(scopedKey);
+
+      const existing = await this.db.getIdempotencyRecord(scopedKey);
+      if (existing) {
+        if (existing.status === StepStatus.SUCCESS) {
+          let output: unknown = null;
+          try {
+            output = existing.output ? JSON.parse(existing.output) : null;
+          } catch (parseError) {
+            this.logger.warn(
+              `  ⚠️ Failed to parse idempotency output for ${stepId}: ${parseError instanceof Error ? parseError.message : String(parseError)}`
+            );
+          }
+          return { status: 'hit', output, error: existing.error || undefined };
+        }
+        if (existing.status === StepStatus.RUNNING) {
+          return { status: 'in-flight' };
+        }
+
+        const claimed = await this.db.markIdempotencyRecordRunning(
+          scopedKey,
+          this.runId,
+          stepId,
+          ttlSeconds
+        );
+        if (claimed) {
+          return { status: 'claimed' };
+        }
+      }
+
+      const inserted = await this.db.insertIdempotencyRecordIfAbsent(
+        scopedKey,
+        this.runId,
+        stepId,
+        StepStatus.RUNNING,
+        ttlSeconds
+      );
+      if (inserted) {
+        return { status: 'claimed' };
+      }
+
+      const current = await this.db.getIdempotencyRecord(scopedKey);
+      if (current?.status === StepStatus.SUCCESS) {
+        let output: unknown = null;
+        try {
+          output = current.output ? JSON.parse(current.output) : null;
+        } catch (parseError) {
+          this.logger.warn(
+            `  ⚠️ Failed to parse idempotency output for ${stepId}: ${parseError instanceof Error ? parseError.message : String(parseError)}`
+          );
+        }
+        return { status: 'hit', output, error: current.error || undefined };
+      }
+      return { status: 'in-flight' };
+    } catch (error) {
+      this.logger.warn(
+        `  ⚠️ Failed to claim idempotency key for ${stepId}: ${error instanceof Error ? error.message : String(error)}`
+      );
+      return { status: 'claimed' };
+    }
+  }
+
+  private async recordIdempotencyResult(
+    scopedKey: string | undefined,
+    stepId: string,
+    status: StepStatusType,
+    output: unknown,
+    error?: string,
+    ttlSeconds?: number
+  ): Promise<void> {
+    if (!scopedKey) return;
+    try {
+      await this.db.storeIdempotencyRecord(
+        scopedKey,
+        this.runId,
+        stepId,
+        status,
+        output,
+        error,
+        ttlSeconds
+      );
+    } catch (err) {
+      this.logger.warn(
+        `  ⚠️ Failed to store idempotency record: ${err instanceof Error ? err.message : String(err)}`
+      );
+    }
+  }
+
   /**
    * Execute a single step instance and return the result
    * Does NOT update global stepContexts
@@ -614,8 +1227,81 @@ export class WorkflowRunner {
   private async executeStepInternal(
     step: Step,
     context: ExpressionContext,
-    stepExecId: string
+    stepExecId: string,
+    idempotencyContext?: {
+      rawKey: string;
+      scopedKey: string;
+      ttlSeconds?: number;
+      claimed: boolean;
+    }
   ): Promise<StepContext> {
+    // Check idempotency key for dedup (scoped per run by default)
+    const dedupEnabled = this.options.dedup !== false;
+    let idempotencyKey: string | undefined = idempotencyContext?.rawKey;
+    let scopedIdempotencyKey: string | undefined = idempotencyContext?.scopedKey;
+    let idempotencyTtlSeconds: number | undefined = idempotencyContext?.ttlSeconds;
+    let idempotencyClaimed = idempotencyContext?.claimed ?? false;
+    if (dedupEnabled && !idempotencyClaimed && step.idempotencyKey) {
+      try {
+        idempotencyKey = ExpressionEvaluator.evaluateString(step.idempotencyKey, context);
+      } catch (error) {
+        this.logger.warn(
+          `  ⚠️ Failed to evaluate idempotencyKey for ${step.id}: ${error instanceof Error ? error.message : String(error)}`
+        );
+      }
+      if (idempotencyKey) {
+        const scope = step.idempotencyScope || 'run';
+        scopedIdempotencyKey = scope === 'run' ? `${this.runId}:${idempotencyKey}` : idempotencyKey;
+        idempotencyTtlSeconds = step.idempotencyTtlSeconds;
+
+        const claim = await this.claimIdempotencyRecord(
+          scopedIdempotencyKey,
+          step.id,
+          idempotencyTtlSeconds
+        );
+        if (claim.status === 'hit') {
+          this.logger.log(`  ⟳ Step ${step.id} skipped (idempotency hit: ${idempotencyKey})`);
+          const output = claim.output;
+          await this.db.completeStep(stepExecId, 'success', output, claim.error || undefined);
+          return {
+            output,
+            outputs:
+              typeof output === 'object' && output !== null && !Array.isArray(output)
+                ? (output as Record<string, unknown>)
+                : {},
+            status: 'success',
+            error: claim.error || undefined,
+          };
+        }
+        if (claim.status === 'in-flight') {
+          const errorMsg = `Idempotency key already in-flight: ${idempotencyKey}`;
+          await this.db.completeStep(
+            stepExecId,
+            StepStatus.FAILED,
+            null,
+            this.redactAtRest ? this.redactor.redact(errorMsg) : errorMsg
+          );
+          return {
+            output: null,
+            outputs: {},
+            status: StepStatus.FAILED,
+            error: errorMsg,
+          };
+        }
+        idempotencyClaimed = true;
+      }
+    }
+
+    const idempotencyContextForRetry =
+      idempotencyClaimed && scopedIdempotencyKey
+        ? {
+            rawKey: idempotencyKey || scopedIdempotencyKey,
+            scopedKey: scopedIdempotencyKey,
+            ttlSeconds: idempotencyTtlSeconds,
+            claimed: true,
+          }
+        : undefined;
+
     let stepToExecute = step;
 
     // Inject few-shot examples if enabled
@@ -639,26 +1325,149 @@ export class WorkflowRunner {
       await this.db.startStep(stepExecId);
     }
 
-    const operation = async () => {
-      const result = await executeStep(stepToExecute, context, this.logger, {
+    const operation = async (attemptContext: ExpressionContext) => {
+      const exec = this.options.executeStep || executeStep;
+      const result = await exec(stepToExecute, attemptContext, this.logger, {
         executeWorkflowFn: this.executeSubWorkflow.bind(this),
         mcpManager: this.mcpManager,
         memoryDb: this.memoryDb,
         workflowDir: this.options.workflowDir,
         dryRun: this.options.dryRun,
+        abortSignal: this.abortSignal,
+        runId: this.runId,
+        stepExecutionId: stepExecId,
+        artifactRoot: this.options.artifactRoot,
+        redactForStorage: this.redactForStorage.bind(this),
+        getAdapter: this.options.getAdapter,
+        executeStep: this.options.executeStep || executeStep,
       });
       if (result.status === 'failed') {
-        throw new Error(result.error || 'Step failed');
+        throw new StepExecutionError(result);
+      }
+      if (result.status === 'success' && stepToExecute.outputSchema) {
+        try {
+          const outputForValidation =
+            stepToExecute.type === 'engine' &&
+            result.output &&
+            typeof result.output === 'object' &&
+            'summary' in result.output
+              ? (result.output as { summary?: unknown }).summary
+              : result.output;
+          this.validateSchema(
+            'output',
+            stepToExecute.outputSchema,
+            outputForValidation,
+            stepToExecute.id
+          );
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error);
+          const outputRetries = stepToExecute.outputRetries || 0;
+          const currentAttempt = (attemptContext.outputRepairAttempts as number) || 0;
+
+          // Only attempt repair for LLM steps with outputRetries configured
+          if (stepToExecute.type === 'llm' && outputRetries > 0 && currentAttempt < outputRetries) {
+            const strategy = stepToExecute.repairStrategy || 'reask';
+            this.logger.log(
+              `  🔄 Output validation failed, attempting ${strategy} repair (${currentAttempt + 1}/${outputRetries})`
+            );
+
+            // Build repair context with validation errors
+            const repairPrompt = this.buildOutputRepairPrompt(
+              stepToExecute,
+              result.output,
+              message,
+              strategy
+            );
+
+            // Create a modified step with repair context
+            const repairStep = {
+              ...stepToExecute,
+              prompt: repairPrompt,
+            };
+
+            // Recursively execute with incremented repair attempt count
+            const repairContext = {
+              ...attemptContext,
+              outputRepairAttempts: currentAttempt + 1,
+            };
+
+            // Execute the repair step
+            const exec = this.options.executeStep || executeStep;
+            const repairResult = await exec(repairStep, repairContext, this.logger, {
+              executeWorkflowFn: this.executeSubWorkflow.bind(this),
+              mcpManager: this.mcpManager,
+              memoryDb: this.memoryDb,
+              workflowDir: this.options.workflowDir,
+              dryRun: this.options.dryRun,
+              abortSignal: this.abortSignal,
+              runId: this.runId,
+              stepExecutionId: stepExecId,
+              artifactRoot: this.options.artifactRoot,
+              redactForStorage: this.redactForStorage.bind(this),
+              executeStep: this.options.executeStep || executeStep,
+            });
+
+            if (repairResult.status === 'failed') {
+              throw new StepExecutionError(repairResult);
+            }
+
+            // Validate the repaired output
+            try {
+              this.validateSchema(
+                'output',
+                stepToExecute.outputSchema,
+                repairResult.output,
+                stepToExecute.id
+              );
+              this.logger.log(
+                `  ✓ Output repair successful after ${currentAttempt + 1} attempt(s)`
+              );
+              return repairResult;
+            } catch (repairError) {
+              // If still failing, either retry again or give up
+              if (currentAttempt + 1 < outputRetries) {
+                // Try again with updated context
+                return operation({
+                  ...attemptContext,
+                  outputRepairAttempts: currentAttempt + 1,
+                });
+              }
+              const repairMessage =
+                repairError instanceof Error ? repairError.message : String(repairError);
+              throw new StepExecutionError({
+                ...repairResult,
+                status: 'failed',
+                error: `Output validation failed after ${outputRetries} repair attempts: ${repairMessage}`,
+              });
+            }
+          }
+
+          throw new StepExecutionError({
+            ...result,
+            status: 'failed',
+            error: message,
+          });
+        }
       }
       return result;
     };
 
     try {
+      if (stepToExecute.inputSchema) {
+        const inputsForValidation = this.buildStepInputs(stepToExecute, context);
+        this.validateSchema(
+          'input',
+          stepToExecute.inputSchema,
+          inputsForValidation,
+          stepToExecute.id
+        );
+      }
+
       const operationWithTimeout = async () => {
         if (step.timeout) {
-          return await withTimeout(operation(), step.timeout, `Step ${step.id}`);
+          return await withTimeout(operation(context), step.timeout, `Step ${step.id}`);
         }
-        return await operation();
+        return await operation(context);
       };
 
       const result = await withRetry(operationWithTimeout, step.retry, async (attempt, error) => {
@@ -666,24 +1475,112 @@ export class WorkflowRunner {
         await this.db.incrementRetry(stepExecId);
       });
 
+      const persistedOutput = this.redactForStorage(result.output);
+      const persistedError = result.error
+        ? this.redactAtRest
+          ? this.redactor.redact(result.error)
+          : result.error
+        : result.error;
+
       if (result.status === StepStatus.SUSPENDED) {
+        if (step.type === 'human') {
+          const existingTimer = await this.db.getTimerByStep(this.runId, step.id);
+          if (!existingTimer) {
+            const timerId = randomUUID();
+            await this.db.createTimer(timerId, this.runId, step.id, 'human');
+          }
+        }
+        if (dedupEnabled && idempotencyClaimed) {
+          await this.recordIdempotencyResult(
+            scopedIdempotencyKey,
+            step.id,
+            StepStatus.SUSPENDED,
+            result.output,
+            result.error,
+            idempotencyTtlSeconds
+          );
+        }
         await this.db.completeStep(
           stepExecId,
           StepStatus.SUSPENDED,
-          result.output,
-          'Waiting for interaction',
+          persistedOutput,
+          this.redactAtRest
+            ? this.redactor.redact('Waiting for interaction')
+            : 'Waiting for interaction',
           result.usage
         );
         return result;
       }
 
+      if (result.status === StepStatus.WAITING) {
+        const wakeAt = getWakeAt(result.output);
+        const waitError = `Waiting until ${wakeAt}`;
+        // Avoid creating duplicate timers for the same step execution
+        const existingTimer = await this.db.getTimerByStep(this.runId, step.id);
+        if (!existingTimer) {
+          const timerId = randomUUID();
+          await this.db.createTimer(timerId, this.runId, step.id, 'sleep', wakeAt);
+        }
+        if (dedupEnabled && idempotencyClaimed) {
+          await this.recordIdempotencyResult(
+            scopedIdempotencyKey,
+            step.id,
+            StepStatus.WAITING,
+            result.output,
+            waitError,
+            idempotencyTtlSeconds
+          );
+        }
+        await this.db.completeStep(
+          stepExecId,
+          StepStatus.WAITING,
+          persistedOutput,
+          this.redactAtRest ? this.redactor.redact(waitError) : waitError,
+          result.usage
+        );
+        result.error = waitError;
+        return result;
+      }
+
       await this.db.completeStep(
         stepExecId,
         result.status,
-        result.output,
-        result.error,
+        persistedOutput,
+        persistedError,
         result.usage
       );
+      if (step.type === 'human') {
+        const existingTimer = await this.db.getTimerByStep(this.runId, step.id);
+        if (existingTimer) {
+          await this.db.completeTimer(existingTimer.id);
+        }
+      }
+
+      // Register compensation if step succeeded and defines one
+      if (result.status === StepStatus.SUCCESS && step.compensate) {
+        try {
+          // Ensure compensation step has an ID
+          const compStep = {
+            ...step.compensate,
+            id: step.compensate.id || `${step.id}-compensate`,
+          };
+          const definition = JSON.stringify(compStep);
+          const compensationId = randomUUID();
+
+          this.logger.log(`  📎 Registering compensation for step ${step.id}`);
+          await this.db.registerCompensation(
+            compensationId,
+            this.runId,
+            step.id,
+            compStep.id,
+            definition
+          );
+        } catch (compError) {
+          this.logger.warn(
+            `  ⚠️ Failed to register compensation for step ${step.id}: ${compError instanceof Error ? compError.message : String(compError)}`
+          );
+        }
+      }
 
       // Auto-Learning logic
       if (step.learn && result.status === StepStatus.SUCCESS) {
@@ -710,10 +1607,22 @@ export class WorkflowRunner {
         outputs = {};
       }
 
+      if (dedupEnabled && idempotencyClaimed) {
+        await this.recordIdempotencyResult(
+          scopedIdempotencyKey,
+          step.id,
+          result.status,
+          result.output,
+          result.error,
+          idempotencyTtlSeconds
+        );
+      }
+
       return {
         output: result.output,
         outputs,
         status: result.status,
+        error: result.error,
         usage: result.usage,
       };
     } catch (error) {
@@ -741,7 +1650,12 @@ export class WorkflowRunner {
               reflexionAttempts: currentAttempt + 1,
             };
 
-            return this.executeStepInternal(newStep, nextContext, stepExecId);
+            return this.executeStepInternal(
+              newStep,
+              nextContext,
+              stepExecId,
+              idempotencyContextForRetry
+            );
           } catch (healError) {
             this.logger.error(
               `  ✗ Reflexion failed: ${healError instanceof Error ? healError.message : String(healError)}`
@@ -777,7 +1691,12 @@ export class WorkflowRunner {
               autoHealAttempts: currentAttempt + 1,
             };
 
-            return this.executeStepInternal(newStep, nextContext, stepExecId);
+            return this.executeStepInternal(
+              newStep,
+              nextContext,
+              stepExecId,
+              idempotencyContextForRetry
+            );
           } catch (healError) {
             this.logger.error(
               `  ✗ Auto-heal failed: ${healError instanceof Error ? healError.message : String(healError)}`
@@ -798,7 +1717,12 @@ export class WorkflowRunner {
             this.logger.log(`  ↻ Retrying step ${step.id} after manual intervention`);
             // We use the modified step if provided, else original
             const stepToRun = action.modifiedStep || step;
-            return this.executeStepInternal(stepToRun, context, stepExecId);
+            return this.executeStepInternal(
+              stepToRun,
+              context,
+              stepExecId,
+              idempotencyContextForRetry
+            );
           }
           if (action.type === 'skip') {
             this.logger.log(`  ⏭️ Skipping step ${step.id} manually`);
@@ -815,16 +1739,68 @@ export class WorkflowRunner {
         }
       }
 
-      const errorMsg = error instanceof Error ? error.message : String(error);
+      const failureResult = error instanceof StepExecutionError ? error.result : null;
+      const errorMsg =
+        failureResult?.error || (error instanceof Error ? error.message : String(error));
       const redactedErrorMsg = this.redactor.redact(errorMsg);
+      const failureOutput = failureResult?.output ?? null;
+      const failureOutputs =
+        typeof failureOutput === 'object' && failureOutput !== null && !Array.isArray(failureOutput)
+          ? (failureOutput as Record<string, unknown>)
+          : {};
+
+      if (step.allowFailure) {
+        this.logger.warn(
+          `  ⚠️ Step ${step.id} failed but allowFailure is true: ${redactedErrorMsg}`
+        );
+        await this.db.completeStep(
+          stepExecId,
+          StepStatus.SUCCESS,
+          this.redactForStorage(failureOutput),
+          this.redactAtRest ? redactedErrorMsg : errorMsg
+        );
+        if (dedupEnabled && idempotencyClaimed) {
+          await this.recordIdempotencyResult(
+            scopedIdempotencyKey,
+            step.id,
+            StepStatus.SUCCESS,
+            failureOutput,
+            errorMsg,
+            idempotencyTtlSeconds
+          );
+        }
+        return {
+          output: failureOutput,
+          outputs: failureOutputs,
+          status: StepStatus.SUCCESS,
+          error: errorMsg,
+        };
+      }
+
       this.logger.error(`  ✗ Step ${step.id} failed: ${redactedErrorMsg}`);
-      await this.db.completeStep(stepExecId, 'failed', null, redactedErrorMsg);
+      await this.db.completeStep(
+        stepExecId,
+        StepStatus.FAILED,
+        this.redactForStorage(failureOutput),
+        this.redactAtRest ? redactedErrorMsg : errorMsg
+      );
+      if (dedupEnabled && idempotencyClaimed) {
+        await this.recordIdempotencyResult(
+          scopedIdempotencyKey,
+          step.id,
+          StepStatus.FAILED,
+          failureOutput,
+          errorMsg,
+          idempotencyTtlSeconds
+        );
+      }
 
       // Return failed context
       return {
-        output: null,
-        outputs: {},
-        status: 'failed',
+        output: failureOutput,
+        outputs: failureOutputs,
+        status: StepStatus.FAILED,
+        error: errorMsg,
       };
     }
   }
@@ -864,7 +1840,7 @@ Do not change the 'id' or 'type' or 'auto_heal' fields.
       agent: auto_heal.agent,
       model: auto_heal.model,
       prompt,
-      schema: {
+      outputSchema: {
         type: 'object',
         description: 'Partial step configuration with fixed values',
         additionalProperties: true,
@@ -875,12 +1851,19 @@ Do not change the 'id' or 'type' or 'auto_heal' fields.
 
     // Execute the agent step
     // We use a fresh context but share secrets/env
-    const result = await executeStep(agentStep, context, this.logger, {
+    const exec = this.options.executeStep || executeStep;
+    const result = await exec(agentStep, context, this.logger, {
       executeWorkflowFn: this.executeSubWorkflow.bind(this),
       mcpManager: this.mcpManager,
       memoryDb: this.memoryDb,
       workflowDir: this.options.workflowDir,
       dryRun: this.options.dryRun,
+      debug: this.options.debug,
+      runId: this.runId,
+      artifactRoot: this.options.artifactRoot,
+      redactForStorage: this.redactForStorage.bind(this),
+      allowInsecure: this.options.allowInsecure,
+      executeStep: this.options.executeStep || executeStep,
     });
 
     if (result.status !== 'success' || !result.output) {
@@ -996,6 +1979,53 @@ Please provide the fixed step configuration as JSON.`;
     }
   }
 
+  /**
+   * Build a repair prompt for output validation failures
+   */
+  private buildOutputRepairPrompt(
+    step: Step,
+    output: unknown,
+    validationError: string,
+    strategy: 'reask' | 'repair' | 'hybrid'
+  ): string {
+    const llmStep = step as import('../parser/schema.ts').LlmStep;
+    const originalPrompt = llmStep.prompt;
+    const outputSchema = step.outputSchema;
+
+    const strategyInstructions = {
+      reask: 'Please try again, carefully following the output format requirements.',
+      repair:
+        'Please fix the output to match the required schema. You may need to restructure, add missing fields, or correct data types.',
+      hybrid:
+        'Please fix the output to match the required schema. If you cannot fix it, regenerate a completely new response.',
+    };
+
+    return `${originalPrompt}
+
+---
+
+**OUTPUT REPAIR REQUIRED**
+
+Your previous response failed validation. Here are the details:
+
+**Your Previous Output:**
+\`\`\`json
+${typeof output === 'string' ? output : JSON.stringify(output, null, 2)}
+\`\`\`
+
+**Validation Error:**
+${validationError}
+
+**Required Output Schema:**
+\`\`\`json
+${JSON.stringify(outputSchema, null, 2)}
+\`\`\`
+
+${strategyInstructions[strategy]}
+
+Please provide a corrected response that exactly matches the required schema.`;
+  }
+
   /**
    * Execute a step (handles foreach if present)
    */
@@ -1011,12 +2041,23 @@ Please provide the fixed step configuration as JSON.`;
       return;
     }
 
+    if (this.options.dryRun && step.type !== 'shell') {
+      this.logger.log(`  ⊘ [DRY RUN] Skipping ${step.type} step ${step.id}`);
+      const stepExecId = randomUUID();
+      await this.db.createStep(stepExecId, this.runId, step.id);
+      await this.db.completeStep(stepExecId, StepStatus.SKIPPED, null);
+      this.stepContexts.set(step.id, { status: StepStatus.SKIPPED });
+      return;
+    }
+
     if (step.foreach) {
       const { ForeachExecutor } = await import('./foreach-executor.ts');
       const executor = new ForeachExecutor(
         this.db,
         this.logger,
-        this.executeStepInternal.bind(this)
+        this.executeStepInternal.bind(this),
+        this.abortSignal,
+        this.resourcePool
       );
 
       const existingContext = this.stepContexts.get(step.id) as ForeachStepContext;
@@ -1038,8 +2079,14 @@ Please provide the fixed step configuration as JSON.`;
         throw new WorkflowSuspendedError(result.error || 'Workflow suspended', step.id, inputType);
       }
 
+      if (result.status === 'waiting') {
+        const wakeAt = getWakeAt(result.output);
+        throw new WorkflowWaitingError(result.error || `Waiting until ${wakeAt}`, step.id, wakeAt);
+      }
+
       if (result.status === 'failed') {
-        throw new Error(`Step ${step.id} failed`);
+        const suffix = result.error ? `: ${result.error}` : '';
+        throw new Error(`Step ${step.id} failed${suffix}`);
       }
     }
   }
@@ -1051,7 +2098,7 @@ Please provide the fixed step configuration as JSON.`;
     step: WorkflowStep,
     context: ExpressionContext
   ): Promise<StepResult> {
-    const workflowPath = WorkflowRegistry.resolvePath(step.path);
+    const workflowPath = WorkflowRegistry.resolvePath(step.path, this.options.workflowDir);
     const workflow = WorkflowParser.loadWorkflow(workflowPath);
     const subWorkflowDir = dirname(workflowPath);
 
@@ -1072,12 +2119,48 @@ Please provide the fixed step configuration as JSON.`;
       mcpManager: this.mcpManager,
       workflowDir: subWorkflowDir,
       depth: this.depth + 1,
+      dedup: this.options.dedup,
+      artifactRoot: this.options.artifactRoot,
     });
 
     try {
       const output = await subRunner.run();
+
+      const rawOutputs =
+        typeof output === 'object' && output !== null && !Array.isArray(output) ? output : {};
+      const mappedOutputs: Record<string, unknown> = {};
+
+      // Handle explicit output mapping
+      if (step.outputMapping) {
+        for (const [alias, mapping] of Object.entries(step.outputMapping)) {
+          let originalKey: string;
+          let defaultValue: unknown;
+
+          if (typeof mapping === 'string') {
+            originalKey = mapping;
+          } else {
+            originalKey = mapping.from;
+            defaultValue = mapping.default;
+          }
+
+          if (originalKey in rawOutputs) {
+            mappedOutputs[alias] = rawOutputs[originalKey];
+          } else if (defaultValue !== undefined) {
+            mappedOutputs[alias] = defaultValue;
+          } else {
+            throw new Error(
+              `Sub-workflow output "${originalKey}" not found (required by mapping "${alias}" in step "${step.id}")`
+            );
+          }
+        }
+      }
+
       return {
-        output,
+        output: {
+          ...mappedOutputs,
+          outputs: rawOutputs, // Namespaced raw outputs
+          __subRunId: subRunner.runId, // Track sub-workflow run ID for rollback
+        },
         status: 'success',
       };
     } catch (error) {
@@ -1114,12 +2197,14 @@ Please provide the fixed step configuration as JSON.`;
         '   Workflows can execute arbitrary shell commands and access your environment.\n'
     );
 
+    this.redactAtRest = ConfigLoader.load().storage?.redact_secrets_at_rest ?? true;
+
     // Apply defaults and validate inputs
     this.applyDefaultsAndValidate();
 
     // Create run record (only for new runs, not for resume)
     if (!isResume) {
-      await this.db.createRun(this.runId, this.workflow.name, this.inputs);
+      await this.db.createRun(this.runId, this.workflow.name, this.redactForStorage(this.inputs));
     }
     await this.db.updateRunStatus(this.runId, 'running');
 
@@ -1144,7 +2229,7 @@ Please provide the fixed step configuration as JSON.`;
         this.logger.log('All steps already completed. Nothing to resume.\n');
         // Evaluate outputs from completed state
         const outputs = this.evaluateOutputs();
-        await this.db.updateRunStatus(this.runId, 'success', outputs);
+        await this.db.updateRunStatus(this.runId, 'success', this.redactForStorage(outputs));
         this.logger.log('✨ Workflow already completed!\n');
         return outputs;
       }
@@ -1176,45 +2261,86 @@ Please provide the fixed step configuration as JSON.`;
         );
       }
 
+      // Register top-level compensation if defined
+      if (this.workflow.compensate) {
+        await this.registerWorkflowCompensation();
+      }
+
       // Execute steps in parallel where possible (respecting dependencies and global concurrency)
       const pendingSteps = new Set(remainingSteps);
       const runningPromises = new Map<string, Promise<void>>();
 
       try {
         while (pendingSteps.size > 0 || runningPromises.size > 0) {
+          // Check for cancellation - drain in-flight steps but don't start new ones
+          if (this.isCanceled) {
+            if (runningPromises.size > 0) {
+              this.logger.log(
+                `⏳ Waiting for ${runningPromises.size} in-flight step(s) to complete...`
+              );
+              await Promise.allSettled(runningPromises.values());
+            }
+            throw new Error('Workflow canceled by user');
+          }
+
           // 1. Find runnable steps (all dependencies met)
           for (const stepId of pendingSteps) {
+            // Don't schedule new steps if canceled
+            if (this.isCanceled) break;
+
             const step = stepMap.get(stepId);
             if (!step) {
               throw new Error(`Step ${stepId} not found in workflow`);
             }
-            const dependenciesMet = step.needs.every((dep: string) => completedSteps.has(dep));
+
+            let dependenciesMet = false;
+            if (step.type === 'join') {
+              dependenciesMet = this.isJoinConditionMet(
+                step as import('../parser/schema.ts').JoinStep,
+                completedSteps
+              );
+            } else {
+              dependenciesMet = step.needs.every((dep: string) => completedSteps.has(dep));
+            }
 
             if (dependenciesMet && runningPromises.size < globalConcurrencyLimit) {
               pendingSteps.delete(stepId);
 
+              // Determine pool for this step
+              const poolName = step.pool || step.type;
+
               // Start execution
               const stepIndex = stepIndices.get(stepId);
-              this.logger.log(
-                `[${stepIndex}/${totalSteps}] ▶ Executing step: ${step.id} (${step.type})`
-              );
-              const promise = this.executeStepWithForeach(step)
-                .then(() => {
+
+              const promise = (async () => {
+                let release: (() => void) | undefined;
+                try {
+                  this.logger.debug?.(
+                    `[${stepIndex}/${totalSteps}] ⏳ Waiting for pool: ${poolName}`
+                  );
+                  release = await this.resourcePool.acquire(poolName, { signal: this.abortSignal });
+
+                  this.logger.log(
+                    `[${stepIndex}/${totalSteps}] ▶ Executing step: ${step.id} (${step.type})`
+                  );
+
+                  await this.executeStepWithForeach(step);
                   completedSteps.add(stepId);
-                  runningPromises.delete(stepId);
                   this.logger.log(`[${stepIndex}/${totalSteps}] ✓ Step ${step.id} completed\n`);
-                })
-                .catch((err) => {
+                } finally {
+                  if (typeof release === 'function') {
+                    release();
+                  }
                   runningPromises.delete(stepId);
-                  throw err; // Fail fast
-                });
+                }
+              })();
 
               runningPromises.set(stepId, promise);
             }
           }
 
-          // 2. Detect deadlock
-          if (runningPromises.size === 0 && pendingSteps.size > 0) {
+          // 2. Detect deadlock (only if not canceled)
+          if (!this.isCanceled && runningPromises.size === 0 && pendingSteps.size > 0) {
             const pendingList = Array.from(pendingSteps).join(', ');
             throw new Error(
               `Deadlock detected in workflow execution. Pending steps: ${pendingList}`
@@ -1231,14 +2357,28 @@ Please provide the fixed step configuration as JSON.`;
         if (runningPromises.size > 0) {
           await Promise.allSettled(runningPromises.values());
         }
+
+        const msg = error instanceof Error ? error.message : String(error);
+
+        // Trigger rollback
+        await this.processCompensations(msg);
+
+        // Re-throw to be caught by the outer block (which calls stop)
+        // Actually, the outer caller usually handles this.
+        // But we want to ensure rollback happens BEFORE final status update if possible.
         throw error;
       }
 
+      // Determine final status
+      const failedSteps = remainingSteps.filter(
+        (id) => this.stepContexts.get(id)?.status === StepStatus.FAILED
+      );
+
       // Evaluate outputs
       const outputs = this.evaluateOutputs();
 
       // Mark run as complete
-      await this.db.updateRunStatus(this.runId, 'success', outputs);
+      await this.db.updateRunStatus(this.runId, 'success', this.redactForStorage(outputs));
 
       this.logger.log('✨ Workflow completed successfully!\n');
 
@@ -1249,9 +2389,33 @@ Please provide the fixed step configuration as JSON.`;
         this.logger.log(`\n⏸  Workflow paused: ${error.message}`);
         throw error;
       }
+
+      if (error instanceof WorkflowWaitingError) {
+        await this.db.updateRunStatus(this.runId, 'paused');
+        this.logger.log(`\n⏳ Workflow waiting: ${error.message}`);
+        throw error;
+      }
+
       const errorMsg = error instanceof Error ? error.message : String(error);
+
+      // Find the failed step from stepContexts
+      for (const [stepId, ctx] of this.stepContexts.entries()) {
+        if (ctx.status === 'failed') {
+          this.lastFailedStep = { id: stepId, error: ctx.error || errorMsg };
+          break;
+        }
+      }
+
+      // Run errors block if defined (before finally, after retries exhausted)
+      await this.runErrors();
+
       this.logger.error(`\n✗ Workflow failed: ${errorMsg}\n`);
-      await this.db.updateRunStatus(this.runId, 'failed', undefined, errorMsg);
+      await this.db.updateRunStatus(
+        this.runId,
+        'failed',
+        undefined,
+        this.redactAtRest ? this.redactor.redact(errorMsg) : errorMsg
+      );
       throw error;
     } finally {
       this.removeSignalHandlers();
@@ -1340,27 +2504,222 @@ Please provide the fixed step configuration as JSON.`;
   }
 
   /**
-   * Evaluate workflow outputs
+   * Execute the errors block if defined (runs after a step exhausts retries, before finally)
    */
-  private evaluateOutputs(): Record<string, unknown> {
-    if (!this.workflow.outputs) {
-      return {};
+  private async runErrors(): Promise<void> {
+    if (!this.workflow.errors || this.workflow.errors.length === 0) {
+      return;
+    }
+
+    if (!this.lastFailedStep) {
+      this.logger.warn('Errors block defined but no failed step context available');
+      return;
+    }
+
+    this.logger.log('\n🔧 Executing errors block...');
+
+    const stepMap = new Map(this.workflow.errors.map((s) => [s.id, s]));
+    const completedErrorsSteps = new Set<string>();
+    const pendingErrorsSteps = new Set(this.workflow.errors.map((s) => s.id));
+    const runningPromises = new Map<string, Promise<void>>();
+    const totalErrorsSteps = this.workflow.errors.length;
+    const errorsStepIndices = new Map(this.workflow.errors.map((s, index) => [s.id, index + 1]));
+
+    try {
+      while (pendingErrorsSteps.size > 0 || runningPromises.size > 0) {
+        for (const stepId of pendingErrorsSteps) {
+          const step = stepMap.get(stepId);
+          if (!step) continue;
+
+          // Dependencies can be from main steps (already in this.stepContexts) or previous errors steps
+          const dependenciesMet = step.needs.every(
+            (dep: string) => this.stepContexts.has(dep) || completedErrorsSteps.has(dep)
+          );
+
+          if (dependenciesMet) {
+            pendingErrorsSteps.delete(stepId);
+
+            const errorsStepIndex = errorsStepIndices.get(stepId);
+            this.logger.log(
+              `[${errorsStepIndex}/${totalErrorsSteps}] ▶ Executing errors step: ${step.id} (${step.type})`
+            );
+            const promise = this.executeStepWithForeach(step)
+              .then(() => {
+                completedErrorsSteps.add(stepId);
+                runningPromises.delete(stepId);
+                this.logger.log(
+                  `[${errorsStepIndex}/${totalErrorsSteps}] ✓ Errors step ${step.id} completed\n`
+                );
+              })
+              .catch((err) => {
+                runningPromises.delete(stepId);
+                this.logger.error(
+                  `  ✗ Errors step ${step.id} failed: ${err instanceof Error ? err.message : String(err)}`
+                );
+                // We continue with other errors steps if possible
+                completedErrorsSteps.add(stepId); // Mark as "done" (even if failed) so dependents can run
+              });
+
+            runningPromises.set(stepId, promise);
+          }
+        }
+
+        if (runningPromises.size === 0 && pendingErrorsSteps.size > 0) {
+          this.logger.error('Deadlock in errors block detected');
+          break;
+        }
+
+        if (runningPromises.size > 0) {
+          await Promise.race(runningPromises.values());
+        }
+      }
+    } catch (error) {
+      // Wait for other parallel steps to settle to avoid unhandled rejections
+      if (runningPromises.size > 0) {
+        await Promise.allSettled(runningPromises.values());
+      }
+      this.logger.error(
+        `Error in errors block: ${error instanceof Error ? error.message : String(error)}`
+      );
     }
+  }
 
+  /**
+   * Evaluate workflow outputs
+   */
+  private evaluateOutputs(): Record<string, unknown> {
     const context = this.buildContext();
     const outputs: Record<string, unknown> = {};
 
-    for (const [key, expression] of Object.entries(this.workflow.outputs)) {
+    if (this.workflow.outputs) {
+      for (const [key, expression] of Object.entries(this.workflow.outputs)) {
+        try {
+          outputs[key] = ExpressionEvaluator.evaluate(expression, context);
+        } catch (error) {
+          this.logger.warn(
+            `Warning: Failed to evaluate output "${key}": ${error instanceof Error ? error.message : String(error)}`
+          );
+          outputs[key] = null;
+        }
+      }
+    }
+
+    // Validate outputs against schema if provided
+    if (this.workflow.outputSchema) {
       try {
-        outputs[key] = ExpressionEvaluator.evaluate(expression, context);
+        this.validateSchema('output', this.workflow.outputSchema, outputs, 'workflow');
       } catch (error) {
-        this.logger.warn(
-          `Warning: Failed to evaluate output "${key}": ${error instanceof Error ? error.message : String(error)}`
+        throw new Error(
+          `Workflow output validation failed: ${error instanceof Error ? error.message : String(error)}`
         );
-        outputs[key] = null;
       }
     }
 
     return outputs;
   }
+
+  /**
+   * Check if a join condition is met based on completed dependencies
+   */
+  private isJoinConditionMet(
+    step: import('../parser/schema.ts').JoinStep,
+    completedSteps: Set<string>
+  ): boolean {
+    const total = step.needs.length;
+    if (total === 0) return true;
+
+    // Count successful/skipped dependencies
+    const successCount = step.needs.filter((dep) => completedSteps.has(dep)).length;
+
+    // Find failed/suspended dependencies (that we've already tried)
+    // If some dependencies failed (and didn't allowFailure), the whole workflow would usually fail.
+    // If allowFailure was true, they are in completedSteps.
+    // So completedSteps effectively represents "done successfully".
+
+    if (step.condition === 'all') {
+      return successCount === total;
+    }
+    if (step.condition === 'any') {
+      // Met if at least one succeeded, OR if all finished and none succeeded?
+      // Actually strictly "any" means at least one success.
+      return successCount > 0;
+    }
+    if (typeof step.condition === 'number') {
+      return successCount >= step.condition;
+    }
+
+    return successCount === total;
+  }
+
+  /**
+   * Register top-level compensation for the workflow
+   */
+  private async registerWorkflowCompensation(): Promise<void> {
+    if (!this.workflow.compensate) return;
+
+    // Check if already registered (for resume)
+    const existing = await this.db.getAllCompensations(this.runId);
+    if (existing.some((c) => c.step_id === 'workflow')) return;
+
+    const compStep = {
+      ...this.workflow.compensate,
+      id: this.workflow.compensate.id || `${this.workflow.name}-compensate`,
+    };
+    const definition = JSON.stringify(compStep);
+    const compensationId = randomUUID();
+
+    this.logger.log(`  📎 Registering top-level compensation for workflow ${this.workflow.name}`);
+    await this.db.registerCompensation(
+      compensationId,
+      this.runId,
+      'workflow', // use 'workflow' as step_id marker
+      compStep.id,
+      definition
+    );
+  }
+
+  /**
+   * Cascade rollback to a child sub-workflow
+   */
+  private async cascadeRollback(subRunId: string, errorReason: string): Promise<void> {
+    this.logger.log(`  📂 Cascading rollback to sub-workflow: ${subRunId}`);
+    try {
+      const runRecord = await this.db.getRun(subRunId);
+      if (!runRecord) {
+        this.logger.warn(`  ⚠️ Could not find run record for sub-workflow ${subRunId}`);
+        return;
+      }
+
+      const workflowPath = WorkflowRegistry.resolvePath(
+        runRecord.workflow_name,
+        this.options.workflowDir
+      );
+      const workflow = WorkflowParser.loadWorkflow(workflowPath);
+
+      const subRunner = new WorkflowRunner(workflow, {
+        resumeRunId: subRunId,
+        dbPath: this.db.dbPath,
+        logger: this.logger,
+        mcpManager: this.mcpManager,
+        workflowDir: dirname(workflowPath),
+        depth: this.depth + 1,
+        allowSuccessResume: true, // Internal workflows might need this
+        resourcePoolManager: this.resourcePool,
+        allowInsecure: this.options.allowInsecure,
+      });
+
+      // Restore sub-workflow state
+      await subRunner.restoreState();
+
+      // Trigger its compensations
+      // We call the private method directly since we're in the same class (different instance)
+      // but TypeScript might complain if it's strictly private.
+      // Actually, in TS, private is accessible by other instances of the same class.
+      await subRunner.processCompensations(errorReason);
+    } catch (error) {
+      this.logger.error(
+        `  ⚠️ Failed to cascade rollback to ${subRunId}: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
 }