keystone-cli 1.0.2 → 1.1.0

package/src/runner/workflow-runner.ts

@@ -1,4 +1,4 @@
-import { randomUUID } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { dirname, join } from 'node:path';
@@ -6,19 +6,26 @@ import { MemoryDb } from '../db/memory-db.ts';
 import { type RunStatus, WorkflowDb } from '../db/workflow-db.ts';
 import type { ExpressionContext } from '../expression/evaluator.ts';
 import { ExpressionEvaluator } from '../expression/evaluator.ts';
-import type { Step, Workflow, WorkflowStep } from '../parser/schema.ts';
+import type { LlmStep, PlanStep, Step, Workflow, WorkflowStep } from '../parser/schema.ts';
 import { WorkflowParser } from '../parser/workflow-parser.ts';
 import { StepStatus, type StepStatusType, WorkflowStatus } from '../types/status.ts';
 import { ConfigLoader } from '../utils/config-loader.ts';
+import { container } from '../utils/container.ts';
 import { extractJson } from '../utils/json-parser.ts';
-import { Redactor } from '../utils/redactor.ts';
+import { ConsoleLogger, type Logger } from '../utils/logger.ts';
+import type { Redactor } from '../utils/redactor.ts';
 import { formatSchemaErrors, validateJsonSchema } from '../utils/schema-validator.ts';
 import { WorkflowRegistry } from '../utils/workflow-registry.ts';
-import { ForeachExecutor } from './foreach-executor.ts';
+import type { EventHandler, StepPhase, WorkflowEvent } from './events.ts';
+import { ForeachExecutor } from './executors/foreach-executor.ts';
+import { type RunnerFactory, executeSubWorkflow } from './executors/subworkflow-executor.ts';
 import { type LLMMessage, getAdapter } from './llm-adapter.ts';
 import { MCPManager } from './mcp-manager.ts';
 import { ResourcePoolManager } from './resource-pool.ts';
 import { withRetry } from './retry.ts';
+import { ContextBuilder } from './services/context-builder.ts';
+import { SecretManager } from './services/secret-manager.ts';
+import { WorkflowValidator } from './services/workflow-validator.ts';
 import {
   type StepResult,
   WorkflowSuspendedError,
@@ -26,8 +33,8 @@ import {
   executeStep,
 } from './step-executor.ts';
 import { withTimeout } from './timeout.ts';
-
-import { ConsoleLogger, type Logger } from '../utils/logger.ts';
+import { WorkflowScheduler } from './workflow-scheduler.ts';
+import { type ForeachStepContext, type StepContext, WorkflowState } from './workflow-state.ts';
 
 /**
  * A logger wrapper that redacts secrets from all log messages
@@ -74,6 +81,22 @@ function getWakeAt(output: unknown): string | undefined {
   return typeof wakeAt === 'string' ? wakeAt : undefined;
 }
 
+const QUALITY_GATE_SCHEMA = {
+  type: 'object',
+  properties: {
+    approved: { type: 'boolean' },
+    issues: { type: 'array', items: { type: 'string' } },
+    suggestions: { type: 'array', items: { type: 'string' } },
+  },
+  required: ['approved'],
+};
+
+type QualityGateReview = {
+  approved: boolean;
+  issues?: string[];
+  suggestions?: string[];
+};
+
 export interface RunOptions {
   inputs?: Record<string, unknown>;
   secrets?: Record<string, string>;
@@ -90,32 +113,20 @@ export interface RunOptions {
   dedup?: boolean;
   getAdapter?: typeof getAdapter;
   executeStep?: typeof executeStep;
+  executeLlmStep?: typeof import('./executors/llm-executor.ts').executeLlmStep;
   depth?: number;
   allowSuccessResume?: boolean;
   resourcePoolManager?: ResourcePoolManager;
   allowInsecure?: boolean;
   artifactRoot?: string;
+  db?: WorkflowDb;
+  memoryDb?: MemoryDb;
+  onEvent?: EventHandler;
+  memoize?: boolean;
+  signal?: AbortSignal;
 }
 
-export interface StepContext {
-  output?: unknown;
-  outputs?: Record<string, unknown>;
-  status: StepStatusType;
-  error?: string;
-  usage?: {
-    prompt_tokens: number;
-    completion_tokens: number;
-    total_tokens: number;
-  };
-}
-
-// Type for foreach results - wraps array to ensure JSON serialization preserves all properties
-export interface ForeachStepContext extends StepContext {
-  items: StepContext[]; // Individual iteration results
-  // output and outputs inherited from StepContext
-  // output: array of output values
-  // outputs: mapped outputs object
-}
+// Redacted StepContext and ForeachStepContext (moved to workflow-state.ts)
 
 /**
  * Main workflow execution engine
@@ -124,16 +135,18 @@ export class WorkflowRunner {
   private workflow: Workflow;
   private db: WorkflowDb;
   private memoryDb: MemoryDb;
+  private contextMemory: Record<string, unknown> = {};
+  private envOverrides: Record<string, string> = {};
   private _runId!: string;
-  private stepContexts: Map<string, StepContext | ForeachStepContext> = new Map();
+  private state!: WorkflowState;
+  private scheduler!: WorkflowScheduler;
   private inputs!: Record<string, unknown>;
-  private secrets: Record<string, string>;
-  private redactor: Redactor;
+  private secretManager: SecretManager;
+  private contextBuilder!: ContextBuilder;
+  private validator!: WorkflowValidator;
   private rawLogger!: Logger;
-  private secretValues: string[] = [];
   private redactAtRest = true;
   private resumeRunId?: string;
-  private restored = false;
   private logger!: Logger;
   private mcpManager: MCPManager;
   private options: RunOptions;
@@ -147,6 +160,7 @@ export class WorkflowRunner {
   private lastFailedStep?: { id: string; error: string };
   private abortController = new AbortController();
   private resourcePool!: ResourcePoolManager;
+  private restored = false;
 
   /**
    * Get the abort signal for cancellation checks
@@ -162,6 +176,27 @@ export class WorkflowRunner {
     return this.abortController.signal.aborted;
   }
 
+  private createStepAbortController(): { controller: AbortController; cleanup: () => void } {
+    const controller = new AbortController();
+    const parentSignal = this.abortSignal;
+    const onAbort = () => {
+      if (!controller.signal.aborted) {
+        controller.abort();
+      }
+    };
+
+    if (parentSignal.aborted) {
+      controller.abort();
+      return { controller, cleanup: () => {} };
+    }
+
+    parentSignal.addEventListener('abort', onAbort, { once: true });
+    return {
+      controller,
+      cleanup: () => parentSignal.removeEventListener('abort', onAbort),
+    };
+  }
+
   constructor(workflow: Workflow, options: RunOptions = {}) {
     this.workflow = workflow;
     this.options = options;
@@ -173,23 +208,54 @@ export class WorkflowRunner {
       );
     }
 
-    this.db = new WorkflowDb(options.dbPath);
-    this.memoryDb = new MemoryDb(options.memoryDbPath);
-    this.secrets = this.loadSecrets();
-    this.redactor = new Redactor(this.secrets, { forcedSecrets: this.secretValues });
-
+    // Use injected instances or resolve from container or create new from paths
+    this.db =
+      options.db ||
+      (options.dbPath
+        ? new WorkflowDb(options.dbPath)
+        : container.resolveOptional<WorkflowDb>('db')) ||
+      new WorkflowDb(options.dbPath);
+
+    this.memoryDb =
+      options.memoryDb ||
+      (options.memoryDbPath
+        ? new MemoryDb(options.memoryDbPath)
+        : container.resolveOptional<MemoryDb>('memoryDb')) ||
+      new MemoryDb(options.memoryDbPath);
+
+    this.secretManager = new SecretManager(options.secrets || {});
     this.initLogger(options);
+    this.initRun(options);
+
+    this.validator = new WorkflowValidator(this.workflow, this.inputs);
+    this.contextBuilder = new ContextBuilder(
+      this.workflow,
+      this.inputs,
+      this.secretManager.getSecretValues(),
+      this.state,
+      this.logger
+    );
     this.mcpManager = options.mcpManager || new MCPManager();
     this.initResourcePool(options);
-    this.initRun(options);
+
+    if (options.signal) {
+      if (options.signal.aborted) {
+        this.abortController.abort();
+      } else {
+        options.signal.addEventListener('abort', () => this.abortController.abort(), {
+          once: true,
+        });
+      }
+    }
 
     this.setupSignalHandlers();
   }
 
   private initLogger(options: RunOptions): void {
-    const rawLogger = options.logger || new ConsoleLogger();
+    const rawLogger =
+      options.logger || container.resolveOptional<Logger>('logger') || new ConsoleLogger();
     this.rawLogger = rawLogger;
-    this.logger = new RedactingLogger(rawLogger, this.redactor);
+    this.logger = new RedactingLogger(rawLogger, this.secretManager.getRedactor());
   }
 
   private initResourcePool(options: RunOptions): void {
@@ -201,7 +267,7 @@ export class WorkflowRunner {
       const workflowPools: Record<string, number> = {};
 
       if (this.workflow.pools) {
-        const baseContext = this.buildContext();
+        const baseContext = this.contextBuilder.buildContext(this.secretManager.getSecrets());
         for (const [name, limit] of Object.entries(this.workflow.pools)) {
           if (typeof limit === 'string') {
             workflowPools[name] = Number(ExpressionEvaluator.evaluate(limit, baseContext));
@@ -227,6 +293,16 @@ export class WorkflowRunner {
       this.inputs = options.inputs || {};
       this._runId = randomUUID();
     }
+
+    this.state = new WorkflowState(
+      this._runId,
+      this.workflow,
+      this.db,
+      this.inputs,
+      this.secretManager.getSecrets(),
+      this.logger
+    );
+    this.scheduler = new WorkflowScheduler(this.workflow, this.state.getCompletedStepIds());
   }
 
   /**
@@ -252,8 +328,6 @@ export class WorkflowRunner {
       throw new Error(`Run ${this.runId} not found`);
     }
 
-    // Only allow resuming failed, paused, canceled, or running (crash recovery) runs
-    // Unless specifically allowed (e.g. for rollback/compensation)
     if (
       run.status !== WorkflowStatus.FAILED &&
       run.status !== WorkflowStatus.PAUSED &&
@@ -268,7 +342,7 @@ export class WorkflowRunner {
 
     if (run.status === WorkflowStatus.RUNNING) {
       this.logger.warn(
-        `⚠️  Resuming a run marked as 'running'. This usually means the previous process crashed or was killed forcefully. Ensure no other instances are running.`
+        `⚠️  Resuming a run marked as 'running'. This usually means the previous process crashed or was killed forcefully.`
       );
     }
 
@@ -276,209 +350,20 @@ export class WorkflowRunner {
       this.logger.log('📋 Resuming a previously canceled run. Completed steps will be skipped.');
     }
 
-    // Restore inputs from the previous run to ensure consistency
-    // Merge with any resumeInputs provided (e.g. answers to human steps)
-    try {
-      if (!run.inputs || run.inputs === 'null' || run.inputs === '') {
-        this.logger.warn(`Run ${this.runId} has no persisted inputs`);
-        // Keep existing inputs
-      } else {
-        const storedInputs = JSON.parse(run.inputs);
-        this.inputs = { ...storedInputs, ...this.inputs };
-      }
-    } catch (error) {
-      this.logger.error(
-        `CRITICAL: Failed to parse inputs from run ${this.runId}. Data may be corrupted. Using default/resume inputs. Error: ${error instanceof Error ? error.message : String(error)}`
-      );
-      // Fallback: preserve existing inputs from resume options
-    }
-
-    // Load all step executions for this run
-    const steps = await this.db.getStepsByRun(this.runId);
-
-    // Group steps by step_id to handle foreach loops (multiple executions per step_id)
-    const stepExecutionsByStepId = new Map<string, typeof steps>();
-    for (const step of steps) {
-      if (!stepExecutionsByStepId.has(step.step_id)) {
-        stepExecutionsByStepId.set(step.step_id, []);
-      }
-      stepExecutionsByStepId.get(step.step_id)?.push(step);
-    }
-
-    // Get topological order to ensure dependencies are restored before dependents
-    const executionOrder = WorkflowParser.topologicalSort(this.workflow);
-    const completedStepIds = new Set<string>();
-
-    // Reconstruct step contexts in topological order
-    for (const stepId of executionOrder) {
-      const stepExecutions = stepExecutionsByStepId.get(stepId);
-      if (!stepExecutions || stepExecutions.length === 0) continue;
-
-      const stepDef = this.workflow.steps.find((s) => s.id === stepId);
-      if (!stepDef) continue;
-
-      const isForeach = !!stepDef.foreach;
-
-      if (isForeach) {
-        // Reconstruct foreach aggregated context
-        const items: StepContext[] = [];
-        const outputs: unknown[] = [];
-        let allSuccess = true;
-
-        // Sort by iteration_index to ensure correct order
-        const sortedExecs = [...stepExecutions].sort(
-          (a, b) => (a.iteration_index ?? 0) - (b.iteration_index ?? 0)
-        );
-
-        for (const exec of sortedExecs) {
-          if (exec.iteration_index === null) continue; // Skip parent step record
-
-          if (exec.status === StepStatus.SUCCESS || exec.status === StepStatus.SKIPPED) {
-            let output: unknown = null;
-            try {
-              output = exec.output ? JSON.parse(exec.output) : null;
-            } catch (error) {
-              this.logger.warn(
-                `Failed to parse output for step ${stepId} iteration ${exec.iteration_index}: ${error}`
-              );
-              output = { error: 'Failed to parse output' };
-            }
-            items[exec.iteration_index] = {
-              output,
-              outputs:
-                typeof output === 'object' && output !== null && !Array.isArray(output)
-                  ? (output as Record<string, unknown>)
-                  : {},
-              status: exec.status as typeof StepStatus.SUCCESS | typeof StepStatus.SKIPPED,
-              error: exec.error || undefined,
-            };
-            outputs[exec.iteration_index] = output;
-          } else {
-            allSuccess = false;
-            // Still populate with placeholder if failed
-            items[exec.iteration_index] = {
-              output: null,
-              outputs: {},
-              status: exec.status as StepStatusType,
-              error: exec.error || undefined,
-            };
-          }
-        }
-
-        // Use persisted foreach items from parent step for deterministic resume
-        // This ensures the resume uses the same array as the initial run
-        let expectedCount = -1;
-        const parentExec = stepExecutions.find((e) => e.iteration_index === null);
-        if (parentExec?.output) {
-          try {
-            const parsed = JSON.parse(parentExec.output);
-            if (parsed.__foreachItems && Array.isArray(parsed.__foreachItems)) {
-              expectedCount = parsed.__foreachItems.length;
-            }
-          } catch (_e) {
-            // ignore parse errors
-          }
-        }
-
-        // Fallback to expression evaluation if persisted items not found
-        if (expectedCount === -1) {
-          try {
-            const baseContext = this.buildContext();
-            const foreachExpr = stepDef.foreach;
-            if (foreachExpr) {
-              const foreachItems = ExpressionEvaluator.evaluate(foreachExpr, baseContext);
-              if (Array.isArray(foreachItems)) {
-                expectedCount = foreachItems.length;
-              }
-            }
-          } catch (e) {
-            // If we can't evaluate yet (dependencies not met?), we can't be sure it's complete
-            allSuccess = false;
-          }
-        }
-
-        // Check if we have all items (no gaps)
-        const hasAllItems =
-          expectedCount !== -1 &&
-          items.length === expectedCount &&
-          !Array.from({ length: expectedCount }).some((_, i) => !items[i]);
-
-        // Determine overall status based on iterations
-        let status: StepContext['status'] = StepStatus.SUCCESS;
-        if (allSuccess && hasAllItems) {
-          status = StepStatus.SUCCESS;
-        } else if (items.some((item) => item?.status === StepStatus.SUSPENDED)) {
-          status = StepStatus.SUSPENDED;
-        } else {
-          status = StepStatus.FAILED;
-        }
+    // Hydrate state from DB
+    await this.state.restore();
 
-        // Always restore what we have to allow partial expression evaluation
-        const mappedOutputs = ForeachExecutor.aggregateOutputs(outputs);
-        this.stepContexts.set(stepId, {
-          output: outputs,
-          outputs: mappedOutputs,
-          status,
-          items,
-        } as ForeachStepContext);
-
-        // Only mark as fully completed if all iterations completed successfully AND we have all items
-        if (status === StepStatus.SUCCESS) {
-          completedStepIds.add(stepId);
-        }
-      } else {
-        // Single execution step
-        const exec = stepExecutions[0];
-        if (
-          exec.status === StepStatus.SUCCESS ||
-          exec.status === StepStatus.SKIPPED ||
-          exec.status === StepStatus.SUSPENDED ||
-          exec.status === StepStatus.WAITING
-        ) {
-          let output: unknown = null;
-          try {
-            output = exec.output ? JSON.parse(exec.output) : null;
-          } catch (error) {
-            this.logger.warn(`Failed to parse output for step ${stepId}: ${error}`);
-            output = { error: 'Failed to parse output' };
-          }
-
-          // If step is WAITING, check if timer has elapsed
-          let effectiveStatus = exec.status as StepContext['status'];
-          if (exec.status === StepStatus.WAITING) {
-            const timer = await this.db.getTimerByStep(this.runId, stepId);
-            const timerId = timer?.id;
-            const wakeAt = timer?.wake_at;
-            if (timerId && wakeAt && new Date(wakeAt) <= new Date()) {
-              // Timer elapsed!
-              await this.db.completeTimer(timerId);
-              await this.db.completeStep(exec.id, StepStatus.SUCCESS, output);
-              effectiveStatus = StepStatus.SUCCESS;
-            }
-          }
-          let effectiveError = exec.error || undefined;
-          if (exec.status === StepStatus.WAITING && effectiveStatus === StepStatus.SUCCESS) {
-            effectiveError = undefined;
-          }
-
-          this.stepContexts.set(stepId, {
-            output,
-            outputs:
-              typeof output === 'object' && output !== null && !Array.isArray(output)
-                ? (output as Record<string, unknown>)
-                : {},
-            status: effectiveStatus,
-            error: effectiveError,
-          });
-          if (effectiveStatus !== StepStatus.SUSPENDED && effectiveStatus !== StepStatus.WAITING) {
-            completedStepIds.add(stepId);
-          }
-        }
+    // Re-initialize scheduler with completed steps from restored state
+    const completedSteps = new Set<string>();
+    for (const [stepId, ctx] of this.state.entries()) {
+      if (ctx.status === StepStatus.SUCCESS || ctx.status === StepStatus.SKIPPED) {
+        completedSteps.add(stepId);
       }
     }
+    this.scheduler = new WorkflowScheduler(this.workflow, completedSteps);
 
     this.restored = true;
-    this.logger.log(`✓ Restored state: ${completedStepIds.size} step(s) already completed`);
+    this.logger.log(`✓ Restored state: ${completedSteps.size} step(s) hydrated`);
   }
 
   /**
@@ -526,33 +411,40 @@ export class WorkflowRunner {
         const stepDef = JSON.parse(compRecord.definition) as Step;
         this.logger.log(`  Running compensation: ${stepDef.id} (undoing ${compRecord.step_id})`);
 
-        await this.db.updateCompensationStatus(compRecord.id, 'running');
+        await this.db.updateCompensationStatus(compRecord.id, StepStatus.RUNNING);
 
         // Build context for compensation
         // It has access to the original step's output via steps.<step_id>.output
-        const context = this.buildContext();
+        const context = this.contextBuilder.buildContext(this.secretManager.getSecrets());
 
         try {
           // Execute the compensation step
           const result = await executeStep(stepDef, context, this.logger, {
             executeWorkflowFn: this.executeSubWorkflow.bind(this),
             mcpManager: this.mcpManager,
+            db: this.db,
             memoryDb: this.memoryDb,
             workflowDir: this.options.workflowDir,
             dryRun: this.options.dryRun,
             runId: this.runId,
             artifactRoot: this.options.artifactRoot,
             redactForStorage: this.redactForStorage.bind(this),
+            emitEvent: this.emitEvent.bind(this),
+            workflowName: this.workflow.name,
           });
 
-          if (result.status === 'success') {
+          if (result.status === StepStatus.SUCCESS) {
             this.logger.log(`  ✓ Compensation ${stepDef.id} succeeded`);
-            await this.db.updateCompensationStatus(compRecord.id, 'success', result.output);
+            await this.db.updateCompensationStatus(
+              compRecord.id,
+              StepStatus.SUCCESS,
+              result.output
+            );
           } else {
             this.logger.error(`  ✗ Compensation ${stepDef.id} failed: ${result.error}`);
             await this.db.updateCompensationStatus(
               compRecord.id,
-              'failed',
+              StepStatus.FAILED,
               result.output,
               result.error
             );
@@ -560,7 +452,7 @@ export class WorkflowRunner {
         } catch (err) {
           const errMsg = err instanceof Error ? err.message : String(err);
           this.logger.error(`  ✗ Compensation ${stepDef.id} crashed: ${errMsg}`);
-          await this.db.updateCompensationStatus(compRecord.id, 'failed', null, errMsg);
+          await this.db.updateCompensationStatus(compRecord.id, StepStatus.FAILED, null, errMsg);
         }
 
         // 2. Recursive rollback for sub-workflows
@@ -574,8 +466,10 @@ export class WorkflowRunner {
             if (subRunId) {
               await this.cascadeRollback(subRunId, errorReason);
             }
-          } catch (_e) {
-            // ignore parse errors
+          } catch (e) {
+            this.logger.warn(
+              `  ⚠️ Failed to parse sub-workflow output for rollback: ${e instanceof Error ? e.message : String(e)}`
+            );
           }
         }
       }
@@ -631,69 +525,36 @@ export class WorkflowRunner {
     }
   }
 
-  /**
-   * Load secrets from environment
-   */
-  private loadSecrets(): Record<string, string> {
-    const secrets: Record<string, string> = { ...(this.options.secrets || {}) };
-
-    // Common non-secret environment variables to exclude from redaction
-    const blocklist = new Set([
-      'USER',
-      'PATH',
-      'SHELL',
-      'HOME',
-      'PWD',
-      'LOGNAME',
-      'LANG',
-      'TERM',
-      'EDITOR',
-      'VISUAL',
-      '_',
-      'SHLVL',
-      'LC_ALL',
-      'DISPLAY',
-      'SSH_AUTH_SOCK',
-      'XPC_FLAGS',
-      'XPC_SERVICE_NAME',
-      'ITERM_SESSION_ID',
-      'ITERM_PROFILE',
-      'TERM_PROGRAM',
-      'TERM_PROGRAM_VERSION',
-      'COLORTERM',
-      'LC_TERMINAL',
-      'LC_TERMINAL_VERSION',
-      'PWD',
-      'OLDPWD',
-      'HOME',
-      'USER',
-      'SHELL',
-      'PATH',
-      'LOGNAME',
-      'TMPDIR',
-      'XDG_CONFIG_HOME',
-      'XDG_DATA_HOME',
-      'XDG_CACHE_HOME',
-      'XDG_RUNTIME_DIR',
-    ]);
-
-    // Bun automatically loads .env file
-    for (const [key, value] of Object.entries(Bun.env)) {
-      if (value && !blocklist.has(key)) {
-        secrets[key] = value;
-      }
-    }
-    return secrets;
+  private redactForStorage<T>(value: T): T {
+    if (!this.redactAtRest) return value;
+    return this.secretManager.getRedactor().redactValue(value) as T;
   }
 
-  private refreshRedactor(): void {
-    this.redactor = new Redactor(this.loadSecrets(), { forcedSecrets: this.secretValues });
-    this.logger = new RedactingLogger(this.rawLogger, this.redactor);
-  }
+  private async calculateStepCacheKey(
+    step: Step,
+    inputs: Record<string, unknown>
+  ): Promise<string | null> {
+    const memoizeEnabled = step.memoize ?? this.options.memoize ?? false;
+    if (!memoizeEnabled) return null;
 
-  private redactForStorage<T>(value: T): T {
-    if (!this.redactAtRest) return value;
-    return this.redactor.redactValue(value) as T;
+    // Only memoize deterministic steps by default unless explicitly requested
+    const cacheableTypes = ['shell', 'file', 'script', 'request', 'engine', 'blueprint'];
+    if (!cacheableTypes.includes(step.type) && step.memoize !== true) return null;
+
+    const data = {
+      type: step.type,
+      inputs,
+      env: step.env,
+      version: 2, // Cache versioning
+    };
+
+    // Use runtime-agnostic hashing
+    // @ts-ignore - Check for Bun environment
+    const hash =
+      typeof Bun !== 'undefined'
+        ? Bun.hash(JSON.stringify(data)).toString(16)
+        : createHash('sha256').update(JSON.stringify(data)).digest('hex');
+    return hash;
   }
 
   private validateSchema(
@@ -768,6 +629,24 @@ export class WorkflowRunner {
           op: step.op,
           allowOutsideCwd: step.allowOutsideCwd,
         });
+      case 'artifact':
+        return stripUndefined({
+          op: step.op,
+          name: ExpressionEvaluator.evaluateString(
+            (step as import('../parser/schema.ts').ArtifactStep).name,
+            context
+          ),
+          paths: (step as import('../parser/schema.ts').ArtifactStep).paths?.map((p) =>
+            ExpressionEvaluator.evaluateString(p, context)
+          ),
+          path: (step as import('../parser/schema.ts').ArtifactStep).path
+            ? ExpressionEvaluator.evaluateString(
+                (step as import('../parser/schema.ts').ArtifactStep).path as string,
+                context
+              )
+            : undefined,
+          allowOutsideCwd: step.allowOutsideCwd,
+        });
       case 'request': {
         let headers: Record<string, string> | undefined;
         if (step.headers) {
@@ -804,9 +683,11 @@ export class WorkflowRunner {
       }
       case 'llm':
         return stripUndefined({
-          agent: step.agent,
-          provider: step.provider,
-          model: step.model,
+          agent: ExpressionEvaluator.evaluateString(step.agent, context),
+          provider: step.provider
+            ? ExpressionEvaluator.evaluateString(step.provider, context)
+            : undefined,
+          model: step.model ? ExpressionEvaluator.evaluateString(step.model, context) : undefined,
           prompt: ExpressionEvaluator.evaluateString(step.prompt, context),
           tools: step.tools,
           maxIterations: step.maxIterations,
@@ -867,6 +748,14 @@ export class WorkflowRunner {
             : undefined,
           limit: step.limit,
         });
+      case 'wait':
+        return stripUndefined({
+          event: ExpressionEvaluator.evaluateString(
+            (step as import('../parser/schema.ts').WaitStep).event,
+            context
+          ),
+          oneShot: (step as import('../parser/schema.ts').WaitStep).oneShot,
+        });
       default:
         return {};
     }
@@ -875,196 +764,6 @@ export class WorkflowRunner {
   /**
    * Collect primitive secret values from structured inputs.
    */
-  private static collectSecretValues(
-    value: unknown,
-    sink: Set<string>,
-    seen: WeakSet<object>
-  ): void {
-    if (value === null || value === undefined) return;
-
-    if (typeof value === 'string') {
-      sink.add(value);
-      return;
-    }
-
-    if (typeof value === 'number' || typeof value === 'boolean' || typeof value === 'bigint') {
-      sink.add(String(value));
-      return;
-    }
-
-    if (typeof value !== 'object') return;
-
-    if (seen.has(value)) return;
-    seen.add(value);
-
-    if (Array.isArray(value)) {
-      for (const item of value) {
-        WorkflowRunner.collectSecretValues(item, sink, seen);
-      }
-      return;
-    }
-
-    for (const item of Object.values(value as Record<string, unknown>)) {
-      WorkflowRunner.collectSecretValues(item, sink, seen);
-    }
-  }
-
-  /**
-   * Apply workflow defaults to inputs and validate types
-   */
-  private applyDefaultsAndValidate(): void {
-    if (!this.workflow.inputs) return;
-
-    const secretValues = new Set<string>();
-
-    for (const [key, config] of Object.entries(this.workflow.inputs)) {
-      // Apply default if missing
-      if (this.inputs[key] === undefined && config.default !== undefined) {
-        this.inputs[key] = config.default;
-      }
-
-      if (config.secret) {
-        if (this.inputs[key] === WorkflowRunner.REDACTED_PLACEHOLDER) {
-          throw new Error(
-            `Secret input "${key}" was redacted at rest. Please provide it again to resume this run.`
-          );
-        }
-      }
-
-      // Validate required inputs
-      if (this.inputs[key] === undefined) {
-        throw new Error(`Missing required input: ${key}`);
-      }
-
-      // Basic type validation
-      const value = this.inputs[key];
-      const type = config.type.toLowerCase();
-
-      if (type === 'string' && typeof value !== 'string') {
-        throw new Error(`Input "${key}" must be a string, got ${typeof value}`);
-      }
-      if (type === 'number' && typeof value !== 'number') {
-        throw new Error(`Input "${key}" must be a number, got ${typeof value}`);
-      }
-      if (type === 'boolean' && typeof value !== 'boolean') {
-        throw new Error(`Input "${key}" must be a boolean, got ${typeof value}`);
-      }
-      if (type === 'array' && !Array.isArray(value)) {
-        throw new Error(`Input "${key}" must be an array, got ${typeof value}`);
-      }
-      if (
-        type === 'object' &&
-        (typeof value !== 'object' || value === null || Array.isArray(value))
-      ) {
-        throw new Error(`Input "${key}" must be an object, got ${typeof value}`);
-      }
-
-      if (config.values) {
-        if (type !== 'string' && type !== 'number' && type !== 'boolean') {
-          throw new Error(`Input "${key}" cannot use enum values with type "${type}"`);
-        }
-        for (const allowed of config.values) {
-          const matchesType =
-            (type === 'string' && typeof allowed === 'string') ||
-            (type === 'number' && typeof allowed === 'number') ||
-            (type === 'boolean' && typeof allowed === 'boolean');
-          if (!matchesType) {
-            throw new Error(
-              `Input "${key}" enum value ${JSON.stringify(allowed)} must be a ${type}`
-            );
-          }
-        }
-        if (!config.values.includes(value as string | number | boolean)) {
-          throw new Error(
-            `Input "${key}" must be one of: ${config.values.map((v) => JSON.stringify(v)).join(', ')}`
-          );
-        }
-      }
-
-      if (config.secret && value !== undefined && value !== WorkflowRunner.REDACTED_PLACEHOLDER) {
-        WorkflowRunner.collectSecretValues(value, secretValues, new WeakSet());
-      }
-    }
-
-    this.secretValues = Array.from(secretValues);
-    this.refreshRedactor();
-  }
-
-  /**
-   * Build expression context for evaluation
-   */
-  private buildContext(item?: unknown, index?: number): ExpressionContext {
-    const stepsContext: Record<
-      string,
-      {
-        output?: unknown;
-        outputs?: Record<string, unknown>;
-        status?: string;
-        error?: string;
-        items?: StepContext[];
-      }
-    > = {};
-
-    for (const [stepId, ctx] of this.stepContexts.entries()) {
-      // For foreach results, include items array for iteration access
-      if ('items' in ctx && ctx.items) {
-        stepsContext[stepId] = {
-          output: ctx.output,
-          outputs: ctx.outputs,
-          status: ctx.status,
-          error: ctx.error,
-          items: ctx.items,
-        };
-      } else {
-        stepsContext[stepId] = {
-          output: ctx.output,
-          outputs: ctx.outputs,
-          status: ctx.status,
-          error: ctx.error,
-        };
-      }
-    }
-
-    const baseContext: ExpressionContext = {
-      inputs: this.inputs,
-      secrets: this.loadSecrets(), // Access secrets from options
-      secretValues: this.secretValues,
-      steps: stepsContext,
-      item,
-      index,
-      env: {},
-      output: item
-        ? undefined
-        : this.stepContexts.get(this.workflow.steps.find((s) => !s.foreach)?.id || '')?.output,
-      last_failed_step: this.lastFailedStep,
-    };
-
-    const resolvedEnv: Record<string, string> = {};
-    for (const [key, value] of Object.entries(process.env)) {
-      if (value !== undefined) {
-        resolvedEnv[key] = value;
-      }
-    }
-
-    if (this.workflow.env) {
-      for (const [key, value] of Object.entries(this.workflow.env)) {
-        try {
-          resolvedEnv[key] = ExpressionEvaluator.evaluateString(value, {
-            ...baseContext,
-            env: resolvedEnv,
-          });
-        } catch (error) {
-          this.logger.warn(
-            `Warning: Failed to evaluate workflow env "${key}": ${error instanceof Error ? error.message : String(error)}`
-          );
-        }
-      }
-    }
-
-    baseContext.env = resolvedEnv;
-    return baseContext;
-  }
-
   /**
    * Evaluate a conditional expression
    */
@@ -1082,10 +781,9 @@ export class WorkflowRunner {
     try {
       return !this.evaluateCondition(step.if, context);
     } catch (error) {
-      this.logger.error(
-        `Warning: Failed to evaluate condition for step ${step.id}: ${error instanceof Error ? error.message : String(error)}`
+      throw new Error(
+        `Failed to evaluate condition for step "${step.id}": ${error instanceof Error ? error.message : String(error)}`
       );
-      return true; // Skip on error
     }
   }
 
@@ -1134,58 +832,30 @@ export class WorkflowRunner {
     try {
       await this.db.clearExpiredIdempotencyRecord(scopedKey);
 
-      const existing = await this.db.getIdempotencyRecord(scopedKey);
-      if (existing) {
-        if (existing.status === StepStatus.SUCCESS) {
+      const result = await this.db.atomicClaimIdempotencyKey(
+        scopedKey,
+        this.runId,
+        stepId,
+        ttlSeconds
+      );
+
+      switch (result.status) {
+        case 'claimed':
+          return { status: 'claimed' };
+        case 'already-running':
+          return { status: 'in-flight' };
+        case 'completed': {
           let output: unknown = null;
           try {
-            output = existing.output ? JSON.parse(existing.output) : null;
+            output = result.record.output ? JSON.parse(result.record.output) : null;
           } catch (parseError) {
             this.logger.warn(
               `  ⚠️ Failed to parse idempotency output for ${stepId}: ${parseError instanceof Error ? parseError.message : String(parseError)}`
             );
           }
-          return { status: 'hit', output, error: existing.error || undefined };
-        }
-        if (existing.status === StepStatus.RUNNING) {
-          return { status: 'in-flight' };
+          return { status: 'hit', output, error: result.record.error || undefined };
         }
-
-        const claimed = await this.db.markIdempotencyRecordRunning(
-          scopedKey,
-          this.runId,
-          stepId,
-          ttlSeconds
-        );
-        if (claimed) {
-          return { status: 'claimed' };
-        }
-      }
-
-      const inserted = await this.db.insertIdempotencyRecordIfAbsent(
-        scopedKey,
-        this.runId,
-        stepId,
-        StepStatus.RUNNING,
-        ttlSeconds
-      );
-      if (inserted) {
-        return { status: 'claimed' };
       }
-
-      const current = await this.db.getIdempotencyRecord(scopedKey);
-      if (current?.status === StepStatus.SUCCESS) {
-        let output: unknown = null;
-        try {
-          output = current.output ? JSON.parse(current.output) : null;
-        } catch (parseError) {
-          this.logger.warn(
-            `  ⚠️ Failed to parse idempotency output for ${stepId}: ${parseError instanceof Error ? parseError.message : String(parseError)}`
-          );
-        }
-        return { status: 'hit', output, error: current.error || undefined };
-      }
-      return { status: 'in-flight' };
     } catch (error) {
       this.logger.warn(
         `  ⚠️ Failed to claim idempotency key for ${stepId}: ${error instanceof Error ? error.message : String(error)}`
@@ -1279,7 +949,7 @@ export class WorkflowRunner {
             stepExecId,
             StepStatus.FAILED,
             null,
-            this.redactAtRest ? this.redactor.redact(errorMsg) : errorMsg
+            this.secretManager.redactAtRest ? this.secretManager.redact(errorMsg) : errorMsg
           );
           return {
             output: null,
@@ -1292,6 +962,26 @@ export class WorkflowRunner {
       }
     }
 
+    // Global step caching (memoization)
+    const inputs = this.contextBuilder.buildStepInputs(step, context);
+    const cacheKey = await this.calculateStepCacheKey(step, inputs);
+    if (cacheKey) {
+      const cached = await this.db.getStepCache(cacheKey);
+      if (cached) {
+        this.logger.log(`  ⚡ Step ${step.id} skipped (global cache hit)`);
+        const output = JSON.parse(cached.output);
+        await this.db.completeStep(stepExecId, StepStatus.SUCCESS, output);
+        return {
+          output,
+          outputs:
+            typeof output === 'object' && output !== null && !Array.isArray(output)
+              ? (output as Record<string, unknown>)
+              : {},
+          status: StepStatus.SUCCESS,
+        };
+      }
+    }
+
     const idempotencyContextForRetry =
       idempotencyClaimed && scopedIdempotencyKey
         ? {
@@ -1325,21 +1015,85 @@ export class WorkflowRunner {
       await this.db.startStep(stepExecId);
     }
 
-    const operation = async (attemptContext: ExpressionContext) => {
+    if (stepToExecute.breakpoint && this.options.debug && !isRecursion) {
+      if (!process.stdin.isTTY) {
+        const message = `Breakpoint hit before step ${stepToExecute.id}`;
+        if (dedupEnabled && idempotencyClaimed) {
+          await this.recordIdempotencyResult(
+            scopedIdempotencyKey,
+            stepToExecute.id,
+            StepStatus.SUSPENDED,
+            null,
+            message,
+            idempotencyTtlSeconds
+          );
+        }
+        await this.db.completeStep(
+          stepExecId,
+          StepStatus.SUSPENDED,
+          null,
+          this.secretManager.redactAtRest ? this.secretManager.redact(message) : message
+        );
+        return {
+          output: null,
+          outputs: {},
+          status: StepStatus.SUSPENDED,
+          error: message,
+        };
+      }
+
+      try {
+        const { DebugRepl } = await import('./debug-repl.ts');
+        const repl = new DebugRepl(
+          context,
+          stepToExecute,
+          undefined,
+          this.logger,
+          process.stdin,
+          process.stdout,
+          {
+            mode: 'breakpoint',
+          }
+        );
+        const action = await repl.start();
+
+        if (action.type === 'skip') {
+          this.logger.log(`  ⏭️ Skipping step ${stepToExecute.id} at breakpoint`);
+          await this.db.completeStep(stepExecId, StepStatus.SKIPPED, null, undefined, undefined);
+          return {
+            output: null,
+            outputs: {},
+            status: StepStatus.SKIPPED,
+          };
+        }
+
+        if (action.type === 'continue' || action.type === 'retry') {
+          stepToExecute = action.modifiedStep || stepToExecute;
+        }
+      } catch (replError) {
+        this.logger.error(`  ✗ Debug REPL error: ${replError}`);
+      }
+    }
+
+    const operation = async (attemptContext: ExpressionContext, abortSignal?: AbortSignal) => {
       const exec = this.options.executeStep || executeStep;
-      const result = await exec(stepToExecute, attemptContext, this.logger, {
+      let result = await exec(stepToExecute, attemptContext, this.logger, {
         executeWorkflowFn: this.executeSubWorkflow.bind(this),
         mcpManager: this.mcpManager,
+        db: this.db,
         memoryDb: this.memoryDb,
         workflowDir: this.options.workflowDir,
         dryRun: this.options.dryRun,
-        abortSignal: this.abortSignal,
+        abortSignal,
         runId: this.runId,
         stepExecutionId: stepExecId,
         artifactRoot: this.options.artifactRoot,
-        redactForStorage: this.redactForStorage.bind(this),
+        redactForStorage: this.secretManager.redactForStorage.bind(this.secretManager),
         getAdapter: this.options.getAdapter,
         executeStep: this.options.executeStep || executeStep,
+        executeLlmStep: this.options.executeLlmStep,
+        emitEvent: this.emitEvent.bind(this),
+        workflowName: this.workflow.name,
       });
       if (result.status === 'failed') {
         throw new StepExecutionError(result);
@@ -1353,7 +1107,7 @@ export class WorkflowRunner {
             'summary' in result.output
               ? (result.output as { summary?: unknown }).summary
               : result.output;
-          this.validateSchema(
+          this.validator.validateSchema(
             'output',
             stepToExecute.outputSchema,
             outputForValidation,
@@ -1363,6 +1117,7 @@ export class WorkflowRunner {
           const message = error instanceof Error ? error.message : String(error);
           const outputRetries = stepToExecute.outputRetries || 0;
           const currentAttempt = (attemptContext.outputRepairAttempts as number) || 0;
+          let handled = false;
 
           // Only attempt repair for LLM steps with outputRetries configured
           if (stepToExecute.type === 'llm' && outputRetries > 0 && currentAttempt < outputRetries) {
@@ -1396,15 +1151,19 @@ export class WorkflowRunner {
             const repairResult = await exec(repairStep, repairContext, this.logger, {
               executeWorkflowFn: this.executeSubWorkflow.bind(this),
               mcpManager: this.mcpManager,
+              db: this.db,
               memoryDb: this.memoryDb,
               workflowDir: this.options.workflowDir,
               dryRun: this.options.dryRun,
-              abortSignal: this.abortSignal,
+              abortSignal,
               runId: this.runId,
               stepExecutionId: stepExecId,
               artifactRoot: this.options.artifactRoot,
-              redactForStorage: this.redactForStorage.bind(this),
+              redactForStorage: this.secretManager.redactForStorage.bind(this.secretManager),
               executeStep: this.options.executeStep || executeStep,
+              executeLlmStep: this.options.executeLlmStep,
+              emitEvent: this.emitEvent.bind(this),
+              workflowName: this.workflow.name,
             });
 
             if (repairResult.status === 'failed') {
@@ -1413,7 +1172,7 @@ export class WorkflowRunner {
 
             // Validate the repaired output
             try {
-              this.validateSchema(
+              this.validator.validateSchema(
                 'output',
                 stepToExecute.outputSchema,
                 repairResult.output,
@@ -1422,15 +1181,19 @@ export class WorkflowRunner {
               this.logger.log(
                 `  ✓ Output repair successful after ${currentAttempt + 1} attempt(s)`
               );
-              return repairResult;
+              result = repairResult;
+              handled = true;
             } catch (repairError) {
               // If still failing, either retry again or give up
               if (currentAttempt + 1 < outputRetries) {
                 // Try again with updated context
-                return operation({
-                  ...attemptContext,
-                  outputRepairAttempts: currentAttempt + 1,
-                });
+                return operation(
+                  {
+                    ...attemptContext,
+                    outputRepairAttempts: currentAttempt + 1,
+                  },
+                  abortSignal
+                );
               }
               const repairMessage =
                 repairError instanceof Error ? repairError.message : String(repairError);
@@ -1442,20 +1205,29 @@ export class WorkflowRunner {
             }
           }
 
-          throw new StepExecutionError({
-            ...result,
-            status: 'failed',
-            error: message,
-          });
+          if (!handled) {
+            throw new StepExecutionError({
+              ...result,
+              status: 'failed',
+              error: message,
+            });
+          }
         }
       }
+      if (
+        result.status === 'success' &&
+        (stepToExecute.type === 'llm' || stepToExecute.type === 'plan') &&
+        stepToExecute.qualityGate
+      ) {
+        result = await this.runQualityGate(stepToExecute, result, attemptContext, abortSignal);
+      }
       return result;
     };
 
     try {
       if (stepToExecute.inputSchema) {
-        const inputsForValidation = this.buildStepInputs(stepToExecute, context);
-        this.validateSchema(
+        const inputsForValidation = this.contextBuilder.buildStepInputs(stepToExecute, context);
+        this.validator.validateSchema(
           'input',
           stepToExecute.inputSchema,
           inputsForValidation,
@@ -1464,10 +1236,18 @@ export class WorkflowRunner {
       }
 
       const operationWithTimeout = async () => {
-        if (step.timeout) {
-          return await withTimeout(operation(context), step.timeout, `Step ${step.id}`);
+        const { controller, cleanup } = this.createStepAbortController();
+        try {
+          const attempt = operation(context, controller.signal);
+          if (step.timeout) {
+            return await withTimeout(attempt, step.timeout, `Step ${step.id}`, {
+              abortController: controller,
+            });
+          }
+          return await attempt;
+        } finally {
+          cleanup();
         }
-        return await operation(context);
       };
 
       const result = await withRetry(operationWithTimeout, step.retry, async (attempt, error) => {
@@ -1475,10 +1255,10 @@ export class WorkflowRunner {
         await this.db.incrementRetry(stepExecId);
       });
 
-      const persistedOutput = this.redactForStorage(result.output);
+      const persistedOutput = this.secretManager.redactForStorage(result.output);
       const persistedError = result.error
-        ? this.redactAtRest
-          ? this.redactor.redact(result.error)
+        ? this.secretManager.redactAtRest
+          ? this.secretManager.redact(result.error)
           : result.error
         : result.error;
 
@@ -1504,8 +1284,8 @@ export class WorkflowRunner {
           stepExecId,
           StepStatus.SUSPENDED,
           persistedOutput,
-          this.redactAtRest
-            ? this.redactor.redact('Waiting for interaction')
+          this.secretManager.redactAtRest
+            ? this.secretManager.redact('Waiting for interaction')
             : 'Waiting for interaction',
           result.usage
         );
@@ -1535,7 +1315,7 @@ export class WorkflowRunner {
           stepExecId,
           StepStatus.WAITING,
           persistedOutput,
-          this.redactAtRest ? this.redactor.redact(waitError) : waitError,
+          this.secretManager.redactAtRest ? this.secretManager.redact(waitError) : waitError,
           result.usage
         );
         result.error = waitError;
@@ -1549,7 +1329,7 @@ export class WorkflowRunner {
         persistedError,
         result.usage
       );
-      if (step.type === 'human') {
+      if (result.status === StepStatus.SUCCESS) {
         const existingTimer = await this.db.getTimerByStep(this.runId, step.id);
         if (existingTimer) {
           await this.db.completeTimer(existingTimer.id);
@@ -1618,14 +1398,25 @@ export class WorkflowRunner {
         );
       }
 
-      return {
+      const finalResult = {
         output: result.output,
         outputs,
         status: result.status,
         error: result.error,
         usage: result.usage,
       };
+
+      // Store in global cache if enabled
+      if (cacheKey && result.status === StepStatus.SUCCESS) {
+        const ttl = step.memoizeTtlSeconds;
+        await this.db.storeStepCache(cacheKey, this.workflow.name, step.id, persistedOutput, ttl);
+      }
+
+      return finalResult;
     } catch (error) {
+      if (error instanceof WorkflowSuspendedError || error instanceof WorkflowWaitingError) {
+        throw error;
+      }
       // Reflexion (Self-Correction) logic
       if (step.reflexion) {
         const { limit = 3, hint } = step.reflexion;
@@ -1742,7 +1533,7 @@ export class WorkflowRunner {
       const failureResult = error instanceof StepExecutionError ? error.result : null;
       const errorMsg =
         failureResult?.error || (error instanceof Error ? error.message : String(error));
-      const redactedErrorMsg = this.redactor.redact(errorMsg);
+      const redactedErrorMsg = this.secretManager.redact(errorMsg);
       const failureOutput = failureResult?.output ?? null;
       const failureOutputs =
         typeof failureOutput === 'object' && failureOutput !== null && !Array.isArray(failureOutput)
@@ -1756,8 +1547,8 @@ export class WorkflowRunner {
         await this.db.completeStep(
           stepExecId,
           StepStatus.SUCCESS,
-          this.redactForStorage(failureOutput),
-          this.redactAtRest ? redactedErrorMsg : errorMsg
+          this.secretManager.redactForStorage(failureOutput),
+          this.secretManager.redactAtRest ? redactedErrorMsg : errorMsg
         );
         if (dedupEnabled && idempotencyClaimed) {
           await this.recordIdempotencyResult(
@@ -1781,8 +1572,8 @@ export class WorkflowRunner {
       await this.db.completeStep(
         stepExecId,
         StepStatus.FAILED,
-        this.redactForStorage(failureOutput),
-        this.redactAtRest ? redactedErrorMsg : errorMsg
+        this.secretManager.redactForStorage(failureOutput),
+        this.secretManager.redactAtRest ? redactedErrorMsg : errorMsg
       );
       if (dedupEnabled && idempotencyClaimed) {
         await this.recordIdempotencyResult(
@@ -1861,9 +1652,11 @@ Do not change the 'id' or 'type' or 'auto_heal' fields.
       debug: this.options.debug,
       runId: this.runId,
       artifactRoot: this.options.artifactRoot,
-      redactForStorage: this.redactForStorage.bind(this),
+      redactForStorage: this.secretManager.redactForStorage.bind(this.secretManager),
       allowInsecure: this.options.allowInsecure,
       executeStep: this.options.executeStep || executeStep,
+      emitEvent: this.emitEvent.bind(this),
+      workflowName: this.workflow.name,
     });
 
     if (result.status !== 'success' || !result.output) {
@@ -1892,11 +1685,9 @@ Do not change the 'id' or 'type' or 'auto_heal' fields.
     let textToEmbed = `Step ID: ${step.id} (${step.type})\n`;
 
     if (step.type === 'llm') {
-      // biome-ignore lint/suspicious/noExplicitAny: generic access
-      textToEmbed += `Task Context/Prompt:\n${(step as any).prompt}\n\n`;
+      textToEmbed += `Task Context/Prompt:\n${(step as LlmStep).prompt}\n\n`;
     } else if (step.type === 'shell') {
-      // biome-ignore lint/suspicious/noExplicitAny: generic access
-      textToEmbed += `Command:\n${(step as any).run}\n\n`;
+      textToEmbed += `Command:\n${(step as unknown as { run: string }).run}\n\n`;
     }
 
     textToEmbed += `Successful Outcome:\n${JSON.stringify(result.output, null, 2)}`;
@@ -1933,8 +1724,7 @@ Rules:
    - Creating missing directories
    - Adjusting flags or arguments`;
 
-    // biome-ignore lint/suspicious/noExplicitAny: generic access
-    const runCommand = (step as any).run;
+    const runCommand = (step as unknown as { run: string }).run;
     const userContent = `The following step failed:
 \`\`\`json
 ${JSON.stringify({ type: step.type, run: runCommand }, null, 2)}
@@ -1952,31 +1742,14 @@ Please provide the fixed step configuration as JSON.`;
       { role: 'user', content: userContent },
     ];
 
-    try {
-      // Use the default model (gpt-4o) or configured default for the Mechanic
-      // We'll use gpt-4o as a strong default for this reasoning task
-      const getAdapterFn = this.options.getAdapter || getAdapter;
-      const { adapter, resolvedModel } = getAdapterFn('gpt-4o');
-      this.logger.log(`  🤖 Mechanic is analyzing the failure using ${resolvedModel}...`);
-
-      const response = await adapter.chat(messages, {
-        model: resolvedModel,
-      });
+    // Use the default model (gpt-4o) or configured default for the Mechanic
+    // We'll use gpt-4o as a strong default for this reasoning task
+    const getAdapterFn = this.options.getAdapter || getAdapter;
+    const { adapter } = getAdapterFn('gpt-4o');
 
-      const content = response.message.content;
-      if (!content) {
-        throw new Error('Mechanic returned empty response');
-      }
+    const response = await adapter.chat(messages);
 
-      try {
-        const fixedConfig = extractJson(content) as Partial<Step>;
-        return fixedConfig;
-      } catch (e) {
-        throw new Error(`Failed to parse Mechanic's response as JSON: ${content}`);
-      }
-    } catch (err) {
-      throw new Error(`Mechanic unavailable: ${err instanceof Error ? err.message : String(err)}`);
-    }
+    return extractJson(response.message.content || '{}') as Partial<Step>;
   }
 
   /**
@@ -2026,18 +1799,233 @@ ${strategyInstructions[strategy]}
 Please provide a corrected response that exactly matches the required schema.`;
   }
 
+  private buildPlanPromptFromStep(step: PlanStep, context: ExpressionContext): string {
+    const goal = ExpressionEvaluator.evaluateString(step.goal, context);
+    const contextText = step.context
+      ? ExpressionEvaluator.evaluateString(step.context, context)
+      : '';
+    const constraintsText = step.constraints
+      ? ExpressionEvaluator.evaluateString(step.constraints, context)
+      : '';
+
+    return `You are a planner. Break the goal into a concise, ordered list of steps.
+
+Goal:
+${goal}
+
+Context:
+${contextText || 'None'}
+
+Constraints:
+${constraintsText || 'None'}
+
+Each step should be small, specific, and independently executable.
+Include any dependencies under "needs" and optional "workflow" or "inputs" when appropriate.
+
+Return only the structured JSON required by the schema.`;
+  }
+
+  private buildQualityGateReviewPrompt(
+    step: LlmStep | PlanStep,
+    output: unknown,
+    gatePrompt: string | undefined,
+    context: ExpressionContext
+  ): string {
+    const reviewContext = {
+      ...context,
+      output,
+    };
+
+    if (gatePrompt) {
+      return ExpressionEvaluator.evaluateString(gatePrompt, reviewContext);
+    }
+
+    const taskDescription =
+      step.type === 'plan' ? this.buildPlanPromptFromStep(step, context) : step.prompt;
+
+    return `Review the output for correctness, completeness, and clarity.
+
+Task:
+${taskDescription}
+
+Output:
+${typeof output === 'string' ? output : JSON.stringify(output, null, 2)}
+
+Identify issues, risks, and missing details. Be specific.
+Return only the structured JSON required by the schema.`;
+  }
+
+  private buildQualityGateRefinePrompt(
+    step: LlmStep | PlanStep,
+    output: unknown,
+    review: QualityGateReview,
+    context: ExpressionContext
+  ): string {
+    const basePrompt =
+      step.type === 'plan' ? this.buildPlanPromptFromStep(step, context) : step.prompt;
+    const reviewText = JSON.stringify(review, null, 2);
+    const outputText = typeof output === 'string' ? output : JSON.stringify(output, null, 2);
+
+    return `${basePrompt}
+
+---
+
+QUALITY REVIEW FAILED
+
+Reviewer feedback:
+${reviewText}
+
+Previous output:
+${outputText}
+
+Revise the output to address the feedback. Return only the corrected output.`;
+  }
+
+  private async runQualityGate(
+    step: LlmStep | PlanStep,
+    result: StepResult,
+    context: ExpressionContext,
+    abortSignal?: AbortSignal
+  ): Promise<StepResult> {
+    const gate = step.qualityGate;
+    if (!gate) return result;
+
+    let attempts = (context.qualityGateAttempts as number) || 0;
+    const maxAttempts = gate.maxAttempts ?? 1;
+    let currentResult = result;
+
+    while (true) {
+      if (abortSignal?.aborted) {
+        throw new Error('Step canceled');
+      }
+
+      const reviewContext = {
+        ...context,
+        output: currentResult.output,
+        qualityGateAttempts: attempts,
+      };
+      const reviewPrompt = this.buildQualityGateReviewPrompt(
+        step,
+        currentResult.output,
+        gate.prompt,
+        reviewContext
+      );
+      const reviewStep: Step = {
+        id: `${step.id}-quality-review`,
+        type: 'llm',
+        agent: gate.agent,
+        provider: gate.provider,
+        model: gate.model,
+        prompt: reviewPrompt,
+        outputSchema: QUALITY_GATE_SCHEMA,
+      } as LlmStep;
+
+      const exec = this.options.executeStep || executeStep;
+      const reviewResult = await exec(reviewStep, reviewContext, this.logger, {
+        executeWorkflowFn: this.executeSubWorkflow.bind(this),
+        mcpManager: this.mcpManager,
+        db: this.db,
+        memoryDb: this.memoryDb,
+        workflowDir: this.options.workflowDir,
+        dryRun: this.options.dryRun,
+        abortSignal,
+        runId: this.runId,
+        artifactRoot: this.options.artifactRoot,
+        redactForStorage: this.secretManager.redactForStorage.bind(this.secretManager),
+        getAdapter: this.options.getAdapter,
+        executeStep: this.options.executeStep || executeStep,
+        emitEvent: this.emitEvent.bind(this),
+        workflowName: this.workflow.name,
+      });
+
+      if (reviewResult.status !== 'success' || !reviewResult.output) {
+        throw new StepExecutionError({
+          ...reviewResult,
+          status: 'failed',
+          error: reviewResult.error || 'Quality gate review failed',
+        });
+      }
+
+      this.validator.validateSchema(
+        'output',
+        QUALITY_GATE_SCHEMA,
+        reviewResult.output,
+        reviewStep.id
+      );
+
+      const review = reviewResult.output as QualityGateReview;
+      if (review.approved) {
+        return currentResult;
+      }
+
+      if (attempts >= maxAttempts) {
+        const issues = review.issues?.join('; ') || 'Quality gate rejected output';
+        throw new StepExecutionError({
+          ...currentResult,
+          status: 'failed',
+          error: `Quality gate rejected: ${issues}`,
+        });
+      }
+
+      attempts += 1;
+      this.logger.log(`  🔍 Quality gate rejected output; refining (${attempts}/${maxAttempts})`);
+
+      const refinePrompt = this.buildQualityGateRefinePrompt(
+        step,
+        currentResult.output,
+        review,
+        context
+      );
+      const refinedStep: Step = {
+        ...step,
+        prompt: refinePrompt,
+      };
+      const refinedContext = {
+        ...context,
+        qualityGateAttempts: attempts,
+      };
+
+      const refinedResult = await exec(refinedStep, refinedContext, this.logger, {
+        executeWorkflowFn: this.executeSubWorkflow.bind(this),
+        mcpManager: this.mcpManager,
+        db: this.db,
+        memoryDb: this.memoryDb,
+        workflowDir: this.options.workflowDir,
+        dryRun: this.options.dryRun,
+        abortSignal,
+        runId: this.runId,
+        artifactRoot: this.options.artifactRoot,
+        redactForStorage: this.secretManager.redactForStorage.bind(this.secretManager),
+        getAdapter: this.options.getAdapter,
+        executeStep: this.options.executeStep || executeStep,
+        emitEvent: this.emitEvent.bind(this),
+        workflowName: this.workflow.name,
+      });
+
+      if (refinedResult.status === 'failed') {
+        throw new StepExecutionError(refinedResult);
+      }
+
+      if (step.outputSchema) {
+        this.validator.validateSchema('output', step.outputSchema, refinedResult.output, step.id);
+      }
+
+      currentResult = refinedResult;
+    }
+  }
+
   /**
    * Execute a step (handles foreach if present)
    */
   private async executeStepWithForeach(step: Step): Promise<void> {
-    const baseContext = this.buildContext();
+    const baseContext = this.contextBuilder.buildContext(this.secretManager.getSecrets());
 
     if (this.shouldSkipStep(step, baseContext)) {
       this.logger.log(`  ⊘ Skipping step ${step.id} (condition not met)`);
       const stepExecId = randomUUID();
       await this.db.createStep(stepExecId, this.runId, step.id);
       await this.db.completeStep(stepExecId, 'skipped', null);
-      this.stepContexts.set(step.id, { status: 'skipped' });
+      this.state.set(step.id, { status: 'skipped' });
       return;
     }
 
@@ -2046,12 +2034,11 @@ Please provide a corrected response that exactly matches the required schema.`;
       const stepExecId = randomUUID();
       await this.db.createStep(stepExecId, this.runId, step.id);
       await this.db.completeStep(stepExecId, StepStatus.SKIPPED, null);
-      this.stepContexts.set(step.id, { status: StepStatus.SKIPPED });
+      this.state.set(step.id, { status: StepStatus.SKIPPED });
       return;
     }
 
     if (step.foreach) {
-      const { ForeachExecutor } = await import('./foreach-executor.ts');
       const executor = new ForeachExecutor(
         this.db,
         this.logger,
@@ -2060,10 +2047,10 @@ Please provide a corrected response that exactly matches the required schema.`;
         this.resourcePool
       );
 
-      const existingContext = this.stepContexts.get(step.id) as ForeachStepContext;
+      const existingContext = this.state.get(step.id) as ForeachStepContext;
       const result = await executor.execute(step, baseContext, this.runId, existingContext);
 
-      this.stepContexts.set(step.id, result);
+      this.state.set(step.id, result);
     } else {
       // Single execution
       const stepExecId = randomUUID();
@@ -2072,7 +2059,7 @@ Please provide a corrected response that exactly matches the required schema.`;
       const result = await this.executeStepInternal(step, baseContext, stepExecId);
 
       // Update global state
-      this.stepContexts.set(step.id, result);
+      this.state.set(step.id, result);
 
       if (result.status === 'suspended') {
         const inputType = step.type === 'human' ? step.inputType : 'text';
@@ -2096,99 +2083,127 @@ Please provide a corrected response that exactly matches the required schema.`;
    */
   private async executeSubWorkflow(
     step: WorkflowStep,
-    context: ExpressionContext
+    context: ExpressionContext,
+    abortSignal?: AbortSignal
   ): Promise<StepResult> {
-    const workflowPath = WorkflowRegistry.resolvePath(step.path, this.options.workflowDir);
-    const workflow = WorkflowParser.loadWorkflow(workflowPath);
-    const subWorkflowDir = dirname(workflowPath);
-
-    // Evaluate inputs for the sub-workflow
-    const inputs: Record<string, unknown> = {};
-    if (step.inputs) {
-      for (const [key, value] of Object.entries(step.inputs)) {
-        inputs[key] = ExpressionEvaluator.evaluate(value, context);
-      }
-    }
+    const factory: RunnerFactory = {
+      create: (workflow, options) => new WorkflowRunner(workflow, options),
+    };
 
-    // Create a new runner for the sub-workflow
-    // We pass the same dbPath to share the state database
-    const subRunner = new WorkflowRunner(workflow, {
-      inputs,
-      dbPath: this.db.dbPath,
-      logger: this.logger,
-      mcpManager: this.mcpManager,
-      workflowDir: subWorkflowDir,
-      depth: this.depth + 1,
-      dedup: this.options.dedup,
-      artifactRoot: this.options.artifactRoot,
+    return executeSubWorkflow(step, context, {
+      runnerFactory: factory,
+      parentWorkflowDir: this.options.workflowDir,
+      parentDbPath: this.db.dbPath,
+      parentLogger: this.logger,
+      parentMcpManager: this.mcpManager,
+      parentDepth: this.depth,
+      parentOptions: this.options,
+      abortSignal,
     });
+  }
 
-    try {
-      const output = await subRunner.run();
-
-      const rawOutputs =
-        typeof output === 'object' && output !== null && !Array.isArray(output) ? output : {};
-      const mappedOutputs: Record<string, unknown> = {};
-
-      // Handle explicit output mapping
-      if (step.outputMapping) {
-        for (const [alias, mapping] of Object.entries(step.outputMapping)) {
-          let originalKey: string;
-          let defaultValue: unknown;
-
-          if (typeof mapping === 'string') {
-            originalKey = mapping;
-          } else {
-            originalKey = mapping.from;
-            defaultValue = mapping.default;
-          }
+  /**
+   * Redact secrets from a value
+   */
+  public redact<T>(value: T): T {
+    return this.secretManager.redactValue(value) as T;
+  }
 
-          if (originalKey in rawOutputs) {
-            mappedOutputs[alias] = rawOutputs[originalKey];
-          } else if (defaultValue !== undefined) {
-            mappedOutputs[alias] = defaultValue;
-          } else {
-            throw new Error(
-              `Sub-workflow output "${originalKey}" not found (required by mapping "${alias}" in step "${step.id}")`
+  private emitEvent(event: WorkflowEvent): void {
+    try {
+      const redactor = this.secretManager.getRedactor();
+      const redacted = redactor.redactValue(event) as WorkflowEvent;
+      if (redacted.type === 'llm.thought') {
+        void this.db
+          .storeThoughtEvent(
+            redacted.runId,
+            redacted.workflow,
+            redacted.stepId,
+            redacted.content,
+            redacted.source
+          )
+          .catch((error) => {
+            this.logger.warn(
+              `  ⚠️ Failed to store thought event: ${error instanceof Error ? error.message : String(error)}`
             );
-          }
-        }
+          });
+      }
+      if (this.options.onEvent) {
+        this.options.onEvent(redacted);
       }
-
-      return {
-        output: {
-          ...mappedOutputs,
-          outputs: rawOutputs, // Namespaced raw outputs
-          __subRunId: subRunner.runId, // Track sub-workflow run ID for rollback
-        },
-        status: 'success',
-      };
     } catch (error) {
-      return {
-        output: null,
-        status: 'failed',
-        error: error instanceof Error ? error.message : String(error),
-      };
+      this.logger.warn(
+        `  ⚠️ Failed to emit event: ${error instanceof Error ? error.message : String(error)}`
+      );
     }
   }
 
-  /**
-   * Redact secrets from a value
-   */
-  public redact<T>(value: T): T {
-    return this.redactor.redactValue(value) as T;
+  private emitStepStart(
+    step: Step,
+    phase: StepPhase,
+    stepIndex?: number,
+    totalSteps?: number
+  ): number {
+    const startedAt = Date.now();
+    this.emitEvent({
+      type: 'step.start',
+      timestamp: new Date(startedAt).toISOString(),
+      runId: this.runId,
+      workflow: this.workflow.name,
+      stepId: step.id,
+      stepType: step.type,
+      phase,
+      stepIndex,
+      totalSteps,
+    });
+    return startedAt;
+  }
+
+  private emitStepEnd(
+    step: Step,
+    phase: StepPhase,
+    startedAt: number,
+    error?: unknown,
+    stepIndex?: number,
+    totalSteps?: number
+  ): void {
+    const endedAt = Date.now();
+    const context = this.state.get(step.id);
+    const status = context?.status || StepStatus.FAILED;
+    const errorMsg =
+      context?.error ||
+      (error instanceof Error ? error.message : error ? String(error) : undefined);
+
+    this.emitEvent({
+      type: 'step.end',
+      timestamp: new Date(endedAt).toISOString(),
+      runId: this.runId,
+      workflow: this.workflow.name,
+      stepId: step.id,
+      stepType: step.type,
+      phase,
+      status,
+      durationMs: endedAt - startedAt,
+      error: status === StepStatus.SUCCESS || status === StepStatus.SKIPPED ? undefined : errorMsg,
+      stepIndex,
+      totalSteps,
+    });
   }
 
   /**
    * Execute the workflow
    */
   async run(): Promise<Record<string, unknown>> {
+    const expressionStrict = ConfigLoader.load().expression?.strict ?? false;
+    ExpressionEvaluator.setStrictMode(expressionStrict);
+    let completionEvent: WorkflowEvent | null = null;
+
     // Handle resume state restoration
     if (this.resumeRunId && !this.restored) {
       await this.restoreState();
     }
 
-    const isResume = !!this.resumeRunId || this.stepContexts.size > 0;
+    const isResume = !!this.resumeRunId || this.state.size > 0;
 
     this.logger.log(`\n🏛️  ${isResume ? 'Resuming' : 'Running'} workflow: ${this.workflow.name}`);
     this.logger.log(`Run ID: ${this.runId}`);
@@ -2197,56 +2212,80 @@ Please provide a corrected response that exactly matches the required schema.`;
         '   Workflows can execute arbitrary shell commands and access your environment.\n'
     );
 
-    this.redactAtRest = ConfigLoader.load().storage?.redact_secrets_at_rest ?? true;
+    this.secretManager.redactAtRest = ConfigLoader.load().storage?.redact_secrets_at_rest ?? true;
 
     // Apply defaults and validate inputs
-    this.applyDefaultsAndValidate();
+    const validated = this.validator.applyDefaultsAndValidate();
+    if (validated.secretValues.length > 0) {
+      this.secretManager.setSecretValues(validated.secretValues);
+      this.logger = new RedactingLogger(this.rawLogger, this.secretManager.getRedactor());
+      this.contextBuilder = new ContextBuilder(
+        this.workflow,
+        this.inputs,
+        this.secretManager.getSecretValues(),
+        this.state,
+        this.logger
+      );
+    }
 
     // Create run record (only for new runs, not for resume)
     if (!isResume) {
-      await this.db.createRun(this.runId, this.workflow.name, this.redactForStorage(this.inputs));
+      await this.db.createRun(
+        this.runId,
+        this.workflow.name,
+        this.secretManager.redactForStorage(this.inputs)
+      );
     }
     await this.db.updateRunStatus(this.runId, 'running');
+    this.emitEvent({
+      type: 'workflow.start',
+      timestamp: new Date().toISOString(),
+      runId: this.runId,
+      workflow: this.workflow.name,
+      inputs: this.secretManager.redactValue(this.inputs),
+    });
 
     try {
-      // Get execution order using topological sort
-      const executionOrder = WorkflowParser.topologicalSort(this.workflow);
-      const stepMap = new Map(this.workflow.steps.map((s) => [s.id, s]));
-
-      // Initialize completedSteps with already completed steps (for resume)
-      // Only include steps that were successful or skipped, so failed steps are retried
-      const completedSteps = new Set<string>();
-      for (const [id, ctx] of this.stepContexts.entries()) {
-        if (ctx.status === 'success' || ctx.status === 'skipped') {
-          completedSteps.add(id);
-        }
-      }
-
-      // Filter out already completed steps from execution order
-      const remainingSteps = executionOrder.filter((stepId) => !completedSteps.has(stepId));
+      // Use scheduler's execution order
+      const executionOrder = this.scheduler.getExecutionOrder();
 
-      if (isResume && remainingSteps.length === 0) {
+      if (isResume && this.scheduler.isComplete()) {
         this.logger.log('All steps already completed. Nothing to resume.\n');
         // Evaluate outputs from completed state
         const outputs = this.evaluateOutputs();
-        await this.db.updateRunStatus(this.runId, 'success', this.redactForStorage(outputs));
+        await this.db.updateRunStatus(
+          this.runId,
+          'success',
+          this.secretManager.redactForStorage(outputs)
+        );
         this.logger.log('✨ Workflow already completed!\n');
+        completionEvent = {
+          type: 'workflow.complete',
+          timestamp: new Date().toISOString(),
+          runId: this.runId,
+          workflow: this.workflow.name,
+          status: WorkflowStatus.SUCCESS,
+          outputs: this.secretManager.redactValue(outputs),
+        };
         return outputs;
       }
 
-      if (isResume && completedSteps.size > 0) {
-        this.logger.log(`Skipping ${completedSteps.size} already completed step(s)\n`);
+      const pendingCount = this.scheduler.getPendingCount();
+      const totalSteps = executionOrder.length;
+      const completedCount = totalSteps - pendingCount;
+
+      if (isResume && completedCount > 0) {
+        this.logger.log(`Skipping ${completedCount} already completed step(s)\n`);
       }
 
       this.logger.log(`Execution order: ${executionOrder.join(' → ')}\n`);
 
-      const totalSteps = executionOrder.length;
       const stepIndices = new Map(executionOrder.map((id, index) => [id, index + 1]));
 
       // Evaluate global concurrency limit
-      let globalConcurrencyLimit = remainingSteps.length;
+      let globalConcurrencyLimit = pendingCount || 10;
       if (this.workflow.concurrency !== undefined) {
-        const baseContext = this.buildContext();
+        const baseContext = this.contextBuilder.buildContext(this.secretManager.getSecrets());
         if (typeof this.workflow.concurrency === 'string') {
           globalConcurrencyLimit = Number(
             ExpressionEvaluator.evaluate(this.workflow.concurrency, baseContext)
@@ -2267,11 +2306,10 @@ Please provide a corrected response that exactly matches the required schema.`;
       }
 
       // Execute steps in parallel where possible (respecting dependencies and global concurrency)
-      const pendingSteps = new Set(remainingSteps);
       const runningPromises = new Map<string, Promise<void>>();
 
       try {
-        while (pendingSteps.size > 0 || runningPromises.size > 0) {
+        while (!this.scheduler.isComplete() || runningPromises.size > 0) {
           // Check for cancellation - drain in-flight steps but don't start new ones
           if (this.isCanceled) {
             if (runningPromises.size > 0) {
@@ -2283,73 +2321,71 @@ Please provide a corrected response that exactly matches the required schema.`;
             throw new Error('Workflow canceled by user');
           }
 
-          // 1. Find runnable steps (all dependencies met)
-          for (const stepId of pendingSteps) {
+          // 1. Find runnable steps from scheduler
+          const runnableSteps = this.scheduler
+            .getRunnableSteps(runningPromises.size, globalConcurrencyLimit)
+            .filter((step) => this.resourcePool.hasCapacity(step.pool || step.type));
+
+          for (const step of runnableSteps) {
             // Don't schedule new steps if canceled
             if (this.isCanceled) break;
 
-            const step = stepMap.get(stepId);
-            if (!step) {
-              throw new Error(`Step ${stepId} not found in workflow`);
-            }
+            const stepId = step.id;
+            this.scheduler.startStep(stepId);
 
-            let dependenciesMet = false;
-            if (step.type === 'join') {
-              dependenciesMet = this.isJoinConditionMet(
-                step as import('../parser/schema.ts').JoinStep,
-                completedSteps
-              );
-            } else {
-              dependenciesMet = step.needs.every((dep: string) => completedSteps.has(dep));
-            }
+            // Determine pool for this step
+            const poolName = step.pool || step.type;
 
-            if (dependenciesMet && runningPromises.size < globalConcurrencyLimit) {
-              pendingSteps.delete(stepId);
-
-              // Determine pool for this step
-              const poolName = step.pool || step.type;
-
-              // Start execution
-              const stepIndex = stepIndices.get(stepId);
-
-              const promise = (async () => {
-                let release: (() => void) | undefined;
-                try {
-                  this.logger.debug?.(
-                    `[${stepIndex}/${totalSteps}] ⏳ Waiting for pool: ${poolName}`
-                  );
-                  release = await this.resourcePool.acquire(poolName, { signal: this.abortSignal });
-
-                  this.logger.log(
-                    `[${stepIndex}/${totalSteps}] ▶ Executing step: ${step.id} (${step.type})`
-                  );
-
-                  await this.executeStepWithForeach(step);
-                  completedSteps.add(stepId);
-                  this.logger.log(`[${stepIndex}/${totalSteps}] ✓ Step ${step.id} completed\n`);
-                } finally {
-                  if (typeof release === 'function') {
-                    release();
-                  }
-                  runningPromises.delete(stepId);
+            // Start execution
+            const stepIndex = stepIndices.get(stepId);
+
+            const promise = (async () => {
+              let release: (() => void) | undefined;
+              const startedAt = this.emitStepStart(step, 'main', stepIndex, totalSteps);
+              try {
+                this.logger.debug?.(
+                  `[${stepIndex}/${totalSteps}] ⏳ Waiting for pool: ${poolName}`
+                );
+                release = await this.resourcePool.acquire(poolName, { signal: this.abortSignal });
+
+                this.logger.log(
+                  `[${stepIndex}/${totalSteps}] ▶ Executing step: ${step.id} (${step.type})`
+                );
+
+                await this.executeStepWithForeach(step);
+                this.emitStepEnd(step, 'main', startedAt, undefined, stepIndex, totalSteps);
+                this.scheduler.markStepComplete(stepId);
+                this.logger.log(`[${stepIndex}/${totalSteps}] ✓ Step ${step.id} completed\n`);
+              } catch (error) {
+                this.emitStepEnd(step, 'main', startedAt, error, stepIndex, totalSteps);
+                throw error;
+              } finally {
+                if (typeof release === 'function') {
+                  release();
                 }
-              })();
+                runningPromises.delete(stepId);
+              }
+            })();
 
-              runningPromises.set(stepId, promise);
-            }
+            runningPromises.set(stepId, promise);
           }
 
           // 2. Detect deadlock (only if not canceled)
-          if (!this.isCanceled && runningPromises.size === 0 && pendingSteps.size > 0) {
-            const pendingList = Array.from(pendingSteps).join(', ');
-            throw new Error(
-              `Deadlock detected in workflow execution. Pending steps: ${pendingList}`
-            );
+          if (!this.isCanceled && runningPromises.size === 0 && !this.scheduler.isComplete()) {
+            // Check if there are ANY steps whose dependencies are met, even if they're blocked by capacity/concurrency
+            const readySteps = this.scheduler.getRunnableSteps(0, Number.MAX_SAFE_INTEGER);
+            if (readySteps.length === 0) {
+              throw new Error(
+                'Deadlock detected in workflow execution. Steps remaining but none runnable (dependency cycles or missing inputs).'
+              );
+            }
           }
 
           // 3. Wait for at least one step to finish before checking again
           if (runningPromises.size > 0) {
             await Promise.race(runningPromises.values());
+            // Yield to event loop to prevent tight loop if multiple steps finish in same tick
+            await Bun.sleep(0);
           }
         }
       } catch (error) {
@@ -2369,37 +2405,61 @@ Please provide a corrected response that exactly matches the required schema.`;
         throw error;
       }
 
-      // Determine final status
-      const failedSteps = remainingSteps.filter(
-        (id) => this.stepContexts.get(id)?.status === StepStatus.FAILED
-      );
-
       // Evaluate outputs
       const outputs = this.evaluateOutputs();
 
       // Mark run as complete
-      await this.db.updateRunStatus(this.runId, 'success', this.redactForStorage(outputs));
+      await this.db.updateRunStatus(
+        this.runId,
+        'success',
+        this.secretManager.redactForStorage(outputs)
+      );
 
       this.logger.log('✨ Workflow completed successfully!\n');
 
+      completionEvent = {
+        type: 'workflow.complete',
+        timestamp: new Date().toISOString(),
+        runId: this.runId,
+        workflow: this.workflow.name,
+        status: WorkflowStatus.SUCCESS,
+        outputs: this.secretManager.redactValue(outputs),
+      };
+
       return outputs;
     } catch (error) {
       if (error instanceof WorkflowSuspendedError) {
         await this.db.updateRunStatus(this.runId, 'paused');
         this.logger.log(`\n⏸  Workflow paused: ${error.message}`);
+        completionEvent = {
+          type: 'workflow.complete',
+          timestamp: new Date().toISOString(),
+          runId: this.runId,
+          workflow: this.workflow.name,
+          status: WorkflowStatus.PAUSED,
+          error: error.message,
+        };
         throw error;
       }
 
       if (error instanceof WorkflowWaitingError) {
         await this.db.updateRunStatus(this.runId, 'paused');
         this.logger.log(`\n⏳ Workflow waiting: ${error.message}`);
+        completionEvent = {
+          type: 'workflow.complete',
+          timestamp: new Date().toISOString(),
+          runId: this.runId,
+          workflow: this.workflow.name,
+          status: WorkflowStatus.PAUSED,
+          error: error.message,
+        };
         throw error;
       }
 
       const errorMsg = error instanceof Error ? error.message : String(error);
 
       // Find the failed step from stepContexts
-      for (const [stepId, ctx] of this.stepContexts.entries()) {
+      for (const [stepId, ctx] of this.state.entries()) {
         if (ctx.status === 'failed') {
           this.lastFailedStep = { id: stepId, error: ctx.error || errorMsg };
           break;
@@ -2414,12 +2474,23 @@ Please provide a corrected response that exactly matches the required schema.`;
         this.runId,
         'failed',
         undefined,
-        this.redactAtRest ? this.redactor.redact(errorMsg) : errorMsg
+        this.secretManager.redactAtRest ? this.secretManager.redact(errorMsg) : errorMsg
       );
+      completionEvent = {
+        type: 'workflow.complete',
+        timestamp: new Date().toISOString(),
+        runId: this.runId,
+        workflow: this.workflow.name,
+        status: WorkflowStatus.FAILED,
+        error: errorMsg,
+      };
       throw error;
     } finally {
       this.removeSignalHandlers();
       await this.runFinally();
+      if (completionEvent) {
+        this.emitEvent(completionEvent);
+      }
       if (!this.options.mcpManager) {
         await this.mcpManager.stopAll();
       }
@@ -2450,9 +2521,9 @@ Please provide a corrected response that exactly matches the required schema.`;
           const step = stepMap.get(stepId);
           if (!step) continue;
 
-          // Dependencies can be from main steps (already in this.stepContexts) or previous finally steps
+          // Dependencies can be from main steps (already in this.state) or previous finally steps
           const dependenciesMet = step.needs.every(
-            (dep: string) => this.stepContexts.has(dep) || completedFinallySteps.has(dep)
+            (dep: string) => this.state.has(dep) || completedFinallySteps.has(dep)
           );
 
           if (dependenciesMet) {
@@ -2462,8 +2533,22 @@ Please provide a corrected response that exactly matches the required schema.`;
             this.logger.log(
               `[${finallyStepIndex}/${totalFinallySteps}] ▶ Executing finally step: ${step.id} (${step.type})`
             );
+            const startedAt = this.emitStepStart(
+              step,
+              'finally',
+              finallyStepIndex,
+              totalFinallySteps
+            );
             const promise = this.executeStepWithForeach(step)
               .then(() => {
+                this.emitStepEnd(
+                  step,
+                  'finally',
+                  startedAt,
+                  undefined,
+                  finallyStepIndex,
+                  totalFinallySteps
+                );
                 completedFinallySteps.add(stepId);
                 runningPromises.delete(stepId);
                 this.logger.log(
@@ -2471,6 +2556,14 @@ Please provide a corrected response that exactly matches the required schema.`;
                 );
               })
               .catch((err) => {
+                this.emitStepEnd(
+                  step,
+                  'finally',
+                  startedAt,
+                  err,
+                  finallyStepIndex,
+                  totalFinallySteps
+                );
                 runningPromises.delete(stepId);
                 this.logger.error(
                   `  ✗ Finally step ${step.id} failed: ${err instanceof Error ? err.message : String(err)}`
@@ -2490,6 +2583,7 @@ Please provide a corrected response that exactly matches the required schema.`;
 
         if (runningPromises.size > 0) {
           await Promise.race(runningPromises.values());
+          await Bun.sleep(0);
         }
       }
     } catch (error) {
@@ -2531,9 +2625,9 @@ Please provide a corrected response that exactly matches the required schema.`;
           const step = stepMap.get(stepId);
           if (!step) continue;
 
-          // Dependencies can be from main steps (already in this.stepContexts) or previous errors steps
+          // Dependencies can be from main steps (already in this.state) or previous errors steps
           const dependenciesMet = step.needs.every(
-            (dep: string) => this.stepContexts.has(dep) || completedErrorsSteps.has(dep)
+            (dep: string) => this.state.has(dep) || completedErrorsSteps.has(dep)
           );
 
           if (dependenciesMet) {
@@ -2543,8 +2637,17 @@ Please provide a corrected response that exactly matches the required schema.`;
             this.logger.log(
               `[${errorsStepIndex}/${totalErrorsSteps}] ▶ Executing errors step: ${step.id} (${step.type})`
             );
+            const startedAt = this.emitStepStart(step, 'errors', errorsStepIndex, totalErrorsSteps);
             const promise = this.executeStepWithForeach(step)
               .then(() => {
+                this.emitStepEnd(
+                  step,
+                  'errors',
+                  startedAt,
+                  undefined,
+                  errorsStepIndex,
+                  totalErrorsSteps
+                );
                 completedErrorsSteps.add(stepId);
                 runningPromises.delete(stepId);
                 this.logger.log(
@@ -2552,6 +2655,7 @@ Please provide a corrected response that exactly matches the required schema.`;
                 );
               })
               .catch((err) => {
+                this.emitStepEnd(step, 'errors', startedAt, err, errorsStepIndex, totalErrorsSteps);
                 runningPromises.delete(stepId);
                 this.logger.error(
                   `  ✗ Errors step ${step.id} failed: ${err instanceof Error ? err.message : String(err)}`
@@ -2571,6 +2675,7 @@ Please provide a corrected response that exactly matches the required schema.`;
 
         if (runningPromises.size > 0) {
           await Promise.race(runningPromises.values());
+          await Bun.sleep(0);
         }
       }
     } catch (error) {
@@ -2588,7 +2693,7 @@ Please provide a corrected response that exactly matches the required schema.`;
    * Evaluate workflow outputs
    */
   private evaluateOutputs(): Record<string, unknown> {
-    const context = this.buildContext();
+    const context = this.contextBuilder.buildContext(this.secretManager.getSecrets());
     const outputs: Record<string, unknown> = {};
 
     if (this.workflow.outputs) {
@@ -2607,7 +2712,7 @@ Please provide a corrected response that exactly matches the required schema.`;
     // Validate outputs against schema if provided
     if (this.workflow.outputSchema) {
       try {
-        this.validateSchema('output', this.workflow.outputSchema, outputs, 'workflow');
+        this.validator.validateSchema('output', this.workflow.outputSchema, outputs, 'workflow');
       } catch (error) {
         throw new Error(
           `Workflow output validation failed: ${error instanceof Error ? error.message : String(error)}`
@@ -2618,39 +2723,6 @@ Please provide a corrected response that exactly matches the required schema.`;
     return outputs;
   }
 
-  /**
-   * Check if a join condition is met based on completed dependencies
-   */
-  private isJoinConditionMet(
-    step: import('../parser/schema.ts').JoinStep,
-    completedSteps: Set<string>
-  ): boolean {
-    const total = step.needs.length;
-    if (total === 0) return true;
-
-    // Count successful/skipped dependencies
-    const successCount = step.needs.filter((dep) => completedSteps.has(dep)).length;
-
-    // Find failed/suspended dependencies (that we've already tried)
-    // If some dependencies failed (and didn't allowFailure), the whole workflow would usually fail.
-    // If allowFailure was true, they are in completedSteps.
-    // So completedSteps effectively represents "done successfully".
-
-    if (step.condition === 'all') {
-      return successCount === total;
-    }
-    if (step.condition === 'any') {
-      // Met if at least one succeeded, OR if all finished and none succeeded?
-      // Actually strictly "any" means at least one success.
-      return successCount > 0;
-    }
-    if (typeof step.condition === 'number') {
-      return successCount >= step.condition;
-    }
-
-    return successCount === total;
-  }
-
   /**
    * Register top-level compensation for the workflow
    */