@donkeylabs/server 2.0.19 → 2.0.21

package/src/core/workflows.ts

@@ -6,12 +6,23 @@
 // - parallel: Run multiple branches concurrently
 // - choice: Conditional branching
 // - pass: Transform data / no-op
+// - isolated: Execute in subprocess to prevent event loop blocking (default)
 
 import type { Events } from "./events";
 import type { Jobs } from "./jobs";
 import type { SSE } from "./sse";
 import type { z } from "zod";
 import type { CoreServices } from "../core";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  createWorkflowSocketServer,
+  type WorkflowSocketServer,
+  type WorkflowEvent,
+  type ProxyRequest,
+} from "./workflow-socket";
+import { isProcessAlive } from "./external-jobs";
+import { WorkflowStateMachine, type StateMachineEvents } from "./workflow-state-machine";
 
 // Type helper for Zod schema inference
 type ZodSchema = z.ZodTypeAny;
@@ -126,6 +137,13 @@ export interface WorkflowDefinition {
   timeout?: number;
   /** Default retry config for all steps */
   defaultRetry?: RetryConfig;
+  /**
+   * Whether to execute this workflow in an isolated subprocess.
+   * Default: true (isolated by default to prevent blocking the event loop)
+   *
+   * Set to false for lightweight workflows that benefit from inline execution.
+   */
+  isolated?: boolean;
 }
 
 // ============================================
@@ -176,6 +194,8 @@ export interface WorkflowInstance {
   parentId?: string;
   /** Branch name if this is a branch instance */
   branchName?: string;
+  /** Custom metadata that persists across steps (JSON-serializable) */
+  metadata?: Record<string, any>;
 }
 
 // ============================================
@@ -197,6 +217,31 @@ export interface WorkflowContext {
   core: CoreServices;
   /** Plugin services - available for business logic in workflow handlers */
   plugins: Record<string, any>;
+  /**
+   * Custom metadata that persists across steps (read-only snapshot).
+   * Use setMetadata() to update values.
+   */
+  metadata: Record<string, any>;
+  /**
+   * Set a metadata value that persists across workflow steps.
+   * Accepts any JSON-serializable value (objects, arrays, primitives).
+   *
+   * @example
+   * await ctx.setMetadata('orderContext', {
+   *   correlationId: 'abc-123',
+   *   customer: { id: 'cust_1', tier: 'premium' },
+   *   flags: { expedited: true }
+   * });
+   */
+  setMetadata(key: string, value: any): Promise<void>;
+  /**
+   * Get a metadata value with type safety.
+   *
+   * @example
+   * interface OrderContext { correlationId: string; customer: { id: string } }
+   * const ctx = ctx.getMetadata<OrderContext>('orderContext');
+   */
+  getMetadata<T = any>(key: string): T | undefined;
 }
 
 // ============================================
@@ -307,11 +352,29 @@ export class WorkflowBuilder {
   private _timeout?: number;
   private _defaultRetry?: RetryConfig;
   private _lastStep?: string;
+  private _isolated = true; // Default to isolated execution
 
   constructor(name: string) {
     this._name = name;
   }
 
+  /**
+   * Set whether to execute this workflow in an isolated subprocess.
+   * Default: true (isolated by default to prevent blocking the event loop)
+   *
+   * @param enabled - Set to false for lightweight workflows that benefit from inline execution
+   * @example
+   * // Heavy workflow - uses default isolation (no call needed)
+   * workflow("data-ingestion").task("process", { ... }).build();
+   *
+   * // Lightweight workflow - opt out of isolation
+   * workflow("quick-validation").isolated(false).task("validate", { ... }).build();
+   */
+  isolated(enabled: boolean = true): this {
+    this._isolated = enabled;
+    return this;
+  }
+
   /** Set the starting step explicitly */
   startAt(stepName: string): this {
     this._startAt = stepName;
@@ -512,6 +575,7 @@ export class WorkflowBuilder {
       startAt: this._startAt,
       timeout: this._timeout,
       defaultRetry: this._defaultRetry,
+      isolated: this._isolated,
     };
   }
 }
@@ -539,11 +603,36 @@ export interface WorkflowsConfig {
   pollInterval?: number;
   /** Core services to pass to step handlers */
   core?: CoreServices;
+  /** Directory for Unix sockets (default: /tmp/donkeylabs-workflows) */
+  socketDir?: string;
+  /** TCP port range for Windows fallback (default: [49152, 65535]) */
+  tcpPortRange?: [number, number];
+  /** Database file path (required for isolated workflows) */
+  dbPath?: string;
+  /** Heartbeat timeout in ms (default: 60000) */
+  heartbeatTimeout?: number;
+}
+
+/** Options for registering a workflow */
+export interface WorkflowRegisterOptions {
+  /**
+   * Module path for isolated workflows.
+   * Required when workflow.isolated !== false and running in isolated mode.
+   * Use `import.meta.url` to get the current module's path.
+   *
+   * @example
+   * workflows.register(myWorkflow, { modulePath: import.meta.url });
+   */
+  modulePath?: string;
 }
 
 export interface Workflows {
-  /** Register a workflow definition */
-  register(definition: WorkflowDefinition): void;
+  /**
+   * Register a workflow definition.
+   * @param definition - The workflow definition to register
+   * @param options - Registration options (modulePath required for isolated workflows)
+   */
+  register(definition: WorkflowDefinition, options?: WorkflowRegisterOptions): void;
   /** Start a new workflow instance */
   start<T = any>(workflowName: string, input: T): Promise<string>;
   /** Get a workflow instance by ID */
@@ -562,44 +651,127 @@ export interface Workflows {
   setCore(core: CoreServices): void;
   /** Set plugin services (called after plugins are initialized) */
   setPlugins(plugins: Record<string, any>): void;
+  /** Update metadata for a workflow instance (used by isolated workflows) */
+  updateMetadata(instanceId: string, key: string, value: any): Promise<void>;
 }
 
 // ============================================
-// Workflow Service Implementation
+// Workflow Service Implementation (Supervisor)
 // ============================================
 
+interface IsolatedProcessInfo {
+  pid: number;
+  timeout?: ReturnType<typeof setTimeout>;
+  heartbeatTimeout?: ReturnType<typeof setTimeout>;
+  lastHeartbeat: number;
+}
+
 class WorkflowsImpl implements Workflows {
   private adapter: WorkflowAdapter;
-  private events?: Events;
+  private eventsService?: Events;
   private jobs?: Jobs;
   private sse?: SSE;
   private core?: CoreServices;
   private plugins: Record<string, any> = {};
   private definitions = new Map<string, WorkflowDefinition>();
-  private running = new Map<string, { timeout?: ReturnType<typeof setTimeout> }>();
+  private running = new Map<string, { timeout?: ReturnType<typeof setTimeout>; sm?: WorkflowStateMachine }>();
   private pollInterval: number;
 
+  // Isolated execution state
+  private socketServer?: WorkflowSocketServer;
+  private socketDir: string;
+  private tcpPortRange: [number, number];
+  private dbPath?: string;
+  private heartbeatTimeoutMs: number;
+  private workflowModulePaths = new Map<string, string>();
+  private isolatedProcesses = new Map<string, IsolatedProcessInfo>();
+
   constructor(config: WorkflowsConfig = {}) {
     this.adapter = config.adapter ?? new MemoryWorkflowAdapter();
-    this.events = config.events;
+    this.eventsService = config.events;
     this.jobs = config.jobs;
     this.sse = config.sse;
     this.core = config.core;
     this.pollInterval = config.pollInterval ?? 1000;
+
+    // Isolated execution config
+    this.socketDir = config.socketDir ?? "/tmp/donkeylabs-workflows";
+    this.tcpPortRange = config.tcpPortRange ?? [49152, 65535];
+    this.dbPath = config.dbPath;
+    this.heartbeatTimeoutMs = config.heartbeatTimeout ?? 60000;
+  }
+
+  private getSocketServer(): WorkflowSocketServer {
+    if (!this.socketServer) {
+      this.socketServer = createWorkflowSocketServer(
+        {
+          socketDir: this.socketDir,
+          tcpPortRange: this.tcpPortRange,
+        },
+        {
+          onEvent: (event) => this.handleIsolatedEvent(event),
+          onProxyCall: (request) => this.handleProxyCall(request),
+          onConnect: (instanceId) => {
+            console.log(`[Workflows] Isolated workflow ${instanceId} connected`);
+          },
+          onDisconnect: (instanceId) => {
+            console.log(`[Workflows] Isolated workflow ${instanceId} disconnected`);
+          },
+          onError: (error, instanceId) => {
+            console.error(`[Workflows] Socket error for ${instanceId}:`, error);
+          },
+        }
+      );
+    }
+    return this.socketServer;
   }
 
   setCore(core: CoreServices): void {
     this.core = core;
+    // Extract DB path if using Kysely adapter (for isolated workflows)
+    if (!this.dbPath && (core.db as any)?.getExecutor) {
+      // Try to get the database path from the Kysely instance
+      // This is a bit hacky but necessary for isolated workflows
+      try {
+        const executor = (core.db as any).getExecutor();
+        const adapter = executor?.adapter;
+        if (adapter?.db?.filename) {
+          this.dbPath = adapter.db.filename;
+        }
+      } catch {
+        // Ignore - dbPath might be set manually
+      }
+    }
   }
 
   setPlugins(plugins: Record<string, any>): void {
     this.plugins = plugins;
   }
 
-  register(definition: WorkflowDefinition): void {
+  async updateMetadata(instanceId: string, key: string, value: any): Promise<void> {
+    const instance = await this.adapter.getInstance(instanceId);
+    if (!instance) return;
+
+    const metadata = { ...(instance.metadata || {}), [key]: value };
+    await this.adapter.updateInstance(instanceId, { metadata });
+  }
+
+  register(definition: WorkflowDefinition, options?: WorkflowRegisterOptions): void {
     if (this.definitions.has(definition.name)) {
       throw new Error(`Workflow "${definition.name}" is already registered`);
     }
+
+    // Store module path for isolated workflows
+    if (options?.modulePath) {
+      this.workflowModulePaths.set(definition.name, options.modulePath);
+    } else if (definition.isolated !== false) {
+      // Warn if isolated workflow has no module path
+      console.warn(
+        `[Workflows] Workflow "${definition.name}" is isolated but no modulePath provided. ` +
+        `Use: workflows.register(myWorkflow, { modulePath: import.meta.url })`
+      );
+    }
+
     this.definitions.set(definition.name, definition);
   }
 
@@ -625,8 +797,35 @@ class WorkflowsImpl implements Workflows {
       input,
     });
 
-    // Start execution
-    this.executeWorkflow(instance.id, definition);
+    // SSE broadcast for real-time monitoring
+    if (this.sse) {
+      this.sse.broadcast(`workflow:${instance.id}`, "started", {
+        workflowName,
+        input,
+      });
+      this.sse.broadcast("workflows:all", "workflow.started", {
+        instanceId: instance.id,
+        workflowName,
+        input,
+      });
+    }
+
+    // Start execution (isolated or inline based on definition.isolated)
+    const isIsolated = definition.isolated !== false;
+    const modulePath = this.workflowModulePaths.get(workflowName);
+
+    if (isIsolated && modulePath && this.dbPath) {
+      // Execute in isolated subprocess
+      this.executeIsolatedWorkflow(instance.id, definition, input, modulePath);
+    } else {
+      // Execute inline using state machine
+      if (isIsolated && !modulePath) {
+        console.warn(
+          `[Workflows] Workflow "${workflowName}" falling back to inline execution (no modulePath)`
+        );
+      }
+      this.startInlineWorkflow(instance.id, definition);
+    }
 
     return instance.id;
   }
@@ -641,8 +840,25 @@ class WorkflowsImpl implements Workflows {
       return false;
     }
 
-    // Clear timeout
+    // Kill isolated process if running
+    const isolatedInfo = this.isolatedProcesses.get(instanceId);
+    if (isolatedInfo) {
+      try {
+        process.kill(isolatedInfo.pid, "SIGTERM");
+      } catch {
+        // Process might already be dead
+      }
+      if (isolatedInfo.timeout) clearTimeout(isolatedInfo.timeout);
+      if (isolatedInfo.heartbeatTimeout) clearTimeout(isolatedInfo.heartbeatTimeout);
+      this.isolatedProcesses.delete(instanceId);
+      await this.getSocketServer().closeSocket(instanceId);
+    }
+
+    // Cancel inline state machine if running
     const runInfo = this.running.get(instanceId);
+    if (runInfo?.sm) {
+      runInfo.sm.cancel(instanceId);
+    }
     if (runInfo?.timeout) {
       clearTimeout(runInfo.timeout);
     }
@@ -686,694 +902,588 @@ class WorkflowsImpl implements Workflows {
       }
 
       console.log(`[Workflows] Resuming workflow instance ${instance.id}`);
-      this.executeWorkflow(instance.id, definition);
+
+      // Check isolation mode and call appropriate method
+      const isIsolated = definition.isolated !== false;
+      const modulePath = this.workflowModulePaths.get(instance.workflowName);
+
+      if (isIsolated && modulePath && this.dbPath) {
+        this.executeIsolatedWorkflow(instance.id, definition, instance.input, modulePath);
+      } else {
+        this.startInlineWorkflow(instance.id, definition);
+      }
     }
   }
 
   async stop(): Promise<void> {
-    // Clear all timeouts
+    // Kill all isolated processes
+    for (const [instanceId, info] of this.isolatedProcesses) {
+      try {
+        process.kill(info.pid, "SIGTERM");
+      } catch {
+        // Process might already be dead
+      }
+      if (info.timeout) clearTimeout(info.timeout);
+      if (info.heartbeatTimeout) clearTimeout(info.heartbeatTimeout);
+    }
+    this.isolatedProcesses.clear();
+
+    // Shutdown socket server
+    if (this.socketServer) {
+      await this.socketServer.shutdown();
+      this.socketServer = undefined;
+    }
+
+    // Clear all inline timeouts and cancel state machines
     for (const [instanceId, runInfo] of this.running) {
+      if (runInfo.sm) {
+        runInfo.sm.cancel(instanceId);
+      }
       if (runInfo.timeout) {
         clearTimeout(runInfo.timeout);
       }
     }
     this.running.clear();
+
+    // Stop adapter (cleanup timers and prevent further DB access)
+    if (this.adapter && typeof (this.adapter as any).stop === "function") {
+      (this.adapter as any).stop();
+    }
   }
 
   // ============================================
-  // Execution Engine
+  // Inline Execution via State Machine
   // ============================================
 
-  private async executeWorkflow(
+  private startInlineWorkflow(
     instanceId: string,
-    definition: WorkflowDefinition
-  ): Promise<void> {
-    const instance = await this.adapter.getInstance(instanceId);
-    if (!instance) return;
-
-    // Mark as running
-    if (instance.status === "pending") {
-      await this.adapter.updateInstance(instanceId, {
-        status: "running",
-        startedAt: new Date(),
-      });
-    }
+    definition: WorkflowDefinition,
+  ): void {
+    const sm = new WorkflowStateMachine({
+      adapter: this.adapter,
+      core: this.core,
+      plugins: this.plugins,
+      events: this.createInlineEventHandler(instanceId),
+      jobs: this.jobs,
+      pollInterval: this.pollInterval,
+    });
 
     // Set up workflow timeout
+    let timeout: ReturnType<typeof setTimeout> | undefined;
     if (definition.timeout) {
-      const timeout = setTimeout(async () => {
-        await this.failWorkflow(instanceId, "Workflow timed out");
+      timeout = setTimeout(async () => {
+        sm.cancel(instanceId);
+        await this.adapter.updateInstance(instanceId, {
+          status: "failed",
+          error: "Workflow timed out",
+          completedAt: new Date(),
+        });
+        await this.emitEvent("workflow.failed", {
+          instanceId,
+          workflowName: definition.name,
+          error: "Workflow timed out",
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${instanceId}`, "failed", { error: "Workflow timed out" });
+          this.sse.broadcast("workflows:all", "workflow.failed", {
+            instanceId,
+            workflowName: definition.name,
+            error: "Workflow timed out",
+          });
+        }
+        this.running.delete(instanceId);
       }, definition.timeout);
-      this.running.set(instanceId, { timeout });
-    } else {
-      this.running.set(instanceId, {});
-    }
-
-    // Execute current step
-    await this.executeStep(instanceId, definition);
-  }
-
-  private async executeStep(
-    instanceId: string,
-    definition: WorkflowDefinition
-  ): Promise<void> {
-    const instance = await this.adapter.getInstance(instanceId);
-    if (!instance || instance.status !== "running") return;
-
-    const stepName = instance.currentStep;
-    if (!stepName) {
-      await this.completeWorkflow(instanceId);
-      return;
     }
 
-    const step = definition.steps.get(stepName);
-    if (!step) {
-      await this.failWorkflow(instanceId, `Step "${stepName}" not found`);
-      return;
-    }
-
-    // Build context
-    const ctx = this.buildContext(instance, definition);
-
-    // Emit step started event
-    await this.emitEvent("workflow.step.started", {
-      instanceId,
-      workflowName: instance.workflowName,
-      stepName,
-      stepType: step.type,
-    });
-
-    // Update step result as running
-    const stepResult: StepResult = {
-      stepName,
-      status: "running",
-      startedAt: new Date(),
-      attempts: (instance.stepResults[stepName]?.attempts ?? 0) + 1,
-    };
-    await this.adapter.updateInstance(instanceId, {
-      stepResults: { ...instance.stepResults, [stepName]: stepResult },
-    });
+    this.running.set(instanceId, { timeout, sm });
 
-    try {
-      let output: any;
-
-      switch (step.type) {
-        case "task":
-          output = await this.executeTaskStep(instanceId, step, ctx, definition);
-          break;
-        case "parallel":
-          output = await this.executeParallelStep(instanceId, step, ctx, definition);
-          break;
-        case "choice":
-          output = await this.executeChoiceStep(instanceId, step, ctx, definition);
-          break;
-        case "pass":
-          output = await this.executePassStep(instanceId, step, ctx);
-          break;
+    // Run the state machine (fire and forget - events handle communication)
+    sm.run(instanceId, definition).then(() => {
+      // Clean up timeout on completion
+      const runInfo = this.running.get(instanceId);
+      if (runInfo?.timeout) {
+        clearTimeout(runInfo.timeout);
       }
-
-      // Step completed successfully
-      await this.completeStep(instanceId, stepName, output, step, definition);
-    } catch (error) {
-      const errorMsg = error instanceof Error ? error.message : String(error);
-      await this.handleStepError(instanceId, stepName, errorMsg, step, definition);
-    }
+      this.running.delete(instanceId);
+    }).catch(() => {
+      // State machine already persisted the failure - just clean up
+      const runInfo = this.running.get(instanceId);
+      if (runInfo?.timeout) {
+        clearTimeout(runInfo.timeout);
+      }
+      this.running.delete(instanceId);
+    });
   }
 
-  private async executeTaskStep(
-    instanceId: string,
-    step: TaskStepDefinition,
-    ctx: WorkflowContext,
-    definition: WorkflowDefinition
-  ): Promise<any> {
-    // Determine which API is being used
-    const useInlineHandler = !!step.handler;
-
-    if (useInlineHandler) {
-      // === NEW API: Inline handler with Zod schemas ===
-      let input: any;
-
-      if (step.inputSchema) {
-        if (typeof step.inputSchema === "function") {
-          // inputSchema is a mapper function: (prev, workflowInput) => input
-          input = step.inputSchema(ctx.prev, ctx.input);
-        } else {
-          // inputSchema is a Zod schema - validate workflow input
-          const parseResult = step.inputSchema.safeParse(ctx.input);
-          if (!parseResult.success) {
-            throw new Error(`Input validation failed: ${parseResult.error.message}`);
-          }
-          input = parseResult.data;
+  /**
+   * Create an event handler that bridges state machine events to Events service + SSE
+   */
+  private createInlineEventHandler(instanceId: string): StateMachineEvents {
+    return {
+      onStepStarted: (id, stepName, stepType) => {
+        this.emitEvent("workflow.step.started", {
+          instanceId: id,
+          stepName,
+          stepType,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${id}`, "step.started", { stepName });
+          this.sse.broadcast("workflows:all", "workflow.step.started", {
+            instanceId: id,
+            stepName,
+          });
         }
-      } else {
-        // No input schema, use workflow input directly
-        input = ctx.input;
-      }
-
-      // Update step with input
-      const instance = await this.adapter.getInstance(instanceId);
-      if (instance) {
-        const stepResult = instance.stepResults[step.name];
-        if (stepResult) {
-          stepResult.input = input;
-          await this.adapter.updateInstance(instanceId, {
-            stepResults: { ...instance.stepResults, [step.name]: stepResult },
+      },
+      onStepCompleted: (id, stepName, output, nextStep) => {
+        this.emitEvent("workflow.step.completed", {
+          instanceId: id,
+          stepName,
+          output,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${id}`, "step.completed", { stepName, output });
+          this.sse.broadcast("workflows:all", "workflow.step.completed", {
+            instanceId: id,
+            stepName,
           });
         }
-      }
-
-      // Execute the inline handler
-      let result = await step.handler!(input, ctx);
-
-      // Validate output if schema provided
-      if (step.outputSchema) {
-        const parseResult = step.outputSchema.safeParse(result);
-        if (!parseResult.success) {
-          throw new Error(`Output validation failed: ${parseResult.error.message}`);
+      },
+      onStepFailed: (id, stepName, error, attempts) => {
+        this.emitEvent("workflow.step.failed", {
+          instanceId: id,
+          stepName,
+          error,
+          attempts,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${id}`, "step.failed", { stepName, error });
+          this.sse.broadcast("workflows:all", "workflow.step.failed", {
+            instanceId: id,
+            stepName,
+            error,
+          });
         }
-        result = parseResult.data;
-      }
-
-      return result;
-    } else {
-      // === LEGACY API: Job-based execution ===
-      if (!this.jobs) {
-        throw new Error("Jobs service not configured");
-      }
-
-      if (!step.job) {
-        throw new Error("Task step requires either 'handler' or 'job'");
-      }
-
-      // Prepare job input
-      const jobInput = step.input ? step.input(ctx) : ctx.input;
-
-      // Update step with input
-      const instance = await this.adapter.getInstance(instanceId);
-      if (instance) {
-        const stepResult = instance.stepResults[step.name];
-        if (stepResult) {
-          stepResult.input = jobInput;
-          await this.adapter.updateInstance(instanceId, {
-            stepResults: { ...instance.stepResults, [step.name]: stepResult },
+      },
+      onStepRetry: (id, stepName, attempt, max, delayMs) => {
+        this.emitEvent("workflow.step.retry", {
+          instanceId: id,
+          stepName,
+          attempt,
+          maxAttempts: max,
+          delay: delayMs,
+        });
+      },
+      onProgress: (id, progress, currentStep, completed, total) => {
+        this.emitEvent("workflow.progress", {
+          instanceId: id,
+          progress,
+          currentStep,
+          completedSteps: completed,
+          totalSteps: total,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${id}`, "progress", {
+            progress,
+            currentStep,
+            completedSteps: completed,
+            totalSteps: total,
+          });
+          this.sse.broadcast("workflows:all", "workflow.progress", {
+            instanceId: id,
+            progress,
+            currentStep,
           });
         }
-      }
-
-      // Enqueue the job
-      const jobId = await this.jobs.enqueue(step.job, {
-        ...jobInput,
-        _workflowInstanceId: instanceId,
-        _workflowStepName: step.name,
-      });
-
-      // Wait for job completion
-      const result = await this.waitForJob(jobId, step.timeout);
-
-      // Transform output if needed
-      return step.output ? step.output(result, ctx) : result;
-    }
+      },
+      onCompleted: (id, output) => {
+        this.emitEvent("workflow.completed", {
+          instanceId: id,
+          output,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${id}`, "completed", { output });
+          this.sse.broadcast("workflows:all", "workflow.completed", {
+            instanceId: id,
+          });
+        }
+      },
+      onFailed: (id, error) => {
+        this.emitEvent("workflow.failed", {
+          instanceId: id,
+          error,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${id}`, "failed", { error });
+          this.sse.broadcast("workflows:all", "workflow.failed", {
+            instanceId: id,
+            error,
+          });
+        }
+      },
+    };
   }
 
-  private async waitForJob(jobId: string, timeout?: number): Promise<any> {
-    if (!this.jobs) {
-      throw new Error("Jobs service not configured");
-    }
-
-    const startTime = Date.now();
-
-    while (true) {
-      const job = await this.jobs.get(jobId);
-
-      if (!job) {
-        throw new Error(`Job ${jobId} not found`);
-      }
-
-      if (job.status === "completed") {
-        return job.result;
-      }
-
-      if (job.status === "failed") {
-        throw new Error(job.error ?? "Job failed");
-      }
-
-      // Check timeout
-      if (timeout && Date.now() - startTime > timeout) {
-        throw new Error("Job timed out");
-      }
-
-      // Wait before polling again
-      await new Promise((resolve) => setTimeout(resolve, this.pollInterval));
-    }
-  }
+  // ============================================
+  // Isolated Execution Engine
+  // ============================================
 
-  private async executeParallelStep(
+  /**
+   * Execute a workflow in an isolated subprocess
+   */
+  private async executeIsolatedWorkflow(
     instanceId: string,
-    step: ParallelStepDefinition,
-    ctx: WorkflowContext,
-    definition: WorkflowDefinition
-  ): Promise<any> {
-    const branchPromises: Promise<{ name: string; result: any }>[] = [];
-    const branchInstanceIds: string[] = [];
-
-    for (const branchDef of step.branches) {
-      // Register branch workflow if not already
-      if (!this.definitions.has(branchDef.name)) {
-        this.definitions.set(branchDef.name, branchDef);
-      }
+    definition: WorkflowDefinition,
+    input: any,
+    modulePath: string
+  ): Promise<void> {
+    const socketServer = this.getSocketServer();
 
-      // Start branch as sub-workflow
-      const branchInstanceId = await this.adapter.createInstance({
-        workflowName: branchDef.name,
-        status: "pending",
-        currentStep: branchDef.startAt,
-        input: ctx.input,
-        stepResults: {},
-        createdAt: new Date(),
-        parentId: instanceId,
-        branchName: branchDef.name,
-      });
+    // Create socket for this workflow instance
+    const { socketPath, tcpPort } = await socketServer.createSocket(instanceId);
 
-      branchInstanceIds.push(branchInstanceId.id);
+    // Mark workflow as running
+    await this.adapter.updateInstance(instanceId, {
+      status: "running",
+      startedAt: new Date(),
+    });
 
-      // Execute branch
-      const branchPromise = (async () => {
-        await this.executeWorkflow(branchInstanceId.id, branchDef);
+    // Get the executor path
+    const currentDir = dirname(fileURLToPath(import.meta.url));
+    const executorPath = join(currentDir, "workflow-executor.ts");
 
-        // Wait for branch completion
-        while (true) {
-          const branchInstance = await this.adapter.getInstance(branchInstanceId.id);
-          if (!branchInstance) {
-            throw new Error(`Branch instance ${branchInstanceId.id} not found`);
-          }
+    // Prepare config for the executor
+    const config = {
+      instanceId,
+      workflowName: definition.name,
+      input,
+      socketPath,
+      tcpPort,
+      modulePath,
+      dbPath: this.dbPath,
+    };
 
-          if (branchInstance.status === "completed") {
-            return { name: branchDef.name, result: branchInstance.output };
-          }
+    // Spawn the subprocess
+    const proc = Bun.spawn(["bun", "run", executorPath], {
+      stdin: "pipe",
+      stdout: "inherit",
+      stderr: "inherit",
+      env: {
+        ...process.env,
+        // Ensure the subprocess can import from the same paths
+        NODE_OPTIONS: process.env.NODE_OPTIONS ?? "",
+      },
+    });
 
-          if (branchInstance.status === "failed") {
-            throw new Error(branchInstance.error ?? `Branch ${branchDef.name} failed`);
-          }
+    // Send config via stdin
+    proc.stdin.write(JSON.stringify(config));
+    proc.stdin.end();
 
-          await new Promise((resolve) => setTimeout(resolve, this.pollInterval));
-        }
-      })();
+    // Track the process
+    this.isolatedProcesses.set(instanceId, {
+      pid: proc.pid,
+      lastHeartbeat: Date.now(),
+    });
 
-      branchPromises.push(branchPromise);
+    // Set up workflow timeout
+    if (definition.timeout) {
+      const timeoutHandle = setTimeout(async () => {
+        await this.handleIsolatedTimeout(instanceId, proc.pid);
+      }, definition.timeout);
+      const info = this.isolatedProcesses.get(instanceId);
+      if (info) info.timeout = timeoutHandle;
     }
 
-    // Track branch instances
-    await this.adapter.updateInstance(instanceId, {
-      branchInstances: {
-        ...((await this.adapter.getInstance(instanceId))?.branchInstances ?? {}),
-        [step.name]: branchInstanceIds,
-      },
-    });
+    // Set up heartbeat timeout
+    this.resetHeartbeatTimeout(instanceId, proc.pid);
 
-    // Wait for all branches
-    if (step.onError === "wait-all") {
-      const results = await Promise.allSettled(branchPromises);
-      const output: Record<string, any> = {};
-      const errors: string[] = [];
-
-      for (const result of results) {
-        if (result.status === "fulfilled") {
-          output[result.value.name] = result.value.result;
-        } else {
-          errors.push(result.reason?.message ?? "Branch failed");
-        }
+    // Handle process exit
+    proc.exited.then(async (exitCode) => {
+      const info = this.isolatedProcesses.get(instanceId);
+      if (info) {
+        if (info.timeout) clearTimeout(info.timeout);
+        if (info.heartbeatTimeout) clearTimeout(info.heartbeatTimeout);
+        this.isolatedProcesses.delete(instanceId);
       }
+      await socketServer.closeSocket(instanceId);
 
-      if (errors.length > 0) {
-        throw new Error(`Parallel branches failed: ${errors.join(", ")}`);
+      // Check if workflow is still running (crashed before completion)
+      const instance = await this.adapter.getInstance(instanceId);
+      if (instance && instance.status === "running") {
+        console.error(`[Workflows] Isolated workflow ${instanceId} crashed with exit code ${exitCode}`);
+        await this.adapter.updateInstance(instanceId, {
+          status: "failed",
+          error: `Subprocess crashed with exit code ${exitCode}`,
+          completedAt: new Date(),
+        });
+        await this.emitEvent("workflow.failed", {
+          instanceId,
+          workflowName: instance.workflowName,
+          error: `Subprocess crashed with exit code ${exitCode}`,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${instanceId}`, "failed", {
+            error: `Subprocess crashed with exit code ${exitCode}`,
+          });
+          this.sse.broadcast("workflows:all", "workflow.failed", {
+            instanceId,
+            workflowName: instance.workflowName,
+            error: `Subprocess crashed with exit code ${exitCode}`,
+          });
+        }
       }
+    });
+  }
 
-      return output;
-    } else {
-      // fail-fast (default)
-      const results = await Promise.all(branchPromises);
-      const output: Record<string, any> = {};
-      for (const result of results) {
-        output[result.name] = result.result;
-      }
-      return output;
+  /**
+   * Handle events from isolated workflow subprocess.
+   * The subprocess owns persistence via its own adapter - we only forward events to SSE/Events.
+   */
+  private async handleIsolatedEvent(event: WorkflowEvent): Promise<void> {
+    const { instanceId, type } = event;
+
+    // Reset heartbeat timeout on any event
+    const info = this.isolatedProcesses.get(instanceId);
+    if (info) {
+      info.lastHeartbeat = Date.now();
+      this.resetHeartbeatTimeout(instanceId, info.pid);
     }
-  }
 
-  private async executeChoiceStep(
-    instanceId: string,
-    step: ChoiceStepDefinition,
-    ctx: WorkflowContext,
-    definition: WorkflowDefinition
-  ): Promise<string> {
-    // Evaluate conditions in order
-    for (const choice of step.choices) {
-      try {
-        if (choice.condition(ctx)) {
-          // Update current step and continue
-          await this.adapter.updateInstance(instanceId, {
-            currentStep: choice.next,
+    switch (type) {
+      case "started":
+      case "heartbeat":
+        // No-op: heartbeat handled above, started is handled by executeIsolatedWorkflow
+        break;
+
+      case "step.started": {
+        await this.emitEvent("workflow.step.started", {
+          instanceId,
+          stepName: event.stepName,
+          stepType: event.stepType,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${instanceId}`, "step.started", {
+            stepName: event.stepName,
           });
-
-          // Mark choice step as complete
-          const instance = await this.adapter.getInstance(instanceId);
-          if (instance) {
-            const stepResult = instance.stepResults[step.name];
-            if (stepResult) {
-              stepResult.status = "completed";
-              stepResult.output = { chosen: choice.next };
-              stepResult.completedAt = new Date();
-              await this.adapter.updateInstance(instanceId, {
-                stepResults: { ...instance.stepResults, [step.name]: stepResult },
-              });
-            }
-          }
-
-          // Emit progress
-          await this.emitEvent("workflow.step.completed", {
+          this.sse.broadcast("workflows:all", "workflow.step.started", {
             instanceId,
-            workflowName: (await this.adapter.getInstance(instanceId))?.workflowName,
-            stepName: step.name,
-            output: { chosen: choice.next },
+            stepName: event.stepName,
           });
-
-          // Execute next step
-          await this.executeStep(instanceId, definition);
-          return choice.next;
         }
-      } catch {
-        // Condition threw, try next
+        break;
       }
-    }
-
-    // No condition matched, use default
-    if (step.default) {
-      await this.adapter.updateInstance(instanceId, {
-        currentStep: step.default,
-      });
 
-      // Mark choice step as complete
-      const instance = await this.adapter.getInstance(instanceId);
-      if (instance) {
-        const stepResult = instance.stepResults[step.name];
-        if (stepResult) {
-          stepResult.status = "completed";
-          stepResult.output = { chosen: step.default };
-          stepResult.completedAt = new Date();
-          await this.adapter.updateInstance(instanceId, {
-            stepResults: { ...instance.stepResults, [step.name]: stepResult },
+      case "step.completed": {
+        await this.emitEvent("workflow.step.completed", {
+          instanceId,
+          stepName: event.stepName,
+          output: event.output,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${instanceId}`, "step.completed", {
+            stepName: event.stepName,
+            output: event.output,
+          });
+          this.sse.broadcast("workflows:all", "workflow.step.completed", {
+            instanceId,
+            stepName: event.stepName,
+            output: event.output,
           });
         }
+        break;
       }
 
-      await this.emitEvent("workflow.step.completed", {
-        instanceId,
-        workflowName: instance?.workflowName,
-        stepName: step.name,
-        output: { chosen: step.default },
-      });
-
-      await this.executeStep(instanceId, definition);
-      return step.default;
-    }
-
-    throw new Error("No choice condition matched and no default specified");
-  }
-
-  private async executePassStep(
-    instanceId: string,
-    step: PassStepDefinition,
-    ctx: WorkflowContext
-  ): Promise<any> {
-    if (step.result !== undefined) {
-      return step.result;
-    }
-
-    if (step.transform) {
-      return step.transform(ctx);
-    }
-
-    return ctx.input;
-  }
+      case "step.failed": {
+        await this.emitEvent("workflow.step.failed", {
+          instanceId,
+          stepName: event.stepName,
+          error: event.error,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${instanceId}`, "step.failed", {
+            stepName: event.stepName,
+            error: event.error,
+          });
+          this.sse.broadcast("workflows:all", "workflow.step.failed", {
+            instanceId,
+            stepName: event.stepName,
+            error: event.error,
+          });
+        }
+        break;
+      }
 
-  private buildContext(instance: WorkflowInstance, definition: WorkflowDefinition): WorkflowContext {
-    // Build steps object with outputs
-    const steps: Record<string, any> = {};
-    for (const [name, result] of Object.entries(instance.stepResults)) {
-      if (result.status === "completed" && result.output !== undefined) {
-        steps[name] = result.output;
+      case "progress": {
+        await this.emitEvent("workflow.progress", {
+          instanceId,
+          progress: event.progress,
+          completedSteps: event.completedSteps,
+          totalSteps: event.totalSteps,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${instanceId}`, "progress", {
+            progress: event.progress,
+            completedSteps: event.completedSteps,
+            totalSteps: event.totalSteps,
+          });
+          this.sse.broadcast("workflows:all", "workflow.progress", {
+            instanceId,
+            progress: event.progress,
+            completedSteps: event.completedSteps,
+            totalSteps: event.totalSteps,
+          });
+        }
+        break;
       }
-    }
 
-    // Find the previous step's output by tracing the workflow path
-    let prev: any = undefined;
-    if (instance.currentStep) {
-      // Find which step comes before current step
-      for (const [stepName, stepDef] of definition.steps) {
-        if (stepDef.next === instance.currentStep && steps[stepName] !== undefined) {
-          prev = steps[stepName];
-          break;
+      case "completed": {
+        // Clean up isolated process tracking
+        this.cleanupIsolatedProcess(instanceId);
+
+        // Subprocess already persisted state - just emit events
+        await this.emitEvent("workflow.completed", {
+          instanceId,
+          output: event.output,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${instanceId}`, "completed", { output: event.output });
+          this.sse.broadcast("workflows:all", "workflow.completed", { instanceId });
         }
+        break;
       }
-      // If no explicit next found, use most recent completed step output
-      if (prev === undefined) {
-        const completedSteps = Object.entries(instance.stepResults)
-          .filter(([, r]) => r.status === "completed" && r.output !== undefined)
-          .sort((a, b) => {
-            const aTime = a[1].completedAt?.getTime() ?? 0;
-            const bTime = b[1].completedAt?.getTime() ?? 0;
-            return bTime - aTime;
+
+      case "failed": {
+        // Clean up isolated process tracking
+        this.cleanupIsolatedProcess(instanceId);
+
+        // Subprocess already persisted state - just emit events
+        await this.emitEvent("workflow.failed", {
+          instanceId,
+          error: event.error,
+        });
+        if (this.sse) {
+          this.sse.broadcast(`workflow:${instanceId}`, "failed", { error: event.error });
+          this.sse.broadcast("workflows:all", "workflow.failed", {
+            instanceId,
+            error: event.error,
           });
-        if (completedSteps.length > 0) {
-          prev = completedSteps[0][1].output;
         }
+        break;
       }
     }
-
-    return {
-      input: instance.input,
-      steps,
-      prev,
-      instance,
-      getStepResult: <T = any>(stepName: string): T | undefined => {
-        return steps[stepName] as T | undefined;
-      },
-      core: this.core!,
-      plugins: this.plugins,
-    };
   }
 
-  private async completeStep(
-    instanceId: string,
-    stepName: string,
-    output: any,
-    step: StepDefinition,
-    definition: WorkflowDefinition
-  ): Promise<void> {
-    const instance = await this.adapter.getInstance(instanceId);
-    if (!instance) return;
-
-    // Update step result
-    const stepResult = instance.stepResults[stepName] ?? {
-      stepName,
-      status: "pending",
-      attempts: 0,
-    };
-    stepResult.status = "completed";
-    stepResult.output = output;
-    stepResult.completedAt = new Date();
-
-    await this.adapter.updateInstance(instanceId, {
-      stepResults: { ...instance.stepResults, [stepName]: stepResult },
-    });
-
-    // Emit step completed event
-    await this.emitEvent("workflow.step.completed", {
-      instanceId,
-      workflowName: instance.workflowName,
-      stepName,
-      output,
-    });
-
-    // Calculate and emit progress
-    const totalSteps = definition.steps.size;
-    const completedSteps = Object.values(instance.stepResults).filter(
-      (r) => r.status === "completed"
-    ).length + 1; // +1 for current step
-    const progress = Math.round((completedSteps / totalSteps) * 100);
-
-    await this.emitEvent("workflow.progress", {
-      instanceId,
-      workflowName: instance.workflowName,
-      progress,
-      currentStep: stepName,
-      completedSteps,
-      totalSteps,
-    });
-
-    // Broadcast via SSE
-    if (this.sse) {
-      this.sse.broadcast(`workflow:${instanceId}`, "progress", {
-        progress,
-        currentStep: stepName,
-        completedSteps,
-        totalSteps,
-      });
-    }
+  /**
+   * Handle proxy calls from isolated subprocess
+   */
+  private async handleProxyCall(request: ProxyRequest): Promise<any> {
+    const { target, service, method, args } = request;
 
-    // Move to next step or complete
-    if (step.end) {
-      await this.completeWorkflow(instanceId, output);
-    } else if (step.next) {
-      await this.adapter.updateInstance(instanceId, {
-        currentStep: step.next,
-      });
-      await this.executeStep(instanceId, definition);
+    if (target === "plugin") {
+      const plugin = this.plugins[service];
+      if (!plugin) {
+        throw new Error(`Plugin "${service}" not found`);
+      }
+      const fn = plugin[method];
+      if (typeof fn !== "function") {
+        throw new Error(`Method "${method}" not found on plugin "${service}"`);
+      }
+      return fn.apply(plugin, args);
+    } else if (target === "core") {
+      if (!this.core) {
+        throw new Error("Core services not available");
+      }
+      const coreService = (this.core as any)[service];
+      if (!coreService) {
+        throw new Error(`Core service "${service}" not found`);
+      }
+      const fn = coreService[method];
+      if (typeof fn !== "function") {
+        throw new Error(`Method "${method}" not found on core service "${service}"`);
+      }
+      return fn.apply(coreService, args);
     } else {
-      // No next step, complete
-      await this.completeWorkflow(instanceId, output);
+      throw new Error(`Unknown proxy target: ${target}`);
     }
   }
 
-  private async handleStepError(
-    instanceId: string,
-    stepName: string,
-    error: string,
-    step: StepDefinition,
-    definition: WorkflowDefinition
-  ): Promise<void> {
-    const instance = await this.adapter.getInstance(instanceId);
-    if (!instance) return;
-
-    const stepResult = instance.stepResults[stepName] ?? {
-      stepName,
-      status: "pending",
-      attempts: 0,
-    };
-
-    // Check retry config
-    const retry = step.retry ?? definition.defaultRetry;
-    if (retry && stepResult.attempts < retry.maxAttempts) {
-      // Retry with backoff
-      const backoffRate = retry.backoffRate ?? 2;
-      const intervalMs = retry.intervalMs ?? 1000;
-      const maxIntervalMs = retry.maxIntervalMs ?? 30000;
-      const delay = Math.min(
-        intervalMs * Math.pow(backoffRate, stepResult.attempts - 1),
-        maxIntervalMs
-      );
-
-      console.log(
-        `[Workflows] Retrying step ${stepName} in ${delay}ms (attempt ${stepResult.attempts}/${retry.maxAttempts})`
-      );
-
-      await this.emitEvent("workflow.step.retry", {
-        instanceId,
-        workflowName: instance.workflowName,
-        stepName,
-        attempt: stepResult.attempts,
-        maxAttempts: retry.maxAttempts,
-        delay,
-        error,
-      });
-
-      // Update step result
-      stepResult.error = error;
-      await this.adapter.updateInstance(instanceId, {
-        stepResults: { ...instance.stepResults, [stepName]: stepResult },
-      });
-
-      // Retry after delay
-      setTimeout(() => {
-        this.executeStep(instanceId, definition);
-      }, delay);
-
-      return;
+  /**
+   * Clean up isolated process tracking
+   */
+  private cleanupIsolatedProcess(instanceId: string): void {
+    const info = this.isolatedProcesses.get(instanceId);
+    if (info) {
+      if (info.timeout) clearTimeout(info.timeout);
+      if (info.heartbeatTimeout) clearTimeout(info.heartbeatTimeout);
+      this.isolatedProcesses.delete(instanceId);
     }
-
-    // No more retries, fail the step
-    stepResult.status = "failed";
-    stepResult.error = error;
-    stepResult.completedAt = new Date();
-
-    await this.adapter.updateInstance(instanceId, {
-      stepResults: { ...instance.stepResults, [stepName]: stepResult },
-    });
-
-    await this.emitEvent("workflow.step.failed", {
-      instanceId,
-      workflowName: instance.workflowName,
-      stepName,
-      error,
-      attempts: stepResult.attempts,
-    });
-
-    // Fail the workflow
-    await this.failWorkflow(instanceId, `Step "${stepName}" failed: ${error}`);
   }
 
-  private async completeWorkflow(instanceId: string, output?: any): Promise<void> {
-    const instance = await this.adapter.getInstance(instanceId);
-    if (!instance) return;
+  /**
+   * Reset heartbeat timeout for an isolated workflow
+   */
+  private resetHeartbeatTimeout(instanceId: string, pid: number): void {
+    const info = this.isolatedProcesses.get(instanceId);
+    if (!info) return;
 
-    // Clear timeout
-    const runInfo = this.running.get(instanceId);
-    if (runInfo?.timeout) {
-      clearTimeout(runInfo.timeout);
+    // Clear existing timeout
+    if (info.heartbeatTimeout) {
+      clearTimeout(info.heartbeatTimeout);
     }
-    this.running.delete(instanceId);
-
-    await this.adapter.updateInstance(instanceId, {
-      status: "completed",
-      output,
-      completedAt: new Date(),
-      currentStep: undefined,
-    });
 
-    await this.emitEvent("workflow.completed", {
-      instanceId,
-      workflowName: instance.workflowName,
-      output,
-    });
+    // Set new timeout
+    info.heartbeatTimeout = setTimeout(async () => {
+      // Check if process is still alive
+      if (!isProcessAlive(pid)) {
+        return; // Process already dead, exit handler will handle it
+      }
 
-    // Broadcast via SSE
-    if (this.sse) {
-      this.sse.broadcast(`workflow:${instanceId}`, "completed", { output });
-    }
+      console.error(`[Workflows] No heartbeat from isolated workflow ${instanceId} for ${this.heartbeatTimeoutMs}ms`);
+      await this.handleIsolatedTimeout(instanceId, pid);
+    }, this.heartbeatTimeoutMs);
   }
 
-  private async failWorkflow(instanceId: string, error: string): Promise<void> {
-    const instance = await this.adapter.getInstance(instanceId);
-    if (!instance) return;
+  /**
+   * Handle timeout for isolated workflow (workflow timeout or heartbeat timeout)
+   */
+  private async handleIsolatedTimeout(instanceId: string, pid: number): Promise<void> {
+    const info = this.isolatedProcesses.get(instanceId);
+    if (!info) return;
 
-    // Clear timeout
-    const runInfo = this.running.get(instanceId);
-    if (runInfo?.timeout) {
-      clearTimeout(runInfo.timeout);
+    // Kill the process
+    try {
+      process.kill(pid, "SIGKILL");
+    } catch {
+      // Process might already be dead
     }
-    this.running.delete(instanceId);
 
+    // Clean up
+    if (info.timeout) clearTimeout(info.timeout);
+    if (info.heartbeatTimeout) clearTimeout(info.heartbeatTimeout);
+    this.isolatedProcesses.delete(instanceId);
+    await this.getSocketServer().closeSocket(instanceId);
+
+    // Fail the workflow
     await this.adapter.updateInstance(instanceId, {
       status: "failed",
-      error,
+      error: "Workflow timed out",
       completedAt: new Date(),
     });
-
     await this.emitEvent("workflow.failed", {
       instanceId,
-      workflowName: instance.workflowName,
-      error,
+      error: "Workflow timed out",
     });
-
-    // Broadcast via SSE
     if (this.sse) {
-      this.sse.broadcast(`workflow:${instanceId}`, "failed", { error });
+      this.sse.broadcast(`workflow:${instanceId}`, "failed", { error: "Workflow timed out" });
+      this.sse.broadcast("workflows:all", "workflow.failed", {
+        instanceId,
+        error: "Workflow timed out",
+      });
     }
   }
 
   private async emitEvent(event: string, data: any): Promise<void> {
-    if (this.events) {
-      await this.events.emit(event, data);
+    if (this.eventsService) {
+      await this.eventsService.emit(event, data);
     }
   }
 }