@pikku/core 0.11.1 → 0.12.0

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -9,21 +9,86 @@ import {
   SerializedError,
 } from '../../types/core.types.js'
 import { QueueService } from '../queue/queue.types.js'
-import {
-  WorkflowAsyncException,
-  WorkflowCancelledException,
-  WorkflowNotFoundError,
-  WorkflowRunNotFound,
-} from './workflow-runner.js'
 import type {
   PikkuWorkflowWire,
   StepState,
   WorkflowRun,
+  WorkflowRunWire,
   WorkflowStatus,
-  WorkflowService,
   WorkflowServiceConfig,
   WorkflowStepOptions,
 } from './workflow.types.js'
+import {
+  continueGraph,
+  executeGraphStep,
+  runWorkflowGraph,
+  runFromMeta,
+} from './graph/graph-runner.js'
+import { WorkflowService } from '../../services/workflow-service.js'
+import { PikkuError, addError } from '../../errors/error-handler.js'
+import { RPCNotFoundError } from '../rpc/rpc-runner.js'
+
+/**
+ * Exception thrown when workflow needs to pause for async step
+ */
+export class WorkflowAsyncException extends Error {
+  constructor(
+    public readonly runId: string,
+    public readonly stepName: string
+  ) {
+    super(`Workflow paused at step: ${stepName}`)
+    this.name = 'WorkflowAsyncException'
+  }
+}
+
+/**
+ * Exception thrown when workflow is cancelled
+ */
+export class WorkflowCancelledException extends Error {
+  constructor(
+    public readonly runId: string,
+    public readonly reason?: string
+  ) {
+    super(reason || 'Workflow cancelled')
+    this.name = 'WorkflowCancelledException'
+  }
+}
+
+/**
+ * Exception thrown when workflow is suspended
+ */
+export class WorkflowSuspendedException extends Error {
+  constructor(
+    public readonly runId: string,
+    public readonly reason: string
+  ) {
+    super(reason || 'Workflow suspended')
+    this.name = 'WorkflowSuspendedException'
+  }
+}
+
+/**
+ * Error class for workflow not found
+ */
+export class WorkflowNotFoundError extends PikkuError {
+  constructor(name: string) {
+    super(`Workflow not found: ${name}`)
+  }
+}
+addError(WorkflowNotFoundError, {
+  status: 404,
+  message: 'Workflow not found.',
+})
+
+export class WorkflowRunNotFoundError extends PikkuError {
+  constructor(runId: string) {
+    super(`Workflow run not found: ${runId}`)
+  }
+}
+addError(WorkflowRunNotFoundError, {
+  status: 404,
+  message: 'Workflow run not found.',
+})
 
 export class WorkflowServiceNotInitialized extends Error {}
 export class WorkflowStepNameNotString extends Error {
@@ -32,6 +97,13 @@ export class WorkflowStepNameNotString extends Error {
   }
 }
 
+const WORKFLOW_END_STATES: ReadonlySet<string> = new Set([
+  'completed',
+  'failed',
+  'cancelled',
+  'suspended',
+])
+
 /**
  * Abstract workflow state service
  * Implementations provide pluggable storage backends (SQLite, PostgreSQL, etc.)
@@ -41,12 +113,34 @@ export abstract class PikkuWorkflowService implements WorkflowService {
   private config: WorkflowServiceConfig | undefined
   private singletonServices: CoreSingletonServices | undefined
   private createWireServices: CreateWireServices | undefined
+  private inlineRuns = new Set<string>()
 
   constructor() {}
 
+  /**
+   * Check if a run is executing inline (without queues)
+   */
+  protected isInline(runId: string): boolean {
+    return this.inlineRuns.has(runId)
+  }
+
+  /**
+   * Register a run as inline (for graph-runner to use)
+   */
+  public registerInlineRun(runId: string): void {
+    this.inlineRuns.add(runId)
+  }
+
+  /**
+   * Unregister a run from inline tracking
+   */
+  public unregisterInlineRun(runId: string): void {
+    this.inlineRuns.delete(runId)
+  }
+
   public setServices(
     singletonServices: CoreSingletonServices,
-    createWireServices: CreateWireServices,
+    createWireServices: CreateWireServices | undefined,
     { workflow }: CoreConfig
   ) {
     this.singletonServices = singletonServices
@@ -58,17 +152,25 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         workflow?.orchestratorQueueName ?? 'pikku-workflow-orchestrator',
       stepWorkerQueueName:
         workflow?.stepWorkerQueueName ?? 'pikku-workflow-step-worker',
-      sleeperRPCName: workflow?.sleeperRPCName ?? 'pikkuWorkflowStepSleeper',
+      sleeperRPCName: workflow?.sleeperRPCName ?? 'pikkuWorkflowSleeper',
     }
   }
 
-  /**
-   * Create a new workflow run
-   * @param name - Workflow name
-   * @param input - Input data for the workflow
-   * @returns Run ID
-   */
-  abstract createRun(workflowName: string, input: any): Promise<string>
+  public async registerWorkflowVersions(): Promise<void> {
+    const allMeta = pikkuState(null, 'workflows', 'meta')
+    for (const [name, meta] of Object.entries(allMeta)) {
+      if (!meta.graphHash) continue
+      await this.upsertWorkflowVersion(name, meta.graphHash, meta, meta.source)
+    }
+  }
+
+  abstract createRun(
+    workflowName: string,
+    input: any,
+    inline: boolean,
+    graphHash: string,
+    wire: WorkflowRunWire
+  ): Promise<string>
 
   /**
    * Get a workflow run by ID
@@ -193,6 +295,81 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    */
   abstract close(): Promise<void>
 
+  // ============================================================================
+  // Workflow Graph Methods
+  // ============================================================================
+
+  /**
+   * Get completed graph state (lightweight - no results)
+   * @param runId - Run ID
+   * @returns Completed node IDs and their branch keys
+   */
+  abstract getCompletedGraphState(runId: string): Promise<{
+    completedNodeIds: string[]
+    failedNodeIds: string[]
+    branchKeys: Record<string, string>
+  }>
+
+  /**
+   * Filter candidate nodes to only those without existing steps
+   * @param runId - Run ID
+   * @param nodeIds - Candidate node IDs to check
+   * @returns Node IDs that don't have a step yet
+   */
+  abstract getNodesWithoutSteps(
+    runId: string,
+    nodeIds: string[]
+  ): Promise<string[]>
+
+  /**
+   * Get results for specific nodes
+   * @param runId - Run ID
+   * @param nodeIds - Node IDs to fetch results for
+   * @returns Map of nodeId to result
+   */
+  abstract getNodeResults(
+    runId: string,
+    nodeIds: string[]
+  ): Promise<Record<string, any>>
+
+  /**
+   * Set the branch key for a graph node step
+   * @param stepId - Step ID
+   * @param branchKey - Branch key selected by graph.branch()
+   */
+  abstract setBranchTaken(stepId: string, branchKey: string): Promise<void>
+
+  /**
+   * Update a state variable in the workflow run's state
+   * @param runId - Run ID
+   * @param name - Variable name
+   * @param value - Value to store
+   */
+  abstract updateRunState(
+    runId: string,
+    name: string,
+    value: unknown
+  ): Promise<void>
+
+  /**
+   * Get the entire state object for a workflow run
+   * @param runId - Run ID
+   * @returns The state object with all variables
+   */
+  abstract getRunState(runId: string): Promise<Record<string, unknown>>
+
+  abstract upsertWorkflowVersion(
+    name: string,
+    graphHash: string,
+    graph: any,
+    source: string
+  ): Promise<void>
+
+  abstract getWorkflowVersion(
+    name: string,
+    graphHash: string
+  ): Promise<{ graph: any; source: string } | null>
+
   // ============================================================================
   // Workflow Lifecycle Methods
   // ============================================================================
@@ -206,12 +383,27 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     await queueService.add(this.getConfig().orchestratorQueueName, { runId })
   }
 
+  public async queueStepWorker(
+    runId: string,
+    stepName: string,
+    rpcName: string,
+    data: any
+  ): Promise<void> {
+    const queueService = this.verifyQueueService()
+    await queueService.add(this.getConfig().stepWorkerQueueName, {
+      runId,
+      stepName,
+      rpcName,
+      data,
+    })
+  }
+
   /**
    * Execute a workflow sleep step completion
    * Sets the step result to null and resumes the workflow
    * @param data - Sleeper input data
    */
-  public async executeWorkflowSleep(
+  public async executeWorkflowSleepCompleted(
     runId: string,
     stepId: string
   ): Promise<void> {
@@ -238,51 +430,141 @@ export abstract class PikkuWorkflowService implements WorkflowService {
 
   /**
    * Start a new workflow run
+   * Automatically detects workflow type (DSL or graph) from meta and executes accordingly
+   * @param options.inline - If true, execute workflow directly without queue service
+   * @param options.startNode - Starting node ID for graph workflows (from wire config)
    */
   public async startWorkflow<I>(
     name: string,
     input: I,
-    rpcService: any
+    wire: WorkflowRunWire,
+    rpcService: any,
+    options?: { inline?: boolean; startNode?: string }
   ): Promise<{ runId: string }> {
-    const registrations = pikkuState('workflows', 'registrations')
+    // Check meta to determine workflow type
+    const meta = pikkuState(null, 'workflows', 'meta')
+    const workflowMeta = meta[name]
+
+    if (!workflowMeta) {
+      throw new WorkflowNotFoundError(name)
+    }
+
+    // Check if this is a graph workflow (source === 'graph')
+    if (workflowMeta.source === 'graph') {
+      const shouldInline =
+        options?.inline || !this.singletonServices?.queueService
+      return runWorkflowGraph(
+        this,
+        name,
+        input,
+        rpcService,
+        shouldInline,
+        options?.startNode,
+        wire
+      )
+    }
+
+    // DSL workflow - check registration exists
+    const registrations = pikkuState(null, 'workflows', 'registrations')
     const workflow = registrations.get(name)
 
     if (!workflow) {
       throw new WorkflowNotFoundError(name)
     }
 
-    // Create workflow run in state
-    const runId = await this.createRun(name, input)
+    if (!workflowMeta.graphHash) {
+      throw new Error(`Missing workflow graphHash for '${name}'`)
+    }
 
-    // If queue service is available, use remote execution (queue-based)
-    // Otherwise, execute directly (inline/synchronous)
-    if (this.singletonServices?.queueService) {
-      // Queue orchestrator to start the workflow
-      await this.resumeWorkflow(runId)
+    const runId = await this.createRun(
+      name,
+      input,
+      options?.inline ?? false,
+      workflowMeta.graphHash,
+      wire
+    )
+
+    if (options?.inline) {
+      this.inlineRuns.add(runId)
+    }
+
+    if (options?.inline || !this.singletonServices?.queueService) {
+      this.runWorkflowJob(runId, rpcService)
+        .catch(() => {})
+        .finally(() => {
+          if (options?.inline) {
+            this.inlineRuns.delete(runId)
+          }
+        })
     } else {
-      // No queue service - execute directly via runWorkflowJob
-      await this.runWorkflowJob(runId, rpcService)
+      await this.resumeWorkflow(runId)
     }
 
     return { runId }
   }
 
+  public async runToCompletion<I>(
+    name: string,
+    input: I,
+    rpcService: any,
+    options?: { pollIntervalMs?: number; wire?: WorkflowRunWire }
+  ): Promise<any> {
+    const pollInterval = options?.pollIntervalMs ?? 1000
+    const { runId } = await this.startWorkflow(
+      name,
+      input,
+      options?.wire ?? { type: 'internal' },
+      rpcService,
+      { inline: true }
+    )
+    while (true) {
+      const run = await this.getRun(runId)
+      if (!run) {
+        throw new WorkflowRunNotFoundError(runId)
+      }
+      if (WORKFLOW_END_STATES.has(run.status)) {
+        if (run.status === 'failed') {
+          throw new Error(run.error?.message || 'Workflow failed')
+        }
+        if (run.status === 'cancelled') {
+          throw new Error('Workflow was cancelled')
+        }
+        return run.output
+      }
+      await new Promise((resolve) => setTimeout(resolve, pollInterval))
+    }
+  }
+
   public async runWorkflowJob(runId: string, rpcService: any): Promise<void> {
     const run = await this.getRun(runId)
     if (!run) {
-      throw new WorkflowRunNotFound(runId)
+      throw new WorkflowRunNotFoundError(runId)
+    }
+
+    const meta = pikkuState(null, 'workflows', 'meta')
+    const workflowMeta = meta[run.workflow]
+
+    if (
+      run.graphHash &&
+      workflowMeta?.graphHash &&
+      run.graphHash !== workflowMeta.graphHash
+    ) {
+      await this.runVersionMismatchFallback(run, workflowMeta, rpcService)
+      return
+    }
+
+    if (workflowMeta?.source === 'graph') {
+      await continueGraph(this, runId, run.workflow)
+      return
     }
 
-    const registrations = pikkuState('workflows', 'registrations')
+    const registrations = pikkuState(null, 'workflows', 'registrations')
     const workflow = registrations.get(run.workflow)
     if (!workflow) {
       throw new WorkflowNotFoundError(run.workflow)
     }
 
     await this.withRunLock(runId, async () => {
-      const meta = pikkuState('workflows', 'meta')
-      const workflowMeta = meta[run.workflow]
-
       const workflowWire = this.createWorkflowWire(
         run.workflow,
         runId,
@@ -292,8 +574,8 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       try {
         const result = await runPikkuFunc(
           'workflow',
-          workflowMeta.workflowName,
-          workflowMeta.pikkuFuncName,
+          workflowMeta.name,
+          workflowMeta.pikkuFuncId,
           {
             singletonServices: this.singletonServices!,
             wire,
@@ -305,13 +587,24 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         await this.updateRunStatus(runId, 'completed', result)
       } catch (error: any) {
         if (error instanceof WorkflowAsyncException) {
-          // Normal - workflow paused for step execution
           throw error
         }
 
-        // Check if it's a WorkflowCancelledException
         if (error instanceof WorkflowCancelledException) {
-          // Workflow was cancelled - status already updated, just rethrow
+          await this.updateRunStatus(runId, 'cancelled', undefined, {
+            message: error.message || 'Workflow cancelled',
+            stack: '',
+            code: 'WORKFLOW_CANCELLED',
+          })
+          throw error
+        }
+
+        if (error instanceof WorkflowSuspendedException) {
+          await this.updateRunStatus(runId, 'suspended', undefined, {
+            message: error.message || 'Workflow suspended',
+            stack: '',
+            code: 'WORKFLOW_SUSPENDED',
+          })
           throw error
         }
 
@@ -326,6 +619,35 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     })
   }
 
+  private async runVersionMismatchFallback(
+    run: WorkflowRun,
+    currentMeta: { source: string },
+    rpcService: any
+  ): Promise<void> {
+    const source = currentMeta.source
+
+    if (source === 'complex') {
+      await this.updateRunStatus(run.id, 'failed', undefined, {
+        message: `Workflow '${run.workflow}' definition changed. Complex workflows with inline steps cannot be migrated.`,
+        stack: '',
+        code: 'VERSION_CONFLICT',
+      })
+      return
+    }
+
+    const version = await this.getWorkflowVersion(run.workflow, run.graphHash!)
+    if (!version) {
+      await this.updateRunStatus(run.id, 'failed', undefined, {
+        message: `Workflow '${run.workflow}' version '${run.graphHash}' not found. Cannot resume with changed definition.`,
+        stack: '',
+        code: 'VERSION_NOT_FOUND',
+      })
+      return
+    }
+
+    await runFromMeta(this, run.id, version.graph, rpcService)
+  }
+
   /**
    * Execute a single workflow step (called by worker)
    * Handles idempotency, RPC execution, result storage, retry logic, and orchestrator triggering
@@ -358,31 +680,58 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         stepState = await this.createRetryAttempt(stepState.stepId, 'running')
       }
 
-      if (stepState.status !== 'pending') {
-        // Mark step as running (pending or failed status)
+      if (stepState.status === 'pending') {
+        // Mark pending step as running before execution
         await this.setStepRunning(stepState.stepId)
       }
 
       try {
-        // Execute RPC with workflow step context
-        const result = await rpcService.rpcWithWire(rpcName, data, {
-          workflowStep: {
+        let result: any
+
+        const run = await this.getRun(runId)
+        if (!run) {
+          throw new Error(`Workflow run not found: ${runId}`)
+        }
+
+        const meta = pikkuState(null, 'workflows', 'meta')
+        const workflowMeta = meta[run.workflow]
+
+        if (workflowMeta?.nodes && stepName in workflowMeta.nodes) {
+          result = await executeGraphStep(
+            this,
+            rpcService,
             runId,
-            stepId: stepState.stepId,
-            attemptCount: stepState.attemptCount,
-          },
-        })
+            stepState.stepId,
+            stepName,
+            rpcName,
+            data,
+            run.workflow
+          )
+        } else {
+          result = await rpcService.rpcWithWire(rpcName, data, {
+            workflowStep: {
+              runId,
+              stepId: stepState.stepId,
+              attemptCount: stepState.attemptCount,
+            },
+          })
+        }
 
         // Store result and mark succeeded
         await this.setStepResult(stepState.stepId, result)
 
         // Resume orchestrator to continue workflow
-        try {
-          await this.resumeWorkflow(runId)
-        } catch (resumeError: any) {
-          throw resumeError
-        }
+        await this.resumeWorkflow(runId)
       } catch (error: any) {
+        if (error instanceof RPCNotFoundError) {
+          await this.setStepError(stepState.stepId, error)
+          await this.updateRunStatus(runId, 'suspended', undefined, {
+            message: `RPC '${rpcName}' not found. Deploy the missing function and resume.`,
+            code: 'RPC_NOT_FOUND',
+          })
+          return
+        }
+
         // Store error and mark failed
         await this.setStepError(stepState.stepId, error)
 
@@ -414,7 +763,8 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     } catch (error: any) {
       if (
         error.name === 'WorkflowAsyncException' ||
-        error.name === 'WorkflowCancelledException'
+        error.name === 'WorkflowCancelledException' ||
+        error.name === 'WorkflowSuspendedException'
       ) {
         return
       }
@@ -451,7 +801,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     let stepState: StepState
     try {
       stepState = await this.getStepState(runId, stepName)
-    } catch (error: any) {
+    } catch {
       // Step doesn't exist - create it
       stepState = await this.insertStepState(
         runId,
@@ -488,8 +838,8 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     // Step is pending - schedule it
     await this.setStepScheduled(stepState.stepId)
 
-    // Enqueue step worker
-    if (this.singletonServices!.queueService) {
+    // Enqueue step worker (unless inline mode)
+    if (!this.isInline(runId) && this.singletonServices!.queueService) {
       // Map step retry options to queue job options
       const retries = stepOptions?.retries ?? 0
       const retryDelay = stepOptions?.retryDelay
@@ -514,45 +864,60 @@ export abstract class PikkuWorkflowService implements WorkflowService {
                 : undefined,
         }
       )
+      // Pause workflow - step will callback when done
+      throw new WorkflowAsyncException(runId, stepName)
     } else {
-      // No queue service - execute locally
+      // Inline or no queue service - execute locally with retry loop
       const retries = stepOptions?.retries ?? 0
       const retryDelay = stepOptions?.retryDelay
-      const attemptCount = stepState.attemptCount
+      let currentStepState = stepState
 
-      try {
-        const result = await rpcService.rpcWithWire(rpcName, data, {
-          workflowStep: {
-            runId,
-            stepId: stepState.stepId,
-            attemptCount: stepState.attemptCount,
-          },
-        })
-        await this.setStepResult(stepState.stepId, result)
-        return result
-      } catch (error: any) {
-        // Record the error (marks step as failed)
-        await this.setStepError(stepState.stepId, error)
-
-        // Check if we should retry
-        if (attemptCount < retries) {
-          // Create a new pending retry attempt (copies metadata from failed step)
-          await this.createRetryAttempt(stepState.stepId, 'pending')
-
-          // Schedule orchestrator to retry after delay
-          await this.scheduleOrchestratorRetry(runId, retryDelay)
+      while (true) {
+        try {
+          await this.setStepRunning(currentStepState.stepId)
+          const result = await rpcService.rpcWithWire(rpcName, data, {
+            workflowStep: {
+              runId,
+              stepId: currentStepState.stepId,
+              attemptCount: currentStepState.attemptCount,
+            },
+          })
+          await this.setStepResult(currentStepState.stepId, result)
+          return result
+        } catch (error: any) {
+          if (error instanceof RPCNotFoundError) {
+            await this.updateRunStatus(runId, 'suspended', undefined, {
+              message: `RPC '${rpcName}' not found. Deploy the missing function and resume.`,
+              code: 'RPC_NOT_FOUND',
+            })
+            throw error
+          }
 
-          // Pause workflow - orchestrator will replay and pick up new attempt
-          throw new WorkflowAsyncException(runId, stepName)
+          // Record the error (marks step as failed)
+          await this.setStepError(currentStepState.stepId, error)
+
+          // Check if we should retry
+          if (currentStepState.attemptCount < retries) {
+            // Create a new pending retry attempt
+            currentStepState = await this.createRetryAttempt(
+              currentStepState.stepId,
+              'pending'
+            )
+
+            // Wait for retry delay if specified
+            if (retryDelay) {
+              await new Promise((resolve) =>
+                setTimeout(resolve, getDurationInMilliseconds(retryDelay))
+              )
+            }
+            // Continue loop to retry
+          } else {
+            // No more retries, fail the workflow
+            throw error
+          }
         }
-
-        // No more retries, fail the workflow
-        throw error
       }
     }
-
-    // Pause workflow - step will callback when done
-    throw new WorkflowAsyncException(runId, stepName)
   }
 
   private async inlineStep(
@@ -565,13 +930,13 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     let stepState: StepState
     try {
       stepState = await this.getStepState(runId, stepName)
-    } catch (error: any) {
+    } catch {
       // Step doesn't exist - create it (inline, no RPC)
       stepState = await this.insertStepState(
         runId,
         stepName,
-        null, // No RPC for inline steps
-        null, // No data for inline steps
+        null,
+        null,
         stepOptions
       )
     }
@@ -584,30 +949,68 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     // Execute inline function
     const retries = stepOptions?.retries ?? this.getConfig().retries
     const retryDelay = stepOptions?.retryDelay ?? this.getConfig().retryDelay
-    const attemptCount = stepState.attemptCount
+    let currentStepState = stepState
 
-    try {
-      const result = await fn()
-      await this.setStepResult(stepState.stepId, result)
-      return result
-    } catch (error: any) {
-      // Record the error (marks step as failed)
-      await this.setStepError(stepState.stepId, error)
+    // Check if we're running inline (in-memory) or remote (queue-based)
+    if (this.isInline(runId)) {
+      // Inline mode - execute with retry loop
+      while (true) {
+        try {
+          await this.setStepRunning(currentStepState.stepId)
+          const result = await fn()
+          await this.setStepResult(currentStepState.stepId, result)
+          return result
+        } catch (error: any) {
+          // Record the error (marks step as failed)
+          await this.setStepError(currentStepState.stepId, error)
+
+          // Check if we should retry
+          if (currentStepState.attemptCount < retries) {
+            // Create a new pending retry attempt
+            currentStepState = await this.createRetryAttempt(
+              currentStepState.stepId,
+              'pending'
+            )
+
+            // Wait for retry delay if specified
+            if (retryDelay) {
+              await new Promise((resolve) =>
+                setTimeout(resolve, getDurationInMilliseconds(retryDelay))
+              )
+            }
+            // Continue loop to retry
+          } else {
+            // No more retries, fail the workflow
+            throw error
+          }
+        }
+      }
+    } else {
+      // Remote mode - use queue-based retry
+      try {
+        await this.setStepRunning(currentStepState.stepId)
+        const result = await fn()
+        await this.setStepResult(currentStepState.stepId, result)
+        return result
+      } catch (error: any) {
+        // Record the error (marks step as failed)
+        await this.setStepError(currentStepState.stepId, error)
 
-      // Check if we should retry
-      if (attemptCount < retries) {
-        // Create a new pending retry attempt (copies metadata from failed step)
-        await this.createRetryAttempt(stepState.stepId, 'pending')
+        // Check if we should retry
+        if (currentStepState.attemptCount < retries) {
+          // Create a new pending retry attempt (copies metadata from failed step)
+          await this.createRetryAttempt(currentStepState.stepId, 'pending')
 
-        // Schedule orchestrator to retry after delay
-        await this.scheduleOrchestratorRetry(runId, retryDelay)
+          // Schedule orchestrator to retry after delay
+          await this.scheduleOrchestratorRetry(runId, retryDelay)
 
-        // Pause workflow - orchestrator will replay and pick up new attempt
-        throw new WorkflowAsyncException(runId, stepName)
-      }
+          // Pause workflow - orchestrator will replay and pick up new attempt
+          throw new WorkflowAsyncException(runId, stepName)
+        }
 
-      // No more retries, fail the workflow
-      throw error
+        // No more retries, fail the workflow
+        throw error
+      }
     }
   }
 
@@ -616,15 +1019,11 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     let stepState: StepState
     try {
       stepState = await this.getStepState(runId, stepName)
-    } catch (error: any) {
+    } catch {
       // Step doesn't exist - create it (sleep step, no RPC)
-      stepState = await this.insertStepState(
-        runId,
-        stepName,
-        null, // No RPC for sleep steps
-        { duration }, // Store duration as data
-        undefined // No retry options for sleep
-      )
+      stepState = await this.insertStepState(runId, stepName, null, {
+        duration,
+      })
     }
 
     if (stepState.status === 'succeeded') {
@@ -640,9 +1039,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     // Step is pending - schedule it
     await this.setStepScheduled(stepState.stepId)
 
-    // Check if we have queue service (remote mode) or inline mode
-    if (this.singletonServices!.schedulerService) {
-      // Remote mode - enqueue sleep worker with delay
+    // Check if inline mode or no scheduler service
+    if (!this.isInline(runId) && this.singletonServices!.schedulerService) {
+      // Remote mode - schedule sleep via scheduler service
       await this.singletonServices!.schedulerService.scheduleRPC(
         duration,
         this.getConfig().sleeperRPCName,
@@ -651,6 +1050,8 @@ export abstract class PikkuWorkflowService implements WorkflowService {
           stepId: stepState.stepId,
         }
       )
+      // Pause workflow - sleep will callback when done
+      throw new WorkflowAsyncException(runId, stepName)
     } else {
       // Inline mode - use setTimeout with actual duration
       await new Promise((resolve) =>
@@ -659,30 +1060,65 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       await this.setStepResult(stepState.stepId, null)
       return
     }
+  }
 
-    // Pause workflow - sleep will callback when done
-    throw new WorkflowAsyncException(runId, stepName)
+  private getSuspendStepName(reason: string): string {
+    if (!reason) {
+      return '__workflow_suspend'
+    }
+    return '__workflow_suspend'
   }
 
-  private async cancelWorkflow(runId: string, reason?: string) {
-    // Update workflow run status to cancelled
-    await this.updateRunStatus(runId, 'cancelled', undefined, {
-      message: reason || 'Workflow cancelled by user',
-      stack: '',
-      code: 'WORKFLOW_CANCELLED',
-    })
+  private async suspendStep(runId: string, reason: string): Promise<void> {
+    const stepName = this.getSuspendStepName(reason)
+    await this.withStepLock(runId, stepName, async () => {
+      let stepState: StepState
+      try {
+        stepState = await this.getStepState(runId, stepName)
+      } catch {
+        stepState = await this.insertStepState(
+          runId,
+          stepName,
+          'pikkuWorkflowSuspend',
+          {
+            reason,
+          }
+        )
+      }
+      if (!stepState.stepId) {
+        stepState = await this.insertStepState(
+          runId,
+          stepName,
+          'pikkuWorkflowSuspend',
+          {
+            reason,
+          }
+        )
+      }
+
+      if (stepState.status === 'succeeded') {
+        return
+      }
 
-    // Throw cancellation exception to stop workflow execution
-    throw new WorkflowCancelledException(runId, reason)
+      if (stepState.status === 'pending') {
+        await this.setStepRunning(stepState.stepId)
+      }
+
+      await this.setStepResult(stepState.stepId, {
+        reason,
+        suspendedAt: new Date().toISOString(),
+      })
+      throw new WorkflowSuspendedException(runId, reason)
+    })
   }
 
   private createWorkflowWire(
-    workflowName: string,
+    name: string,
     runId: string,
     rpcService: any
   ): PikkuWorkflowWire {
     const workflowWire: PikkuWorkflowWire = {
-      workflowName,
+      name,
       runId,
       getRun: async () => (await this.getRun(runId)) as WorkflowRun,
 
@@ -723,9 +1159,8 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         )
       },
 
-      // Implement workflow.cancel()
-      cancel: async (reason?: string): Promise<void> => {
-        await this.cancelWorkflow(runId, reason)
+      suspend: async (reason: string) => {
+        await this.suspendStep(runId, reason || 'Workflow suspended')
       },
     }
     return workflowWire