@pikku/core 0.11.2 → 0.12.0

package/src/wirings/workflow/pikku-workflow-service.ts

@@ -13,13 +13,20 @@ import type {
   PikkuWorkflowWire,
   StepState,
   WorkflowRun,
+  WorkflowRunWire,
   WorkflowStatus,
   WorkflowServiceConfig,
   WorkflowStepOptions,
 } from './workflow.types.js'
-import { executeGraphStep, runWorkflowGraph } from './graph/graph-runner.js'
+import {
+  continueGraph,
+  executeGraphStep,
+  runWorkflowGraph,
+  runFromMeta,
+} from './graph/graph-runner.js'
 import { WorkflowService } from '../../services/workflow-service.js'
-import { PikkuError } from '../../errors/error-handler.js'
+import { PikkuError, addError } from '../../errors/error-handler.js'
+import { RPCNotFoundError } from '../rpc/rpc-runner.js'
 
 /**
  * Exception thrown when workflow needs to pause for async step
@@ -47,6 +54,19 @@ export class WorkflowCancelledException extends Error {
   }
 }
 
+/**
+ * Exception thrown when workflow is suspended
+ */
+export class WorkflowSuspendedException extends Error {
+  constructor(
+    public readonly runId: string,
+    public readonly reason: string
+  ) {
+    super(reason || 'Workflow suspended')
+    this.name = 'WorkflowSuspendedException'
+  }
+}
+
 /**
  * Error class for workflow not found
  */
@@ -55,15 +75,20 @@ export class WorkflowNotFoundError extends PikkuError {
     super(`Workflow not found: ${name}`)
   }
 }
+addError(WorkflowNotFoundError, {
+  status: 404,
+  message: 'Workflow not found.',
+})
 
-/**
- * Error class for workflow not found
- */
-export class WorkflowRunNotFound extends PikkuError {
+export class WorkflowRunNotFoundError extends PikkuError {
   constructor(runId: string) {
     super(`Workflow run not found: ${runId}`)
   }
 }
+addError(WorkflowRunNotFoundError, {
+  status: 404,
+  message: 'Workflow run not found.',
+})
 
 export class WorkflowServiceNotInitialized extends Error {}
 export class WorkflowStepNameNotString extends Error {
@@ -72,6 +97,13 @@ export class WorkflowStepNameNotString extends Error {
   }
 }
 
+const WORKFLOW_END_STATES: ReadonlySet<string> = new Set([
+  'completed',
+  'failed',
+  'cancelled',
+  'suspended',
+])
+
 /**
  * Abstract workflow state service
  * Implementations provide pluggable storage backends (SQLite, PostgreSQL, etc.)
@@ -108,7 +140,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
 
   public setServices(
     singletonServices: CoreSingletonServices,
-    createWireServices: CreateWireServices,
+    createWireServices: CreateWireServices | undefined,
     { workflow }: CoreConfig
   ) {
     this.singletonServices = singletonServices
@@ -124,16 +156,20 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     }
   }
 
-  /**
-   * Create a new workflow run
-   * @param name - Workflow name
-   * @param input - Input data for the workflow
-   * @returns Run ID
-   */
+  public async registerWorkflowVersions(): Promise<void> {
+    const allMeta = pikkuState(null, 'workflows', 'meta')
+    for (const [name, meta] of Object.entries(allMeta)) {
+      if (!meta.graphHash) continue
+      await this.upsertWorkflowVersion(name, meta.graphHash, meta, meta.source)
+    }
+  }
+
   abstract createRun(
     workflowName: string,
     input: any,
-    inline?: boolean
+    inline: boolean,
+    graphHash: string,
+    wire: WorkflowRunWire
   ): Promise<string>
 
   /**
@@ -270,6 +306,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    */
   abstract getCompletedGraphState(runId: string): Promise<{
     completedNodeIds: string[]
+    failedNodeIds: string[]
     branchKeys: Record<string, string>
   }>
 
@@ -321,6 +358,18 @@ export abstract class PikkuWorkflowService implements WorkflowService {
    */
   abstract getRunState(runId: string): Promise<Record<string, unknown>>
 
+  abstract upsertWorkflowVersion(
+    name: string,
+    graphHash: string,
+    graph: any,
+    source: string
+  ): Promise<void>
+
+  abstract getWorkflowVersion(
+    name: string,
+    graphHash: string
+  ): Promise<{ graph: any; source: string } | null>
+
   // ============================================================================
   // Workflow Lifecycle Methods
   // ============================================================================
@@ -334,6 +383,21 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     await queueService.add(this.getConfig().orchestratorQueueName, { runId })
   }
 
+  public async queueStepWorker(
+    runId: string,
+    stepName: string,
+    rpcName: string,
+    data: any
+  ): Promise<void> {
+    const queueService = this.verifyQueueService()
+    await queueService.add(this.getConfig().stepWorkerQueueName, {
+      runId,
+      stepName,
+      rpcName,
+      data,
+    })
+  }
+
   /**
    * Execute a workflow sleep step completion
    * Sets the step result to null and resumes the workflow
@@ -373,6 +437,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
   public async startWorkflow<I>(
     name: string,
     input: I,
+    wire: WorkflowRunWire,
     rpcService: any,
     options?: { inline?: boolean; startNode?: string }
   ): Promise<{ runId: string }> {
@@ -386,14 +451,16 @@ export abstract class PikkuWorkflowService implements WorkflowService {
 
     // Check if this is a graph workflow (source === 'graph')
     if (workflowMeta.source === 'graph') {
-      // Graph workflows use the graph scheduler
+      const shouldInline =
+        options?.inline || !this.singletonServices?.queueService
       return runWorkflowGraph(
         this,
         name,
         input,
         rpcService,
-        options?.inline,
-        options?.startNode
+        shouldInline,
+        options?.startNode,
+        wire
       )
     }
 
@@ -405,36 +472,90 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       throw new WorkflowNotFoundError(name)
     }
 
-    // Create workflow run in state
-    const runId = await this.createRun(name, input, options?.inline)
+    if (!workflowMeta.graphHash) {
+      throw new Error(`Missing workflow graphHash for '${name}'`)
+    }
+
+    const runId = await this.createRun(
+      name,
+      input,
+      options?.inline ?? false,
+      workflowMeta.graphHash,
+      wire
+    )
 
-    // Register inline run for fast lookup
     if (options?.inline) {
       this.inlineRuns.add(runId)
     }
 
-    try {
-      // If inline mode or no queue service, execute directly
-      // Otherwise, use remote execution (queue-based)
-      if (options?.inline || !this.singletonServices?.queueService) {
-        await this.runWorkflowJob(runId, rpcService)
-      } else {
-        await this.resumeWorkflow(runId)
-      }
+    if (options?.inline || !this.singletonServices?.queueService) {
+      this.runWorkflowJob(runId, rpcService)
+        .catch(() => {})
+        .finally(() => {
+          if (options?.inline) {
+            this.inlineRuns.delete(runId)
+          }
+        })
+    } else {
+      await this.resumeWorkflow(runId)
+    }
+
+    return { runId }
+  }
 
-      return { runId }
-    } finally {
-      // Clean up inline tracking
-      if (options?.inline) {
-        this.inlineRuns.delete(runId)
+  public async runToCompletion<I>(
+    name: string,
+    input: I,
+    rpcService: any,
+    options?: { pollIntervalMs?: number; wire?: WorkflowRunWire }
+  ): Promise<any> {
+    const pollInterval = options?.pollIntervalMs ?? 1000
+    const { runId } = await this.startWorkflow(
+      name,
+      input,
+      options?.wire ?? { type: 'internal' },
+      rpcService,
+      { inline: true }
+    )
+    while (true) {
+      const run = await this.getRun(runId)
+      if (!run) {
+        throw new WorkflowRunNotFoundError(runId)
+      }
+      if (WORKFLOW_END_STATES.has(run.status)) {
+        if (run.status === 'failed') {
+          throw new Error(run.error?.message || 'Workflow failed')
+        }
+        if (run.status === 'cancelled') {
+          throw new Error('Workflow was cancelled')
+        }
+        return run.output
       }
+      await new Promise((resolve) => setTimeout(resolve, pollInterval))
     }
   }
 
   public async runWorkflowJob(runId: string, rpcService: any): Promise<void> {
     const run = await this.getRun(runId)
     if (!run) {
-      throw new WorkflowRunNotFound(runId)
+      throw new WorkflowRunNotFoundError(runId)
+    }
+
+    const meta = pikkuState(null, 'workflows', 'meta')
+    const workflowMeta = meta[run.workflow]
+
+    if (
+      run.graphHash &&
+      workflowMeta?.graphHash &&
+      run.graphHash !== workflowMeta.graphHash
+    ) {
+      await this.runVersionMismatchFallback(run, workflowMeta, rpcService)
+      return
+    }
+
+    if (workflowMeta?.source === 'graph') {
+      await continueGraph(this, runId, run.workflow)
+      return
     }
 
     const registrations = pikkuState(null, 'workflows', 'registrations')
@@ -444,9 +565,6 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     }
 
     await this.withRunLock(runId, async () => {
-      const meta = pikkuState(null, 'workflows', 'meta')
-      const workflowMeta = meta[run.workflow]
-
       const workflowWire = this.createWorkflowWire(
         run.workflow,
         runId,
@@ -457,7 +575,7 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         const result = await runPikkuFunc(
           'workflow',
           workflowMeta.name,
-          workflowMeta.pikkuFuncName,
+          workflowMeta.pikkuFuncId,
           {
             singletonServices: this.singletonServices!,
             wire,
@@ -469,13 +587,10 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         await this.updateRunStatus(runId, 'completed', result)
       } catch (error: any) {
         if (error instanceof WorkflowAsyncException) {
-          // Normal - workflow paused for step execution
           throw error
         }
 
-        // Check if it's a WorkflowCancelledException
         if (error instanceof WorkflowCancelledException) {
-          // Workflow was cancelled - update status and rethrow
           await this.updateRunStatus(runId, 'cancelled', undefined, {
             message: error.message || 'Workflow cancelled',
             stack: '',
@@ -484,6 +599,15 @@ export abstract class PikkuWorkflowService implements WorkflowService {
           throw error
         }
 
+        if (error instanceof WorkflowSuspendedException) {
+          await this.updateRunStatus(runId, 'suspended', undefined, {
+            message: error.message || 'Workflow suspended',
+            stack: '',
+            code: 'WORKFLOW_SUSPENDED',
+          })
+          throw error
+        }
+
         await this.updateRunStatus(runId, 'failed', undefined, {
           message: error.message,
           stack: error.stack,
@@ -495,6 +619,35 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     })
   }
 
+  private async runVersionMismatchFallback(
+    run: WorkflowRun,
+    currentMeta: { source: string },
+    rpcService: any
+  ): Promise<void> {
+    const source = currentMeta.source
+
+    if (source === 'complex') {
+      await this.updateRunStatus(run.id, 'failed', undefined, {
+        message: `Workflow '${run.workflow}' definition changed. Complex workflows with inline steps cannot be migrated.`,
+        stack: '',
+        code: 'VERSION_CONFLICT',
+      })
+      return
+    }
+
+    const version = await this.getWorkflowVersion(run.workflow, run.graphHash!)
+    if (!version) {
+      await this.updateRunStatus(run.id, 'failed', undefined, {
+        message: `Workflow '${run.workflow}' version '${run.graphHash}' not found. Cannot resume with changed definition.`,
+        stack: '',
+        code: 'VERSION_NOT_FOUND',
+      })
+      return
+    }
+
+    await runFromMeta(this, run.id, version.graph, rpcService)
+  }
+
   /**
    * Execute a single workflow step (called by worker)
    * Handles idempotency, RPC execution, result storage, retry logic, and orchestrator triggering
@@ -527,24 +680,23 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         stepState = await this.createRetryAttempt(stepState.stepId, 'running')
       }
 
-      if (stepState.status !== 'pending') {
-        // Mark step as running (pending or failed status)
+      if (stepState.status === 'pending') {
+        // Mark pending step as running before execution
         await this.setStepRunning(stepState.stepId)
       }
 
       try {
         let result: any
 
-        // Check if this is a graph node step (step name starts with 'node:')
-        if (stepName.startsWith('node:')) {
-          // Get the graph name from the workflow run
-          const run = await this.getRun(runId)
-          if (!run) {
-            throw new Error(`Workflow run not found: ${runId}`)
-          }
-          const graphName = run.workflow
+        const run = await this.getRun(runId)
+        if (!run) {
+          throw new Error(`Workflow run not found: ${runId}`)
+        }
+
+        const meta = pikkuState(null, 'workflows', 'meta')
+        const workflowMeta = meta[run.workflow]
 
-          // Execute as graph step with graph wire context
+        if (workflowMeta?.nodes && stepName in workflowMeta.nodes) {
           result = await executeGraphStep(
             this,
             rpcService,
@@ -553,10 +705,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
             stepName,
             rpcName,
             data,
-            graphName
+            run.workflow
           )
         } else {
-          // Execute RPC with workflow step context (regular workflow)
           result = await rpcService.rpcWithWire(rpcName, data, {
             workflowStep: {
               runId,
@@ -572,6 +723,15 @@ export abstract class PikkuWorkflowService implements WorkflowService {
         // Resume orchestrator to continue workflow
         await this.resumeWorkflow(runId)
       } catch (error: any) {
+        if (error instanceof RPCNotFoundError) {
+          await this.setStepError(stepState.stepId, error)
+          await this.updateRunStatus(runId, 'suspended', undefined, {
+            message: `RPC '${rpcName}' not found. Deploy the missing function and resume.`,
+            code: 'RPC_NOT_FOUND',
+          })
+          return
+        }
+
         // Store error and mark failed
         await this.setStepError(stepState.stepId, error)
 
@@ -603,7 +763,8 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     } catch (error: any) {
       if (
         error.name === 'WorkflowAsyncException' ||
-        error.name === 'WorkflowCancelledException'
+        error.name === 'WorkflowCancelledException' ||
+        error.name === 'WorkflowSuspendedException'
       ) {
         return
       }
@@ -724,6 +885,14 @@ export abstract class PikkuWorkflowService implements WorkflowService {
           await this.setStepResult(currentStepState.stepId, result)
           return result
         } catch (error: any) {
+          if (error instanceof RPCNotFoundError) {
+            await this.updateRunStatus(runId, 'suspended', undefined, {
+              message: `RPC '${rpcName}' not found. Deploy the missing function and resume.`,
+              code: 'RPC_NOT_FOUND',
+            })
+            throw error
+          }
+
           // Record the error (marks step as failed)
           await this.setStepError(currentStepState.stepId, error)
 
@@ -766,8 +935,8 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       stepState = await this.insertStepState(
         runId,
         stepName,
-        null, // No RPC for inline steps
-        null, // No data for inline steps
+        null,
+        null,
         stepOptions
       )
     }
@@ -852,13 +1021,9 @@ export abstract class PikkuWorkflowService implements WorkflowService {
       stepState = await this.getStepState(runId, stepName)
     } catch {
       // Step doesn't exist - create it (sleep step, no RPC)
-      stepState = await this.insertStepState(
-        runId,
-        stepName,
-        null, // No RPC for sleep steps
-        { duration }, // Store duration as data
-        undefined // No retry options for sleep
-      )
+      stepState = await this.insertStepState(runId, stepName, null, {
+        duration,
+      })
     }
 
     if (stepState.status === 'succeeded') {
@@ -897,13 +1062,63 @@ export abstract class PikkuWorkflowService implements WorkflowService {
     }
   }
 
+  private getSuspendStepName(reason: string): string {
+    if (!reason) {
+      return '__workflow_suspend'
+    }
+    return '__workflow_suspend'
+  }
+
+  private async suspendStep(runId: string, reason: string): Promise<void> {
+    const stepName = this.getSuspendStepName(reason)
+    await this.withStepLock(runId, stepName, async () => {
+      let stepState: StepState
+      try {
+        stepState = await this.getStepState(runId, stepName)
+      } catch {
+        stepState = await this.insertStepState(
+          runId,
+          stepName,
+          'pikkuWorkflowSuspend',
+          {
+            reason,
+          }
+        )
+      }
+      if (!stepState.stepId) {
+        stepState = await this.insertStepState(
+          runId,
+          stepName,
+          'pikkuWorkflowSuspend',
+          {
+            reason,
+          }
+        )
+      }
+
+      if (stepState.status === 'succeeded') {
+        return
+      }
+
+      if (stepState.status === 'pending') {
+        await this.setStepRunning(stepState.stepId)
+      }
+
+      await this.setStepResult(stepState.stepId, {
+        reason,
+        suspendedAt: new Date().toISOString(),
+      })
+      throw new WorkflowSuspendedException(runId, reason)
+    })
+  }
+
   private createWorkflowWire(
-    workflowName: string,
+    name: string,
     runId: string,
     rpcService: any
   ): PikkuWorkflowWire {
     const workflowWire: PikkuWorkflowWire = {
-      workflowName,
+      name,
       runId,
       getRun: async () => (await this.getRun(runId)) as WorkflowRun,
 
@@ -943,6 +1158,10 @@ export abstract class PikkuWorkflowService implements WorkflowService {
           getDurationInMilliseconds(duration)
         )
       },
+
+      suspend: async (reason: string) => {
+        await this.suspendStep(runId, reason || 'Workflow suspended')
+      },
     }
     return workflowWire
   }

package/src/wirings/workflow/workflow-helpers.ts

@@ -0,0 +1,79 @@
+import { WorkflowRunNotFoundError } from './pikku-workflow-service.js'
+
+export function workflow<
+  TWorkflowMap extends Record<string, { input: any; output: any }>,
+>(
+  workflowName: string & keyof TWorkflowMap,
+  options?: { pollIntervalMs?: number }
+): { func: (services: any, data: any, wire: any) => Promise<any> } {
+  return {
+    func: async (services: any, data: any, { rpc }: any) => {
+      return services.workflowService.runToCompletion(
+        workflowName,
+        data,
+        rpc,
+        options
+      )
+    },
+  }
+}
+
+export function workflowStart<
+  TWorkflowMap extends Record<string, { input: any; output: any }>,
+>(
+  workflowName: string & keyof TWorkflowMap
+): {
+  func: (services: any, data: any, wire: any) => Promise<{ runId: string }>
+} {
+  return {
+    func: async (_services: any, data: any, { rpc }: any) => {
+      return rpc.startWorkflow(workflowName, data)
+    },
+  }
+}
+
+export function workflowStatus<
+  TWorkflowMap extends Record<string, { input: any; output: any }>,
+>(
+  _workflowName: string & keyof TWorkflowMap
+): {
+  func: (
+    services: any,
+    data: { runId: string }
+  ) => Promise<{
+    id: string
+    status: 'running' | 'suspended' | 'completed' | 'failed' | 'cancelled'
+    output?: any
+    error?: { message?: string }
+  }>
+} {
+  return {
+    func: async (services: any, data: { runId: string }) => {
+      const run = await services.workflowService.getRun(data.runId)
+      if (!run) {
+        throw new WorkflowRunNotFoundError(data.runId)
+      }
+      return {
+        id: run.id,
+        status: run.status,
+        output: run.output,
+        error: run.error ? { message: run.error.message } : undefined,
+      }
+    },
+  }
+}
+
+export function graphStart<
+  TGraphsMap extends Record<string, Record<string, any>>,
+>(
+  graphName: string & keyof TGraphsMap,
+  startNode: string
+): {
+  func: (services: any, data: any, wire: any) => Promise<{ runId: string }>
+} {
+  return {
+    func: async (_services: any, data: any, { rpc }: any) => {
+      return rpc.startWorkflow(graphName, data, { startNode })
+    },
+  }
+}