@donkeylabs/server 0.4.7 → 0.4.8

package/docs/workflows.md

@@ -0,0 +1,509 @@
+# Workflows
+
+Workflows provide step function / state machine orchestration for complex multi-step processes. Built on top of the Jobs service, workflows support sequential tasks, parallel execution, conditional branching, retries, and real-time progress via SSE.
+
+## Overview
+
+Use workflows when you need to:
+- Orchestrate multi-step processes (order processing, onboarding, data pipelines)
+- Run steps in parallel and wait for all to complete
+- Make decisions based on previous step outputs
+- Track progress across long-running processes
+- Automatically retry failed steps with backoff
+
+## Quick Start
+
+### 1. Define a Workflow
+
+```typescript
+import { workflow } from "@donkeylabs/server";
+
+const orderWorkflow = workflow("process-order")
+  .task("validate", {
+    job: "validate-order",
+    input: (ctx) => ({ orderId: ctx.input.orderId }),
+  })
+  .choice("check-inventory", {
+    choices: [
+      {
+        condition: (ctx) => ctx.steps.validate.inStock,
+        next: "fulfill",
+      },
+    ],
+    default: "backorder",
+  })
+  .parallel("fulfill", {
+    branches: [
+      workflow.branch("shipping")
+        .task("ship", { job: "create-shipment" })
+        .build(),
+      workflow.branch("notification")
+        .task("notify", { job: "send-confirmation" })
+        .build(),
+    ],
+    next: "complete",
+  })
+  .task("backorder", {
+    job: "create-backorder",
+    next: "complete",
+  })
+  .pass("complete", { end: true })
+  .build();
+```
+
+### 2. Register and Start
+
+```typescript
+// Register the workflow
+ctx.core.workflows.register(orderWorkflow);
+
+// Start an instance
+const instanceId = await ctx.core.workflows.start("process-order", {
+  orderId: "ORD-123",
+  customerId: "CUST-456",
+});
+```
+
+### 3. Track Progress
+
+```typescript
+// Via Events
+ctx.core.events.on("workflow.progress", (data) => {
+  console.log(`${data.workflowName}: ${data.progress}%`);
+});
+
+// Via SSE (client subscribes to workflow:${instanceId})
+ctx.core.events.on("workflow.progress", (data) => {
+  ctx.core.sse.broadcast(`workflow:${data.instanceId}`, "progress", data);
+});
+```
+
+## Step Types
+
+### Task
+
+Executes a job (in-process or external) and waits for completion.
+
+```typescript
+workflow("example")
+  .task("step-name", {
+    // Required: job to execute
+    job: "my-job-name",
+
+    // Optional: transform workflow context to job input
+    input: (ctx) => ({
+      orderId: ctx.input.orderId,
+      previousResult: ctx.steps.previousStep,
+    }),
+
+    // Optional: transform job result to step output
+    output: (result, ctx) => ({
+      processed: true,
+      data: result.data,
+    }),
+
+    // Optional: retry configuration
+    retry: {
+      maxAttempts: 3,
+      intervalMs: 1000,
+      backoffRate: 2,
+      maxIntervalMs: 30000,
+    },
+
+    // Optional: step timeout in ms
+    timeout: 60000,
+
+    // Control flow (one of these)
+    next: "next-step",  // Go to specific step
+    end: true,          // End workflow (mutually exclusive with next)
+  })
+```
+
+### Parallel
+
+Runs multiple workflow branches concurrently.
+
+```typescript
+workflow("example")
+  .parallel("parallel-step", {
+    // Required: branches to execute
+    branches: [
+      workflow.branch("branch-a")
+        .task("task-a1", { job: "job-a1" })
+        .task("task-a2", { job: "job-a2" })
+        .build(),
+
+      workflow.branch("branch-b")
+        .task("task-b1", { job: "job-b1" })
+        .build(),
+    ],
+
+    // Optional: error handling
+    onError: "fail-fast", // Stop all on first error (default)
+    // onError: "wait-all", // Wait for all branches, collect errors
+
+    // Control flow
+    next: "next-step",
+  })
+```
+
+The output of a parallel step is an object with each branch's output:
+
+```typescript
+{
+  "branch-a": { /* branch-a output */ },
+  "branch-b": { /* branch-b output */ }
+}
+```
+
+### Choice
+
+Conditional branching based on workflow context.
+
+```typescript
+workflow("example")
+  .choice("decision-point", {
+    // Evaluated in order, first match wins
+    choices: [
+      {
+        condition: (ctx) => ctx.steps.validate.amount > 1000,
+        next: "large-order-flow",
+      },
+      {
+        condition: (ctx) => ctx.steps.validate.isPriority,
+        next: "priority-flow",
+      },
+    ],
+
+    // Fallback if no conditions match
+    default: "standard-flow",
+  })
+```
+
+### Pass
+
+Transform data or create a no-op step.
+
+```typescript
+workflow("example")
+  // Transform data
+  .pass("transform", {
+    transform: (ctx) => ({
+      summary: {
+        input: ctx.input,
+        results: ctx.steps,
+      },
+    }),
+    next: "next-step",
+  })
+
+  // Static result
+  .pass("static", {
+    result: { status: "initialized" },
+    next: "next-step",
+  })
+
+  // End marker (shorthand)
+  .end("done")
+```
+
+## Workflow Context
+
+Every step receives a `WorkflowContext` with:
+
+```typescript
+interface WorkflowContext {
+  /** Original workflow input */
+  input: any;
+
+  /** Results from completed steps (keyed by step name) */
+  steps: Record<string, any>;
+
+  /** Current workflow instance */
+  instance: WorkflowInstance;
+
+  /** Type-safe step result getter */
+  getStepResult<T>(stepName: string): T | undefined;
+}
+```
+
+Example usage in step configuration:
+
+```typescript
+.task("process", {
+  job: "process-data",
+  input: (ctx) => ({
+    // Access original input
+    orderId: ctx.input.orderId,
+
+    // Access previous step output
+    validationResult: ctx.steps.validate,
+
+    // Type-safe access
+    amount: ctx.getStepResult<{ amount: number }>("calculate")?.amount,
+  }),
+})
+```
+
+## Retry Configuration
+
+Configure retries at the step level or set defaults for the entire workflow:
+
+```typescript
+// Default retry for all steps
+workflow("example")
+  .defaultRetry({
+    maxAttempts: 3,
+    intervalMs: 1000,
+    backoffRate: 2,
+    maxIntervalMs: 30000,
+  })
+  .task("step1", { job: "job1" }) // Uses default
+  .task("step2", {
+    job: "job2",
+    retry: { maxAttempts: 5 }, // Override
+  })
+```
+
+Retry parameters:
+- `maxAttempts`: Maximum retry attempts (including first try)
+- `intervalMs`: Initial delay between retries (default: 1000)
+- `backoffRate`: Multiplier for each retry (default: 2)
+- `maxIntervalMs`: Maximum delay cap (default: 30000)
+- `retryOn`: Array of error messages to retry on (default: all errors)
+
+## Workflow Timeout
+
+Set a timeout for the entire workflow:
+
+```typescript
+workflow("example")
+  .timeout(3600000) // 1 hour max
+  .task("step1", { job: "job1" })
+  .task("step2", { job: "job2" })
+  .build();
+```
+
+## Events
+
+Workflows emit events at key points:
+
+| Event | Data | Description |
+|-------|------|-------------|
+| `workflow.started` | `{ instanceId, workflowName, input }` | Workflow started |
+| `workflow.progress` | `{ instanceId, workflowName, progress, currentStep, completedSteps, totalSteps }` | Progress update |
+| `workflow.completed` | `{ instanceId, workflowName, output }` | Workflow completed |
+| `workflow.failed` | `{ instanceId, workflowName, error }` | Workflow failed |
+| `workflow.cancelled` | `{ instanceId, workflowName }` | Workflow cancelled |
+| `workflow.step.started` | `{ instanceId, workflowName, stepName, stepType }` | Step started |
+| `workflow.step.completed` | `{ instanceId, workflowName, stepName, output }` | Step completed |
+| `workflow.step.failed` | `{ instanceId, workflowName, stepName, error, attempts }` | Step failed |
+| `workflow.step.retry` | `{ instanceId, workflowName, stepName, attempt, maxAttempts, delay, error }` | Step retrying |
+
+### Example: SSE Progress Broadcasting
+
+```typescript
+// Broadcast all workflow events to SSE
+const workflowEvents = [
+  "workflow.progress",
+  "workflow.completed",
+  "workflow.failed",
+  "workflow.step.started",
+  "workflow.step.completed",
+];
+
+for (const event of workflowEvents) {
+  ctx.core.events.on(event, (data) => {
+    ctx.core.sse.broadcast(`workflow:${data.instanceId}`, event, data);
+  });
+}
+```
+
+## API Reference
+
+### Workflows Service
+
+```typescript
+interface Workflows {
+  /** Register a workflow definition */
+  register(definition: WorkflowDefinition): void;
+
+  /** Start a new workflow instance */
+  start<T = any>(workflowName: string, input: T): Promise<string>;
+
+  /** Get a workflow instance by ID */
+  getInstance(instanceId: string): Promise<WorkflowInstance | null>;
+
+  /** Cancel a running workflow */
+  cancel(instanceId: string): Promise<boolean>;
+
+  /** Get all instances of a workflow */
+  getInstances(workflowName: string, status?: WorkflowStatus): Promise<WorkflowInstance[]>;
+
+  /** Resume workflows after server restart */
+  resume(): Promise<void>;
+
+  /** Stop the workflow service */
+  stop(): Promise<void>;
+}
+```
+
+### Workflow Instance
+
+```typescript
+interface WorkflowInstance {
+  id: string;
+  workflowName: string;
+  status: "pending" | "running" | "completed" | "failed" | "cancelled" | "timed_out";
+  currentStep?: string;
+  input: any;
+  output?: any;
+  error?: string;
+  stepResults: Record<string, StepResult>;
+  createdAt: Date;
+  startedAt?: Date;
+  completedAt?: Date;
+}
+
+interface StepResult {
+  stepName: string;
+  status: "pending" | "running" | "completed" | "failed" | "skipped";
+  input?: any;
+  output?: any;
+  error?: string;
+  startedAt?: Date;
+  completedAt?: Date;
+  attempts: number;
+}
+```
+
+## Persistence
+
+By default, workflows use an in-memory adapter. For production, implement a `WorkflowAdapter`:
+
+```typescript
+interface WorkflowAdapter {
+  createInstance(instance: Omit<WorkflowInstance, "id">): Promise<WorkflowInstance>;
+  getInstance(instanceId: string): Promise<WorkflowInstance | null>;
+  updateInstance(instanceId: string, updates: Partial<WorkflowInstance>): Promise<void>;
+  deleteInstance(instanceId: string): Promise<boolean>;
+  getInstancesByWorkflow(workflowName: string, status?: WorkflowStatus): Promise<WorkflowInstance[]>;
+  getRunningInstances(): Promise<WorkflowInstance[]>;
+}
+```
+
+Configure via `ServerConfig`:
+
+```typescript
+const server = new AppServer({
+  db: createDatabase(),
+  workflows: {
+    adapter: new MyDatabaseWorkflowAdapter(db),
+    pollInterval: 1000, // How often to check job completion
+  },
+});
+```
+
+## Server Restart Resilience
+
+Workflows automatically resume after server restart:
+
+1. On startup, `workflows.resume()` is called
+2. All instances with `status: "running"` are retrieved
+3. Execution continues from the current step
+
+For this to work properly:
+- Use a persistent adapter (not in-memory) in production
+- Jobs should be idempotent when possible
+- The Jobs service must also support restart resilience
+
+## Complete Example
+
+```typescript
+import { AppServer, workflow, createDatabase } from "@donkeylabs/server";
+
+// Define workflow
+const onboardingWorkflow = workflow("user-onboarding")
+  .timeout(86400000) // 24 hour max
+  .defaultRetry({ maxAttempts: 3 })
+
+  .task("create-account", {
+    job: "create-user-account",
+    input: (ctx) => ctx.input,
+  })
+
+  .task("send-welcome-email", {
+    job: "send-email",
+    input: (ctx) => ({
+      to: ctx.input.email,
+      template: "welcome",
+      userId: ctx.steps["create-account"].userId,
+    }),
+  })
+
+  .choice("check-plan", {
+    choices: [
+      {
+        condition: (ctx) => ctx.input.plan === "enterprise",
+        next: "enterprise-setup",
+      },
+    ],
+    default: "standard-setup",
+  })
+
+  .task("enterprise-setup", {
+    job: "setup-enterprise",
+    input: (ctx) => ({ userId: ctx.steps["create-account"].userId }),
+    next: "complete",
+  })
+
+  .task("standard-setup", {
+    job: "setup-standard",
+    input: (ctx) => ({ userId: ctx.steps["create-account"].userId }),
+    next: "complete",
+  })
+
+  .pass("complete", {
+    transform: (ctx) => ({
+      userId: ctx.steps["create-account"].userId,
+      plan: ctx.input.plan,
+      setupComplete: true,
+    }),
+    end: true,
+  })
+  .build();
+
+// Setup server
+const server = new AppServer({ db: createDatabase() });
+
+// Register jobs that workflows use
+server.getCore().jobs.register("create-user-account", async (data) => {
+  // ... create user
+  return { userId: "user-123" };
+});
+
+server.getCore().jobs.register("send-email", async (data) => {
+  // ... send email
+  return { sent: true };
+});
+
+// ... register other jobs
+
+// Register workflow
+server.getCore().workflows.register(onboardingWorkflow);
+
+// Start workflow from a route
+router.route("onboard").typed({
+  input: z.object({
+    email: z.string().email(),
+    name: z.string(),
+    plan: z.enum(["free", "pro", "enterprise"]),
+  }),
+  handle: async (input, ctx) => {
+    const instanceId = await ctx.core.workflows.start("user-onboarding", input);
+    return { instanceId };
+  },
+});
+
+await server.start();
+```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@donkeylabs/server",
-  "version": "0.4.7",
+  "version": "0.4.8",
   "type": "module",
   "description": "Type-safe plugin system for building RPC-style APIs with Bun",
   "main": "./src/index.ts",