@donkeylabs/server 2.0.18 → 2.0.20

package/docs/workflows.md

@@ -305,6 +305,21 @@ interface WorkflowContext {
 
   /** Type-safe step result getter */
   getStepResult<T>(stepName: string): T | undefined;
+
+  /** Core services (logger, events, cache, jobs, sse, etc.) */
+  core: CoreServices;
+
+  /** Plugin services - access your plugins' service methods */
+  plugins: Record<string, any>;
+
+  /** Custom metadata that persists across steps (read-only snapshot) */
+  metadata: Record<string, any>;
+
+  /** Set a metadata value that persists across workflow steps */
+  setMetadata(key: string, value: any): Promise<void>;
+
+  /** Get a typed metadata value */
+  getMetadata<T>(key: string): T | undefined;
 }
 ```
 
@@ -320,11 +335,59 @@ Example usage in step configuration:
   handler: async (input, ctx) => {
     // Access any step's output
     const calcResult = ctx.getStepResult<{ amount: number }>("calculate");
+
+    // Access plugin services
+    const order = await ctx.plugins.orders.getById(input.orderId);
+
+    // Use core services
+    ctx.core.logger.info("Processing order", { orderId: input.orderId });
+
     return { processed: true, amount: calcResult?.amount };
   },
 })
 ```
 
+### Cross-Step Metadata
+
+Use metadata to share data across workflow steps that isn't part of the normal step output flow:
+
+```typescript
+.task("validate", {
+  inputSchema: z.object({ orderId: z.string() }),
+  handler: async (input, ctx) => {
+    // Store correlation ID for logging/tracing across steps
+    await ctx.setMetadata("correlationId", crypto.randomUUID());
+
+    // Store complex context that multiple steps need
+    await ctx.setMetadata("orderContext", {
+      customer: await ctx.plugins.customers.getByOrder(input.orderId),
+      flags: { expedited: false, giftWrap: false },
+    });
+
+    return { valid: true };
+  },
+})
+.task("fulfill", {
+  handler: async (input, ctx) => {
+    // Read metadata from previous steps
+    const correlationId = ctx.getMetadata<string>("correlationId");
+    const orderCtx = ctx.getMetadata<{ customer: Customer; flags: object }>("orderContext");
+
+    ctx.core.logger.info("Fulfilling order", { correlationId, customer: orderCtx?.customer.id });
+
+    // Update metadata for downstream steps
+    await ctx.setMetadata("orderContext", {
+      ...orderCtx,
+      flags: { ...orderCtx?.flags, fulfilled: true },
+    });
+
+    return { shipped: true };
+  },
+})
+```
+
+Metadata is persisted to the database and survives server restarts.
+
 ## Retry Configuration
 
 Configure retries at the step level or set defaults for the entire workflow:
@@ -440,6 +503,8 @@ interface WorkflowInstance {
   output?: any;
   error?: string;
   stepResults: Record<string, StepResult>;
+  /** Custom metadata that persists across steps */
+  metadata?: Record<string, any>;
   createdAt: Date;
   startedAt?: Date;
   completedAt?: Date;
@@ -459,7 +524,28 @@ interface StepResult {
 
 ## Persistence
 
-By default, workflows use an in-memory adapter. For production, implement a `WorkflowAdapter`:
+By default, workflows use a **Kysely database adapter** that stores workflow instances in the same database as your application. This provides automatic persistence and restart resilience.
+
+The framework automatically creates and manages the `__donkeylabs_workflow_instances__` table with proper migrations.
+
+### Built-in Kysely Adapter
+
+The `KyselyWorkflowAdapter` is automatically configured when you provide a database connection:
+
+```typescript
+const server = new AppServer({
+  db: createDatabase(),  // Workflows will automatically persist to this database
+  workflows: {
+    pollInterval: 1000,     // How often to check job completion
+    cleanupDays: 30,        // Auto-cleanup completed workflows older than 30 days (0 to disable)
+    cleanupInterval: 3600000, // Cleanup check interval (default: 1 hour)
+  },
+});
+```
+
+### Custom Adapter
+
+For custom storage backends, implement the `WorkflowAdapter` interface:
 
 ```typescript
 interface WorkflowAdapter {
@@ -469,6 +555,7 @@ interface WorkflowAdapter {
   deleteInstance(instanceId: string): Promise<boolean>;
   getInstancesByWorkflow(workflowName: string, status?: WorkflowStatus): Promise<WorkflowInstance[]>;
   getRunningInstances(): Promise<WorkflowInstance[]>;
+  getAllInstances(options?: GetAllWorkflowsOptions): Promise<WorkflowInstance[]>;
 }
 ```
 
@@ -478,8 +565,8 @@ Configure via `ServerConfig`:
 const server = new AppServer({
   db: createDatabase(),
   workflows: {
-    adapter: new MyDatabaseWorkflowAdapter(db),
-    pollInterval: 1000, // How often to check job completion
+    adapter: new MyCustomWorkflowAdapter(),
+    pollInterval: 1000,
   },
 });
 ```

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@donkeylabs/server",
-  "version": "2.0.18",
+  "version": "2.0.20",
   "type": "module",
   "description": "Type-safe plugin system for building RPC-style APIs with Bun",
   "main": "./src/index.ts",
@@ -30,6 +30,10 @@
       "types": "./src/process-client.ts",
       "import": "./src/process-client.ts"
     },
+    "./testing": {
+      "types": "./src/testing/index.ts",
+      "import": "./src/testing/index.ts"
+    },
     "./context": {
       "types": "./context.d.ts"
     },
@@ -68,7 +72,10 @@
     "kysely": "^0.27.0 || ^0.28.0",
     "zod": "^3.20.0",
     "@aws-sdk/client-s3": "^3.0.0",
-    "@aws-sdk/s3-request-presigner": "^3.0.0"
+    "@aws-sdk/s3-request-presigner": "^3.0.0",
+    "@playwright/test": "^1.40.0",
+    "pg": "^8.0.0",
+    "mysql2": "^3.0.0"
   },
   "peerDependenciesMeta": {
     "@aws-sdk/client-s3": {
@@ -76,6 +83,15 @@
     },
     "@aws-sdk/s3-request-presigner": {
       "optional": true
+    },
+    "@playwright/test": {
+      "optional": true
+    },
+    "pg": {
+      "optional": true
+    },
+    "mysql2": {
+      "optional": true
     }
   },
   "dependencies": {

package/src/admin/routes.ts

@@ -397,6 +397,159 @@ export function createAdminRouter(config: AdminRouteContext) {
     })
   );
 
+  // Workflows get route - full instance details with step results
+  router.route("workflows.get").typed(
+    defineRoute({
+      input: z.object({ instanceId: z.string() }),
+      output: z.object({
+        id: z.string(),
+        workflowName: z.string(),
+        status: z.string(),
+        currentStep: z.string().nullable(),
+        input: z.any(),
+        output: z.any().nullable(),
+        error: z.string().nullable(),
+        stepResults: z.record(
+          z.object({
+            stepName: z.string(),
+            status: z.string(),
+            input: z.any().optional(),
+            output: z.any().optional(),
+            error: z.string().optional(),
+            startedAt: z.string().nullable(),
+            completedAt: z.string().nullable(),
+            attempts: z.number(),
+          })
+        ),
+        metadata: z.record(z.any()).nullable(),
+        createdAt: z.string(),
+        startedAt: z.string().nullable(),
+        completedAt: z.string().nullable(),
+        parentId: z.string().nullable(),
+        branchName: z.string().nullable(),
+      }).nullable(),
+      handle: async (input, ctx) => {
+        if (!checkAuth(ctx)) {
+          throw ctx.errors.Forbidden("Unauthorized");
+        }
+        const instance = await ctx.core.workflows.getInstance(input.instanceId);
+        if (!instance) {
+          return null;
+        }
+
+        // Transform step results with proper date serialization
+        const stepResults: Record<string, any> = {};
+        for (const [key, result] of Object.entries(instance.stepResults)) {
+          stepResults[key] = {
+            stepName: result.stepName,
+            status: result.status,
+            input: result.input,
+            output: result.output,
+            error: result.error,
+            startedAt: result.startedAt?.toISOString() ?? null,
+            completedAt: result.completedAt?.toISOString() ?? null,
+            attempts: result.attempts,
+          };
+        }
+
+        return {
+          id: instance.id,
+          workflowName: instance.workflowName,
+          status: instance.status,
+          currentStep: instance.currentStep ?? null,
+          input: instance.input,
+          output: instance.output ?? null,
+          error: instance.error ?? null,
+          stepResults,
+          metadata: instance.metadata ?? null,
+          createdAt: instance.createdAt.toISOString(),
+          startedAt: instance.startedAt?.toISOString() ?? null,
+          completedAt: instance.completedAt?.toISOString() ?? null,
+          parentId: instance.parentId ?? null,
+          branchName: instance.branchName ?? null,
+        };
+      },
+    })
+  );
+
+  // Workflows subscribe route - SSE for real-time workflow events
+  // Subscribes to workflow:{instanceId} channel for progress, completed, and failed events
+  router.route("workflows.subscribe").sse({
+    input: z.object({
+      instanceId: z.string(),
+    }),
+    events: {
+      progress: z.object({
+        progress: z.number(),
+        currentStep: z.string().optional(),
+        completedSteps: z.number(),
+        totalSteps: z.number(),
+      }),
+      "step.started": z.object({
+        stepName: z.string(),
+      }),
+      "step.completed": z.object({
+        stepName: z.string(),
+        output: z.any().optional(),
+      }),
+      "step.failed": z.object({
+        stepName: z.string(),
+        error: z.string(),
+      }),
+      completed: z.object({
+        output: z.any().optional(),
+      }),
+      failed: z.object({
+        error: z.string(),
+      }),
+    },
+    handle: (input, ctx) => {
+      if (!checkAuth(ctx)) {
+        return [];
+      }
+      // Subscribe to the workflow-specific channel
+      return [`workflow:${input.instanceId}`];
+    },
+  });
+
+  // Workflows subscribe all route - SSE for all workflow events (useful for dashboard)
+  // Subscribes to workflow:* pattern for all workflow updates
+  router.route("workflows.subscribeAll").sse({
+    input: z.object({
+      // Optional filter by workflow name
+      workflowName: z.string().optional(),
+    }),
+    events: {
+      "workflow.started": z.object({
+        instanceId: z.string(),
+        workflowName: z.string(),
+      }),
+      "workflow.progress": z.object({
+        instanceId: z.string(),
+        workflowName: z.string(),
+        progress: z.number(),
+        currentStep: z.string().optional(),
+      }),
+      "workflow.completed": z.object({
+        instanceId: z.string(),
+        workflowName: z.string(),
+      }),
+      "workflow.failed": z.object({
+        instanceId: z.string(),
+        workflowName: z.string(),
+        error: z.string(),
+      }),
+    },
+    handle: (input, ctx) => {
+      if (!checkAuth(ctx)) {
+        return [];
+      }
+      // Subscribe to the global workflow events channel
+      // Events are emitted via ctx.core.events and need to be bridged to SSE
+      return ["workflows:all"];
+    },
+  });
+
   // Audit list route
   router.route("audit.list").typed(
     defineRoute({

package/src/core/cron.ts

@@ -35,8 +35,16 @@ export interface Cron {
 // Supports: * (any), specific values, ranges (1-5), steps (*/5)
 // Format: second minute hour dayOfMonth month dayOfWeek
 // Also supports 5-field format (minute hour dayOfMonth month dayOfWeek)
+//
+// Day-of-month/day-of-week semantics:
+// - Standard cron uses OR semantics when both are restricted (not *)
+// - If BOTH day-of-month and day-of-week are explicitly set, job runs if EITHER matches
+// - If one is *, the other is used exclusively
 class CronExpression {
   private fields: [number[], number[], number[], number[], number[], number[]];
+  // Track if day fields were explicitly set (not *)
+  private dayOfMonthRestricted: boolean;
+  private dayOfWeekRestricted: boolean;
 
   constructor(expression: string) {
     const parts = expression.trim().split(/\s+/);
@@ -44,6 +52,8 @@ class CronExpression {
     // Support both 5-field and 6-field cron
     if (parts.length === 5) {
       // minute hour dayOfMonth month dayOfWeek
+      this.dayOfMonthRestricted = parts[2] !== "*";
+      this.dayOfWeekRestricted = parts[4] !== "*";
       this.fields = [
         [0], // seconds (always 0)
         this.parseField(parts[0]!, 0, 59),  // minutes
@@ -54,6 +64,8 @@ class CronExpression {
       ];
     } else if (parts.length === 6) {
       // second minute hour dayOfMonth month dayOfWeek
+      this.dayOfMonthRestricted = parts[3] !== "*";
+      this.dayOfWeekRestricted = parts[5] !== "*";
       this.fields = [
         this.parseField(parts[0]!, 0, 59),  // seconds
         this.parseField(parts[1]!, 0, 59),  // minutes
@@ -99,31 +111,142 @@ class CronExpression {
     const month = date.getMonth() + 1;
     const dayOfWeek = date.getDay();
 
-    return (
-      this.fields[0].includes(second) &&
-      this.fields[1].includes(minute) &&
-      this.fields[2].includes(hour) &&
-      this.fields[3].includes(dayOfMonth) &&
-      this.fields[4].includes(month) &&
-      this.fields[5].includes(dayOfWeek)
-    );
+    // Check time fields (always AND)
+    if (!this.fields[0].includes(second)) return false;
+    if (!this.fields[1].includes(minute)) return false;
+    if (!this.fields[2].includes(hour)) return false;
+    if (!this.fields[4].includes(month)) return false;
+
+    // Day matching uses standard cron semantics:
+    // - If both day-of-month AND day-of-week are restricted (not *), use OR
+    // - Otherwise, use AND (one or both are unrestricted)
+    const domMatches = this.fields[3].includes(dayOfMonth);
+    const dowMatches = this.fields[5].includes(dayOfWeek);
+
+    if (this.dayOfMonthRestricted && this.dayOfWeekRestricted) {
+      // Both restricted: OR semantics - job runs if EITHER matches
+      return domMatches || dowMatches;
+    } else {
+      // One or both unrestricted: AND semantics
+      return domMatches && dowMatches;
+    }
   }
 
+  /**
+   * Get the next run time using an optimized jump algorithm.
+   * Instead of iterating second-by-second (which could be 31M iterations),
+   * this jumps directly to the next valid value for each field.
+   */
   getNextRun(from: Date = new Date()): Date {
     const next = new Date(from);
     next.setMilliseconds(0);
     next.setSeconds(next.getSeconds() + 1);
 
-    // Search up to 1 year ahead
-    const maxIterations = 366 * 24 * 60 * 60;
-    for (let i = 0; i < maxIterations; i++) {
-      if (this.matches(next)) {
-        return next;
+    const [seconds, minutes, hours, daysOfMonth, months, daysOfWeek] = this.fields;
+
+    // Maximum iterations to prevent infinite loops (covers 4 years to handle leap years)
+    const maxYearIterations = 4;
+    const startYear = next.getFullYear();
+
+    // Iterate through potential dates (worst case: a few hundred iterations)
+    for (let yearOffset = 0; yearOffset <= maxYearIterations; yearOffset++) {
+      // Try each valid month
+      for (const month of months) {
+        const targetMonth = month - 1; // JS months are 0-indexed
+
+        // Skip months in the past
+        if (next.getFullYear() === startYear + yearOffset) {
+          if (targetMonth < next.getMonth()) continue;
+        }
+
+        // Set to this month
+        if (targetMonth !== next.getMonth() || next.getFullYear() !== startYear + yearOffset) {
+          next.setFullYear(startYear + yearOffset, targetMonth, 1);
+          next.setHours(0, 0, 0, 0);
+        }
+
+        // Get days in this month
+        const daysInMonth = new Date(next.getFullYear(), targetMonth + 1, 0).getDate();
+
+        // Try each valid day of month
+        for (const dayOfMonth of daysOfMonth) {
+          if (dayOfMonth > daysInMonth) continue; // Skip invalid days for this month
+
+          // Check if this day matches day-of-week constraint
+          // Use OR semantics if both are restricted, AND otherwise
+          const testDate = new Date(next.getFullYear(), targetMonth, dayOfMonth);
+          const dayOfWeek = testDate.getDay();
+          const domMatches = daysOfMonth.includes(dayOfMonth);
+          const dowMatches = daysOfWeek.includes(dayOfWeek);
+
+          let dayMatches: boolean;
+          if (this.dayOfMonthRestricted && this.dayOfWeekRestricted) {
+            // Both restricted: OR semantics
+            dayMatches = domMatches || dowMatches;
+          } else {
+            // One or both unrestricted: AND semantics
+            dayMatches = domMatches && dowMatches;
+          }
+          if (!dayMatches) continue;
+
+          // Skip days in the past
+          if (testDate < new Date(from.getFullYear(), from.getMonth(), from.getDate())) continue;
+
+          // Set to this day
+          if (dayOfMonth !== next.getDate()) {
+            next.setDate(dayOfMonth);
+            next.setHours(0, 0, 0, 0);
+          }
+
+          // Try each valid hour
+          for (const hour of hours) {
+            // Skip hours in the past for today
+            if (next.getFullYear() === from.getFullYear() &&
+                next.getMonth() === from.getMonth() &&
+                next.getDate() === from.getDate() &&
+                hour < from.getHours()) continue;
+
+            if (hour !== next.getHours()) {
+              next.setHours(hour, 0, 0, 0);
+            }
+
+            // Try each valid minute
+            for (const minute of minutes) {
+              // Skip minutes in the past for this hour
+              if (next.getFullYear() === from.getFullYear() &&
+                  next.getMonth() === from.getMonth() &&
+                  next.getDate() === from.getDate() &&
+                  next.getHours() === from.getHours() &&
+                  minute < from.getMinutes()) continue;
+
+              if (minute !== next.getMinutes()) {
+                next.setMinutes(minute, 0, 0);
+              }
+
+              // Try each valid second
+              for (const second of seconds) {
+                // Skip seconds in the past for this minute
+                if (next.getFullYear() === from.getFullYear() &&
+                    next.getMonth() === from.getMonth() &&
+                    next.getDate() === from.getDate() &&
+                    next.getHours() === from.getHours() &&
+                    next.getMinutes() === from.getMinutes() &&
+                    second <= from.getSeconds()) continue;
+
+                next.setSeconds(second);
+
+                // Verify the date is still valid (handles edge cases like month rollover)
+                if (next > from && this.matches(next)) {
+                  return next;
+                }
+              }
+            }
+          }
+        }
       }
-      next.setSeconds(next.getSeconds() + 1);
     }
 
-    throw new Error("Could not find next run time within 1 year");
+    throw new Error("Could not find next run time within 4 years");
   }
 }
 
@@ -220,6 +343,9 @@ class CronImpl implements Cron {
     if (this.running) return;
     this.running = true;
 
+    // Catch-up check for missed runs since last shutdown
+    this.catchUpMissedRuns();
+
     // Check every second
     this.timer = setInterval(() => {
       const now = new Date();
@@ -241,6 +367,49 @@ class CronImpl implements Cron {
     }, 1000);
   }
 
+  /**
+   * Catch up on missed runs since last shutdown.
+   * For each task with a lastRun timestamp, execute any runs that were missed.
+   * Limited to 10 catch-up runs per task to avoid flooding.
+   */
+  private catchUpMissedRuns(): void {
+    const now = new Date();
+    const maxCatchUp = 10;
+
+    for (const task of this.tasks.values()) {
+      if (!task.enabled || !task.lastRun) continue;
+
+      try {
+        const cronExpr = new CronExpression(task.expression);
+        let missedRun = cronExpr.getNextRun(task.lastRun);
+        let catchUpCount = 0;
+
+        // Execute any missed runs (up to limit to avoid flooding)
+        while (missedRun && missedRun < now && catchUpCount < maxCatchUp) {
+          console.log(`[Cron] Catching up missed run for "${task.name}" at ${missedRun.toISOString()}`);
+
+          // Execute the handler asynchronously
+          Promise.resolve(task.handler()).catch(err => {
+            console.error(`[Cron] Catch-up task "${task.name}" failed:`, err);
+          });
+
+          task.lastRun = missedRun;
+          missedRun = cronExpr.getNextRun(missedRun);
+          catchUpCount++;
+        }
+
+        // Update nextRun to the next scheduled time
+        task.nextRun = cronExpr.getNextRun(now);
+
+        if (catchUpCount > 0) {
+          console.log(`[Cron] Caught up ${catchUpCount} missed runs for "${task.name}"`);
+        }
+      } catch (err) {
+        console.error(`[Cron] Error catching up task "${task.name}":`, err);
+      }
+    }
+  }
+
   async stop(): Promise<void> {
     this.running = false;
     if (this.timer) {

package/src/core/index.ts

@@ -132,6 +132,7 @@ export {
 export {
   type Workflows,
   type WorkflowsConfig,
+  type WorkflowRegisterOptions,
   type WorkflowDefinition,
   type WorkflowInstance,
   type WorkflowStatus,
@@ -154,6 +155,30 @@ export {
   createWorkflows,
 } from "./workflows";
 
+export {
+  type WorkflowSocketServer,
+  type WorkflowSocketServerOptions,
+  type WorkflowSocketConfig,
+  type WorkflowEvent,
+  type WorkflowEventType,
+  type ProxyRequest,
+  type ProxyResponse,
+  type WorkflowMessage,
+  createWorkflowSocketServer,
+  isWorkflowEvent,
+  isProxyRequest,
+  parseWorkflowMessage,
+} from "./workflow-socket";
+
+export {
+  type ProxyConnection,
+  WorkflowProxyConnection,
+  createPluginProxy,
+  createCoreProxy,
+  createPluginsProxy,
+  createCoreServicesProxy,
+} from "./workflow-proxy";
+
 export {
   type Processes,
   type ProcessesConfig,