@donkeylabs/server 2.0.27 → 2.0.29

package/docs/jobs.md

@@ -192,6 +192,26 @@ const cancelled = await ctx.core.jobs.cancel(jobId);
 
 ---
 
+## External Jobs (Subprocess)
+
+External jobs run in a separate process and are monitored by a watchdog.
+
+```ts
+ctx.core.jobs.registerExternal("batchWorker", {
+  command: "bun",
+  args: ["./workers/batch-worker.ts"],
+  heartbeatTimeout: 30000,
+  timeout: 10 * 60 * 1000,
+  killGraceMs: 5000,
+});
+```
+
+Watchdog events:
+- `job.watchdog.stale`
+- `job.watchdog.killed`
+
+---
+
 ## Event Integration
 
 Jobs automatically emit events on completion and failure:

package/docs/processes.md

@@ -8,6 +8,7 @@ Processes provide:
 - Long-running daemon management (start, stop, restart)
 - Typed event communication from process to server
 - Automatic heartbeat monitoring
+- Watchdog termination for unresponsive processes
 - Connection resilience with auto-reconnection
 - Metadata passing to spawned processes
 - Cross-platform support (Unix sockets / TCP on Windows)
@@ -89,6 +90,13 @@ server.getCore().processes.define("video-encoder", {
 
   // Heartbeat configuration
   heartbeatTimeout: 30000, // 30 seconds
+
+  // Optional hard limits (requires stats for memory/CPU)
+  limits: {
+    maxRuntimeMs: 60_000,
+    maxMemoryMb: 512,
+    maxCpuPercent: 90,
+  },
 });
 ```
 
@@ -221,6 +229,25 @@ const client = await ProcessClient.connect({
 });
 ```
 
+---
+
+## Hard Limits
+
+Processes can be terminated automatically when limits are exceeded:
+
+- `maxRuntimeMs` always enforced by the server watchdog
+- `maxMemoryMb` and `maxCpuPercent` require `ProcessClient` stats enabled
+
+```ts
+const client = await ProcessClient.connect({
+  stats: { enabled: true, interval: 5000 },
+});
+```
+
+Watchdog events:
+- `process.watchdog.stale`
+- `process.watchdog.killed`
+
 ### Properties
 
 ```typescript

package/docs/workflows.md

@@ -285,6 +285,75 @@ workflow("example")
   .end("done")
 ```
 
+### Poll
+
+Use a poll step for wait → check loops that persist across restarts.
+
+```typescript
+workflow("batch.status")
+  .poll("wait-for-result", {
+    interval: 5000,
+    timeout: 600000,
+    maxAttempts: 120,
+    check: async (input, ctx) => {
+      const status = await fetchStatus(input.operationId);
+      if (status.state === "FAILED") throw new Error(status.error);
+      if (status.state === "SUCCEEDED") {
+        return { done: true, result: status.data };
+      }
+      return { done: false };
+    },
+  })
+  .build();
+```
+
+Each poll cycle emits `workflow.step.poll` events and persists progress to the instance.
+
+---
+
+## Watchdog and Subprocess Settings
+
+You can tune subprocess termination and SQLite pragmas used by isolated workflows:
+
+```ts
+const server = new AppServer({
+  db,
+  workflows: {
+    killGraceMs: 5000,
+    sqlitePragmas: {
+      busyTimeout: 5000,
+      journalMode: "WAL",
+      synchronous: "NORMAL",
+    },
+  },
+});
+```
+
+Watchdog events:
+- `workflow.watchdog.stale` (heartbeat missed)
+- `workflow.watchdog.killed` (process terminated)
+
+### Loop
+
+Use a loop step to jump back to a previous step until a condition is false.
+
+```typescript
+workflow("loop-example")
+  .task("increment", {
+    handler: async (input) => ({ count: (input.count ?? 0) + 1 }),
+  })
+  .loop("repeat", {
+    condition: (ctx) => ctx.steps.increment.count < 3,
+    target: "increment",
+    interval: 1000,
+    maxIterations: 10,
+    timeout: 30000,
+  })
+  .build();
+```
+
+Each loop iteration emits `workflow.step.loop` and persists loop counters to the instance.
+
 ## Workflow Context
 
 Every step receives a `WorkflowContext` with:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@donkeylabs/server",
-  "version": "2.0.27",
+  "version": "2.0.29",
   "type": "module",
   "description": "Type-safe plugin system for building RPC-style APIs with Bun",
   "main": "./src/index.ts",

package/src/admin/routes.ts

@@ -505,6 +505,17 @@ export function createAdminRouter(config: AdminRouteContext) {
         stepName: z.string(),
         error: z.string(),
       }),
+      "step.poll": z.object({
+        stepName: z.string(),
+        pollCount: z.number(),
+        done: z.boolean(),
+        result: z.any().optional(),
+      }),
+      "step.loop": z.object({
+        stepName: z.string(),
+        loopCount: z.number(),
+        target: z.string(),
+      }),
       completed: z.object({
         output: z.any().optional(),
       }),
@@ -548,6 +559,19 @@ export function createAdminRouter(config: AdminRouteContext) {
         workflowName: z.string(),
         error: z.string(),
       }),
+      "workflow.step.poll": z.object({
+        instanceId: z.string(),
+        stepName: z.string(),
+        pollCount: z.number(),
+        done: z.boolean(),
+        result: z.any().optional(),
+      }),
+      "workflow.step.loop": z.object({
+        instanceId: z.string(),
+        stepName: z.string(),
+        loopCount: z.number(),
+        target: z.string(),
+      }),
     },
     handle: (input, ctx) => {
       if (!checkAuth(ctx)) {

package/src/core/external-job-socket.ts

@@ -201,6 +201,23 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
 
     let buffer = "";
 
+    const queue: AnyExternalJobMessage[] = [];
+    let processing = false;
+
+    const processQueue = async () => {
+      if (processing) return;
+      processing = true;
+      while (queue.length > 0) {
+        const message = queue.shift()!;
+        try {
+          await this.onMessage(message);
+        } catch (err) {
+          this.onError?.(err instanceof Error ? err : new Error(String(err)), jobId);
+        }
+      }
+      processing = false;
+    };
+
     socket.on("data", (data) => {
       buffer += data.toString();
 
@@ -213,11 +230,13 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
 
         const message = parseJobMessage(line);
         if (message) {
-          this.onMessage(message);
+          queue.push(message);
         } else {
           this.onError?.(new Error(`Invalid message: ${line}`), jobId);
         }
       }
+
+      processQueue().catch(() => undefined);
     });
 
     socket.on("error", (err) => {

package/src/core/external-jobs.ts

@@ -80,6 +80,8 @@ export interface ExternalJobConfig {
   heartbeatTimeout?: number;
   /** Job timeout in milliseconds (optional) */
   timeout?: number;
+  /** Grace period before SIGKILL when terminating (ms, default: 5000) */
+  killGraceMs?: number;
 }
 
 // ============================================
@@ -120,6 +122,8 @@ export interface ExternalJobsConfig {
   defaultHeartbeatTimeout?: number;
   /** Heartbeat check interval in ms (default: 10000) */
   heartbeatCheckInterval?: number;
+  /** Default grace period before SIGKILL when terminating (ms, default: 5000) */
+  killGraceMs?: number;
 }
 
 // ============================================

package/src/core/index.ts

@@ -134,6 +134,7 @@ export {
 export {
   type Workflows,
   type WorkflowsConfig,
+  type SqlitePragmaConfig,
   type WorkflowRegisterOptions,
   type WorkflowDefinition,
   type WorkflowInstance,
@@ -149,6 +150,9 @@ export {
   type ChoiceStepDefinition,
   type ChoiceCondition,
   type PassStepDefinition,
+  type PollStepDefinition,
+  type PollStepResult,
+  type LoopStepDefinition,
   type RetryConfig,
   type GetAllWorkflowsOptions,
   type PluginMetadata,

package/src/core/jobs.ts

@@ -273,7 +273,10 @@ class JobsImpl implements Jobs {
   private externalConfigs = new Map<string, ExternalJobConfig>();
   private externalConfig: ExternalJobsConfig;
   private socketServer: ExternalJobSocketServer | null = null;
-  private externalProcesses = new Map<string, { pid: number; timeout?: ReturnType<typeof setTimeout> }>();
+  private externalProcesses = new Map<
+    string,
+    { pid: number; timeout?: ReturnType<typeof setTimeout>; killTimer?: ReturnType<typeof setTimeout> }
+  >();
 
   constructor(config: JobsConfig = {}) {
     this.events = config.events;
@@ -521,6 +524,7 @@ class JobsImpl implements Jobs {
 
         const config = this.externalConfigs.get(job.name);
         const heartbeatTimeout = config?.heartbeatTimeout ?? this.externalConfig.defaultHeartbeatTimeout ?? 30000;
+        const killGraceMs = config?.killGraceMs ?? this.externalConfig.killGraceMs ?? 5000;
         const timeSinceHeartbeat = now - job.lastHeartbeat.getTime();
 
         if (timeSinceHeartbeat > heartbeatTimeout) {
@@ -533,36 +537,22 @@ class JobsImpl implements Jobs {
               name: job.name,
               timeSinceHeartbeat,
             });
-          }
-
-          // If stale for 2x timeout, kill the process
-          if (timeSinceHeartbeat > heartbeatTimeout * 2) {
-            console.error(`[Jobs] Killing stale external job ${job.id}`);
-
-            if (job.pid) {
-              try {
-                process.kill(job.pid, "SIGKILL");
-              } catch {
-                // Process may already be dead
-              }
-            }
-
-            await this.adapter.update(job.id, {
-              status: "failed",
-              error: "Heartbeat timeout - job process unresponsive",
-              completedAt: new Date(),
-              processState: "orphaned",
+            await this.events.emit("job.watchdog.stale", {
+              jobId: job.id,
+              name: job.name,
+              timeSinceHeartbeat,
             });
+          }
 
-            await this.cleanupExternalJob(job.id);
-
-            if (this.events) {
-              await this.events.emit("job.failed", {
-                jobId: job.id,
-                name: job.name,
-                error: "Heartbeat timeout",
-              });
-            }
+          const procInfo = this.externalProcesses.get(job.id);
+          if (job.pid && !procInfo?.killTimer) {
+            console.error(`[Jobs] Terminating stale external job ${job.id}`);
+            await this.terminateExternalProcess(
+              job.id,
+              job.pid,
+              killGraceMs,
+              "Heartbeat timeout - job process unresponsive"
+            );
           }
         }
       }
@@ -764,6 +754,9 @@ class JobsImpl implements Jobs {
     if (procInfo?.timeout) {
       clearTimeout(procInfo.timeout);
     }
+    if (procInfo?.killTimer) {
+      clearTimeout(procInfo.killTimer);
+    }
     this.externalProcesses.delete(jobId);
 
     // Close the socket
@@ -901,27 +894,13 @@ class JobsImpl implements Jobs {
       if (config.timeout) {
         const timeout = setTimeout(async () => {
           console.warn(`[Jobs] External job ${job.id} timed out after ${config.timeout}ms`);
-          try {
-            process.kill(proc.pid, "SIGTERM");
-          } catch {
-            // Process may already be dead
-          }
-
-          await this.adapter.update(job.id, {
-            status: "failed",
-            error: `Job timed out after ${config.timeout}ms`,
-            completedAt: new Date(),
-          });
-
-          await this.cleanupExternalJob(job.id);
-
-          if (this.events) {
-            await this.events.emit("job.failed", {
-              jobId: job.id,
-              name: job.name,
-              error: "Timeout",
-            });
-          }
+          const killGraceMs = config.killGraceMs ?? this.externalConfig.killGraceMs ?? 5000;
+          await this.terminateExternalProcess(
+            job.id,
+            proc.pid,
+            killGraceMs,
+            `Job timed out after ${config.timeout}ms`
+          );
         }, config.timeout);
 
         const procInfo = this.externalProcesses.get(job.id);
@@ -998,6 +977,73 @@ class JobsImpl implements Jobs {
     }
   }
 
+  private async terminateExternalProcess(
+    jobId: string,
+    pid: number,
+    killGraceMs: number,
+    error: string
+  ): Promise<void> {
+    try {
+      process.kill(pid, "SIGTERM");
+    } catch {
+      return;
+    }
+
+    if (killGraceMs <= 0) {
+      try {
+        process.kill(pid, "SIGKILL");
+      } catch {
+        // ignore
+      }
+      await this.handleExternalFailure(jobId, error);
+      return;
+    }
+
+    const timer = setTimeout(async () => {
+      try {
+        process.kill(pid, 0);
+        process.kill(pid, "SIGKILL");
+      } catch {
+        // ignore
+      }
+
+      await this.handleExternalFailure(jobId, error);
+    }, killGraceMs);
+
+    const procInfo = this.externalProcesses.get(jobId);
+    if (procInfo) {
+      procInfo.killTimer = timer;
+    }
+  }
+
+  private async handleExternalFailure(jobId: string, error: string): Promise<void> {
+    await this.adapter.update(jobId, {
+      status: "failed",
+      error,
+      completedAt: new Date(),
+      processState: "orphaned",
+    });
+
+    const job = await this.adapter.get(jobId);
+    if (this.events && job) {
+      await this.events.emit("job.watchdog.killed", {
+        jobId,
+        name: job.name,
+        reason: error,
+      });
+    }
+
+    await this.cleanupExternalJob(jobId);
+
+    if (this.events && job) {
+      await this.events.emit("job.failed", {
+        jobId,
+        name: job.name,
+        error,
+      });
+    }
+  }
+
   private streamProcessOutput(
     jobId: string,
     jobName: string,

package/src/core/process-socket.ts

@@ -207,6 +207,23 @@ export class ProcessSocketServerImpl implements ProcessSocketServer {
 
     let buffer = "";
 
+    const queue: ProcessMessage[] = [];
+    let processing = false;
+
+    const processQueue = async () => {
+      if (processing) return;
+      processing = true;
+      while (queue.length > 0) {
+        const message = queue.shift()!;
+        try {
+          await this.onMessage(message);
+        } catch (err) {
+          this.onError?.(err instanceof Error ? err : new Error(String(err)), processId);
+        }
+      }
+      processing = false;
+    };
+
     socket.on("data", (data) => {
       buffer += data.toString();
 
@@ -219,11 +236,13 @@ export class ProcessSocketServerImpl implements ProcessSocketServer {
 
         const message = this.parseMessage(line);
         if (message) {
-          this.onMessage(message);
+          queue.push(message);
         } else {
           this.onError?.(new Error(`Invalid message: ${line}`), processId);
         }
       }
+
+      processQueue().catch(() => undefined);
     });
 
     socket.on("error", (err) => {

package/src/core/processes.ts

@@ -61,6 +61,15 @@ export interface ProcessConfig {
     /** Timeout before considering unhealthy in ms (default: 60000) */
     timeoutMs?: number;
   };
+  /** Hard limits for the process (optional) */
+  limits?: {
+    /** Max runtime in ms before termination */
+    maxRuntimeMs?: number;
+    /** Max memory (RSS) in MB before termination (requires stats enabled) */
+    maxMemoryMb?: number;
+    /** Max CPU percent before termination (requires stats enabled) */
+    maxCpuPercent?: number;
+  };
 }
 
 export interface ManagedProcess {
@@ -171,6 +180,8 @@ export interface ProcessesConfig {
   heartbeatCheckInterval?: number;
   /** Enable auto-reconnect to orphaned processes on startup (default: true) */
   autoRecoverOrphans?: boolean;
+  /** Grace period before SIGKILL when stopping/killing (ms, default: 5000) */
+  killGraceMs?: number;
 }
 
 // ============================================
@@ -251,6 +262,8 @@ export class ProcessesImpl implements Processes {
   private events?: Events;
   private heartbeatCheckInterval: number;
   private autoRecoverOrphans: boolean;
+  private killGraceMs: number;
+  private runtimeLimitTimers = new Map<string, ReturnType<typeof setTimeout>>();
 
   // Track running Bun subprocesses
   private subprocesses = new Map<string, Subprocess>();
@@ -266,6 +279,7 @@ export class ProcessesImpl implements Processes {
     this.events = config.events;
     this.heartbeatCheckInterval = config.heartbeatCheckInterval ?? 10000;
     this.autoRecoverOrphans = config.autoRecoverOrphans ?? true;
+    this.killGraceMs = config.killGraceMs ?? 5000;
 
     // Create socket server with callbacks
     this.socketServer = createProcessSocketServer(config.socket ?? {}, {
@@ -361,6 +375,21 @@ export class ProcessesImpl implements Processes {
       // Set up exit handler for crash detection
       proc.exited.then((exitCode) => this.handleExit(process.id, exitCode));
 
+      const maxRuntimeMs = config.limits?.maxRuntimeMs;
+      if (maxRuntimeMs && maxRuntimeMs > 0) {
+        const timer = setTimeout(async () => {
+          console.warn(`[Processes] Max runtime exceeded for ${name} (${process.id})`);
+          await this.emitEvent("process.limits_exceeded", {
+            processId: process.id,
+            name,
+            reason: "maxRuntimeMs",
+            limit: maxRuntimeMs,
+          });
+          await this.stop(process.id);
+        }, maxRuntimeMs);
+        this.runtimeLimitTimers.set(process.id, timer);
+      }
+
       console.log(`[Processes] Spawned ${name} (${process.id}) with PID ${proc.pid}`);
       return process.id;
     } catch (error) {
@@ -395,7 +424,7 @@ export class ProcessesImpl implements Processes {
         // Wait for process to exit (with timeout)
         const exitPromise = subprocess.exited;
         const timeoutPromise = new Promise<null>((resolve) =>
-          setTimeout(() => resolve(null), 5000)
+          setTimeout(() => resolve(null), this.killGraceMs)
         );
 
         const result = await Promise.race([exitPromise, timeoutPromise]);
@@ -412,6 +441,11 @@ export class ProcessesImpl implements Processes {
     // Cleanup
     await this.socketServer.closeSocket(processId);
     this.subprocesses.delete(processId);
+    const runtimeTimer = this.runtimeLimitTimers.get(processId);
+    if (runtimeTimer) {
+      clearTimeout(runtimeTimer);
+      this.runtimeLimitTimers.delete(processId);
+    }
 
     await this.adapter.update(processId, {
       status: "stopped",
@@ -443,6 +477,11 @@ export class ProcessesImpl implements Processes {
     // Cleanup
     await this.socketServer.closeSocket(processId);
     this.subprocesses.delete(processId);
+    const runtimeTimer = this.runtimeLimitTimers.get(processId);
+    if (runtimeTimer) {
+      clearTimeout(runtimeTimer);
+      this.runtimeLimitTimers.delete(processId);
+    }
 
     await this.adapter.update(processId, {
       status: "stopped",
@@ -590,6 +629,47 @@ export class ProcessesImpl implements Processes {
         await definition.onStats(proc, stats);
       }
 
+      const limits = proc.config.limits;
+      if (limits) {
+        if (limits.maxMemoryMb && stats.memory.rss / 1e6 > limits.maxMemoryMb) {
+          console.warn(`[Processes] Memory limit exceeded for ${proc.name} (${proc.id})`);
+          await this.emitEvent("process.limits_exceeded", {
+            processId,
+            name: proc.name,
+            reason: "maxMemoryMb",
+            limit: limits.maxMemoryMb,
+            value: stats.memory.rss / 1e6,
+          });
+          await this.emitEvent("process.watchdog.killed", {
+            processId,
+            name: proc.name,
+            reason: "maxMemoryMb",
+            value: stats.memory.rss / 1e6,
+          });
+          await this.stop(proc.id);
+          return;
+        }
+
+        if (limits.maxCpuPercent && stats.cpu.percent > limits.maxCpuPercent) {
+          console.warn(`[Processes] CPU limit exceeded for ${proc.name} (${proc.id})`);
+          await this.emitEvent("process.limits_exceeded", {
+            processId,
+            name: proc.name,
+            reason: "maxCpuPercent",
+            limit: limits.maxCpuPercent,
+            value: stats.cpu.percent,
+          });
+          await this.emitEvent("process.watchdog.killed", {
+            processId,
+            name: proc.name,
+            reason: "maxCpuPercent",
+            value: stats.cpu.percent,
+          });
+          await this.stop(proc.id);
+          return;
+        }
+      }
+
       return;
     }
 
@@ -835,16 +915,27 @@ export class ProcessesImpl implements Processes {
             processId: proc.id,
             name: proc.name,
           });
+          await this.emitEvent("process.watchdog.stale", {
+            processId: proc.id,
+            name: proc.name,
+            reason: "heartbeat",
+            timeoutMs,
+          });
 
           const definition = this.definitions.get(proc.name);
           if (definition?.onUnhealthy) {
             await definition.onUnhealthy(proc);
           }
 
-          // If heartbeat is way overdue (2x timeout), kill and restart
+          // If heartbeat is way overdue (2x timeout), stop and restart
           if (now - lastHeartbeat > timeoutMs * 2) {
-            console.warn(`[Processes] Killing unresponsive process ${proc.name} (${proc.id})`);
-            await this.kill(proc.id);
+            console.warn(`[Processes] Stopping unresponsive process ${proc.name} (${proc.id})`);
+            await this.stop(proc.id);
+            await this.emitEvent("process.watchdog.killed", {
+              processId: proc.id,
+              name: proc.name,
+              reason: "heartbeat",
+            });
             // handleExit will trigger auto-restart if configured
           }
         }