@donkeylabs/server 2.0.28 → 2.0.30

package/docs/jobs.md

@@ -192,6 +192,33 @@ const cancelled = await ctx.core.jobs.cancel(jobId);
 
 ---
 
+## External Jobs (Subprocess)
+
+External jobs run in a separate process and are monitored by a watchdog.
+
+```ts
+ctx.core.jobs.registerExternal("batchWorker", {
+  command: "bun",
+  args: ["./workers/batch-worker.ts"],
+  heartbeatTimeout: 30000,
+  timeout: 10 * 60 * 1000,
+  killGraceMs: 5000,
+});
+
+// Disable in-process timers when using the watchdog subprocess
+const server = new AppServer({
+  db,
+  watchdog: { enabled: true },
+  jobs: { external: { useWatchdog: true } },
+});
+```
+
+Watchdog events:
+- `job.watchdog.stale`
+- `job.watchdog.killed`
+
+---
+
 ## Event Integration
 
 Jobs automatically emit events on completion and failure:

package/docs/processes.md

@@ -8,6 +8,7 @@ Processes provide:
 - Long-running daemon management (start, stop, restart)
 - Typed event communication from process to server
 - Automatic heartbeat monitoring
+- Watchdog termination for unresponsive processes
 - Connection resilience with auto-reconnection
 - Metadata passing to spawned processes
 - Cross-platform support (Unix sockets / TCP on Windows)
@@ -89,6 +90,13 @@ server.getCore().processes.define("video-encoder", {
 
   // Heartbeat configuration
   heartbeatTimeout: 30000, // 30 seconds
+
+  // Optional hard limits (requires stats for memory/CPU)
+  limits: {
+    maxRuntimeMs: 60_000,
+    maxMemoryMb: 512,
+    maxCpuPercent: 90,
+  },
 });
 ```
 
@@ -221,6 +229,27 @@ const client = await ProcessClient.connect({
 });
 ```
 
+---
+
+## Hard Limits
+
+Processes can be terminated automatically when limits are exceeded:
+
+- `maxRuntimeMs` always enforced by the server watchdog
+- `maxMemoryMb` and `maxCpuPercent` require `ProcessClient` stats enabled
+
+```ts
+const client = await ProcessClient.connect({
+  stats: { enabled: true, interval: 5000 },
+});
+```
+
+Watchdog events:
+- `process.watchdog.stale`
+- `process.watchdog.killed`
+
+When `watchdog.enabled` is true, heartbeat monitoring runs in the watchdog subprocess.
+
 ### Properties
 
 ```typescript

package/docs/workflows.md

@@ -309,6 +309,38 @@ workflow("batch.status")
 
 Each poll cycle emits `workflow.step.poll` events and persists progress to the instance.
 
+---
+
+## Watchdog and Subprocess Settings
+
+You can tune subprocess termination and SQLite pragmas used by isolated workflows:
+
+```ts
+const server = new AppServer({
+  db,
+  watchdog: {
+    enabled: true,
+    intervalMs: 5000,
+    services: ["workflows", "jobs", "processes"],
+    killGraceMs: 5000,
+  },
+  workflows: {
+    killGraceMs: 5000,
+    sqlitePragmas: {
+      busyTimeout: 5000,
+      journalMode: "WAL",
+      synchronous: "NORMAL",
+    },
+  },
+});
+```
+
+When `watchdog.enabled` is true, workflow heartbeat timers run in the watchdog subprocess instead of the main server.
+
+Watchdog events:
+- `workflow.watchdog.stale` (heartbeat missed)
+- `workflow.watchdog.killed` (process terminated)
+
 ### Loop
 
 Use a loop step to jump back to a previous step until a condition is false.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@donkeylabs/server",
-  "version": "2.0.28",
+  "version": "2.0.30",
   "type": "module",
   "description": "Type-safe plugin system for building RPC-style APIs with Bun",
   "main": "./src/index.ts",

package/src/core/external-job-socket.ts

@@ -201,6 +201,23 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
 
     let buffer = "";
 
+    const queue: AnyExternalJobMessage[] = [];
+    let processing = false;
+
+    const processQueue = async () => {
+      if (processing) return;
+      processing = true;
+      while (queue.length > 0) {
+        const message = queue.shift()!;
+        try {
+          await this.onMessage(message);
+        } catch (err) {
+          this.onError?.(err instanceof Error ? err : new Error(String(err)), jobId);
+        }
+      }
+      processing = false;
+    };
+
     socket.on("data", (data) => {
       buffer += data.toString();
 
@@ -213,11 +230,13 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
 
         const message = parseJobMessage(line);
         if (message) {
-          this.onMessage(message);
+          queue.push(message);
         } else {
           this.onError?.(new Error(`Invalid message: ${line}`), jobId);
         }
       }
+
+      processQueue().catch(() => undefined);
     });
 
     socket.on("error", (err) => {

package/src/core/external-jobs.ts

@@ -80,6 +80,8 @@ export interface ExternalJobConfig {
   heartbeatTimeout?: number;
   /** Job timeout in milliseconds (optional) */
   timeout?: number;
+  /** Grace period before SIGKILL when terminating (ms, default: 5000) */
+  killGraceMs?: number;
 }
 
 // ============================================
@@ -120,6 +122,10 @@ export interface ExternalJobsConfig {
   defaultHeartbeatTimeout?: number;
   /** Heartbeat check interval in ms (default: 10000) */
   heartbeatCheckInterval?: number;
+  /** Default grace period before SIGKILL when terminating (ms, default: 5000) */
+  killGraceMs?: number;
+  /** Disable in-process watchdog timers (use external watchdog instead) */
+  useWatchdog?: boolean;
 }
 
 // ============================================

package/src/core/index.ts

@@ -134,6 +134,7 @@ export {
 export {
   type Workflows,
   type WorkflowsConfig,
+  type SqlitePragmaConfig,
   type WorkflowRegisterOptions,
   type WorkflowDefinition,
   type WorkflowInstance,

package/src/core/jobs.ts

@@ -141,6 +141,8 @@ export interface Jobs {
   getRunningExternal(): Promise<Job[]>;
   /** Get all jobs with optional filtering (for admin dashboard) */
   getAll(options?: GetAllJobsOptions): Promise<Job[]>;
+  /** Get external job config snapshot for watchdog */
+  getExternalJobConfigs(): Record<string, ExternalJobConfig>;
   /** Start the job processing loop */
   start(): void;
   /** Stop the job processing and cleanup */
@@ -273,7 +275,10 @@ class JobsImpl implements Jobs {
   private externalConfigs = new Map<string, ExternalJobConfig>();
   private externalConfig: ExternalJobsConfig;
   private socketServer: ExternalJobSocketServer | null = null;
-  private externalProcesses = new Map<string, { pid: number; timeout?: ReturnType<typeof setTimeout> }>();
+  private externalProcesses = new Map<
+    string,
+    { pid: number; timeout?: ReturnType<typeof setTimeout>; killTimer?: ReturnType<typeof setTimeout> }
+  >();
 
   constructor(config: JobsConfig = {}) {
     this.events = config.events;
@@ -322,6 +327,14 @@ class JobsImpl implements Jobs {
     this.externalConfigs.set(name, config);
   }
 
+  getExternalJobConfigs(): Record<string, ExternalJobConfig> {
+    const snapshot: Record<string, ExternalJobConfig> = {};
+    for (const [name, config] of this.externalConfigs.entries()) {
+      snapshot[name] = { ...config };
+    }
+    return snapshot;
+  }
+
   private isExternalJob(name: string): boolean {
     return this.externalConfigs.has(name);
   }
@@ -419,7 +432,9 @@ class JobsImpl implements Jobs {
     // Initialize socket server for external jobs
     if (this.externalConfigs.size > 0) {
       this.initializeSocketServer();
-      this.startHeartbeatMonitor();
+      if (!this.externalConfig.useWatchdog) {
+        this.startHeartbeatMonitor();
+      }
       // Attempt to reconnect to orphaned jobs from previous run
       this.reconnectOrphanedJobs();
     }
@@ -521,6 +536,7 @@ class JobsImpl implements Jobs {
 
         const config = this.externalConfigs.get(job.name);
         const heartbeatTimeout = config?.heartbeatTimeout ?? this.externalConfig.defaultHeartbeatTimeout ?? 30000;
+        const killGraceMs = config?.killGraceMs ?? this.externalConfig.killGraceMs ?? 5000;
         const timeSinceHeartbeat = now - job.lastHeartbeat.getTime();
 
         if (timeSinceHeartbeat > heartbeatTimeout) {
@@ -533,36 +549,22 @@ class JobsImpl implements Jobs {
               name: job.name,
               timeSinceHeartbeat,
             });
-          }
-
-          // If stale for 2x timeout, kill the process
-          if (timeSinceHeartbeat > heartbeatTimeout * 2) {
-            console.error(`[Jobs] Killing stale external job ${job.id}`);
-
-            if (job.pid) {
-              try {
-                process.kill(job.pid, "SIGKILL");
-              } catch {
-                // Process may already be dead
-              }
-            }
-
-            await this.adapter.update(job.id, {
-              status: "failed",
-              error: "Heartbeat timeout - job process unresponsive",
-              completedAt: new Date(),
-              processState: "orphaned",
+            await this.events.emit("job.watchdog.stale", {
+              jobId: job.id,
+              name: job.name,
+              timeSinceHeartbeat,
             });
+          }
 
-            await this.cleanupExternalJob(job.id);
-
-            if (this.events) {
-              await this.events.emit("job.failed", {
-                jobId: job.id,
-                name: job.name,
-                error: "Heartbeat timeout",
-              });
-            }
+          const procInfo = this.externalProcesses.get(job.id);
+          if (job.pid && !procInfo?.killTimer) {
+            console.error(`[Jobs] Terminating stale external job ${job.id}`);
+            await this.terminateExternalProcess(
+              job.id,
+              job.pid,
+              killGraceMs,
+              "Heartbeat timeout - job process unresponsive"
+            );
           }
         }
       }
@@ -764,6 +766,9 @@ class JobsImpl implements Jobs {
     if (procInfo?.timeout) {
       clearTimeout(procInfo.timeout);
     }
+    if (procInfo?.killTimer) {
+      clearTimeout(procInfo.killTimer);
+    }
     this.externalProcesses.delete(jobId);
 
     // Close the socket
@@ -898,30 +903,16 @@ class JobsImpl implements Jobs {
       proc.stdin.end();
 
       // Set up process timeout if configured
-      if (config.timeout) {
+      if (config.timeout && !this.externalConfig.useWatchdog) {
         const timeout = setTimeout(async () => {
           console.warn(`[Jobs] External job ${job.id} timed out after ${config.timeout}ms`);
-          try {
-            process.kill(proc.pid, "SIGTERM");
-          } catch {
-            // Process may already be dead
-          }
-
-          await this.adapter.update(job.id, {
-            status: "failed",
-            error: `Job timed out after ${config.timeout}ms`,
-            completedAt: new Date(),
-          });
-
-          await this.cleanupExternalJob(job.id);
-
-          if (this.events) {
-            await this.events.emit("job.failed", {
-              jobId: job.id,
-              name: job.name,
-              error: "Timeout",
-            });
-          }
+          const killGraceMs = config.killGraceMs ?? this.externalConfig.killGraceMs ?? 5000;
+          await this.terminateExternalProcess(
+            job.id,
+            proc.pid,
+            killGraceMs,
+            `Job timed out after ${config.timeout}ms`
+          );
         }, config.timeout);
 
         const procInfo = this.externalProcesses.get(job.id);
@@ -998,6 +989,73 @@ class JobsImpl implements Jobs {
     }
   }
 
+  private async terminateExternalProcess(
+    jobId: string,
+    pid: number,
+    killGraceMs: number,
+    error: string
+  ): Promise<void> {
+    try {
+      process.kill(pid, "SIGTERM");
+    } catch {
+      return;
+    }
+
+    if (killGraceMs <= 0) {
+      try {
+        process.kill(pid, "SIGKILL");
+      } catch {
+        // ignore
+      }
+      await this.handleExternalFailure(jobId, error);
+      return;
+    }
+
+    const timer = setTimeout(async () => {
+      try {
+        process.kill(pid, 0);
+        process.kill(pid, "SIGKILL");
+      } catch {
+        // ignore
+      }
+
+      await this.handleExternalFailure(jobId, error);
+    }, killGraceMs);
+
+    const procInfo = this.externalProcesses.get(jobId);
+    if (procInfo) {
+      procInfo.killTimer = timer;
+    }
+  }
+
+  private async handleExternalFailure(jobId: string, error: string): Promise<void> {
+    await this.adapter.update(jobId, {
+      status: "failed",
+      error,
+      completedAt: new Date(),
+      processState: "orphaned",
+    });
+
+    const job = await this.adapter.get(jobId);
+    if (this.events && job) {
+      await this.events.emit("job.watchdog.killed", {
+        jobId,
+        name: job.name,
+        reason: error,
+      });
+    }
+
+    await this.cleanupExternalJob(jobId);
+
+    if (this.events && job) {
+      await this.events.emit("job.failed", {
+        jobId,
+        name: job.name,
+        error,
+      });
+    }
+  }
+
   private streamProcessOutput(
     jobId: string,
     jobName: string,

package/src/core/process-socket.ts

@@ -207,6 +207,23 @@ export class ProcessSocketServerImpl implements ProcessSocketServer {
 
     let buffer = "";
 
+    const queue: ProcessMessage[] = [];
+    let processing = false;
+
+    const processQueue = async () => {
+      if (processing) return;
+      processing = true;
+      while (queue.length > 0) {
+        const message = queue.shift()!;
+        try {
+          await this.onMessage(message);
+        } catch (err) {
+          this.onError?.(err instanceof Error ? err : new Error(String(err)), processId);
+        }
+      }
+      processing = false;
+    };
+
     socket.on("data", (data) => {
       buffer += data.toString();
 
@@ -219,11 +236,13 @@ export class ProcessSocketServerImpl implements ProcessSocketServer {
 
         const message = this.parseMessage(line);
         if (message) {
-          this.onMessage(message);
+          queue.push(message);
         } else {
           this.onError?.(new Error(`Invalid message: ${line}`), processId);
         }
       }
+
+      processQueue().catch(() => undefined);
     });
 
     socket.on("error", (err) => {

package/src/core/processes.ts

@@ -61,6 +61,15 @@ export interface ProcessConfig {
     /** Timeout before considering unhealthy in ms (default: 60000) */
     timeoutMs?: number;
   };
+  /** Hard limits for the process (optional) */
+  limits?: {
+    /** Max runtime in ms before termination */
+    maxRuntimeMs?: number;
+    /** Max memory (RSS) in MB before termination (requires stats enabled) */
+    maxMemoryMb?: number;
+    /** Max CPU percent before termination (requires stats enabled) */
+    maxCpuPercent?: number;
+  };
 }
 
 export interface ManagedProcess {
@@ -171,6 +180,10 @@ export interface ProcessesConfig {
   heartbeatCheckInterval?: number;
   /** Enable auto-reconnect to orphaned processes on startup (default: true) */
   autoRecoverOrphans?: boolean;
+  /** Grace period before SIGKILL when stopping/killing (ms, default: 5000) */
+  killGraceMs?: number;
+  /** Disable in-process watchdog timers (use external watchdog instead) */
+  useWatchdog?: boolean;
 }
 
 // ============================================
@@ -251,6 +264,9 @@ export class ProcessesImpl implements Processes {
   private events?: Events;
   private heartbeatCheckInterval: number;
   private autoRecoverOrphans: boolean;
+  private killGraceMs: number;
+  private runtimeLimitTimers = new Map<string, ReturnType<typeof setTimeout>>();
+  private useWatchdog: boolean;
 
   // Track running Bun subprocesses
   private subprocesses = new Map<string, Subprocess>();
@@ -266,6 +282,8 @@ export class ProcessesImpl implements Processes {
     this.events = config.events;
     this.heartbeatCheckInterval = config.heartbeatCheckInterval ?? 10000;
     this.autoRecoverOrphans = config.autoRecoverOrphans ?? true;
+    this.killGraceMs = config.killGraceMs ?? 5000;
+    this.useWatchdog = config.useWatchdog ?? false;
 
     // Create socket server with callbacks
     this.socketServer = createProcessSocketServer(config.socket ?? {}, {
@@ -361,6 +379,21 @@ export class ProcessesImpl implements Processes {
       // Set up exit handler for crash detection
       proc.exited.then((exitCode) => this.handleExit(process.id, exitCode));
 
+      const maxRuntimeMs = config.limits?.maxRuntimeMs;
+      if (!this.useWatchdog && maxRuntimeMs && maxRuntimeMs > 0) {
+        const timer = setTimeout(async () => {
+          console.warn(`[Processes] Max runtime exceeded for ${name} (${process.id})`);
+          await this.emitEvent("process.limits_exceeded", {
+            processId: process.id,
+            name,
+            reason: "maxRuntimeMs",
+            limit: maxRuntimeMs,
+          });
+          await this.stop(process.id);
+        }, maxRuntimeMs);
+        this.runtimeLimitTimers.set(process.id, timer);
+      }
+
       console.log(`[Processes] Spawned ${name} (${process.id}) with PID ${proc.pid}`);
       return process.id;
     } catch (error) {
@@ -395,7 +428,7 @@ export class ProcessesImpl implements Processes {
         // Wait for process to exit (with timeout)
         const exitPromise = subprocess.exited;
         const timeoutPromise = new Promise<null>((resolve) =>
-          setTimeout(() => resolve(null), 5000)
+          setTimeout(() => resolve(null), this.killGraceMs)
         );
 
         const result = await Promise.race([exitPromise, timeoutPromise]);
@@ -412,6 +445,11 @@ export class ProcessesImpl implements Processes {
     // Cleanup
     await this.socketServer.closeSocket(processId);
     this.subprocesses.delete(processId);
+    const runtimeTimer = this.runtimeLimitTimers.get(processId);
+    if (runtimeTimer) {
+      clearTimeout(runtimeTimer);
+      this.runtimeLimitTimers.delete(processId);
+    }
 
     await this.adapter.update(processId, {
       status: "stopped",
@@ -443,6 +481,11 @@ export class ProcessesImpl implements Processes {
     // Cleanup
     await this.socketServer.closeSocket(processId);
     this.subprocesses.delete(processId);
+    const runtimeTimer = this.runtimeLimitTimers.get(processId);
+    if (runtimeTimer) {
+      clearTimeout(runtimeTimer);
+      this.runtimeLimitTimers.delete(processId);
+    }
 
     await this.adapter.update(processId, {
       status: "stopped",
@@ -590,6 +633,47 @@ export class ProcessesImpl implements Processes {
         await definition.onStats(proc, stats);
       }
 
+      const limits = proc.config.limits;
+      if (limits) {
+        if (limits.maxMemoryMb && stats.memory.rss / 1e6 > limits.maxMemoryMb) {
+          console.warn(`[Processes] Memory limit exceeded for ${proc.name} (${proc.id})`);
+          await this.emitEvent("process.limits_exceeded", {
+            processId,
+            name: proc.name,
+            reason: "maxMemoryMb",
+            limit: limits.maxMemoryMb,
+            value: stats.memory.rss / 1e6,
+          });
+          await this.emitEvent("process.watchdog.killed", {
+            processId,
+            name: proc.name,
+            reason: "maxMemoryMb",
+            value: stats.memory.rss / 1e6,
+          });
+          await this.stop(proc.id);
+          return;
+        }
+
+        if (limits.maxCpuPercent && stats.cpu.percent > limits.maxCpuPercent) {
+          console.warn(`[Processes] CPU limit exceeded for ${proc.name} (${proc.id})`);
+          await this.emitEvent("process.limits_exceeded", {
+            processId,
+            name: proc.name,
+            reason: "maxCpuPercent",
+            limit: limits.maxCpuPercent,
+            value: stats.cpu.percent,
+          });
+          await this.emitEvent("process.watchdog.killed", {
+            processId,
+            name: proc.name,
+            reason: "maxCpuPercent",
+            value: stats.cpu.percent,
+          });
+          await this.stop(proc.id);
+          return;
+        }
+      }
+
       return;
     }
 
@@ -815,6 +899,7 @@ export class ProcessesImpl implements Processes {
   }
 
   private startHeartbeatMonitor(): void {
+    if (this.useWatchdog) return;
     this.heartbeatMonitor = setInterval(async () => {
       if (this.isShuttingDown) return;
 
@@ -835,16 +920,27 @@ export class ProcessesImpl implements Processes {
             processId: proc.id,
             name: proc.name,
           });
+          await this.emitEvent("process.watchdog.stale", {
+            processId: proc.id,
+            name: proc.name,
+            reason: "heartbeat",
+            timeoutMs,
+          });
 
           const definition = this.definitions.get(proc.name);
           if (definition?.onUnhealthy) {
             await definition.onUnhealthy(proc);
           }
 
-          // If heartbeat is way overdue (2x timeout), kill and restart
+          // If heartbeat is way overdue (2x timeout), stop and restart
           if (now - lastHeartbeat > timeoutMs * 2) {
-            console.warn(`[Processes] Killing unresponsive process ${proc.name} (${proc.id})`);
-            await this.kill(proc.id);
+            console.warn(`[Processes] Stopping unresponsive process ${proc.name} (${proc.id})`);
+            await this.stop(proc.id);
+            await this.emitEvent("process.watchdog.killed", {
+              processId: proc.id,
+              name: proc.name,
+              reason: "heartbeat",
+            });
             // handleExit will trigger auto-restart if configured
           }
         }

package/src/core/subprocess-bootstrap.ts

@@ -32,6 +32,11 @@ export interface SubprocessPluginMetadata {
 export interface SubprocessBootstrapOptions {
   dbPath: string;
   coreConfig?: Record<string, any>;
+  sqlitePragmas?: {
+    busyTimeout?: number;
+    synchronous?: "OFF" | "NORMAL" | "FULL" | "EXTRA";
+    journalMode?: "DELETE" | "TRUNCATE" | "PERSIST" | "MEMORY" | "WAL" | "OFF";
+  };
   pluginMetadata: SubprocessPluginMetadata;
   startServices?: {
     cron?: boolean;
@@ -53,7 +58,15 @@ export async function bootstrapSubprocess(
   options: SubprocessBootstrapOptions
 ): Promise<SubprocessBootstrapResult> {
   const sqlite = new Database(options.dbPath);
-  sqlite.run("PRAGMA busy_timeout = 5000");
+  const pragmas = options.sqlitePragmas;
+  const busyTimeout = pragmas?.busyTimeout ?? 5000;
+  sqlite.run(`PRAGMA busy_timeout = ${busyTimeout}`);
+  if (pragmas?.journalMode) {
+    sqlite.run(`PRAGMA journal_mode = ${pragmas.journalMode}`);
+  }
+  if (pragmas?.synchronous) {
+    sqlite.run(`PRAGMA synchronous = ${pragmas.synchronous}`);
+  }
 
   const db = new Kysely<any>({
     dialect: new BunSqliteDialect({ database: sqlite }),