@donkeylabs/server 2.0.29 → 2.0.31

package/docs/jobs.md

@@ -204,6 +204,13 @@ ctx.core.jobs.registerExternal("batchWorker", {
   timeout: 10 * 60 * 1000,
   killGraceMs: 5000,
 });
+
+// Disable in-process timers when using the watchdog subprocess
+const server = new AppServer({
+  db,
+  watchdog: { enabled: true },
+  jobs: { external: { useWatchdog: true } },
+});
 ```
 
 Watchdog events:

package/docs/processes.md

@@ -248,6 +248,8 @@ Watchdog events:
 - `process.watchdog.stale`
 - `process.watchdog.killed`
 
+When `watchdog.enabled` is true, heartbeat monitoring runs in the watchdog subprocess.
+
 ### Properties
 
 ```typescript

package/docs/workflows.md

@@ -89,6 +89,19 @@ const instanceId = await ctx.core.workflows.start("process-order", {
 });
 ```
 
+### Concurrency Guard
+
+Limit concurrent instances per workflow name:
+
+```ts
+const server = new AppServer({
+  db,
+  workflows: {
+    concurrentWorkflows: 1, // 0 = unlimited
+  },
+});
+```
+
 ### 3. Track Progress
 
 ```typescript
@@ -318,6 +331,12 @@ You can tune subprocess termination and SQLite pragmas used by isolated workflow
 ```ts
 const server = new AppServer({
   db,
+  watchdog: {
+    enabled: true,
+    intervalMs: 5000,
+    services: ["workflows", "jobs", "processes"],
+    killGraceMs: 5000,
+  },
   workflows: {
     killGraceMs: 5000,
     sqlitePragmas: {
@@ -329,6 +348,8 @@ const server = new AppServer({
 });
 ```
 
+When `watchdog.enabled` is true, workflow heartbeat timers run in the watchdog subprocess instead of the main server.
+
 Watchdog events:
 - `workflow.watchdog.stale` (heartbeat missed)
 - `workflow.watchdog.killed` (process terminated)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@donkeylabs/server",
-  "version": "2.0.29",
+  "version": "2.0.31",
   "type": "module",
   "description": "Type-safe plugin system for building RPC-style APIs with Bun",
   "main": "./src/index.ts",

package/src/core/external-jobs.ts

@@ -124,6 +124,8 @@ export interface ExternalJobsConfig {
   heartbeatCheckInterval?: number;
   /** Default grace period before SIGKILL when terminating (ms, default: 5000) */
   killGraceMs?: number;
+  /** Disable in-process watchdog timers (use external watchdog instead) */
+  useWatchdog?: boolean;
 }
 
 // ============================================

package/src/core/jobs.ts

@@ -141,6 +141,8 @@ export interface Jobs {
   getRunningExternal(): Promise<Job[]>;
   /** Get all jobs with optional filtering (for admin dashboard) */
   getAll(options?: GetAllJobsOptions): Promise<Job[]>;
+  /** Get external job config snapshot for watchdog */
+  getExternalJobConfigs(): Record<string, ExternalJobConfig>;
   /** Start the job processing loop */
   start(): void;
   /** Stop the job processing and cleanup */
@@ -325,6 +327,14 @@ class JobsImpl implements Jobs {
     this.externalConfigs.set(name, config);
   }
 
+  getExternalJobConfigs(): Record<string, ExternalJobConfig> {
+    const snapshot: Record<string, ExternalJobConfig> = {};
+    for (const [name, config] of this.externalConfigs.entries()) {
+      snapshot[name] = { ...config };
+    }
+    return snapshot;
+  }
+
   private isExternalJob(name: string): boolean {
     return this.externalConfigs.has(name);
   }
@@ -422,7 +432,9 @@ class JobsImpl implements Jobs {
     // Initialize socket server for external jobs
     if (this.externalConfigs.size > 0) {
       this.initializeSocketServer();
-      this.startHeartbeatMonitor();
+      if (!this.externalConfig.useWatchdog) {
+        this.startHeartbeatMonitor();
+      }
       // Attempt to reconnect to orphaned jobs from previous run
       this.reconnectOrphanedJobs();
     }
@@ -891,7 +903,7 @@ class JobsImpl implements Jobs {
       proc.stdin.end();
 
       // Set up process timeout if configured
-      if (config.timeout) {
+      if (config.timeout && !this.externalConfig.useWatchdog) {
         const timeout = setTimeout(async () => {
           console.warn(`[Jobs] External job ${job.id} timed out after ${config.timeout}ms`);
           const killGraceMs = config.killGraceMs ?? this.externalConfig.killGraceMs ?? 5000;

package/src/core/processes.ts

@@ -182,6 +182,8 @@ export interface ProcessesConfig {
   autoRecoverOrphans?: boolean;
   /** Grace period before SIGKILL when stopping/killing (ms, default: 5000) */
   killGraceMs?: number;
+  /** Disable in-process watchdog timers (use external watchdog instead) */
+  useWatchdog?: boolean;
 }
 
 // ============================================
@@ -264,6 +266,7 @@ export class ProcessesImpl implements Processes {
   private autoRecoverOrphans: boolean;
   private killGraceMs: number;
   private runtimeLimitTimers = new Map<string, ReturnType<typeof setTimeout>>();
+  private useWatchdog: boolean;
 
   // Track running Bun subprocesses
   private subprocesses = new Map<string, Subprocess>();
@@ -280,6 +283,7 @@ export class ProcessesImpl implements Processes {
     this.heartbeatCheckInterval = config.heartbeatCheckInterval ?? 10000;
     this.autoRecoverOrphans = config.autoRecoverOrphans ?? true;
     this.killGraceMs = config.killGraceMs ?? 5000;
+    this.useWatchdog = config.useWatchdog ?? false;
 
     // Create socket server with callbacks
     this.socketServer = createProcessSocketServer(config.socket ?? {}, {
@@ -376,7 +380,7 @@ export class ProcessesImpl implements Processes {
       proc.exited.then((exitCode) => this.handleExit(process.id, exitCode));
 
       const maxRuntimeMs = config.limits?.maxRuntimeMs;
-      if (maxRuntimeMs && maxRuntimeMs > 0) {
+      if (!this.useWatchdog && maxRuntimeMs && maxRuntimeMs > 0) {
         const timer = setTimeout(async () => {
           console.warn(`[Processes] Max runtime exceeded for ${name} (${process.id})`);
           await this.emitEvent("process.limits_exceeded", {
@@ -895,6 +899,7 @@ export class ProcessesImpl implements Processes {
   }
 
   private startHeartbeatMonitor(): void {
+    if (this.useWatchdog) return;
     this.heartbeatMonitor = setInterval(async () => {
       if (this.isShuttingDown) return;
 

package/src/core/watchdog-executor.ts

@@ -0,0 +1,80 @@
+import { Kysely } from "kysely";
+import { BunSqliteDialect } from "kysely-bun-sqlite";
+import Database from "bun:sqlite";
+import { ProcessClient } from "./process-client";
+import { KyselyWorkflowAdapter } from "./workflow-adapter-kysely";
+import { KyselyJobAdapter } from "./job-adapter-kysely";
+import { SqliteProcessAdapter } from "./process-adapter-sqlite";
+import { WatchdogRunner, type WatchdogRunnerConfig } from "./watchdog-runner";
+
+interface WatchdogConfig extends WatchdogRunnerConfig {
+  intervalMs: number;
+  workflows?: { dbPath?: string };
+  jobs?: { dbPath?: string };
+  processes?: { dbPath?: string };
+  sqlitePragmas?: {
+    busyTimeout?: number;
+    synchronous?: "OFF" | "NORMAL" | "FULL" | "EXTRA";
+    journalMode?: "DELETE" | "TRUNCATE" | "PERSIST" | "MEMORY" | "WAL" | "OFF";
+  };
+}
+
+const raw = process.env.DONKEYLABS_WATCHDOG_CONFIG;
+if (!raw) {
+  throw new Error("Missing DONKEYLABS_WATCHDOG_CONFIG");
+}
+
+const config: WatchdogConfig = JSON.parse(raw);
+const client = await ProcessClient.connect();
+
+const workflowAdapter = config.workflows?.dbPath
+  ? new KyselyWorkflowAdapter(createDb(config.workflows.dbPath, config.sqlitePragmas), {
+      cleanupDays: 0,
+    })
+  : undefined;
+const jobAdapter = config.jobs?.dbPath
+  ? new KyselyJobAdapter(createDb(config.jobs.dbPath, config.sqlitePragmas), {
+      cleanupDays: 0,
+    })
+  : undefined;
+const processAdapter = config.processes?.dbPath
+  ? new SqliteProcessAdapter({ path: config.processes.dbPath, cleanupDays: 0 })
+  : undefined;
+
+const runner = new WatchdogRunner(config, {
+  workflowsAdapter: workflowAdapter,
+  jobsAdapter: jobAdapter,
+  processesAdapter: processAdapter,
+  emit: async (event, data) => {
+    await client.emit(event, data);
+  },
+});
+
+const interval = Math.max(1000, config.intervalMs);
+const timer = setInterval(() => {
+  runner.runOnce().catch(() => undefined);
+}, interval);
+
+process.on("SIGTERM", async () => {
+  clearInterval(timer);
+  client.disconnect();
+});
+
+function createDb(
+  dbPath: string,
+  pragmas?: { busyTimeout?: number; synchronous?: string; journalMode?: string }
+): Kysely<any> {
+  const sqlite = new Database(dbPath);
+  const busyTimeout = pragmas?.busyTimeout ?? 5000;
+  sqlite.run(`PRAGMA busy_timeout = ${busyTimeout}`);
+  if (pragmas?.journalMode) {
+    sqlite.run(`PRAGMA journal_mode = ${pragmas.journalMode}`);
+  }
+  if (pragmas?.synchronous) {
+    sqlite.run(`PRAGMA synchronous = ${pragmas.synchronous}`);
+  }
+
+  return new Kysely<any>({
+    dialect: new BunSqliteDialect({ database: sqlite }),
+  });
+}

package/src/core/watchdog-runner.ts

@@ -0,0 +1,276 @@
+import type { WorkflowAdapter, WorkflowInstance } from "./workflows";
+import type { JobAdapter, Job } from "./jobs";
+import type { ProcessAdapter } from "./process-adapter-sqlite";
+
+export type WatchdogService = "workflows" | "jobs" | "processes";
+
+export interface WatchdogRunnerConfig {
+  services: WatchdogService[];
+  killGraceMs: number;
+  workflowHeartbeatTimeoutMs: number;
+  jobDefaults: {
+    heartbeatTimeoutMs: number;
+    killGraceMs: number;
+  };
+  jobConfigs: Record<string, { heartbeatTimeout?: number; timeout?: number; killGraceMs?: number }>;
+}
+
+export interface WatchdogRunnerDeps {
+  workflowsAdapter?: WorkflowAdapter;
+  jobsAdapter?: JobAdapter;
+  processesAdapter?: ProcessAdapter;
+  now?: () => number;
+  killProcess?: (pid: number, signal: NodeJS.Signals) => void;
+  isProcessAlive?: (pid: number) => boolean;
+  emit?: (event: string, data: Record<string, any>) => Promise<void>;
+}
+
+export class WatchdogRunner {
+  private config: WatchdogRunnerConfig;
+  private deps: WatchdogRunnerDeps;
+
+  constructor(config: WatchdogRunnerConfig, deps: WatchdogRunnerDeps) {
+    this.config = config;
+    this.deps = deps;
+  }
+
+  async runOnce(): Promise<void> {
+    if (this.config.services.includes("workflows") && this.deps.workflowsAdapter) {
+      await this.checkWorkflows();
+    }
+    if (this.config.services.includes("jobs") && this.deps.jobsAdapter) {
+      await this.checkJobs();
+    }
+    if (this.config.services.includes("processes") && this.deps.processesAdapter) {
+      await this.checkProcesses();
+    }
+  }
+
+  private async checkWorkflows(): Promise<void> {
+    const adapter = this.deps.workflowsAdapter!;
+    const now = this.now();
+    const instances = await adapter.getRunningInstances();
+
+    for (const instance of instances) {
+      const info = this.getWatchdogMetadata(instance);
+      if (!info?.pid) continue;
+
+      const last = info.lastHeartbeat ?? instance.startedAt?.getTime() ?? 0;
+      if (now - last <= this.config.workflowHeartbeatTimeoutMs) continue;
+
+      await this.emit("workflow.watchdog.stale", {
+        instanceId: instance.id,
+        pid: info.pid,
+        timeoutMs: this.config.workflowHeartbeatTimeoutMs,
+      });
+
+      await this.killProcessWithGrace(info.pid, this.config.killGraceMs);
+
+      await adapter.updateInstance(instance.id, {
+        status: "failed",
+        error: "Watchdog killed unresponsive workflow",
+        completedAt: new Date(),
+      });
+
+      await this.emit("workflow.watchdog.killed", {
+        instanceId: instance.id,
+        pid: info.pid,
+        reason: "heartbeat",
+      });
+    }
+  }
+
+  private async checkJobs(): Promise<void> {
+    const adapter = this.deps.jobsAdapter!;
+    const now = this.now();
+    const jobs = await adapter.getRunningExternal();
+
+    for (const job of jobs) {
+      if (!job.pid) continue;
+      const config = this.config.jobConfigs[job.name] ?? {};
+      const heartbeatTimeout =
+        config.heartbeatTimeout ?? this.config.jobDefaults.heartbeatTimeoutMs;
+      const killGraceMs = config.killGraceMs ?? this.config.jobDefaults.killGraceMs;
+      const lastHeartbeat = job.lastHeartbeat?.getTime() ?? job.startedAt?.getTime() ?? 0;
+
+      if (now - lastHeartbeat > heartbeatTimeout) {
+        await this.emit("job.watchdog.stale", {
+          jobId: job.id,
+          name: job.name,
+          pid: job.pid,
+          timeoutMs: heartbeatTimeout,
+        });
+
+        await this.killProcessWithGrace(job.pid, killGraceMs);
+
+        await adapter.update(job.id, {
+          status: "failed",
+          error: "Watchdog killed unresponsive job",
+          completedAt: new Date(),
+          processState: "orphaned",
+        });
+
+        await this.emit("job.watchdog.killed", {
+          jobId: job.id,
+          name: job.name,
+          pid: job.pid,
+          reason: "heartbeat",
+        });
+        continue;
+      }
+
+      if (config.timeout && job.startedAt) {
+        if (now - job.startedAt.getTime() > config.timeout) {
+          await this.emit("job.watchdog.stale", {
+            jobId: job.id,
+            name: job.name,
+            pid: job.pid,
+            timeoutMs: config.timeout,
+            reason: "timeout",
+          });
+
+          await this.killProcessWithGrace(job.pid, killGraceMs);
+
+          await adapter.update(job.id, {
+            status: "failed",
+            error: "Watchdog killed job after timeout",
+            completedAt: new Date(),
+            processState: "orphaned",
+          });
+
+          await this.emit("job.watchdog.killed", {
+            jobId: job.id,
+            name: job.name,
+            pid: job.pid,
+            reason: "timeout",
+          });
+        }
+      }
+    }
+  }
+
+  private async checkProcesses(): Promise<void> {
+    const adapter = this.deps.processesAdapter!;
+    const now = this.now();
+    const running = await adapter.getRunning();
+
+    for (const proc of running) {
+      if (!proc.pid) continue;
+      const heartbeatTimeout = proc.config.heartbeat?.timeoutMs;
+      const lastHeartbeat = proc.lastHeartbeat?.getTime() ?? proc.startedAt?.getTime() ?? 0;
+
+      if (heartbeatTimeout && now - lastHeartbeat > heartbeatTimeout) {
+        await this.emit("process.watchdog.stale", {
+          processId: proc.id,
+          name: proc.name,
+          pid: proc.pid,
+          timeoutMs: heartbeatTimeout,
+        });
+
+        await this.killProcessWithGrace(proc.pid, this.config.killGraceMs);
+
+        await adapter.update(proc.id, {
+          status: "crashed",
+          error: "Watchdog killed unresponsive process",
+          stoppedAt: new Date(),
+        });
+
+        await this.emit("process.watchdog.killed", {
+          processId: proc.id,
+          name: proc.name,
+          pid: proc.pid,
+          reason: "heartbeat",
+        });
+        continue;
+      }
+
+      const maxRuntimeMs = proc.config.limits?.maxRuntimeMs;
+      if (maxRuntimeMs && proc.startedAt) {
+        if (now - proc.startedAt.getTime() > maxRuntimeMs) {
+          await this.emit("process.watchdog.stale", {
+            processId: proc.id,
+            name: proc.name,
+            pid: proc.pid,
+            timeoutMs: maxRuntimeMs,
+            reason: "maxRuntimeMs",
+          });
+
+          await this.killProcessWithGrace(proc.pid, this.config.killGraceMs);
+
+          await adapter.update(proc.id, {
+            status: "crashed",
+            error: "Watchdog killed process after max runtime",
+            stoppedAt: new Date(),
+          });
+
+          await this.emit("process.watchdog.killed", {
+            processId: proc.id,
+            name: proc.name,
+            pid: proc.pid,
+            reason: "maxRuntimeMs",
+          });
+        }
+      }
+    }
+  }
+
+  private getWatchdogMetadata(instance: WorkflowInstance): { pid?: number; lastHeartbeat?: number } | null {
+    const meta = instance.metadata as any;
+    if (!meta || typeof meta !== "object") return null;
+    const info = meta.__watchdog;
+    if (!info || typeof info !== "object") return null;
+    return {
+      pid: typeof info.pid === "number" ? info.pid : undefined,
+      lastHeartbeat: info.lastHeartbeat ? new Date(info.lastHeartbeat).getTime() : undefined,
+    };
+  }
+
+  private now(): number {
+    return this.deps.now ? this.deps.now() : Date.now();
+  }
+
+  private async emit(event: string, data: Record<string, any>): Promise<void> {
+    if (this.deps.emit) {
+      await this.deps.emit(event, data);
+    }
+  }
+
+  private async killProcessWithGrace(pid: number, graceMs: number): Promise<void> {
+    const kill = this.deps.killProcess ?? process.kill;
+    try {
+      kill(pid, "SIGTERM");
+    } catch {
+      return;
+    }
+
+    if (graceMs <= 0) {
+      try {
+        kill(pid, "SIGKILL");
+      } catch {
+        // ignore
+      }
+      return;
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, graceMs));
+
+    try {
+      const isAlive = this.deps.isProcessAlive
+        ? this.deps.isProcessAlive(pid)
+        : (() => {
+            try {
+              process.kill(pid, 0);
+              return true;
+            } catch {
+              return false;
+            }
+          })();
+
+      if (isAlive) {
+        kill(pid, "SIGKILL");
+      }
+    } catch {
+      // ignore
+    }
+  }
+}

package/src/core/workflows.ts

@@ -764,6 +764,10 @@ export interface WorkflowsConfig {
   killGraceMs?: number;
   /** SQLite pragmas for isolated subprocess connections */
   sqlitePragmas?: SqlitePragmaConfig;
+  /** Disable in-process watchdog timers (use external watchdog instead) */
+  useWatchdog?: boolean;
+  /** Max concurrent instances per workflow name (0 = unlimited, default: 0) */
+  concurrentWorkflows?: number;
   /** Resume strategy for orphaned workflows (default: "blocking") */
   resumeStrategy?: WorkflowResumeStrategy;
 }
@@ -825,6 +829,8 @@ export interface Workflows {
   updateMetadata(instanceId: string, key: string, value: any): Promise<void>;
   /** Set plugin metadata for local instantiation in isolated workflows */
   setPluginMetadata(metadata: PluginMetadata): void;
+  /** Get resolved SQLite db path (for watchdog) */
+  getDbPath(): string | undefined;
 }
 
 export interface PluginMetadata {
@@ -866,6 +872,8 @@ class WorkflowsImpl implements Workflows {
   private readyTimeoutMs: number;
   private killGraceMs: number;
   private sqlitePragmas?: SqlitePragmaConfig;
+  private useWatchdog: boolean;
+  private concurrentWorkflows: number;
   private resumeStrategy!: WorkflowResumeStrategy;
   private workflowModulePaths = new Map<string, string>();
   private isolatedProcesses = new Map<string, IsolatedProcessInfo>();
@@ -902,6 +910,8 @@ class WorkflowsImpl implements Workflows {
     this.readyTimeoutMs = config.readyTimeout ?? 10000;
     this.killGraceMs = config.killGraceMs ?? 5000;
     this.sqlitePragmas = config.sqlitePragmas;
+    this.useWatchdog = config.useWatchdog ?? false;
+    this.concurrentWorkflows = config.concurrentWorkflows ?? 0;
     this.resumeStrategy = config.resumeStrategy ?? "blocking";
   }
 
@@ -962,6 +972,10 @@ class WorkflowsImpl implements Workflows {
     this.pluginCustomErrors = metadata.customErrors;
   }
 
+  getDbPath(): string | undefined {
+    return this.dbPath;
+  }
+
   async updateMetadata(instanceId: string, key: string, value: any): Promise<void> {
     const instance = await this.adapter.getInstance(instanceId);
     if (!instance) return;
@@ -996,6 +1010,16 @@ class WorkflowsImpl implements Workflows {
       throw new Error(`Workflow "${workflowName}" is not registered`);
     }
 
+    if (this.concurrentWorkflows > 0) {
+      const running = await this.adapter.getInstancesByWorkflow(workflowName, "running");
+      const pending = await this.adapter.getInstancesByWorkflow(workflowName, "pending");
+      if (running.length + pending.length >= this.concurrentWorkflows) {
+        throw new Error(
+          `Workflow "${workflowName}" has reached its concurrency limit (${this.concurrentWorkflows})`
+        );
+      }
+    }
+
     const instance = await this.adapter.createInstance({
       workflowName,
       status: "pending",
@@ -1517,6 +1541,17 @@ class WorkflowsImpl implements Workflows {
     // Set up heartbeat timeout
     this.resetHeartbeatTimeout(instanceId, proc.pid);
 
+    const instance = await this.adapter.getInstance(instanceId);
+    const metadata = { ...(instance?.metadata ?? {}) } as Record<string, any>;
+    metadata.__watchdog = {
+      ...(metadata.__watchdog ?? {}),
+      pid: proc.pid,
+      socketPath,
+      tcpPort,
+      lastHeartbeat: new Date().toISOString(),
+    };
+    await this.adapter.updateInstance(instanceId, { metadata });
+
     const exitBeforeReady = proc.exited.then((exitCode) => {
       throw new Error(`Subprocess exited before ready (code ${exitCode})`);
     });
@@ -1590,7 +1625,8 @@ class WorkflowsImpl implements Workflows {
 
       case "started":
       case "heartbeat":
-        // No-op: heartbeat handled above, started is handled by executeIsolatedWorkflow
+        // Update heartbeat tracking metadata
+        await this.updateWatchdogHeartbeat(instanceId);
         break;
 
       case "step.started": {
@@ -1958,6 +1994,17 @@ class WorkflowsImpl implements Workflows {
     this.rejectIsolatedReady(instanceId, new Error("Isolated workflow cleaned up"));
   }
 
+  private async updateWatchdogHeartbeat(instanceId: string): Promise<void> {
+    const instance = await this.adapter.getInstance(instanceId);
+    if (!instance) return;
+    const metadata = { ...(instance.metadata ?? {}) } as Record<string, any>;
+    metadata.__watchdog = {
+      ...(metadata.__watchdog ?? {}),
+      lastHeartbeat: new Date().toISOString(),
+    };
+    await this.adapter.updateInstance(instanceId, { metadata });
+  }
+
   private async markOrphanedAsFailed(
     instances: WorkflowInstance[],
     reason: string
@@ -1990,6 +2037,7 @@ class WorkflowsImpl implements Workflows {
    * Reset heartbeat timeout for an isolated workflow
    */
   private resetHeartbeatTimeout(instanceId: string, pid: number): void {
+    if (this.useWatchdog) return;
     const info = this.isolatedProcesses.get(instanceId);
     if (!info) return;
 

package/src/server.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import { mkdir, writeFile } from "node:fs/promises";
-import { dirname } from "node:path";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
 import { PluginManager, type CoreServices, type ConfiguredPlugin } from "./core";
 import { type IRouter, type RouteDefinition, type ServerContext, type HandlerRegistry } from "./router";
 import { Handlers } from "./handlers";
@@ -87,6 +88,13 @@ export interface ServerConfig {
    */
   workflowsResumeStrategy?: "blocking" | "background" | "skip";
   processes?: ProcessesConfig;
+  /** Watchdog subprocess configuration */
+  watchdog?: {
+    enabled?: boolean;
+    intervalMs?: number;
+    services?: ("workflows" | "jobs" | "processes")[];
+    killGraceMs?: number;
+  };
   audit?: AuditConfig;
   websocket?: WebSocketConfig;
   storage?: StorageConfig;
@@ -221,6 +229,9 @@ export class AppServer {
   private generateModeSetup = false;
   private initMode: "adapter" | "server" = "server";
   private workflowsResumeStrategy?: "blocking" | "background" | "skip";
+  private watchdogConfig?: ServerConfig["watchdog"];
+  private watchdogStarted = false;
+  private options: ServerConfig;
 
   // Custom services registry
   private serviceFactories = new Map<string, ServiceFactory<any>>();
@@ -228,11 +239,13 @@ export class AppServer {
   private generateModeTimer?: ReturnType<typeof setTimeout>;
 
   constructor(options: ServerConfig) {
+    this.options = options;
     // Port priority: explicit config > PORT env var > default 3000
     const envPort = process.env.PORT ? parseInt(process.env.PORT, 10) : undefined;
     this.port = options.port ?? envPort ?? 3000;
     this.maxPortAttempts = options.maxPortAttempts ?? 5;
     this.workflowsResumeStrategy = options.workflowsResumeStrategy ?? options.workflows?.resumeStrategy;
+    this.watchdogConfig = options.watchdog;
 
     // Determine if we should use legacy databases
     const useLegacy = options.useLegacyCoreDatabases ?? false;
@@ -286,6 +299,10 @@ export class AppServer {
       events,
       logger,
       adapter: jobAdapter,
+      external: {
+        ...options.jobs?.external,
+        useWatchdog: options.watchdog?.enabled ? true : options.jobs?.external?.useWatchdog,
+      },
       // Disable built-in persistence when using Kysely adapter
       persist: useLegacy ? options.jobs?.persist : false,
     });
@@ -297,12 +314,14 @@ export class AppServer {
       jobs,
       sse,
       adapter: workflowAdapter,
+      useWatchdog: options.watchdog?.enabled ? true : options.workflows?.useWatchdog,
     });
 
     // Processes - still uses its own adapter pattern but can use Kysely
     const processes = createProcesses({
       ...options.processes,
       events,
+      useWatchdog: options.watchdog?.enabled ? true : options.processes?.useWatchdog,
     });
 
     // New services
@@ -1055,6 +1074,7 @@ ${factoryFunction}
       await this.coreServices.workflows.resume();
     }
     this.coreServices.processes.start();
+    await this.startWatchdog();
     logger.info("Background services started (cron, jobs, workflows, processes)");
 
     for (const router of this.routers) {
@@ -1073,6 +1093,69 @@ ${factoryFunction}
     await this.runReadyHandlers();
   }
 
+  private async startWatchdog(): Promise<void> {
+    if (!this.watchdogConfig?.enabled) return;
+    if (this.watchdogStarted) return;
+
+    const executorPath = join(dirname(fileURLToPath(import.meta.url)), "core", "watchdog-executor.ts");
+    const services = this.watchdogConfig.services ?? ["workflows", "jobs", "processes"];
+    const workflowsDbPath = this.coreServices.workflows.getDbPath?.();
+    const jobsDbPath = (this.options.jobs?.dbPath ?? workflowsDbPath ?? ".donkeylabs/jobs.db") as string;
+    const processesDbPath = (this.options.processes?.adapter?.path ?? ".donkeylabs/processes.db") as string;
+
+    const config = {
+      intervalMs: this.watchdogConfig.intervalMs ?? 5000,
+      services,
+      killGraceMs: this.watchdogConfig.killGraceMs ?? 5000,
+      workflowHeartbeatTimeoutMs: this.options.workflows?.heartbeatTimeout ?? 60000,
+      jobDefaults: {
+        heartbeatTimeoutMs: this.options.jobs?.external?.defaultHeartbeatTimeout ?? 30000,
+        killGraceMs: this.options.jobs?.external?.killGraceMs ?? this.watchdogConfig.killGraceMs ?? 5000,
+      },
+      jobConfigs: this.coreServices.jobs.getExternalJobConfigs(),
+      workflows: workflowsDbPath ? { dbPath: workflowsDbPath } : undefined,
+      jobs: jobsDbPath ? { dbPath: jobsDbPath } : undefined,
+      processes: processesDbPath ? { dbPath: processesDbPath } : undefined,
+      sqlitePragmas: this.options.workflows?.sqlitePragmas,
+    };
+
+    try {
+      this.coreServices.processes.register({
+        name: "__watchdog",
+        config: {
+          command: "bun",
+          args: ["run", executorPath],
+          env: {
+            DONKEYLABS_WATCHDOG_CONFIG: JSON.stringify(config),
+          },
+          heartbeat: { intervalMs: 5000, timeoutMs: 30000 },
+        },
+      });
+    } catch {
+      // Already registered
+    }
+
+    await this.coreServices.processes.spawn("__watchdog", {
+      metadata: { role: "watchdog" },
+    });
+
+    this.coreServices.events.on("process.event", async (data: any) => {
+      if (data?.name !== "__watchdog") return;
+      if (!data.event) return;
+
+      await this.coreServices.events.emit(data.event, data.data ?? {});
+
+      if (data.event.startsWith("workflow.watchdog")) {
+        const instanceId = data.data?.instanceId;
+        if (instanceId && this.coreServices.sse) {
+          this.coreServices.sse.broadcast(`workflow:${instanceId}`, data.event, data.data ?? {});
+        }
+      }
+    });
+
+    this.watchdogStarted = true;
+  }
+
   /**
    * Handle a single API request. Used by adapters.
    * Returns null if the route is not found.