@donkeylabs/server 2.0.28 → 2.0.30

package/src/core/watchdog-executor.ts

@@ -0,0 +1,80 @@
+import { Kysely } from "kysely";
+import { BunSqliteDialect } from "kysely-bun-sqlite";
+import Database from "bun:sqlite";
+import { ProcessClient } from "./process-client";
+import { KyselyWorkflowAdapter } from "./workflow-adapter-kysely";
+import { KyselyJobAdapter } from "./job-adapter-kysely";
+import { SqliteProcessAdapter } from "./process-adapter-sqlite";
+import { WatchdogRunner, type WatchdogRunnerConfig } from "./watchdog-runner";
+
+interface WatchdogConfig extends WatchdogRunnerConfig {
+  intervalMs: number;
+  workflows?: { dbPath?: string };
+  jobs?: { dbPath?: string };
+  processes?: { dbPath?: string };
+  sqlitePragmas?: {
+    busyTimeout?: number;
+    synchronous?: "OFF" | "NORMAL" | "FULL" | "EXTRA";
+    journalMode?: "DELETE" | "TRUNCATE" | "PERSIST" | "MEMORY" | "WAL" | "OFF";
+  };
+}
+
+const raw = process.env.DONKEYLABS_WATCHDOG_CONFIG;
+if (!raw) {
+  throw new Error("Missing DONKEYLABS_WATCHDOG_CONFIG");
+}
+
+const config: WatchdogConfig = JSON.parse(raw);
+const client = await ProcessClient.connect();
+
+const workflowAdapter = config.workflows?.dbPath
+  ? new KyselyWorkflowAdapter(createDb(config.workflows.dbPath, config.sqlitePragmas), {
+      cleanupDays: 0,
+    })
+  : undefined;
+const jobAdapter = config.jobs?.dbPath
+  ? new KyselyJobAdapter(createDb(config.jobs.dbPath, config.sqlitePragmas), {
+      cleanupDays: 0,
+    })
+  : undefined;
+const processAdapter = config.processes?.dbPath
+  ? new SqliteProcessAdapter({ path: config.processes.dbPath, cleanupDays: 0 })
+  : undefined;
+
+const runner = new WatchdogRunner(config, {
+  workflowsAdapter: workflowAdapter,
+  jobsAdapter: jobAdapter,
+  processesAdapter: processAdapter,
+  emit: async (event, data) => {
+    await client.emit(event, data);
+  },
+});
+
+const interval = Math.max(1000, config.intervalMs);
+const timer = setInterval(() => {
+  runner.runOnce().catch(() => undefined);
+}, interval);
+
+process.on("SIGTERM", async () => {
+  clearInterval(timer);
+  client.disconnect();
+});
+
+function createDb(
+  dbPath: string,
+  pragmas?: { busyTimeout?: number; synchronous?: string; journalMode?: string }
+): Kysely<any> {
+  const sqlite = new Database(dbPath);
+  const busyTimeout = pragmas?.busyTimeout ?? 5000;
+  sqlite.run(`PRAGMA busy_timeout = ${busyTimeout}`);
+  if (pragmas?.journalMode) {
+    sqlite.run(`PRAGMA journal_mode = ${pragmas.journalMode}`);
+  }
+  if (pragmas?.synchronous) {
+    sqlite.run(`PRAGMA synchronous = ${pragmas.synchronous}`);
+  }
+
+  return new Kysely<any>({
+    dialect: new BunSqliteDialect({ database: sqlite }),
+  });
+}

package/src/core/watchdog-runner.ts

@@ -0,0 +1,276 @@
+import type { WorkflowAdapter, WorkflowInstance } from "./workflows";
+import type { JobAdapter, Job } from "./jobs";
+import type { ProcessAdapter } from "./process-adapter-sqlite";
+
+export type WatchdogService = "workflows" | "jobs" | "processes";
+
+export interface WatchdogRunnerConfig {
+  services: WatchdogService[];
+  killGraceMs: number;
+  workflowHeartbeatTimeoutMs: number;
+  jobDefaults: {
+    heartbeatTimeoutMs: number;
+    killGraceMs: number;
+  };
+  jobConfigs: Record<string, { heartbeatTimeout?: number; timeout?: number; killGraceMs?: number }>;
+}
+
+export interface WatchdogRunnerDeps {
+  workflowsAdapter?: WorkflowAdapter;
+  jobsAdapter?: JobAdapter;
+  processesAdapter?: ProcessAdapter;
+  now?: () => number;
+  killProcess?: (pid: number, signal: NodeJS.Signals) => void;
+  isProcessAlive?: (pid: number) => boolean;
+  emit?: (event: string, data: Record<string, any>) => Promise<void>;
+}
+
+export class WatchdogRunner {
+  private config: WatchdogRunnerConfig;
+  private deps: WatchdogRunnerDeps;
+
+  constructor(config: WatchdogRunnerConfig, deps: WatchdogRunnerDeps) {
+    this.config = config;
+    this.deps = deps;
+  }
+
+  async runOnce(): Promise<void> {
+    if (this.config.services.includes("workflows") && this.deps.workflowsAdapter) {
+      await this.checkWorkflows();
+    }
+    if (this.config.services.includes("jobs") && this.deps.jobsAdapter) {
+      await this.checkJobs();
+    }
+    if (this.config.services.includes("processes") && this.deps.processesAdapter) {
+      await this.checkProcesses();
+    }
+  }
+
+  private async checkWorkflows(): Promise<void> {
+    const adapter = this.deps.workflowsAdapter!;
+    const now = this.now();
+    const instances = await adapter.getRunningInstances();
+
+    for (const instance of instances) {
+      const info = this.getWatchdogMetadata(instance);
+      if (!info?.pid) continue;
+
+      const last = info.lastHeartbeat ?? instance.startedAt?.getTime() ?? 0;
+      if (now - last <= this.config.workflowHeartbeatTimeoutMs) continue;
+
+      await this.emit("workflow.watchdog.stale", {
+        instanceId: instance.id,
+        pid: info.pid,
+        timeoutMs: this.config.workflowHeartbeatTimeoutMs,
+      });
+
+      await this.killProcessWithGrace(info.pid, this.config.killGraceMs);
+
+      await adapter.updateInstance(instance.id, {
+        status: "failed",
+        error: "Watchdog killed unresponsive workflow",
+        completedAt: new Date(),
+      });
+
+      await this.emit("workflow.watchdog.killed", {
+        instanceId: instance.id,
+        pid: info.pid,
+        reason: "heartbeat",
+      });
+    }
+  }
+
+  private async checkJobs(): Promise<void> {
+    const adapter = this.deps.jobsAdapter!;
+    const now = this.now();
+    const jobs = await adapter.getRunningExternal();
+
+    for (const job of jobs) {
+      if (!job.pid) continue;
+      const config = this.config.jobConfigs[job.name] ?? {};
+      const heartbeatTimeout =
+        config.heartbeatTimeout ?? this.config.jobDefaults.heartbeatTimeoutMs;
+      const killGraceMs = config.killGraceMs ?? this.config.jobDefaults.killGraceMs;
+      const lastHeartbeat = job.lastHeartbeat?.getTime() ?? job.startedAt?.getTime() ?? 0;
+
+      if (now - lastHeartbeat > heartbeatTimeout) {
+        await this.emit("job.watchdog.stale", {
+          jobId: job.id,
+          name: job.name,
+          pid: job.pid,
+          timeoutMs: heartbeatTimeout,
+        });
+
+        await this.killProcessWithGrace(job.pid, killGraceMs);
+
+        await adapter.update(job.id, {
+          status: "failed",
+          error: "Watchdog killed unresponsive job",
+          completedAt: new Date(),
+          processState: "orphaned",
+        });
+
+        await this.emit("job.watchdog.killed", {
+          jobId: job.id,
+          name: job.name,
+          pid: job.pid,
+          reason: "heartbeat",
+        });
+        continue;
+      }
+
+      if (config.timeout && job.startedAt) {
+        if (now - job.startedAt.getTime() > config.timeout) {
+          await this.emit("job.watchdog.stale", {
+            jobId: job.id,
+            name: job.name,
+            pid: job.pid,
+            timeoutMs: config.timeout,
+            reason: "timeout",
+          });
+
+          await this.killProcessWithGrace(job.pid, killGraceMs);
+
+          await adapter.update(job.id, {
+            status: "failed",
+            error: "Watchdog killed job after timeout",
+            completedAt: new Date(),
+            processState: "orphaned",
+          });
+
+          await this.emit("job.watchdog.killed", {
+            jobId: job.id,
+            name: job.name,
+            pid: job.pid,
+            reason: "timeout",
+          });
+        }
+      }
+    }
+  }
+
+  private async checkProcesses(): Promise<void> {
+    const adapter = this.deps.processesAdapter!;
+    const now = this.now();
+    const running = await adapter.getRunning();
+
+    for (const proc of running) {
+      if (!proc.pid) continue;
+      const heartbeatTimeout = proc.config.heartbeat?.timeoutMs;
+      const lastHeartbeat = proc.lastHeartbeat?.getTime() ?? proc.startedAt?.getTime() ?? 0;
+
+      if (heartbeatTimeout && now - lastHeartbeat > heartbeatTimeout) {
+        await this.emit("process.watchdog.stale", {
+          processId: proc.id,
+          name: proc.name,
+          pid: proc.pid,
+          timeoutMs: heartbeatTimeout,
+        });
+
+        await this.killProcessWithGrace(proc.pid, this.config.killGraceMs);
+
+        await adapter.update(proc.id, {
+          status: "crashed",
+          error: "Watchdog killed unresponsive process",
+          stoppedAt: new Date(),
+        });
+
+        await this.emit("process.watchdog.killed", {
+          processId: proc.id,
+          name: proc.name,
+          pid: proc.pid,
+          reason: "heartbeat",
+        });
+        continue;
+      }
+
+      const maxRuntimeMs = proc.config.limits?.maxRuntimeMs;
+      if (maxRuntimeMs && proc.startedAt) {
+        if (now - proc.startedAt.getTime() > maxRuntimeMs) {
+          await this.emit("process.watchdog.stale", {
+            processId: proc.id,
+            name: proc.name,
+            pid: proc.pid,
+            timeoutMs: maxRuntimeMs,
+            reason: "maxRuntimeMs",
+          });
+
+          await this.killProcessWithGrace(proc.pid, this.config.killGraceMs);
+
+          await adapter.update(proc.id, {
+            status: "crashed",
+            error: "Watchdog killed process after max runtime",
+            stoppedAt: new Date(),
+          });
+
+          await this.emit("process.watchdog.killed", {
+            processId: proc.id,
+            name: proc.name,
+            pid: proc.pid,
+            reason: "maxRuntimeMs",
+          });
+        }
+      }
+    }
+  }
+
+  private getWatchdogMetadata(instance: WorkflowInstance): { pid?: number; lastHeartbeat?: number } | null {
+    const meta = instance.metadata as any;
+    if (!meta || typeof meta !== "object") return null;
+    const info = meta.__watchdog;
+    if (!info || typeof info !== "object") return null;
+    return {
+      pid: typeof info.pid === "number" ? info.pid : undefined,
+      lastHeartbeat: info.lastHeartbeat ? new Date(info.lastHeartbeat).getTime() : undefined,
+    };
+  }
+
+  private now(): number {
+    return this.deps.now ? this.deps.now() : Date.now();
+  }
+
+  private async emit(event: string, data: Record<string, any>): Promise<void> {
+    if (this.deps.emit) {
+      await this.deps.emit(event, data);
+    }
+  }
+
+  private async killProcessWithGrace(pid: number, graceMs: number): Promise<void> {
+    const kill = this.deps.killProcess ?? process.kill;
+    try {
+      kill(pid, "SIGTERM");
+    } catch {
+      return;
+    }
+
+    if (graceMs <= 0) {
+      try {
+        kill(pid, "SIGKILL");
+      } catch {
+        // ignore
+      }
+      return;
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, graceMs));
+
+    try {
+      const isAlive = this.deps.isProcessAlive
+        ? this.deps.isProcessAlive(pid)
+        : (() => {
+            try {
+              process.kill(pid, 0);
+              return true;
+            } catch {
+              return false;
+            }
+          })();
+
+      if (isAlive) {
+        kill(pid, "SIGKILL");
+      }
+    } catch {
+      // ignore
+    }
+  }
+}

package/src/core/workflow-executor.ts

@@ -25,6 +25,11 @@ interface ExecutorConfig {
   pluginModulePaths: Record<string, string>;
   pluginConfigs: Record<string, any>;
   coreConfig?: Record<string, any>;
+  sqlitePragmas?: {
+    busyTimeout?: number;
+    synchronous?: "OFF" | "NORMAL" | "FULL" | "EXTRA";
+    journalMode?: "DELETE" | "TRUNCATE" | "PERSIST" | "MEMORY" | "WAL" | "OFF";
+  };
 }
 
 // ============================================
@@ -47,6 +52,7 @@ async function main(): Promise<void> {
     pluginModulePaths,
     pluginConfigs,
     coreConfig,
+    sqlitePragmas,
   } = config;
 
   const socket = await connectToSocket(socketPath, tcpPort);
@@ -71,6 +77,7 @@ async function main(): Promise<void> {
     const bootstrap = await bootstrapSubprocess({
       dbPath,
       coreConfig,
+      sqlitePragmas,
       pluginMetadata: {
         names: pluginNames,
         modulePaths: pluginModulePaths,

package/src/core/workflow-socket.ts

@@ -248,7 +248,24 @@ export class WorkflowSocketServerImpl implements WorkflowSocketServer {
 
     let buffer = "";
 
-    socket.on("data", async (data) => {
+    const queue: WorkflowMessage[] = [];
+    let processing = false;
+
+    const processQueue = async () => {
+      if (processing) return;
+      processing = true;
+      while (queue.length > 0) {
+        const message = queue.shift()!;
+        try {
+          await this.handleMessage(instanceId, message);
+        } catch (err) {
+          this.onError?.(err instanceof Error ? err : new Error(String(err)), instanceId);
+        }
+      }
+      processing = false;
+    };
+
+    socket.on("data", (data) => {
       buffer += data.toString();
 
       // Process complete messages (newline-delimited JSON)
@@ -260,11 +277,13 @@ export class WorkflowSocketServerImpl implements WorkflowSocketServer {
 
         try {
           const message = JSON.parse(line) as WorkflowMessage;
-          await this.handleMessage(instanceId, message);
+          queue.push(message);
         } catch (err) {
           this.onError?.(new Error(`Invalid message: ${line}`), instanceId);
         }
       }
+
+      processQueue().catch(() => undefined);
     });
 
     socket.on("error", (err) => {

package/src/core/workflows.ts

@@ -760,12 +760,24 @@ export interface WorkflowsConfig {
   heartbeatTimeout?: number;
   /** Timeout waiting for isolated subprocess readiness (ms, default: 10000) */
   readyTimeout?: number;
+  /** Grace period before SIGKILL when terminating isolated subprocesses (ms, default: 5000) */
+  killGraceMs?: number;
+  /** SQLite pragmas for isolated subprocess connections */
+  sqlitePragmas?: SqlitePragmaConfig;
+  /** Disable in-process watchdog timers (use external watchdog instead) */
+  useWatchdog?: boolean;
   /** Resume strategy for orphaned workflows (default: "blocking") */
   resumeStrategy?: WorkflowResumeStrategy;
 }
 
 export type WorkflowResumeStrategy = "blocking" | "background" | "skip";
 
+export interface SqlitePragmaConfig {
+  busyTimeout?: number;
+  synchronous?: "OFF" | "NORMAL" | "FULL" | "EXTRA";
+  journalMode?: "DELETE" | "TRUNCATE" | "PERSIST" | "MEMORY" | "WAL" | "OFF";
+}
+
 /** Options for registering a workflow */
 export interface WorkflowRegisterOptions {
   /**
@@ -815,6 +827,8 @@ export interface Workflows {
   updateMetadata(instanceId: string, key: string, value: any): Promise<void>;
   /** Set plugin metadata for local instantiation in isolated workflows */
   setPluginMetadata(metadata: PluginMetadata): void;
+  /** Get resolved SQLite db path (for watchdog) */
+  getDbPath(): string | undefined;
 }
 
 export interface PluginMetadata {
@@ -854,6 +868,9 @@ class WorkflowsImpl implements Workflows {
   private dbPath?: string;
   private heartbeatTimeoutMs: number;
   private readyTimeoutMs: number;
+  private killGraceMs: number;
+  private sqlitePragmas?: SqlitePragmaConfig;
+  private useWatchdog: boolean;
   private resumeStrategy!: WorkflowResumeStrategy;
   private workflowModulePaths = new Map<string, string>();
   private isolatedProcesses = new Map<string, IsolatedProcessInfo>();
@@ -888,6 +905,9 @@ class WorkflowsImpl implements Workflows {
     this.dbPath = config.dbPath;
     this.heartbeatTimeoutMs = config.heartbeatTimeout ?? 60000;
     this.readyTimeoutMs = config.readyTimeout ?? 10000;
+    this.killGraceMs = config.killGraceMs ?? 5000;
+    this.sqlitePragmas = config.sqlitePragmas;
+    this.useWatchdog = config.useWatchdog ?? false;
     this.resumeStrategy = config.resumeStrategy ?? "blocking";
   }
 
@@ -948,6 +968,10 @@ class WorkflowsImpl implements Workflows {
     this.pluginCustomErrors = metadata.customErrors;
   }
 
+  getDbPath(): string | undefined {
+    return this.dbPath;
+  }
+
   async updateMetadata(instanceId: string, key: string, value: any): Promise<void> {
     const instance = await this.adapter.getInstance(instanceId);
     if (!instance) return;
@@ -1049,11 +1073,7 @@ class WorkflowsImpl implements Workflows {
     // Kill isolated process if running
     const isolatedInfo = this.isolatedProcesses.get(instanceId);
     if (isolatedInfo) {
-      try {
-        process.kill(isolatedInfo.pid, "SIGTERM");
-      } catch {
-        // Process might already be dead
-      }
+      await killProcessWithGrace(isolatedInfo.pid, this.killGraceMs);
       if (isolatedInfo.timeout) clearTimeout(isolatedInfo.timeout);
       if (isolatedInfo.heartbeatTimeout) clearTimeout(isolatedInfo.heartbeatTimeout);
       this.isolatedProcesses.delete(instanceId);
@@ -1470,6 +1490,7 @@ class WorkflowsImpl implements Workflows {
       pluginModulePaths: this.pluginModulePaths,
       pluginConfigs,
       coreConfig,
+      sqlitePragmas: this.sqlitePragmas,
     };
 
     // Spawn the subprocess
@@ -1506,6 +1527,17 @@ class WorkflowsImpl implements Workflows {
     // Set up heartbeat timeout
     this.resetHeartbeatTimeout(instanceId, proc.pid);
 
+    const instance = await this.adapter.getInstance(instanceId);
+    const metadata = { ...(instance?.metadata ?? {}) } as Record<string, any>;
+    metadata.__watchdog = {
+      ...(metadata.__watchdog ?? {}),
+      pid: proc.pid,
+      socketPath,
+      tcpPort,
+      lastHeartbeat: new Date().toISOString(),
+    };
+    await this.adapter.updateInstance(instanceId, { metadata });
+
     const exitBeforeReady = proc.exited.then((exitCode) => {
       throw new Error(`Subprocess exited before ready (code ${exitCode})`);
     });
@@ -1579,7 +1611,8 @@ class WorkflowsImpl implements Workflows {
 
       case "started":
       case "heartbeat":
-        // No-op: heartbeat handled above, started is handled by executeIsolatedWorkflow
+        // Update heartbeat tracking metadata
+        await this.updateWatchdogHeartbeat(instanceId);
         break;
 
       case "step.started": {
@@ -1947,6 +1980,17 @@ class WorkflowsImpl implements Workflows {
     this.rejectIsolatedReady(instanceId, new Error("Isolated workflow cleaned up"));
   }
 
+  private async updateWatchdogHeartbeat(instanceId: string): Promise<void> {
+    const instance = await this.adapter.getInstance(instanceId);
+    if (!instance) return;
+    const metadata = { ...(instance.metadata ?? {}) } as Record<string, any>;
+    metadata.__watchdog = {
+      ...(metadata.__watchdog ?? {}),
+      lastHeartbeat: new Date().toISOString(),
+    };
+    await this.adapter.updateInstance(instanceId, { metadata });
+  }
+
   private async markOrphanedAsFailed(
     instances: WorkflowInstance[],
     reason: string
@@ -1979,6 +2023,7 @@ class WorkflowsImpl implements Workflows {
    * Reset heartbeat timeout for an isolated workflow
    */
   private resetHeartbeatTimeout(instanceId: string, pid: number): void {
+    if (this.useWatchdog) return;
     const info = this.isolatedProcesses.get(instanceId);
     if (!info) return;
 
@@ -1995,6 +2040,11 @@ class WorkflowsImpl implements Workflows {
       }
 
       console.error(`[Workflows] No heartbeat from isolated workflow ${instanceId} for ${this.heartbeatTimeoutMs}ms`);
+      await this.emitEvent("workflow.watchdog.stale", {
+        instanceId,
+        reason: "heartbeat",
+        timeoutMs: this.heartbeatTimeoutMs,
+      });
       await this.handleIsolatedTimeout(instanceId, pid);
     }, this.heartbeatTimeoutMs);
   }
@@ -2006,12 +2056,12 @@ class WorkflowsImpl implements Workflows {
     const info = this.isolatedProcesses.get(instanceId);
     if (!info) return;
 
-    // Kill the process
-    try {
-      process.kill(pid, "SIGKILL");
-    } catch {
-      // Process might already be dead
-    }
+    await killProcessWithGrace(pid, this.killGraceMs);
+    await this.emitEvent("workflow.watchdog.killed", {
+      instanceId,
+      reason: "timeout",
+      timeoutMs: this.heartbeatTimeoutMs,
+    });
 
     // Clean up
     if (info.timeout) clearTimeout(info.timeout);
@@ -2148,3 +2198,29 @@ function isPlainObject(value: Record<string, any>): boolean {
 export function createWorkflows(config?: WorkflowsConfig): Workflows {
   return new WorkflowsImpl(config);
 }
+
+async function killProcessWithGrace(pid: number, graceMs: number): Promise<void> {
+  try {
+    process.kill(pid, "SIGTERM");
+  } catch {
+    return;
+  }
+
+  if (graceMs <= 0) {
+    try {
+      process.kill(pid, "SIGKILL");
+    } catch {
+      return;
+    }
+    return;
+  }
+
+  await new Promise((resolve) => setTimeout(resolve, graceMs));
+
+  try {
+    process.kill(pid, 0);
+    process.kill(pid, "SIGKILL");
+  } catch {
+    // Process already exited
+  }
+}