@donkeylabs/server 0.4.8 → 0.5.1

package/src/client/base.ts

@@ -248,19 +248,21 @@ export class ApiClientBase<TEvents extends Record<string, any> = Record<string,
   }
 
   /**
-   * Make a raw request (for non-JSON endpoints)
+   * Make a raw request (for non-JSON endpoints like streaming)
    */
   protected async rawRequest(
     route: string,
-    init: RequestInit = {}
+    init?: RequestInit
   ): Promise<Response> {
     const fetchFn = this.options.fetch || fetch;
+    const requestInit = init ?? {};
 
     return fetchFn(`${this.baseUrl}/${route}`, {
-      ...init,
+      method: "POST",
+      ...requestInit,
       headers: {
         ...this.options.headers,
-        ...init.headers,
+        ...requestInit.headers,
       },
       credentials: this.options.credentials,
     });

package/src/core/external-job-socket.ts

@@ -34,12 +34,18 @@ export interface SocketServerOptions {
 export interface ExternalJobSocketServer {
   /** Create a new socket for a job (returns socket path or TCP port) */
   createSocket(jobId: string): Promise<{ socketPath?: string; tcpPort?: number }>;
-  /** Close a specific job's socket */
+  /** Close a specific job's socket and release reservations */
   closeSocket(jobId: string): Promise<void>;
   /** Get all active job connections */
   getActiveConnections(): string[];
   /** Attempt to reconnect to an existing socket */
   reconnect(jobId: string, socketPath?: string, tcpPort?: number): Promise<boolean>;
+  /** Reserve a socket path/port for an orphaned job (prevents reuse until released) */
+  reserve(jobId: string, socketPath?: string, tcpPort?: number): void;
+  /** Release reservation for a job (called when job is cleaned up) */
+  release(jobId: string): void;
+  /** Check if a socket path or port is reserved */
+  isReserved(socketPath?: string, tcpPort?: number): boolean;
   /** Shutdown all sockets and cleanup */
   shutdown(): Promise<void>;
   /** Clean orphaned socket files from a previous run */
@@ -68,6 +74,14 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
   private tcpPorts = new Map<string, number>();
   // Track used TCP ports
   private usedPorts = new Set<number>();
+  // Track reserved socket paths (for jobs that might reconnect)
+  private reservedSocketPaths = new Set<string>();
+  // Track reserved TCP ports (for jobs that might reconnect)
+  private reservedTcpPorts = new Set<number>();
+  // Map jobId -> reserved socket path (for release by jobId)
+  private jobReservedSocketPath = new Map<string, string>();
+  // Map jobId -> reserved TCP port (for release by jobId)
+  private jobReservedTcpPort = new Map<string, number>();
 
   private isWindows = process.platform === "win32";
 
@@ -96,6 +110,11 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
   private async createUnixServer(jobId: string): Promise<{ socketPath: string }> {
     const socketPath = join(this.socketDir, `job_${jobId}.sock`);
 
+    // Check if this socket path is reserved by another job
+    if (this.reservedSocketPaths.has(socketPath) && !this.jobReservedSocketPath.has(jobId)) {
+      throw new Error(`Socket path ${socketPath} is reserved by another job`);
+    }
+
     // Remove existing socket file if it exists
     if (existsSync(socketPath)) {
       await unlink(socketPath);
@@ -148,12 +167,14 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
     // Try random ports within range
     for (let i = 0; i < 100; i++) {
       const port = minPort + Math.floor(Math.random() * (maxPort - minPort));
-      if (!this.usedPorts.has(port)) {
-        // Check if port is actually available
-        const isAvailable = await this.checkPortAvailable(port);
-        if (isAvailable) {
-          return port;
-        }
+      // Skip if port is already in use or reserved by another job
+      if (this.usedPorts.has(port) || this.reservedTcpPorts.has(port)) {
+        continue;
+      }
+      // Check if port is actually available
+      const isAvailable = await this.checkPortAvailable(port);
+      if (isAvailable) {
+        return port;
       }
     }
 
@@ -241,12 +262,62 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
       this.usedPorts.delete(port);
       this.tcpPorts.delete(jobId);
     }
+
+    // Release any reservations for this job
+    this.release(jobId);
   }
 
   getActiveConnections(): string[] {
     return Array.from(this.clientSockets.keys());
   }
 
+  reserve(jobId: string, socketPath?: string, tcpPort?: number): void {
+    if (socketPath) {
+      this.reservedSocketPaths.add(socketPath);
+      this.jobReservedSocketPath.set(jobId, socketPath);
+    }
+    if (tcpPort) {
+      this.reservedTcpPorts.add(tcpPort);
+      this.jobReservedTcpPort.set(jobId, tcpPort);
+    }
+  }
+
+  release(jobId: string): void {
+    // Release socket path reservation
+    const socketPath = this.jobReservedSocketPath.get(jobId);
+    if (socketPath) {
+      this.reservedSocketPaths.delete(socketPath);
+      this.jobReservedSocketPath.delete(jobId);
+    }
+    // Also check socketPaths map (for active jobs)
+    const activeSocketPath = this.socketPaths.get(jobId);
+    if (activeSocketPath) {
+      this.reservedSocketPaths.delete(activeSocketPath);
+    }
+
+    // Release TCP port reservation
+    const tcpPort = this.jobReservedTcpPort.get(jobId);
+    if (tcpPort) {
+      this.reservedTcpPorts.delete(tcpPort);
+      this.jobReservedTcpPort.delete(jobId);
+    }
+    // Also check tcpPorts map (for active jobs)
+    const activeTcpPort = this.tcpPorts.get(jobId);
+    if (activeTcpPort) {
+      this.reservedTcpPorts.delete(activeTcpPort);
+    }
+  }
+
+  isReserved(socketPath?: string, tcpPort?: number): boolean {
+    if (socketPath && this.reservedSocketPaths.has(socketPath)) {
+      return true;
+    }
+    if (tcpPort && this.reservedTcpPorts.has(tcpPort)) {
+      return true;
+    }
+    return false;
+  }
+
   async reconnect(
     jobId: string,
     socketPath?: string,
@@ -257,19 +328,67 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
       return true;
     }
 
-    // For Unix sockets, check if the socket file exists
-    if (socketPath && existsSync(socketPath)) {
-      // The external process should still be connected to this socket
-      // We need to create a server to listen for the existing connection
-      // However, this is tricky because the original server is gone
-      // The reconnection strategy depends on the external process behavior
-
-      // For now, we'll just check if the socket exists
-      // The external process should attempt to reconnect if configured
-      return false;
+    // For Unix sockets, recreate the server on the same path
+    // The external process should be retrying to connect
+    if (socketPath && !this.isWindows) {
+      try {
+        // Remove old socket file if it exists
+        if (existsSync(socketPath)) {
+          await unlink(socketPath);
+        }
+
+        // Create new server on the same path
+        return new Promise((resolve) => {
+          const server = createNetServer((socket) => {
+            this.handleConnection(jobId, socket);
+          });
+
+          server.on("error", (err) => {
+            this.onError?.(err, jobId);
+            resolve(false);
+          });
+
+          server.listen(socketPath, () => {
+            this.servers.set(jobId, server);
+            this.socketPaths.set(jobId, socketPath);
+            console.log(`[SocketServer] Recreated socket for job ${jobId} at ${socketPath}`);
+            // Return true - the server is ready, external process should reconnect
+            resolve(true);
+          });
+        });
+      } catch (err) {
+        this.onError?.(err as Error, jobId);
+        return false;
+      }
+    }
+
+    // For TCP, recreate the server on the same port
+    if (tcpPort && this.isWindows) {
+      try {
+        return new Promise((resolve) => {
+          const server = createNetServer((socket) => {
+            this.handleConnection(jobId, socket);
+          });
+
+          server.on("error", (err) => {
+            this.onError?.(err, jobId);
+            resolve(false);
+          });
+
+          server.listen(tcpPort, "127.0.0.1", () => {
+            this.servers.set(jobId, server);
+            this.tcpPorts.set(jobId, tcpPort);
+            this.usedPorts.add(tcpPort);
+            console.log(`[SocketServer] Recreated TCP server for job ${jobId} on port ${tcpPort}`);
+            resolve(true);
+          });
+        });
+      } catch (err) {
+        this.onError?.(err as Error, jobId);
+        return false;
+      }
     }
 
-    // For TCP, we can't easily reconnect without the process knowing
     return false;
   }
 
@@ -319,9 +438,11 @@ export class ExternalJobSocketServerImpl implements ExternalJobSocketServer {
         const match = file.match(/^job_(.+)\.sock$/);
         if (match) {
           const jobId = match[1]!;
-          if (!activeJobIds.has(jobId)) {
-            // This socket file doesn't correspond to any active job
-            const socketPath = join(this.socketDir, file);
+          const socketPath = join(this.socketDir, file);
+
+          // Don't clean if job is active or socket path is reserved
+          if (!activeJobIds.has(jobId) && !this.reservedSocketPaths.has(socketPath)) {
+            // This socket file doesn't correspond to any active job and isn't reserved
             await unlink(socketPath).catch(() => {});
           }
         }

package/src/core/index.ts

@@ -47,6 +47,11 @@ export {
   createJobs,
 } from "./jobs";
 
+export {
+  SqliteJobAdapter,
+  type SqliteJobAdapterConfig,
+} from "./job-adapter-sqlite";
+
 export {
   type ExternalJobConfig,
   type ExternalJob,

package/src/core/job-adapter-sqlite.ts

@@ -0,0 +1,287 @@
+/**
+ * Built-in SQLite Job Adapter
+ *
+ * Provides automatic persistence for jobs, enabling server restart resilience
+ * for external jobs without requiring user configuration.
+ */
+
+import { Database } from "bun:sqlite";
+import { mkdir } from "node:fs/promises";
+import { dirname } from "node:path";
+import type { Job, JobAdapter, JobStatus } from "./jobs";
+import type { ExternalJobProcessState } from "./external-jobs";
+
+export interface SqliteJobAdapterConfig {
+  /** Path to SQLite database file (default: .donkeylabs/jobs.db) */
+  path?: string;
+  /** Auto-cleanup completed jobs older than N days (default: 7, 0 to disable) */
+  cleanupDays?: number;
+  /** Cleanup interval in ms (default: 3600000 = 1 hour) */
+  cleanupInterval?: number;
+}
+
+export class SqliteJobAdapter implements JobAdapter {
+  private db: Database;
+  private initialized = false;
+  private cleanupTimer?: ReturnType<typeof setInterval>;
+  private cleanupDays: number;
+
+  constructor(config: SqliteJobAdapterConfig = {}) {
+    const dbPath = config.path ?? ".donkeylabs/jobs.db";
+    this.cleanupDays = config.cleanupDays ?? 7;
+
+    // Ensure directory exists
+    this.ensureDir(dbPath);
+
+    this.db = new Database(dbPath);
+    this.init();
+
+    // Start cleanup timer
+    if (this.cleanupDays > 0) {
+      const interval = config.cleanupInterval ?? 3600000; // 1 hour
+      this.cleanupTimer = setInterval(() => this.cleanup(), interval);
+      // Run cleanup on startup
+      this.cleanup();
+    }
+  }
+
+  private ensureDir(dbPath: string): void {
+    const dir = dirname(dbPath);
+    if (dir && dir !== ".") {
+      // Sync mkdir for constructor
+      try {
+        Bun.spawnSync(["mkdir", "-p", dir]);
+      } catch {
+        // Directory may already exist
+      }
+    }
+  }
+
+  private init(): void {
+    if (this.initialized) return;
+
+    this.db.run(`
+      CREATE TABLE IF NOT EXISTS jobs (
+        id TEXT PRIMARY KEY,
+        name TEXT NOT NULL,
+        data TEXT NOT NULL,
+        status TEXT NOT NULL DEFAULT 'pending',
+        created_at TEXT NOT NULL,
+        run_at TEXT,
+        started_at TEXT,
+        completed_at TEXT,
+        result TEXT,
+        error TEXT,
+        attempts INTEGER NOT NULL DEFAULT 0,
+        max_attempts INTEGER NOT NULL DEFAULT 3,
+        -- External job fields
+        external INTEGER DEFAULT 0,
+        pid INTEGER,
+        socket_path TEXT,
+        tcp_port INTEGER,
+        last_heartbeat TEXT,
+        process_state TEXT
+      )
+    `);
+
+    // Indexes for efficient queries
+    this.db.run(`CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)`);
+    this.db.run(`CREATE INDEX IF NOT EXISTS idx_jobs_name ON jobs(name)`);
+    this.db.run(`CREATE INDEX IF NOT EXISTS idx_jobs_external ON jobs(external, status)`);
+    this.db.run(`CREATE INDEX IF NOT EXISTS idx_jobs_scheduled ON jobs(status, run_at)`);
+
+    this.initialized = true;
+  }
+
+  async create(job: Omit<Job, "id">): Promise<Job> {
+    const id = `job_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`;
+
+    this.db.run(
+      `INSERT INTO jobs (
+        id, name, data, status, created_at, run_at, attempts, max_attempts,
+        external, process_state
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      [
+        id,
+        job.name,
+        JSON.stringify(job.data),
+        job.status,
+        job.createdAt.toISOString(),
+        job.runAt?.toISOString() ?? null,
+        job.attempts,
+        job.maxAttempts,
+        job.external ? 1 : 0,
+        job.processState ?? null,
+      ]
+    );
+
+    return { ...job, id };
+  }
+
+  async get(jobId: string): Promise<Job | null> {
+    const row = this.db.query(`SELECT * FROM jobs WHERE id = ?`).get(jobId) as any;
+    if (!row) return null;
+    return this.rowToJob(row);
+  }
+
+  async update(jobId: string, updates: Partial<Job>): Promise<void> {
+    const sets: string[] = [];
+    const values: any[] = [];
+
+    if (updates.status !== undefined) {
+      sets.push("status = ?");
+      values.push(updates.status);
+    }
+    if (updates.startedAt !== undefined) {
+      sets.push("started_at = ?");
+      values.push(updates.startedAt?.toISOString() ?? null);
+    }
+    if (updates.completedAt !== undefined) {
+      sets.push("completed_at = ?");
+      values.push(updates.completedAt?.toISOString() ?? null);
+    }
+    if (updates.result !== undefined) {
+      sets.push("result = ?");
+      values.push(JSON.stringify(updates.result));
+    }
+    if (updates.error !== undefined) {
+      sets.push("error = ?");
+      values.push(updates.error);
+    }
+    if (updates.attempts !== undefined) {
+      sets.push("attempts = ?");
+      values.push(updates.attempts);
+    }
+    // External job fields
+    if (updates.pid !== undefined) {
+      sets.push("pid = ?");
+      values.push(updates.pid);
+    }
+    if (updates.socketPath !== undefined) {
+      sets.push("socket_path = ?");
+      values.push(updates.socketPath);
+    }
+    if (updates.tcpPort !== undefined) {
+      sets.push("tcp_port = ?");
+      values.push(updates.tcpPort);
+    }
+    if (updates.lastHeartbeat !== undefined) {
+      sets.push("last_heartbeat = ?");
+      values.push(updates.lastHeartbeat?.toISOString() ?? null);
+    }
+    if (updates.processState !== undefined) {
+      sets.push("process_state = ?");
+      values.push(updates.processState);
+    }
+
+    if (sets.length === 0) return;
+
+    values.push(jobId);
+    this.db.run(`UPDATE jobs SET ${sets.join(", ")} WHERE id = ?`, values);
+  }
+
+  async delete(jobId: string): Promise<boolean> {
+    const result = this.db.run(`DELETE FROM jobs WHERE id = ?`, [jobId]);
+    return result.changes > 0;
+  }
+
+  async getPending(limit: number = 100): Promise<Job[]> {
+    const rows = this.db
+      .query(`SELECT * FROM jobs WHERE status = 'pending' ORDER BY created_at LIMIT ?`)
+      .all(limit) as any[];
+    return rows.map((r) => this.rowToJob(r));
+  }
+
+  async getScheduledReady(now: Date): Promise<Job[]> {
+    const rows = this.db
+      .query(`SELECT * FROM jobs WHERE status = 'scheduled' AND run_at <= ? ORDER BY run_at`)
+      .all(now.toISOString()) as any[];
+    return rows.map((r) => this.rowToJob(r));
+  }
+
+  async getByName(name: string, status?: JobStatus): Promise<Job[]> {
+    let query = `SELECT * FROM jobs WHERE name = ?`;
+    const params: any[] = [name];
+
+    if (status) {
+      query += ` AND status = ?`;
+      params.push(status);
+    }
+
+    query += ` ORDER BY created_at DESC`;
+
+    const rows = this.db.query(query).all(...params) as any[];
+    return rows.map((r) => this.rowToJob(r));
+  }
+
+  async getRunningExternal(): Promise<Job[]> {
+    const rows = this.db
+      .query(`SELECT * FROM jobs WHERE external = 1 AND status = 'running'`)
+      .all() as any[];
+    return rows.map((r) => this.rowToJob(r));
+  }
+
+  async getOrphanedExternal(): Promise<Job[]> {
+    // Get external jobs that were running when server died
+    const rows = this.db
+      .query(
+        `SELECT * FROM jobs WHERE external = 1 AND status = 'running'
+         AND (process_state = 'running' OR process_state = 'orphaned' OR process_state = 'spawning')`
+      )
+      .all() as any[];
+    return rows.map((r) => this.rowToJob(r));
+  }
+
+  private rowToJob(row: any): Job {
+    return {
+      id: row.id,
+      name: row.name,
+      data: JSON.parse(row.data),
+      status: row.status as JobStatus,
+      createdAt: new Date(row.created_at),
+      runAt: row.run_at ? new Date(row.run_at) : undefined,
+      startedAt: row.started_at ? new Date(row.started_at) : undefined,
+      completedAt: row.completed_at ? new Date(row.completed_at) : undefined,
+      result: row.result ? JSON.parse(row.result) : undefined,
+      error: row.error ?? undefined,
+      attempts: row.attempts,
+      maxAttempts: row.max_attempts,
+      // External job fields
+      external: row.external === 1 ? true : undefined,
+      pid: row.pid ?? undefined,
+      socketPath: row.socket_path ?? undefined,
+      tcpPort: row.tcp_port ?? undefined,
+      lastHeartbeat: row.last_heartbeat ? new Date(row.last_heartbeat) : undefined,
+      processState: row.process_state as ExternalJobProcessState | undefined,
+    };
+  }
+
+  /** Clean up old completed/failed jobs */
+  private cleanup(): void {
+    if (this.cleanupDays <= 0) return;
+
+    try {
+      const cutoff = new Date();
+      cutoff.setDate(cutoff.getDate() - this.cleanupDays);
+
+      const result = this.db.run(
+        `DELETE FROM jobs WHERE (status = 'completed' OR status = 'failed') AND completed_at < ?`,
+        [cutoff.toISOString()]
+      );
+
+      if (result.changes > 0) {
+        console.log(`[Jobs] Cleaned up ${result.changes} old jobs`);
+      }
+    } catch (err) {
+      console.error("[Jobs] Cleanup error:", err);
+    }
+  }
+
+  /** Stop the adapter and cleanup timer */
+  stop(): void {
+    if (this.cleanupTimer) {
+      clearInterval(this.cleanupTimer);
+      this.cleanupTimer = undefined;
+    }
+  }
+}

package/src/core/jobs.ts

@@ -25,6 +25,7 @@ import {
   createExternalJobSocketServer,
   type ExternalJobSocketServer,
 } from "./external-job-socket";
+import { SqliteJobAdapter } from "./job-adapter-sqlite";
 
 export type JobStatus = "pending" | "running" | "completed" | "failed" | "scheduled";
 
@@ -82,6 +83,13 @@ export interface JobsConfig {
   maxAttempts?: number; // Default retry attempts, default 3
   /** External jobs configuration */
   external?: ExternalJobsConfig;
+  /**
+   * Use SQLite for persistence (default: true when external jobs are used)
+   * Set to false to use MemoryJobAdapter (not recommended for production)
+   */
+  persist?: boolean;
+  /** SQLite database path (default: .donkeylabs/jobs.db) */
+  dbPath?: string;
 }
 
 export interface Jobs {
@@ -188,6 +196,7 @@ export class MemoryJobAdapter implements JobAdapter {
 
 class JobsImpl implements Jobs {
   private adapter: JobAdapter;
+  private sqliteAdapter?: SqliteJobAdapter;
   private events?: Events;
   private handlers = new Map<string, JobHandler>();
   private running = false;
@@ -197,6 +206,8 @@ class JobsImpl implements Jobs {
   private concurrency: number;
   private pollInterval: number;
   private defaultMaxAttempts: number;
+  private usePersistence: boolean;
+  private dbPath?: string;
 
   // External jobs support
   private externalConfigs = new Map<string, ExternalJobConfig>();
@@ -205,12 +216,23 @@ class JobsImpl implements Jobs {
   private externalProcesses = new Map<string, { pid: number; timeout?: ReturnType<typeof setTimeout> }>();
 
   constructor(config: JobsConfig = {}) {
-    this.adapter = config.adapter ?? new MemoryJobAdapter();
     this.events = config.events;
     this.concurrency = config.concurrency ?? 5;
     this.pollInterval = config.pollInterval ?? 1000;
     this.defaultMaxAttempts = config.maxAttempts ?? 3;
     this.externalConfig = config.external ?? {};
+    this.usePersistence = config.persist ?? true; // Default to SQLite persistence
+    this.dbPath = config.dbPath;
+
+    // Use provided adapter, or create SQLite adapter if persistence enabled
+    if (config.adapter) {
+      this.adapter = config.adapter;
+    } else if (this.usePersistence) {
+      this.sqliteAdapter = new SqliteJobAdapter({ path: this.dbPath });
+      this.adapter = this.sqliteAdapter;
+    } else {
+      this.adapter = new MemoryJobAdapter();
+    }
   }
 
   register<T = any, R = any>(name: string, handler: JobHandler<T, R>): void {
@@ -362,6 +384,11 @@ class JobsImpl implements Jobs {
       this.socketServer = null;
     }
 
+    // Stop SQLite adapter cleanup timer
+    if (this.sqliteAdapter) {
+      this.sqliteAdapter.stop();
+    }
+
     // Wait for active in-process jobs to complete (with timeout)
     const maxWait = 30000; // 30 seconds
     const startTime = Date.now();
@@ -476,6 +503,9 @@ class JobsImpl implements Jobs {
           console.log(`[Jobs] Found orphaned job ${job.id} with PID ${job.pid}, attempting reconnect`);
           activeJobIds.add(job.id);
 
+          // Reserve the socket path/port to prevent new jobs from using it
+          this.socketServer?.reserve(job.id, job.socketPath, job.tcpPort);
+
           // Try to reconnect to the socket
           const reconnected = await this.socketServer?.reconnect(
             job.id,
@@ -496,7 +526,7 @@ class JobsImpl implements Jobs {
               });
             }
           } else {
-            // Mark as orphaned, but keep tracking
+            // Mark as orphaned, but keep tracking (reservation remains)
             await this.adapter.update(job.id, { processState: "orphaned" });
 
             if (this.events) {
@@ -507,7 +537,7 @@ class JobsImpl implements Jobs {
             }
           }
         } else {
-          // Process is dead, mark job as failed
+          // Process is dead, mark job as failed and release any reservations
           console.log(`[Jobs] Orphaned job ${job.id} process (PID ${job.pid}) is dead`);
           await this.adapter.update(job.id, {
             status: "failed",
@@ -515,6 +545,9 @@ class JobsImpl implements Jobs {
             completedAt: new Date(),
           });
 
+          // Release reservation since the job is done
+          this.socketServer?.release(job.id);
+
           if (this.events) {
             await this.events.emit("job.failed", {
               jobId: job.id,