@donkeylabs/server 2.0.19 → 2.0.21

package/src/core/job-adapter-kysely.ts

@@ -46,6 +46,7 @@ export class KyselyJobAdapter implements JobAdapter {
   private db: Kysely<Database>;
   private cleanupTimer?: ReturnType<typeof setInterval>;
   private cleanupDays: number;
+  private stopped = false;
 
   constructor(db: Kysely<any>, config: KyselyJobAdapterConfig = {}) {
     this.db = db as Kysely<Database>;
@@ -58,7 +59,16 @@ export class KyselyJobAdapter implements JobAdapter {
     }
   }
 
+  /** Check if adapter is stopped (for safe database access) */
+  private checkStopped(): boolean {
+    return this.stopped;
+  }
+
   async create(job: Omit<Job, "id">): Promise<Job> {
+    if (this.checkStopped()) {
+      throw new Error("JobAdapter has been stopped");
+    }
+
     const id = `job_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`;
 
     await this.db
@@ -89,17 +99,27 @@ export class KyselyJobAdapter implements JobAdapter {
   }
 
   async get(jobId: string): Promise<Job | null> {
-    const row = await this.db
-      .selectFrom("__donkeylabs_jobs__")
-      .selectAll()
-      .where("id", "=", jobId)
-      .executeTakeFirst();
+    if (this.checkStopped()) return null;
 
-    if (!row) return null;
-    return this.rowToJob(row);
+    try {
+      const row = await this.db
+        .selectFrom("__donkeylabs_jobs__")
+        .selectAll()
+        .where("id", "=", jobId)
+        .executeTakeFirst();
+
+      if (!row) return null;
+      return this.rowToJob(row);
+    } catch (err: any) {
+      // Silently ignore errors if adapter was stopped during query
+      if (this.stopped && err?.message?.includes("destroyed")) return null;
+      throw err;
+    }
   }
 
   async update(jobId: string, updates: Partial<Job>): Promise<void> {
+    if (this.checkStopped()) return;
+
     const updateData: Partial<JobsTable> = {};
 
     if (updates.status !== undefined) {
@@ -139,14 +159,22 @@ export class KyselyJobAdapter implements JobAdapter {
 
     if (Object.keys(updateData).length === 0) return;
 
-    await this.db
-      .updateTable("__donkeylabs_jobs__")
-      .set(updateData)
-      .where("id", "=", jobId)
-      .execute();
+    try {
+      await this.db
+        .updateTable("__donkeylabs_jobs__")
+        .set(updateData)
+        .where("id", "=", jobId)
+        .execute();
+    } catch (err: any) {
+      // Silently ignore errors if adapter was stopped during query
+      if (this.stopped && err?.message?.includes("destroyed")) return;
+      throw err;
+    }
   }
 
   async delete(jobId: string): Promise<boolean> {
+    if (this.checkStopped()) return false;
+
     // Check if exists first since BunSqliteDialect doesn't report numDeletedRows properly
     const exists = await this.db
       .selectFrom("__donkeylabs_jobs__")
@@ -165,91 +193,139 @@ export class KyselyJobAdapter implements JobAdapter {
   }
 
   async getPending(limit: number = 100): Promise<Job[]> {
-    const rows = await this.db
-      .selectFrom("__donkeylabs_jobs__")
-      .selectAll()
-      .where("status", "=", "pending")
-      .orderBy("created_at", "asc")
-      .limit(limit)
-      .execute();
+    if (this.checkStopped()) return [];
+
+    try {
+      const rows = await this.db
+        .selectFrom("__donkeylabs_jobs__")
+        .selectAll()
+        .where("status", "=", "pending")
+        .orderBy("created_at", "asc")
+        .limit(limit)
+        .execute();
 
-    return rows.map((r) => this.rowToJob(r));
+      return rows.map((r) => this.rowToJob(r));
+    } catch (err: any) {
+      // Silently ignore errors if adapter was stopped during query
+      if (this.stopped && err?.message?.includes("destroyed")) return [];
+      throw err;
+    }
   }
 
   async getScheduledReady(now: Date): Promise<Job[]> {
-    const rows = await this.db
-      .selectFrom("__donkeylabs_jobs__")
-      .selectAll()
-      .where("status", "=", "scheduled")
-      .where("run_at", "<=", now.toISOString())
-      .orderBy("run_at", "asc")
-      .execute();
+    if (this.checkStopped()) return [];
 
-    return rows.map((r) => this.rowToJob(r));
+    try {
+      const rows = await this.db
+        .selectFrom("__donkeylabs_jobs__")
+        .selectAll()
+        .where("status", "=", "scheduled")
+        .where("run_at", "<=", now.toISOString())
+        .orderBy("run_at", "asc")
+        .execute();
+
+      return rows.map((r) => this.rowToJob(r));
+    } catch (err: any) {
+      // Silently ignore errors if adapter was stopped during query
+      if (this.stopped && err?.message?.includes("destroyed")) return [];
+      throw err;
+    }
   }
 
   async getByName(name: string, status?: JobStatus): Promise<Job[]> {
-    let query = this.db
-      .selectFrom("__donkeylabs_jobs__")
-      .selectAll()
-      .where("name", "=", name);
+    if (this.checkStopped()) return [];
 
-    if (status) {
-      query = query.where("status", "=", status);
-    }
+    try {
+      let query = this.db
+        .selectFrom("__donkeylabs_jobs__")
+        .selectAll()
+        .where("name", "=", name);
+
+      if (status) {
+        query = query.where("status", "=", status);
+      }
 
-    const rows = await query.orderBy("created_at", "desc").execute();
-    return rows.map((r) => this.rowToJob(r));
+      const rows = await query.orderBy("created_at", "desc").execute();
+      return rows.map((r) => this.rowToJob(r));
+    } catch (err: any) {
+      // Silently ignore errors if adapter was stopped during query
+      if (this.stopped && err?.message?.includes("destroyed")) return [];
+      throw err;
+    }
   }
 
   async getRunningExternal(): Promise<Job[]> {
-    const rows = await this.db
-      .selectFrom("__donkeylabs_jobs__")
-      .selectAll()
-      .where("external", "=", 1)
-      .where("status", "=", "running")
-      .execute();
+    if (this.checkStopped()) return [];
+
+    try {
+      const rows = await this.db
+        .selectFrom("__donkeylabs_jobs__")
+        .selectAll()
+        .where("external", "=", 1)
+        .where("status", "=", "running")
+        .execute();
 
-    return rows.map((r) => this.rowToJob(r));
+      return rows.map((r) => this.rowToJob(r));
+    } catch (err: any) {
+      // Silently ignore errors if adapter was stopped during query
+      if (this.stopped && err?.message?.includes("destroyed")) return [];
+      throw err;
+    }
   }
 
   async getOrphanedExternal(): Promise<Job[]> {
-    const rows = await this.db
-      .selectFrom("__donkeylabs_jobs__")
-      .selectAll()
-      .where("external", "=", 1)
-      .where("status", "=", "running")
-      .where((eb) =>
-        eb.or([
-          eb("process_state", "=", "running"),
-          eb("process_state", "=", "orphaned"),
-          eb("process_state", "=", "spawning"),
-        ])
-      )
-      .execute();
+    if (this.checkStopped()) return [];
 
-    return rows.map((r) => this.rowToJob(r));
+    try {
+      const rows = await this.db
+        .selectFrom("__donkeylabs_jobs__")
+        .selectAll()
+        .where("external", "=", 1)
+        .where("status", "=", "running")
+        .where((eb) =>
+          eb.or([
+            eb("process_state", "=", "running"),
+            eb("process_state", "=", "orphaned"),
+            eb("process_state", "=", "spawning"),
+          ])
+        )
+        .execute();
+
+      return rows.map((r) => this.rowToJob(r));
+    } catch (err: any) {
+      // Silently ignore errors if adapter was stopped during query
+      if (this.stopped && err?.message?.includes("destroyed")) return [];
+      throw err;
+    }
   }
 
   async getAll(options: GetAllJobsOptions = {}): Promise<Job[]> {
-    const { status, name, limit = 100, offset = 0 } = options;
+    if (this.checkStopped()) return [];
 
-    let query = this.db.selectFrom("__donkeylabs_jobs__").selectAll();
+    try {
+      const { status, name, limit = 100, offset = 0 } = options;
 
-    if (status) {
-      query = query.where("status", "=", status);
-    }
-    if (name) {
-      query = query.where("name", "=", name);
-    }
+      let query = this.db.selectFrom("__donkeylabs_jobs__").selectAll();
 
-    const rows = await query
-      .orderBy("created_at", "desc")
-      .limit(limit)
-      .offset(offset)
-      .execute();
+      if (status) {
+        query = query.where("status", "=", status);
+      }
+      if (name) {
+        query = query.where("name", "=", name);
+      }
 
-    return rows.map((r) => this.rowToJob(r));
+      const rows = await query
+        .orderBy("created_at", "desc")
+        .limit(limit)
+        .offset(offset)
+        .execute();
+
+      return rows.map((r) => this.rowToJob(r));
+    } catch (err: any) {
+      // Silently ignore errors if adapter was stopped during query
+      if (this.stopped && err?.message?.includes("destroyed")) return [];
+      throw err;
+    }
   }
 
   private rowToJob(row: JobsTable): Job {
@@ -278,7 +354,7 @@ export class KyselyJobAdapter implements JobAdapter {
 
   /** Clean up old completed/failed jobs */
   private async cleanup(): Promise<void> {
-    if (this.cleanupDays <= 0) return;
+    if (this.cleanupDays <= 0 || this.checkStopped()) return;
 
     try {
       const cutoff = new Date();
@@ -305,9 +381,36 @@ export class KyselyJobAdapter implements JobAdapter {
 
   /** Stop the adapter and cleanup timer */
   stop(): void {
+    this.stopped = true;
     if (this.cleanupTimer) {
       clearInterval(this.cleanupTimer);
       this.cleanupTimer = undefined;
     }
   }
+
+  /** Atomically claim a pending job (returns true if successfully claimed) */
+  async claim(jobId: string): Promise<boolean> {
+    if (this.checkStopped()) return false;
+
+    try {
+      // Use WHERE status = 'pending' for atomicity - only one process can claim
+      const result = await this.db
+        .updateTable("__donkeylabs_jobs__")
+        .set({
+          status: "running",
+          started_at: new Date().toISOString(),
+        })
+        .where("id", "=", jobId)
+        .where("status", "=", "pending")
+        .execute();
+
+      // Kysely returns numUpdatedRows for updates
+      const numUpdated = Number(result[0]?.numUpdatedRows ?? 0);
+      return numUpdated > 0;
+    } catch (err: any) {
+      // Silently ignore errors if adapter was stopped during query
+      if (this.stopped && err?.message?.includes("destroyed")) return false;
+      throw err;
+    }
+  }
 }

package/src/core/job-adapter-sqlite.ts

@@ -305,4 +305,14 @@ export class SqliteJobAdapter implements JobAdapter {
       this.cleanupTimer = undefined;
     }
   }
+
+  /** Atomically claim a pending job (returns true if successfully claimed) */
+  async claim(jobId: string): Promise<boolean> {
+    // Use WHERE status = 'pending' for atomicity - only one process can claim
+    const result = this.db.run(
+      `UPDATE jobs SET status = 'running', started_at = ? WHERE id = ? AND status = 'pending'`,
+      [new Date().toISOString(), jobId]
+    );
+    return result.changes > 0;
+  }
 }

package/src/core/jobs.ts

@@ -87,6 +87,8 @@ export interface JobAdapter {
   getOrphanedExternal(): Promise<Job[]>;
   /** Get all jobs with optional filtering (for admin dashboard) */
   getAll(options?: GetAllJobsOptions): Promise<Job[]>;
+  /** Atomically claim a pending job (returns true if successfully claimed) */
+  claim(jobId: string): Promise<boolean>;
 }
 
 export interface JobsConfig {
@@ -104,6 +106,12 @@ export interface JobsConfig {
   persist?: boolean;
   /** SQLite database path (default: .donkeylabs/jobs.db) */
   dbPath?: string;
+  /**
+   * Retry backoff configuration.
+   * Set to false to disable backoff (immediate retry).
+   * Default: exponential backoff starting at 1000ms, max 300000ms (5 min)
+   */
+  retryBackoff?: false | { baseMs?: number; maxMs?: number };
 }
 
 export interface Jobs {
@@ -225,6 +233,14 @@ export class MemoryJobAdapter implements JobAdapter {
     // Apply pagination
     return results.slice(offset, offset + limit);
   }
+
+  async claim(jobId: string): Promise<boolean> {
+    const job = this.jobs.get(jobId);
+    if (!job || job.status !== "pending") return false;
+    job.status = "running";
+    job.startedAt = new Date();
+    return true;
+  }
 }
 
 class JobsImpl implements Jobs {
@@ -241,6 +257,8 @@ class JobsImpl implements Jobs {
   private defaultMaxAttempts: number;
   private usePersistence: boolean;
   private dbPath?: string;
+  private tickRunning = false; // Reentrancy guard for tick()
+  private retryBackoff: false | { baseMs: number; maxMs: number };
 
   // External jobs support
   private externalConfigs = new Map<string, ExternalJobConfig>();
@@ -256,6 +274,18 @@ class JobsImpl implements Jobs {
     this.externalConfig = config.external ?? {};
     this.usePersistence = config.persist ?? true; // Default to SQLite persistence
     this.dbPath = config.dbPath;
+    // Configure retry backoff
+    if (config.retryBackoff === false) {
+      this.retryBackoff = false;
+    } else if (config.retryBackoff) {
+      this.retryBackoff = {
+        baseMs: config.retryBackoff.baseMs ?? 1000,
+        maxMs: config.retryBackoff.maxMs ?? 300000,
+      };
+    } else {
+      // Default: exponential backoff
+      this.retryBackoff = { baseMs: 1000, maxMs: 300000 };
+    }
 
     // Use provided adapter, or create SQLite adapter if persistence enabled
     if (config.adapter) {
@@ -402,6 +432,12 @@ class JobsImpl implements Jobs {
       this.heartbeatTimer = null;
     }
 
+    // Wait for any in-progress tick to complete
+    const tickWaitStart = Date.now();
+    while (this.tickRunning && Date.now() - tickWaitStart < 5000) {
+      await new Promise(resolve => setTimeout(resolve, 10));
+    }
+
     // Cleanup external job processes
     for (const [jobId, procInfo] of this.externalProcesses) {
       if (procInfo.timeout) {
@@ -426,6 +462,12 @@ class JobsImpl implements Jobs {
       this.sqliteAdapter.stop();
     }
 
+    // Stop adapter (cleanup timers and prevent further DB access)
+    // This handles KyselyJobAdapter and other adapters with stop() method
+    if (this.adapter && typeof (this.adapter as any).stop === "function") {
+      (this.adapter as any).stop();
+    }
+
     // Wait for active in-process jobs to complete (with timeout)
     const maxWait = 30000; // 30 seconds
     const startTime = Date.now();
@@ -721,30 +763,59 @@ class JobsImpl implements Jobs {
   private async tick(): Promise<void> {
     if (!this.running) return;
 
+    // Reentrancy guard - prevent concurrent tick execution
+    if (this.tickRunning) return;
+    this.tickRunning = true;
+
     try {
+      // Check running state before each async operation to exit quickly on stop()
+      if (!this.running) return;
+
       // Process scheduled jobs that are ready
       const now = new Date();
       const scheduledReady = await this.adapter.getScheduledReady(now);
+
+      if (!this.running) return;
+
       for (const job of scheduledReady) {
+        if (!this.running) return;
         await this.adapter.update(job.id, { status: "pending" });
       }
 
+      if (!this.running) return;
+
       // Process pending jobs
       const availableSlots = this.concurrency - this.activeJobs;
       if (availableSlots <= 0) return;
 
       const pending = await this.adapter.getPending(availableSlots);
+
+      if (!this.running) return;
+
       for (const job of pending) {
+        if (!this.running) break;
         if (this.activeJobs >= this.concurrency) break;
 
+        // Atomic claim - prevent double execution
+        const claimed = await this.adapter.claim(job.id);
+        if (!claimed) continue; // Another process claimed it
+
         if (job.external) {
           this.processExternalJob(job);
         } else {
           this.processJob(job);
         }
       }
-    } catch (err) {
-      console.error("[Jobs] Tick error:", err);
+    } catch (err: any) {
+      // Suppress "driver destroyed" errors which happen during test cleanup
+      // when the database is garbage collected before the tick completes
+      const isDriverDestroyed = err?.message?.includes("driver has already been destroyed");
+      // Only log if we're still running and it's not a driver destroyed error
+      if (this.running && !isDriverDestroyed) {
+        console.error("[Jobs] Tick error:", err);
+      }
+    } finally {
+      this.tickRunning = false; // Release guard
     }
   }
 
@@ -770,10 +841,9 @@ class JobsImpl implements Jobs {
 
       const { socketPath, tcpPort } = await this.socketServer!.createSocket(job.id);
 
-      // Update job with socket info
+      // Job is already claimed (status=running, startedAt set)
+      // Update with socket info and other external job fields
       await this.adapter.update(job.id, {
-        status: "running",
-        startedAt,
         attempts: job.attempts + 1,
         socketPath,
         tcpPort,
@@ -856,9 +926,20 @@ class JobsImpl implements Jobs {
         const currentJob = await this.adapter.get(job.id);
         if (currentJob?.status === "running") {
           if (code === 0) {
-            // Process exited cleanly but didn't send completion message
-            // This might be ok, or might indicate an issue
-            console.warn(`[Jobs] External job ${job.id} exited with code 0 but no completion message`);
+            // Process exited cleanly but didn't send completion message - mark as completed
+            console.warn(`[Jobs] External job ${job.id} exited with code 0 but no completion message, marking completed`);
+            await this.adapter.update(job.id, {
+              status: "completed",
+              completedAt: new Date(),
+            });
+
+            if (this.events) {
+              await this.events.emit("job.completed", {
+                jobId: job.id,
+                name: job.name,
+                result: undefined,
+              });
+            }
           } else {
             // Process failed
             await this.adapter.update(job.id, {
@@ -972,12 +1053,11 @@ class JobsImpl implements Jobs {
     }
 
     this.activeJobs++;
-    const startedAt = new Date();
 
     try {
+      // Job is already claimed (status=running, startedAt set)
+      // Just update attempts
       await this.adapter.update(job.id, {
-        status: "running",
-        startedAt,
         attempts: job.attempts + 1,
       });
 
@@ -1006,12 +1086,27 @@ class JobsImpl implements Jobs {
       const attempts = job.attempts + 1;
 
       if (attempts < job.maxAttempts) {
-        // Retry later
-        await this.adapter.update(job.id, {
-          status: "pending",
-          attempts,
-          error,
-        });
+        // Retry with optional exponential backoff
+        if (this.retryBackoff === false) {
+          // No backoff - immediate retry
+          await this.adapter.update(job.id, {
+            status: "pending",
+            attempts,
+            error,
+          });
+        } else {
+          // Exponential backoff: delay = min(baseMs * 2^(attempts-1), maxMs)
+          const { baseMs, maxMs } = this.retryBackoff;
+          const backoffMs = Math.min(baseMs * Math.pow(2, attempts - 1), maxMs);
+          const runAt = new Date(Date.now() + backoffMs);
+
+          await this.adapter.update(job.id, {
+            status: "scheduled",
+            runAt,
+            attempts,
+            error,
+          });
+        }
       } else {
         // Max attempts reached, mark as failed
         await this.adapter.update(job.id, {

package/src/core/migrations/workflows/002_add_metadata_column.ts

@@ -0,0 +1,28 @@
+/**
+ * Core Migration: Add metadata column to workflow instances
+ *
+ * Adds a metadata column to store custom JSON data that persists across workflow steps.
+ */
+
+import { sql, type Kysely } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+  // SQLite doesn't support IF NOT EXISTS for ALTER TABLE ADD COLUMN
+  // Check if column exists first
+  const tableInfo = await sql<{ name: string }>`
+    PRAGMA table_info(__donkeylabs_workflow_instances__)
+  `.execute(db);
+
+  const hasMetadataColumn = tableInfo.rows.some((row) => row.name === "metadata");
+
+  if (!hasMetadataColumn) {
+    await sql`
+      ALTER TABLE __donkeylabs_workflow_instances__ ADD COLUMN metadata TEXT
+    `.execute(db);
+  }
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  // SQLite doesn't support DROP COLUMN directly
+  // In practice, we don't need to remove it - the column can stay
+}