@nicnocquee/dataqueue 1.25.0 → 1.26.0-beta.20260223202259

package/src/queue.ts

@@ -16,10 +16,9 @@ import {
   JobEventType,
   TagQueryMode,
   WaitpointRecord,
+  AddJobOptions,
 } from './types.js';
 import { PostgresBackend } from './backends/postgres.js';
-import { randomUUID } from 'crypto';
-import { log } from './log-context.js';
 
 /* Thin wrappers — every function creates a lightweight backend wrapper
    around the given pool and forwards the call.  The class itself holds
@@ -36,7 +35,14 @@ export const recordJobEvent = async (
 export const addJob = async <PayloadMap, T extends keyof PayloadMap & string>(
   pool: Pool,
   job: JobOptions<PayloadMap, T>,
-): Promise<number> => new PostgresBackend(pool).addJob(job);
+  options?: AddJobOptions,
+): Promise<number> => new PostgresBackend(pool).addJob(job, options);
+
+export const addJobs = async <PayloadMap, T extends keyof PayloadMap & string>(
+  pool: Pool,
+  jobs: JobOptions<PayloadMap, T>[],
+  options?: AddJobOptions,
+): Promise<number[]> => new PostgresBackend(pool).addJobs(jobs, options);
 
 export const getJob = async <PayloadMap, T extends keyof PayloadMap & string>(
   pool: Pool,
@@ -94,7 +100,9 @@ export const retryJob = async (pool: Pool, jobId: number): Promise<void> =>
 export const cleanupOldJobs = async (
   pool: Pool,
   daysToKeep = 30,
-): Promise<number> => new PostgresBackend(pool).cleanupOldJobs(daysToKeep);
+  batchSize = 1000,
+): Promise<number> =>
+  new PostgresBackend(pool).cleanupOldJobs(daysToKeep, batchSize);
 
 export const cancelJob = async (pool: Pool, jobId: number): Promise<void> =>
   new PostgresBackend(pool).cancelJob(jobId);
@@ -109,6 +117,9 @@ export const editJob = async <PayloadMap, T extends keyof PayloadMap & string>(
     runAt?: Date | null;
     timeoutMs?: number | null;
     tags?: string[] | null;
+    retryDelay?: number | null;
+    retryBackoff?: boolean | null;
+    retryDelayMax?: number | null;
   },
 ): Promise<void> => new PostgresBackend(pool).editJob(jobId, updates);
 
@@ -134,6 +145,9 @@ export const editAllPendingJobs = async <
     runAt?: Date | null;
     timeoutMs?: number;
     tags?: string[];
+    retryDelay?: number | null;
+    retryBackoff?: boolean | null;
+    retryDelayMax?: number | null;
   },
 ): Promise<number> =>
   new PostgresBackend(pool).editAllPendingJobs(filters, updates);
@@ -214,12 +228,9 @@ export const updateProgress = async (
   progress: number,
 ): Promise<void> => new PostgresBackend(pool).updateProgress(jobId, progress);
 
-// ── Wait support functions (PostgreSQL-only) ─────────────────────────────────
+// ── Wait support functions (backward-compatible delegates) ────────────────────
 
-/**
- * Transition a job to 'waiting' status with wait_until and/or wait_token_id.
- * Saves step_data so the handler can resume from where it left off.
- */
+/** @deprecated Use backend.waitJob() directly. Delegates to PostgresBackend. */
 export const waitJob = async (
   pool: Pool,
   jobId: number,
@@ -228,266 +239,37 @@ export const waitJob = async (
     waitTokenId?: string;
     stepData: Record<string, any>;
   },
-): Promise<void> => {
-  const client = await pool.connect();
-  try {
-    const result = await client.query(
-      `
-      UPDATE job_queue
-      SET status = 'waiting',
-          wait_until = $2,
-          wait_token_id = $3,
-          step_data = $4,
-          locked_at = NULL,
-          locked_by = NULL,
-          updated_at = NOW()
-      WHERE id = $1 AND status = 'processing'
-    `,
-      [
-        jobId,
-        options.waitUntil ?? null,
-        options.waitTokenId ?? null,
-        JSON.stringify(options.stepData),
-      ],
-    );
-    if (result.rowCount === 0) {
-      log(
-        `Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`,
-      );
-      return;
-    }
-    await recordJobEvent(pool, jobId, JobEventType.Waiting, {
-      waitUntil: options.waitUntil?.toISOString() ?? null,
-      waitTokenId: options.waitTokenId ?? null,
-    });
-    log(`Job ${jobId} set to waiting`);
-  } catch (error) {
-    log(`Error setting job ${jobId} to waiting: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
+): Promise<void> => new PostgresBackend(pool).waitJob(jobId, options);
 
-/**
- * Update step_data for a job. Called after each ctx.run() step completes
- * to persist intermediate progress.
- */
+/** @deprecated Use backend.updateStepData() directly. Delegates to PostgresBackend. */
 export const updateStepData = async (
   pool: Pool,
   jobId: number,
   stepData: Record<string, any>,
-): Promise<void> => {
-  const client = await pool.connect();
-  try {
-    await client.query(
-      `UPDATE job_queue SET step_data = $2, updated_at = NOW() WHERE id = $1`,
-      [jobId, JSON.stringify(stepData)],
-    );
-  } catch (error) {
-    log(`Error updating step_data for job ${jobId}: ${error}`);
-    // Best-effort: do not throw to avoid killing the running handler
-  } finally {
-    client.release();
-  }
-};
+): Promise<void> => new PostgresBackend(pool).updateStepData(jobId, stepData);
 
-/**
- * Parse a timeout string like '10m', '1h', '24h', '7d' into milliseconds.
- */
-/**
- * Maximum allowed timeout in milliseconds (~365 days).
- * Prevents overflow to Infinity when computing Date offsets.
- */
-const MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1000;
-
-function parseTimeoutString(timeout: string): number {
-  const match = timeout.match(/^(\d+)(s|m|h|d)$/);
-  if (!match) {
-    throw new Error(
-      `Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`,
-    );
-  }
-  const value = parseInt(match[1], 10);
-  const unit = match[2];
-  let ms: number;
-  switch (unit) {
-    case 's':
-      ms = value * 1000;
-      break;
-    case 'm':
-      ms = value * 60 * 1000;
-      break;
-    case 'h':
-      ms = value * 60 * 60 * 1000;
-      break;
-    case 'd':
-      ms = value * 24 * 60 * 60 * 1000;
-      break;
-    default:
-      throw new Error(`Unknown timeout unit: "${unit}"`);
-  }
-  if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS) {
-    throw new Error(
-      `Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`,
-    );
-  }
-  return ms;
-}
-
-/**
- * Create a waitpoint token in the database.
- * The token can be used to pause a job until an external signal completes it.
- *
- * @param pool - The database pool
- * @param jobId - The job ID to associate with the token (null if created outside a handler)
- * @param options - Optional timeout and tags
- * @returns The created waitpoint token
- */
+/** @deprecated Use backend.createWaitpoint() directly. Delegates to PostgresBackend. */
 export const createWaitpoint = async (
   pool: Pool,
   jobId: number | null,
   options?: { timeout?: string; tags?: string[] },
-): Promise<{ id: string }> => {
-  const client = await pool.connect();
-  try {
-    const id = `wp_${randomUUID()}`;
-    let timeoutAt: Date | null = null;
-
-    if (options?.timeout) {
-      const ms = parseTimeoutString(options.timeout);
-      timeoutAt = new Date(Date.now() + ms);
-    }
+): Promise<{ id: string }> =>
+  new PostgresBackend(pool).createWaitpoint(jobId, options);
 
-    await client.query(
-      `INSERT INTO waitpoints (id, job_id, status, timeout_at, tags) VALUES ($1, $2, 'waiting', $3, $4)`,
-      [id, jobId, timeoutAt, options?.tags ?? null],
-    );
-
-    log(`Created waitpoint ${id} for job ${jobId}`);
-    return { id };
-  } catch (error) {
-    log(`Error creating waitpoint: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
-
-/**
- * Complete a waitpoint token, optionally providing output data.
- * This also moves the associated job from 'waiting' back to 'pending' so
- * it gets picked up by the polling loop.
- */
+/** @deprecated Use backend.completeWaitpoint() directly. Delegates to PostgresBackend. */
 export const completeWaitpoint = async (
   pool: Pool,
   tokenId: string,
   data?: any,
-): Promise<void> => {
-  const client = await pool.connect();
-  try {
-    await client.query('BEGIN');
-
-    // Update the waitpoint
-    const wpResult = await client.query(
-      `UPDATE waitpoints SET status = 'completed', output = $2, completed_at = NOW()
-       WHERE id = $1 AND status = 'waiting'
-       RETURNING job_id`,
-      [tokenId, data != null ? JSON.stringify(data) : null],
-    );
-
-    if (wpResult.rows.length === 0) {
-      await client.query('ROLLBACK');
-      log(`Waitpoint ${tokenId} not found or already completed`);
-      return;
-    }
-
-    const jobId = wpResult.rows[0].job_id;
-
-    // Move the associated job back to 'pending' so it gets picked up
-    if (jobId != null) {
-      await client.query(
-        `UPDATE job_queue
-         SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
-         WHERE id = $1 AND status = 'waiting'`,
-        [jobId],
-      );
-    }
-
-    await client.query('COMMIT');
-    log(`Completed waitpoint ${tokenId} for job ${jobId}`);
-  } catch (error) {
-    await client.query('ROLLBACK');
-    log(`Error completing waitpoint ${tokenId}: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
+): Promise<void> => new PostgresBackend(pool).completeWaitpoint(tokenId, data);
 
-/**
- * Retrieve a waitpoint token by its ID.
- */
+/** @deprecated Use backend.getWaitpoint() directly. Delegates to PostgresBackend. */
 export const getWaitpoint = async (
   pool: Pool,
   tokenId: string,
-): Promise<WaitpointRecord | null> => {
-  const client = await pool.connect();
-  try {
-    const result = await client.query(
-      `SELECT id, job_id AS "jobId", status, output, timeout_at AS "timeoutAt", created_at AS "createdAt", completed_at AS "completedAt", tags FROM waitpoints WHERE id = $1`,
-      [tokenId],
-    );
-    if (result.rows.length === 0) return null;
-    return result.rows[0] as WaitpointRecord;
-  } catch (error) {
-    log(`Error getting waitpoint ${tokenId}: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
-
-/**
- * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
- * Should be called periodically (e.g., alongside reclaimStuckJobs).
- */
-export const expireTimedOutWaitpoints = async (pool: Pool): Promise<number> => {
-  const client = await pool.connect();
-  try {
-    await client.query('BEGIN');
-
-    // Find and expire timed-out waitpoints
-    const result = await client.query(
-      `UPDATE waitpoints
-       SET status = 'timed_out'
-       WHERE status = 'waiting' AND timeout_at IS NOT NULL AND timeout_at <= NOW()
-       RETURNING id, job_id`,
-    );
-
-    // Move associated jobs back to 'pending'
-    for (const row of result.rows) {
-      if (row.job_id != null) {
-        await client.query(
-          `UPDATE job_queue
-           SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
-           WHERE id = $1 AND status = 'waiting'`,
-          [row.job_id],
-        );
-      }
-    }
+): Promise<WaitpointRecord | null> =>
+  new PostgresBackend(pool).getWaitpoint(tokenId);
 
-    await client.query('COMMIT');
-    const count = result.rowCount || 0;
-    if (count > 0) {
-      log(`Expired ${count} timed-out waitpoints`);
-    }
-    return count;
-  } catch (error) {
-    await client.query('ROLLBACK');
-    log(`Error expiring timed-out waitpoints: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
+/** @deprecated Use backend.expireTimedOutWaitpoints() directly. Delegates to PostgresBackend. */
+export const expireTimedOutWaitpoints = async (pool: Pool): Promise<number> =>
+  new PostgresBackend(pool).expireTimedOutWaitpoints();

package/src/supervisor.test.ts

@@ -0,0 +1,340 @@
+import { describe, expect, it, beforeEach, afterEach, vi } from 'vitest';
+import { createSupervisor } from './supervisor.js';
+import type { QueueBackend } from './backend.js';
+import type { SupervisorOptions } from './types.js';
+
+/**
+ * Builds a fake {@link QueueBackend} with only the methods the supervisor
+ * calls, each backed by a vi.fn() that resolves to 0 by default.
+ */
+function createFakeBackend(overrides: Partial<QueueBackend> = {}) {
+  return {
+    reclaimStuckJobs: vi.fn().mockResolvedValue(0),
+    cleanupOldJobs: vi.fn().mockResolvedValue(0),
+    cleanupOldJobEvents: vi.fn().mockResolvedValue(0),
+    expireTimedOutWaitpoints: vi.fn().mockResolvedValue(0),
+    ...overrides,
+  } as unknown as QueueBackend;
+}
+
+describe('createSupervisor', () => {
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+  });
+
+  describe('start (one-shot)', () => {
+    it('runs all maintenance tasks and returns results', async () => {
+      // Setup
+      const backend = createFakeBackend({
+        reclaimStuckJobs: vi.fn().mockResolvedValue(3),
+        cleanupOldJobs: vi.fn().mockResolvedValue(15),
+        cleanupOldJobEvents: vi.fn().mockResolvedValue(7),
+        expireTimedOutWaitpoints: vi.fn().mockResolvedValue(2),
+      });
+      const supervisor = createSupervisor(backend);
+
+      // Act
+      const result = await supervisor.start();
+
+      // Assert
+      expect(result).toEqual({
+        reclaimedJobs: 3,
+        cleanedUpJobs: 15,
+        cleanedUpEvents: 7,
+        expiredTokens: 2,
+      });
+      expect(backend.reclaimStuckJobs).toHaveBeenCalledWith(10);
+      expect(backend.cleanupOldJobs).toHaveBeenCalledWith(30, 1000);
+      expect(backend.cleanupOldJobEvents).toHaveBeenCalledWith(30, 1000);
+      expect(backend.expireTimedOutWaitpoints).toHaveBeenCalledOnce();
+    });
+
+    it('passes custom option values to backend methods', async () => {
+      // Setup
+      const backend = createFakeBackend();
+      const supervisor = createSupervisor(backend, {
+        stuckJobsTimeoutMinutes: 20,
+        cleanupJobsDaysToKeep: 60,
+        cleanupEventsDaysToKeep: 14,
+        cleanupBatchSize: 500,
+      });
+
+      // Act
+      await supervisor.start();
+
+      // Assert
+      expect(backend.reclaimStuckJobs).toHaveBeenCalledWith(20);
+      expect(backend.cleanupOldJobs).toHaveBeenCalledWith(60, 500);
+      expect(backend.cleanupOldJobEvents).toHaveBeenCalledWith(14, 500);
+    });
+
+    it('skips reclaimStuckJobs when disabled', async () => {
+      // Setup
+      const backend = createFakeBackend();
+      const supervisor = createSupervisor(backend, {
+        reclaimStuckJobs: false,
+      });
+
+      // Act
+      const result = await supervisor.start();
+
+      // Assert
+      expect(backend.reclaimStuckJobs).not.toHaveBeenCalled();
+      expect(result.reclaimedJobs).toBe(0);
+      expect(backend.cleanupOldJobs).toHaveBeenCalledOnce();
+    });
+
+    it('skips cleanupOldJobs when cleanupJobsDaysToKeep is 0', async () => {
+      // Setup
+      const backend = createFakeBackend();
+      const supervisor = createSupervisor(backend, {
+        cleanupJobsDaysToKeep: 0,
+      });
+
+      // Act
+      const result = await supervisor.start();
+
+      // Assert
+      expect(backend.cleanupOldJobs).not.toHaveBeenCalled();
+      expect(result.cleanedUpJobs).toBe(0);
+      expect(backend.reclaimStuckJobs).toHaveBeenCalledOnce();
+    });
+
+    it('skips cleanupOldJobEvents when cleanupEventsDaysToKeep is 0', async () => {
+      // Setup
+      const backend = createFakeBackend();
+      const supervisor = createSupervisor(backend, {
+        cleanupEventsDaysToKeep: 0,
+      });
+
+      // Act
+      const result = await supervisor.start();
+
+      // Assert
+      expect(backend.cleanupOldJobEvents).not.toHaveBeenCalled();
+      expect(result.cleanedUpEvents).toBe(0);
+    });
+
+    it('skips expireTimedOutTokens when disabled', async () => {
+      // Setup
+      const backend = createFakeBackend();
+      const supervisor = createSupervisor(backend, {
+        expireTimedOutTokens: false,
+      });
+
+      // Act
+      const result = await supervisor.start();
+
+      // Assert
+      expect(backend.expireTimedOutWaitpoints).not.toHaveBeenCalled();
+      expect(result.expiredTokens).toBe(0);
+    });
+
+    it('calls onError and continues when a task throws', async () => {
+      // Setup
+      const onError = vi.fn();
+      const taskError = new Error('reclaim failed');
+      const backend = createFakeBackend({
+        reclaimStuckJobs: vi.fn().mockRejectedValue(taskError),
+        cleanupOldJobs: vi.fn().mockResolvedValue(5),
+        cleanupOldJobEvents: vi
+          .fn()
+          .mockRejectedValue(new Error('events boom')),
+        expireTimedOutWaitpoints: vi.fn().mockResolvedValue(1),
+      });
+      const supervisor = createSupervisor(backend, { onError });
+
+      // Act
+      const result = await supervisor.start();
+
+      // Assert
+      expect(onError).toHaveBeenCalledTimes(2);
+      expect(onError).toHaveBeenCalledWith(taskError);
+      expect(result.reclaimedJobs).toBe(0);
+      expect(result.cleanedUpJobs).toBe(5);
+      expect(result.cleanedUpEvents).toBe(0);
+      expect(result.expiredTokens).toBe(1);
+    });
+
+    it('wraps non-Error throws in an Error object', async () => {
+      // Setup
+      const onError = vi.fn();
+      const backend = createFakeBackend({
+        reclaimStuckJobs: vi.fn().mockRejectedValue('string error'),
+      });
+      const supervisor = createSupervisor(backend, { onError });
+
+      // Act
+      await supervisor.start();
+
+      // Assert
+      expect(onError).toHaveBeenCalledWith(expect.any(Error));
+      expect(onError.mock.calls[0][0].message).toBe('string error');
+    });
+  });
+
+  describe('startInBackground / stop', () => {
+    it('polls on interval and can be stopped', async () => {
+      // Setup
+      vi.useFakeTimers();
+      const backend = createFakeBackend();
+      const supervisor = createSupervisor(backend, { intervalMs: 1000 });
+
+      // Act
+      supervisor.startInBackground();
+      expect(supervisor.isRunning()).toBe(true);
+
+      // First run is immediate (microtask)
+      await vi.advanceTimersByTimeAsync(0);
+      expect(backend.reclaimStuckJobs).toHaveBeenCalledTimes(1);
+
+      // Advance to trigger second run
+      await vi.advanceTimersByTimeAsync(1000);
+      expect(backend.reclaimStuckJobs).toHaveBeenCalledTimes(2);
+
+      // Advance to trigger third run
+      await vi.advanceTimersByTimeAsync(1000);
+      expect(backend.reclaimStuckJobs).toHaveBeenCalledTimes(3);
+
+      // Stop
+      supervisor.stop();
+      expect(supervisor.isRunning()).toBe(false);
+
+      // No more runs after stop
+      await vi.advanceTimersByTimeAsync(5000);
+      expect(backend.reclaimStuckJobs).toHaveBeenCalledTimes(3);
+    });
+
+    it('does nothing when called while already running', async () => {
+      // Setup
+      vi.useFakeTimers();
+      const backend = createFakeBackend();
+      const supervisor = createSupervisor(backend, { intervalMs: 1000 });
+
+      // Act
+      supervisor.startInBackground();
+      supervisor.startInBackground(); // second call should be ignored
+      await vi.advanceTimersByTimeAsync(0);
+
+      // Assert -- only one loop running, single call
+      expect(backend.reclaimStuckJobs).toHaveBeenCalledTimes(1);
+
+      supervisor.stop();
+    });
+  });
+
+  describe('stopAndDrain', () => {
+    it('waits for current maintenance run to finish', async () => {
+      // Setup
+      vi.useFakeTimers();
+      let resolveTask!: () => void;
+      const slowTask = new Promise<number>((resolve) => {
+        resolveTask = () => resolve(2);
+      });
+      const backend = createFakeBackend({
+        reclaimStuckJobs: vi.fn().mockReturnValue(slowTask),
+      });
+      const supervisor = createSupervisor(backend, { intervalMs: 1000 });
+
+      // Act
+      supervisor.startInBackground();
+      // Let the loop start (but reclaimStuckJobs is blocked)
+      await vi.advanceTimersByTimeAsync(0);
+
+      let drained = false;
+      const drainPromise = supervisor.stopAndDrain().then(() => {
+        drained = true;
+      });
+
+      // Assert -- not drained yet because slowTask is still pending
+      expect(drained).toBe(false);
+      expect(supervisor.isRunning()).toBe(false);
+
+      // Resolve the slow task
+      resolveTask();
+      await drainPromise;
+
+      // Assert -- now drained
+      expect(drained).toBe(true);
+    });
+
+    it('resolves immediately when no maintenance run is in progress', async () => {
+      // Setup
+      const backend = createFakeBackend();
+      const supervisor = createSupervisor(backend);
+
+      // Act & Assert
+      await expect(supervisor.stopAndDrain()).resolves.toBeUndefined();
+    });
+
+    it('resolves after timeout if maintenance run hangs', async () => {
+      // Setup
+      vi.useFakeTimers();
+      const neverResolve = new Promise<number>(() => {});
+      const backend = createFakeBackend({
+        reclaimStuckJobs: vi.fn().mockReturnValue(neverResolve),
+      });
+      const supervisor = createSupervisor(backend, { intervalMs: 5000 });
+
+      // Act
+      supervisor.startInBackground();
+      await vi.advanceTimersByTimeAsync(0);
+
+      let drained = false;
+      const drainPromise = supervisor.stopAndDrain(500).then(() => {
+        drained = true;
+      });
+
+      // Assert -- not drained yet
+      expect(drained).toBe(false);
+
+      // Advance past the drain timeout
+      await vi.advanceTimersByTimeAsync(500);
+      await drainPromise;
+
+      // Assert -- drained by timeout
+      expect(drained).toBe(true);
+    });
+  });
+
+  describe('isRunning', () => {
+    it('returns false before start', () => {
+      // Setup
+      const backend = createFakeBackend();
+      const supervisor = createSupervisor(backend);
+
+      // Assert
+      expect(supervisor.isRunning()).toBe(false);
+    });
+
+    it('returns true after startInBackground', async () => {
+      // Setup
+      vi.useFakeTimers();
+      const backend = createFakeBackend();
+      const supervisor = createSupervisor(backend);
+
+      // Act
+      supervisor.startInBackground();
+
+      // Assert
+      expect(supervisor.isRunning()).toBe(true);
+
+      supervisor.stop();
+    });
+
+    it('returns false after stop', async () => {
+      // Setup
+      vi.useFakeTimers();
+      const backend = createFakeBackend();
+      const supervisor = createSupervisor(backend);
+
+      // Act
+      supervisor.startInBackground();
+      supervisor.stop();
+
+      // Assert
+      expect(supervisor.isRunning()).toBe(false);
+    });
+  });
+});