@nicnocquee/dataqueue 1.31.0 → 1.33.0

package/src/processor.ts

@@ -1,5 +1,4 @@
 import { Worker } from 'worker_threads';
-import { Pool } from 'pg';
 import {
   JobRecord,
   ProcessorOptions,
@@ -15,69 +14,8 @@ import {
   WaitTokenResult,
 } from './types.js';
 import { QueueBackend } from './backend.js';
-import { PostgresBackend } from './backends/postgres.js';
-import {
-  waitJob,
-  updateStepData,
-  createWaitpoint,
-  getWaitpoint,
-} from './queue.js';
 import { log, setLogContext } from './log-context.js';
 
-/**
- * Try to extract the underlying pg Pool from a QueueBackend.
- * Returns null for non-PostgreSQL backends.
- */
-function tryExtractPool(backend: QueueBackend): Pool | null {
-  if (backend instanceof PostgresBackend) {
-    return backend.getPool();
-  }
-  return null;
-}
-
-/**
- * Build a JobContext without wait support (for non-PostgreSQL backends).
- * prolong/onTimeout work normally; wait-related methods throw helpful errors.
- */
-function buildBasicContext(
-  backend: QueueBackend,
-  jobId: number,
-  baseCtx: {
-    prolong: JobContext['prolong'];
-    onTimeout: JobContext['onTimeout'];
-  },
-): JobContext {
-  const waitError = () =>
-    new Error(
-      'Wait features (waitFor, waitUntil, createToken, waitForToken, ctx.run) are currently only supported with the PostgreSQL backend.',
-    );
-  return {
-    prolong: baseCtx.prolong,
-    onTimeout: baseCtx.onTimeout,
-    run: async <T>(_stepName: string, fn: () => Promise<T>): Promise<T> => {
-      // Without PostgreSQL, just execute the function directly (no persistence)
-      return fn();
-    },
-    waitFor: async () => {
-      throw waitError();
-    },
-    waitUntil: async () => {
-      throw waitError();
-    },
-    createToken: async () => {
-      throw waitError();
-    },
-    waitForToken: async () => {
-      throw waitError();
-    },
-    setProgress: async (percent: number) => {
-      if (percent < 0 || percent > 100)
-        throw new Error('Progress must be between 0 and 100');
-      await backend.updateProgress(jobId, Math.round(percent));
-    },
-  };
-}
-
 /**
  * Validates that a handler can be serialized for worker thread execution.
  * Throws an error with helpful message if serialization fails.
@@ -388,7 +326,7 @@ function createNoOpContext(
  * Marks pending waits as completed and fetches token outputs.
  */
 async function resolveCompletedWaits(
-  pool: Pool,
+  backend: QueueBackend,
   stepData: Record<string, any>,
 ): Promise<void> {
   for (const key of Object.keys(stepData)) {
@@ -401,7 +339,7 @@ async function resolveCompletedWaits(
       stepData[key] = { ...entry, completed: true };
     } else if (entry.type === 'token' && entry.tokenId) {
       // Token-based wait -- fetch the waitpoint result
-      const wp = await getWaitpoint(pool, entry.tokenId);
+      const wp = await backend.getWaitpoint(entry.tokenId);
       if (wp && wp.status === 'completed') {
         stepData[key] = {
           ...entry,
@@ -422,10 +360,10 @@ async function resolveCompletedWaits(
 
 /**
  * Build the extended JobContext with step tracking and wait support.
+ * Works with any QueueBackend (Postgres or Redis).
  */
 function buildWaitContext(
   backend: QueueBackend,
-  pool: Pool,
   jobId: number,
   stepData: Record<string, any>,
   baseCtx: {
@@ -455,7 +393,7 @@ function buildWaitContext(
 
       // Persist step result
       stepData[stepName] = { __completed: true, result };
-      await updateStepData(pool, jobId, stepData);
+      await backend.updateStepData(jobId, stepData);
 
       return result;
     },
@@ -498,7 +436,7 @@ function buildWaitContext(
     },
 
     createToken: async (options?) => {
-      const token = await createWaitpoint(pool, jobId, options);
+      const token = await backend.createWaitpoint(jobId, options);
       return token;
     },
 
@@ -517,7 +455,7 @@ function buildWaitContext(
       }
 
       // Check if the token is already completed (e.g., completed while job was still processing)
-      const wp = await getWaitpoint(pool, tokenId);
+      const wp = await backend.getWaitpoint(tokenId);
       if (wp && wp.status === 'completed') {
         const result: WaitTokenResult<T> = {
           ok: true,
@@ -529,7 +467,7 @@ function buildWaitContext(
           completed: true,
           result,
         };
-        await updateStepData(pool, jobId, stepData);
+        await backend.updateStepData(jobId, stepData);
         return result;
       }
       if (wp && wp.status === 'timed_out') {
@@ -543,7 +481,7 @@ function buildWaitContext(
           completed: true,
           result,
         };
-        await updateStepData(pool, jobId, stepData);
+        await backend.updateStepData(jobId, stepData);
         return result;
       }
 
@@ -591,17 +529,14 @@ export async function processJobWithHandlers<
   // Load step data (may contain completed steps from previous invocations)
   const stepData: Record<string, any> = { ...(job.stepData || {}) };
 
-  // Try to get pool for wait features (PostgreSQL-only)
-  const pool = tryExtractPool(backend);
-
   // If resuming from a wait, resolve any pending wait entries
   const hasStepHistory = Object.keys(stepData).some((k) =>
     k.startsWith('__wait_'),
   );
-  if (hasStepHistory && pool) {
-    await resolveCompletedWaits(pool, stepData);
+  if (hasStepHistory) {
+    await resolveCompletedWaits(backend, stepData);
     // Persist the resolved step data
-    await updateStepData(pool, job.id, stepData);
+    await backend.updateStepData(job.id, stepData);
   }
 
   // Per-job timeout logic
@@ -685,10 +620,8 @@ export async function processJobWithHandlers<
             },
           };
 
-      // Build context: full wait support for PostgreSQL, basic for others
-      const ctx = pool
-        ? buildWaitContext(backend, pool, job.id, stepData, baseCtx)
-        : buildBasicContext(backend, job.id, baseCtx);
+      // Build context: full wait support for all backends
+      const ctx = buildWaitContext(backend, job.id, stepData, baseCtx);
 
       // If forceKillOnTimeout was set but timeoutMs was missing, warn
       if (forceKillOnTimeout && !hasTimeout) {
@@ -720,22 +653,10 @@ export async function processJobWithHandlers<
 
     // Check if this is a WaitSignal (not a real error)
     if (error instanceof WaitSignal) {
-      if (!pool) {
-        // Wait signals should never happen with non-PostgreSQL backends
-        // since the context methods throw, but guard just in case
-        await backend.failJob(
-          job.id,
-          new Error(
-            'WaitSignal received but wait features require the PostgreSQL backend.',
-          ),
-          FailureReason.HandlerError,
-        );
-        return;
-      }
       log(
         `Job ${job.id} entering wait: type=${error.type}, waitUntil=${error.waitUntil?.toISOString() ?? 'none'}, tokenId=${error.tokenId ?? 'none'}`,
       );
-      await waitJob(pool, job.id, {
+      await backend.waitJob(job.id, {
         waitUntil: error.waitUntil,
         waitTokenId: error.tokenId,
         stepData: error.stepData,

package/src/queue.test.ts

@@ -141,6 +141,35 @@ describe('queue integration', () => {
     expect(job).toBeNull();
   });
 
+  it('should cleanup old completed jobs in batches', async () => {
+    // Add and complete 5 jobs
+    const ids: number[] = [];
+    for (let i = 0; i < 5; i++) {
+      const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(
+        pool,
+        {
+          jobType: 'email',
+          payload: { to: `batch-${i}@example.com` },
+        },
+      );
+      await queue.getNextBatch(pool, 'worker-batch-cleanup', 1);
+      await queue.completeJob(pool, jobId);
+      ids.push(jobId);
+    }
+    // Manually backdate all 5
+    await pool.query(
+      `UPDATE job_queue SET updated_at = NOW() - INTERVAL '31 days' WHERE id = ANY($1::int[])`,
+      [ids],
+    );
+    // Cleanup with batchSize=2 so it takes multiple iterations
+    const deleted = await queue.cleanupOldJobs(pool, 30, 2);
+    expect(deleted).toBe(5);
+    for (const id of ids) {
+      const job = await queue.getJob(pool, id);
+      expect(job).toBeNull();
+    }
+  });
+
   it('should cancel a scheduled job', async () => {
     const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
       jobType: 'email',

package/src/queue.ts

@@ -18,8 +18,6 @@ import {
   WaitpointRecord,
 } from './types.js';
 import { PostgresBackend } from './backends/postgres.js';
-import { randomUUID } from 'crypto';
-import { log } from './log-context.js';
 
 /* Thin wrappers — every function creates a lightweight backend wrapper
    around the given pool and forwards the call.  The class itself holds
@@ -94,7 +92,9 @@ export const retryJob = async (pool: Pool, jobId: number): Promise<void> =>
 export const cleanupOldJobs = async (
   pool: Pool,
   daysToKeep = 30,
-): Promise<number> => new PostgresBackend(pool).cleanupOldJobs(daysToKeep);
+  batchSize = 1000,
+): Promise<number> =>
+  new PostgresBackend(pool).cleanupOldJobs(daysToKeep, batchSize);
 
 export const cancelJob = async (pool: Pool, jobId: number): Promise<void> =>
   new PostgresBackend(pool).cancelJob(jobId);
@@ -214,12 +214,9 @@ export const updateProgress = async (
   progress: number,
 ): Promise<void> => new PostgresBackend(pool).updateProgress(jobId, progress);
 
-// ── Wait support functions (PostgreSQL-only) ─────────────────────────────────
+// ── Wait support functions (backward-compatible delegates) ────────────────────
 
-/**
- * Transition a job to 'waiting' status with wait_until and/or wait_token_id.
- * Saves step_data so the handler can resume from where it left off.
- */
+/** @deprecated Use backend.waitJob() directly. Delegates to PostgresBackend. */
 export const waitJob = async (
   pool: Pool,
   jobId: number,
@@ -228,266 +225,37 @@ export const waitJob = async (
     waitTokenId?: string;
     stepData: Record<string, any>;
   },
-): Promise<void> => {
-  const client = await pool.connect();
-  try {
-    const result = await client.query(
-      `
-      UPDATE job_queue
-      SET status = 'waiting',
-          wait_until = $2,
-          wait_token_id = $3,
-          step_data = $4,
-          locked_at = NULL,
-          locked_by = NULL,
-          updated_at = NOW()
-      WHERE id = $1 AND status = 'processing'
-    `,
-      [
-        jobId,
-        options.waitUntil ?? null,
-        options.waitTokenId ?? null,
-        JSON.stringify(options.stepData),
-      ],
-    );
-    if (result.rowCount === 0) {
-      log(
-        `Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`,
-      );
-      return;
-    }
-    await recordJobEvent(pool, jobId, JobEventType.Waiting, {
-      waitUntil: options.waitUntil?.toISOString() ?? null,
-      waitTokenId: options.waitTokenId ?? null,
-    });
-    log(`Job ${jobId} set to waiting`);
-  } catch (error) {
-    log(`Error setting job ${jobId} to waiting: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
+): Promise<void> => new PostgresBackend(pool).waitJob(jobId, options);
 
-/**
- * Update step_data for a job. Called after each ctx.run() step completes
- * to persist intermediate progress.
- */
+/** @deprecated Use backend.updateStepData() directly. Delegates to PostgresBackend. */
 export const updateStepData = async (
   pool: Pool,
   jobId: number,
   stepData: Record<string, any>,
-): Promise<void> => {
-  const client = await pool.connect();
-  try {
-    await client.query(
-      `UPDATE job_queue SET step_data = $2, updated_at = NOW() WHERE id = $1`,
-      [jobId, JSON.stringify(stepData)],
-    );
-  } catch (error) {
-    log(`Error updating step_data for job ${jobId}: ${error}`);
-    // Best-effort: do not throw to avoid killing the running handler
-  } finally {
-    client.release();
-  }
-};
-
-/**
- * Parse a timeout string like '10m', '1h', '24h', '7d' into milliseconds.
- */
-/**
- * Maximum allowed timeout in milliseconds (~365 days).
- * Prevents overflow to Infinity when computing Date offsets.
- */
-const MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1000;
+): Promise<void> => new PostgresBackend(pool).updateStepData(jobId, stepData);
 
-function parseTimeoutString(timeout: string): number {
-  const match = timeout.match(/^(\d+)(s|m|h|d)$/);
-  if (!match) {
-    throw new Error(
-      `Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`,
-    );
-  }
-  const value = parseInt(match[1], 10);
-  const unit = match[2];
-  let ms: number;
-  switch (unit) {
-    case 's':
-      ms = value * 1000;
-      break;
-    case 'm':
-      ms = value * 60 * 1000;
-      break;
-    case 'h':
-      ms = value * 60 * 60 * 1000;
-      break;
-    case 'd':
-      ms = value * 24 * 60 * 60 * 1000;
-      break;
-    default:
-      throw new Error(`Unknown timeout unit: "${unit}"`);
-  }
-  if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS) {
-    throw new Error(
-      `Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`,
-    );
-  }
-  return ms;
-}
-
-/**
- * Create a waitpoint token in the database.
- * The token can be used to pause a job until an external signal completes it.
- *
- * @param pool - The database pool
- * @param jobId - The job ID to associate with the token (null if created outside a handler)
- * @param options - Optional timeout and tags
- * @returns The created waitpoint token
- */
+/** @deprecated Use backend.createWaitpoint() directly. Delegates to PostgresBackend. */
 export const createWaitpoint = async (
   pool: Pool,
   jobId: number | null,
   options?: { timeout?: string; tags?: string[] },
-): Promise<{ id: string }> => {
-  const client = await pool.connect();
-  try {
-    const id = `wp_${randomUUID()}`;
-    let timeoutAt: Date | null = null;
-
-    if (options?.timeout) {
-      const ms = parseTimeoutString(options.timeout);
-      timeoutAt = new Date(Date.now() + ms);
-    }
-
-    await client.query(
-      `INSERT INTO waitpoints (id, job_id, status, timeout_at, tags) VALUES ($1, $2, 'waiting', $3, $4)`,
-      [id, jobId, timeoutAt, options?.tags ?? null],
-    );
+): Promise<{ id: string }> =>
+  new PostgresBackend(pool).createWaitpoint(jobId, options);
 
-    log(`Created waitpoint ${id} for job ${jobId}`);
-    return { id };
-  } catch (error) {
-    log(`Error creating waitpoint: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
-
-/**
- * Complete a waitpoint token, optionally providing output data.
- * This also moves the associated job from 'waiting' back to 'pending' so
- * it gets picked up by the polling loop.
- */
+/** @deprecated Use backend.completeWaitpoint() directly. Delegates to PostgresBackend. */
 export const completeWaitpoint = async (
   pool: Pool,
   tokenId: string,
   data?: any,
-): Promise<void> => {
-  const client = await pool.connect();
-  try {
-    await client.query('BEGIN');
-
-    // Update the waitpoint
-    const wpResult = await client.query(
-      `UPDATE waitpoints SET status = 'completed', output = $2, completed_at = NOW()
-       WHERE id = $1 AND status = 'waiting'
-       RETURNING job_id`,
-      [tokenId, data != null ? JSON.stringify(data) : null],
-    );
-
-    if (wpResult.rows.length === 0) {
-      await client.query('ROLLBACK');
-      log(`Waitpoint ${tokenId} not found or already completed`);
-      return;
-    }
-
-    const jobId = wpResult.rows[0].job_id;
-
-    // Move the associated job back to 'pending' so it gets picked up
-    if (jobId != null) {
-      await client.query(
-        `UPDATE job_queue
-         SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
-         WHERE id = $1 AND status = 'waiting'`,
-        [jobId],
-      );
-    }
-
-    await client.query('COMMIT');
-    log(`Completed waitpoint ${tokenId} for job ${jobId}`);
-  } catch (error) {
-    await client.query('ROLLBACK');
-    log(`Error completing waitpoint ${tokenId}: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
+): Promise<void> => new PostgresBackend(pool).completeWaitpoint(tokenId, data);
 
-/**
- * Retrieve a waitpoint token by its ID.
- */
+/** @deprecated Use backend.getWaitpoint() directly. Delegates to PostgresBackend. */
 export const getWaitpoint = async (
   pool: Pool,
   tokenId: string,
-): Promise<WaitpointRecord | null> => {
-  const client = await pool.connect();
-  try {
-    const result = await client.query(
-      `SELECT id, job_id AS "jobId", status, output, timeout_at AS "timeoutAt", created_at AS "createdAt", completed_at AS "completedAt", tags FROM waitpoints WHERE id = $1`,
-      [tokenId],
-    );
-    if (result.rows.length === 0) return null;
-    return result.rows[0] as WaitpointRecord;
-  } catch (error) {
-    log(`Error getting waitpoint ${tokenId}: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
-
-/**
- * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
- * Should be called periodically (e.g., alongside reclaimStuckJobs).
- */
-export const expireTimedOutWaitpoints = async (pool: Pool): Promise<number> => {
-  const client = await pool.connect();
-  try {
-    await client.query('BEGIN');
-
-    // Find and expire timed-out waitpoints
-    const result = await client.query(
-      `UPDATE waitpoints
-       SET status = 'timed_out'
-       WHERE status = 'waiting' AND timeout_at IS NOT NULL AND timeout_at <= NOW()
-       RETURNING id, job_id`,
-    );
-
-    // Move associated jobs back to 'pending'
-    for (const row of result.rows) {
-      if (row.job_id != null) {
-        await client.query(
-          `UPDATE job_queue
-           SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
-           WHERE id = $1 AND status = 'waiting'`,
-          [row.job_id],
-        );
-      }
-    }
+): Promise<WaitpointRecord | null> =>
+  new PostgresBackend(pool).getWaitpoint(tokenId);
 
-    await client.query('COMMIT');
-    const count = result.rowCount || 0;
-    if (count > 0) {
-      log(`Expired ${count} timed-out waitpoints`);
-    }
-    return count;
-  } catch (error) {
-    await client.query('ROLLBACK');
-    log(`Error expiring timed-out waitpoints: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
+/** @deprecated Use backend.expireTimedOutWaitpoints() directly. Delegates to PostgresBackend. */
+export const expireTimedOutWaitpoints = async (pool: Pool): Promise<number> =>
+  new PostgresBackend(pool).expireTimedOutWaitpoints();

package/src/types.ts

@@ -485,6 +485,23 @@ export interface PostgresJobQueueConfig {
     user?: string;
     password?: string;
     ssl?: DatabaseSSLConfig;
+    /**
+     * Maximum number of clients in the pool (default: 10).
+     * Increase when running multiple processors in the same process.
+     */
+    max?: number;
+    /**
+     * Minimum number of idle clients in the pool (default: 0).
+     */
+    min?: number;
+    /**
+     * Milliseconds a client must sit idle before being closed (default: 10000).
+     */
+    idleTimeoutMillis?: number;
+    /**
+     * Milliseconds to wait for a connection before throwing (default: 0, no timeout).
+     */
+    connectionTimeoutMillis?: number;
   };
   verbose?: boolean;
 }
@@ -690,12 +707,21 @@ export interface JobQueue<PayloadMap> {
   retryJob: (jobId: number) => Promise<void>;
   /**
    * Cleanup jobs that are older than the specified number of days.
+   * Deletes in batches for scale safety.
+   * @param daysToKeep - Number of days to retain completed jobs (default 30).
+   * @param batchSize - Number of rows to delete per batch (default 1000 for PostgreSQL, 200 for Redis).
    */
-  cleanupOldJobs: (daysToKeep?: number) => Promise<number>;
+  cleanupOldJobs: (daysToKeep?: number, batchSize?: number) => Promise<number>;
   /**
    * Cleanup job events that are older than the specified number of days.
+   * Deletes in batches for scale safety.
+   * @param daysToKeep - Number of days to retain events (default 30).
+   * @param batchSize - Number of rows to delete per batch (default 1000).
    */
-  cleanupOldJobEvents: (daysToKeep?: number) => Promise<number>;
+  cleanupOldJobEvents: (
+    daysToKeep?: number,
+    batchSize?: number,
+  ) => Promise<number>;
   /**
    * Cancel a job given its ID.
    * - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
@@ -779,8 +805,6 @@ export interface JobQueue<PayloadMap> {
    * Tokens can be completed externally to resume a waiting job.
    * Can be called outside of handlers (e.g., from an API route).
    *
-   * **PostgreSQL backend only.** Throws if the backend is Redis.
-   *
    * @param options - Optional token configuration (timeout, tags).
    * @returns A token object with `id`.
    */
@@ -790,8 +814,6 @@ export interface JobQueue<PayloadMap> {
    * Complete a waitpoint token, resuming the associated waiting job.
    * Can be called from anywhere (API routes, external services, etc.).
    *
-   * **PostgreSQL backend only.** Throws if the backend is Redis.
-   *
    * @param tokenId - The ID of the token to complete.
    * @param data - Optional data to pass to the waiting handler.
    */
@@ -800,8 +822,6 @@ export interface JobQueue<PayloadMap> {
   /**
    * Retrieve a waitpoint token by its ID.
    *
-   * **PostgreSQL backend only.** Throws if the backend is Redis.
-   *
    * @param tokenId - The ID of the token to retrieve.
    * @returns The token record, or null if not found.
    */
@@ -811,8 +831,6 @@ export interface JobQueue<PayloadMap> {
    * Expire timed-out waitpoint tokens and resume their associated jobs.
    * Call this periodically (e.g., alongside `reclaimStuckJobs`).
    *
-   * **PostgreSQL backend only.** Throws if the backend is Redis.
-   *
    * @returns The number of tokens that were expired.
    */
   expireTimedOutTokens: () => Promise<number>;