@nicnocquee/dataqueue 1.30.0 → 1.32.0

package/src/queue.ts

@@ -18,8 +18,6 @@ import {
   WaitpointRecord,
 } from './types.js';
 import { PostgresBackend } from './backends/postgres.js';
-import { randomUUID } from 'crypto';
-import { log } from './log-context.js';
 
 /* Thin wrappers — every function creates a lightweight backend wrapper
    around the given pool and forwards the call.  The class itself holds
@@ -94,7 +92,9 @@ export const retryJob = async (pool: Pool, jobId: number): Promise<void> =>
 export const cleanupOldJobs = async (
   pool: Pool,
   daysToKeep = 30,
-): Promise<number> => new PostgresBackend(pool).cleanupOldJobs(daysToKeep);
+  batchSize = 1000,
+): Promise<number> =>
+  new PostgresBackend(pool).cleanupOldJobs(daysToKeep, batchSize);
 
 export const cancelJob = async (pool: Pool, jobId: number): Promise<void> =>
   new PostgresBackend(pool).cancelJob(jobId);
@@ -214,12 +214,9 @@ export const updateProgress = async (
   progress: number,
 ): Promise<void> => new PostgresBackend(pool).updateProgress(jobId, progress);
 
-// ── Wait support functions (PostgreSQL-only) ─────────────────────────────────
+// ── Wait support functions (backward-compatible delegates) ────────────────────
 
-/**
- * Transition a job to 'waiting' status with wait_until and/or wait_token_id.
- * Saves step_data so the handler can resume from where it left off.
- */
+/** @deprecated Use backend.waitJob() directly. Delegates to PostgresBackend. */
 export const waitJob = async (
   pool: Pool,
   jobId: number,
@@ -228,266 +225,37 @@ export const waitJob = async (
     waitTokenId?: string;
     stepData: Record<string, any>;
   },
-): Promise<void> => {
-  const client = await pool.connect();
-  try {
-    const result = await client.query(
-      `
-      UPDATE job_queue
-      SET status = 'waiting',
-          wait_until = $2,
-          wait_token_id = $3,
-          step_data = $4,
-          locked_at = NULL,
-          locked_by = NULL,
-          updated_at = NOW()
-      WHERE id = $1 AND status = 'processing'
-    `,
-      [
-        jobId,
-        options.waitUntil ?? null,
-        options.waitTokenId ?? null,
-        JSON.stringify(options.stepData),
-      ],
-    );
-    if (result.rowCount === 0) {
-      log(
-        `Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`,
-      );
-      return;
-    }
-    await recordJobEvent(pool, jobId, JobEventType.Waiting, {
-      waitUntil: options.waitUntil?.toISOString() ?? null,
-      waitTokenId: options.waitTokenId ?? null,
-    });
-    log(`Job ${jobId} set to waiting`);
-  } catch (error) {
-    log(`Error setting job ${jobId} to waiting: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
+): Promise<void> => new PostgresBackend(pool).waitJob(jobId, options);
 
-/**
- * Update step_data for a job. Called after each ctx.run() step completes
- * to persist intermediate progress.
- */
+/** @deprecated Use backend.updateStepData() directly. Delegates to PostgresBackend. */
 export const updateStepData = async (
   pool: Pool,
   jobId: number,
   stepData: Record<string, any>,
-): Promise<void> => {
-  const client = await pool.connect();
-  try {
-    await client.query(
-      `UPDATE job_queue SET step_data = $2, updated_at = NOW() WHERE id = $1`,
-      [jobId, JSON.stringify(stepData)],
-    );
-  } catch (error) {
-    log(`Error updating step_data for job ${jobId}: ${error}`);
-    // Best-effort: do not throw to avoid killing the running handler
-  } finally {
-    client.release();
-  }
-};
-
-/**
- * Parse a timeout string like '10m', '1h', '24h', '7d' into milliseconds.
- */
-/**
- * Maximum allowed timeout in milliseconds (~365 days).
- * Prevents overflow to Infinity when computing Date offsets.
- */
-const MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1000;
+): Promise<void> => new PostgresBackend(pool).updateStepData(jobId, stepData);
 
-function parseTimeoutString(timeout: string): number {
-  const match = timeout.match(/^(\d+)(s|m|h|d)$/);
-  if (!match) {
-    throw new Error(
-      `Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`,
-    );
-  }
-  const value = parseInt(match[1], 10);
-  const unit = match[2];
-  let ms: number;
-  switch (unit) {
-    case 's':
-      ms = value * 1000;
-      break;
-    case 'm':
-      ms = value * 60 * 1000;
-      break;
-    case 'h':
-      ms = value * 60 * 60 * 1000;
-      break;
-    case 'd':
-      ms = value * 24 * 60 * 60 * 1000;
-      break;
-    default:
-      throw new Error(`Unknown timeout unit: "${unit}"`);
-  }
-  if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS) {
-    throw new Error(
-      `Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`,
-    );
-  }
-  return ms;
-}
-
-/**
- * Create a waitpoint token in the database.
- * The token can be used to pause a job until an external signal completes it.
- *
- * @param pool - The database pool
- * @param jobId - The job ID to associate with the token (null if created outside a handler)
- * @param options - Optional timeout and tags
- * @returns The created waitpoint token
- */
+/** @deprecated Use backend.createWaitpoint() directly. Delegates to PostgresBackend. */
 export const createWaitpoint = async (
   pool: Pool,
   jobId: number | null,
   options?: { timeout?: string; tags?: string[] },
-): Promise<{ id: string }> => {
-  const client = await pool.connect();
-  try {
-    const id = `wp_${randomUUID()}`;
-    let timeoutAt: Date | null = null;
-
-    if (options?.timeout) {
-      const ms = parseTimeoutString(options.timeout);
-      timeoutAt = new Date(Date.now() + ms);
-    }
-
-    await client.query(
-      `INSERT INTO waitpoints (id, job_id, status, timeout_at, tags) VALUES ($1, $2, 'waiting', $3, $4)`,
-      [id, jobId, timeoutAt, options?.tags ?? null],
-    );
+): Promise<{ id: string }> =>
+  new PostgresBackend(pool).createWaitpoint(jobId, options);
 
-    log(`Created waitpoint ${id} for job ${jobId}`);
-    return { id };
-  } catch (error) {
-    log(`Error creating waitpoint: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
-
-/**
- * Complete a waitpoint token, optionally providing output data.
- * This also moves the associated job from 'waiting' back to 'pending' so
- * it gets picked up by the polling loop.
- */
+/** @deprecated Use backend.completeWaitpoint() directly. Delegates to PostgresBackend. */
 export const completeWaitpoint = async (
   pool: Pool,
   tokenId: string,
   data?: any,
-): Promise<void> => {
-  const client = await pool.connect();
-  try {
-    await client.query('BEGIN');
-
-    // Update the waitpoint
-    const wpResult = await client.query(
-      `UPDATE waitpoints SET status = 'completed', output = $2, completed_at = NOW()
-       WHERE id = $1 AND status = 'waiting'
-       RETURNING job_id`,
-      [tokenId, data != null ? JSON.stringify(data) : null],
-    );
-
-    if (wpResult.rows.length === 0) {
-      await client.query('ROLLBACK');
-      log(`Waitpoint ${tokenId} not found or already completed`);
-      return;
-    }
-
-    const jobId = wpResult.rows[0].job_id;
-
-    // Move the associated job back to 'pending' so it gets picked up
-    if (jobId != null) {
-      await client.query(
-        `UPDATE job_queue
-         SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
-         WHERE id = $1 AND status = 'waiting'`,
-        [jobId],
-      );
-    }
-
-    await client.query('COMMIT');
-    log(`Completed waitpoint ${tokenId} for job ${jobId}`);
-  } catch (error) {
-    await client.query('ROLLBACK');
-    log(`Error completing waitpoint ${tokenId}: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
+): Promise<void> => new PostgresBackend(pool).completeWaitpoint(tokenId, data);
 
-/**
- * Retrieve a waitpoint token by its ID.
- */
+/** @deprecated Use backend.getWaitpoint() directly. Delegates to PostgresBackend. */
 export const getWaitpoint = async (
   pool: Pool,
   tokenId: string,
-): Promise<WaitpointRecord | null> => {
-  const client = await pool.connect();
-  try {
-    const result = await client.query(
-      `SELECT id, job_id AS "jobId", status, output, timeout_at AS "timeoutAt", created_at AS "createdAt", completed_at AS "completedAt", tags FROM waitpoints WHERE id = $1`,
-      [tokenId],
-    );
-    if (result.rows.length === 0) return null;
-    return result.rows[0] as WaitpointRecord;
-  } catch (error) {
-    log(`Error getting waitpoint ${tokenId}: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
-
-/**
- * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
- * Should be called periodically (e.g., alongside reclaimStuckJobs).
- */
-export const expireTimedOutWaitpoints = async (pool: Pool): Promise<number> => {
-  const client = await pool.connect();
-  try {
-    await client.query('BEGIN');
-
-    // Find and expire timed-out waitpoints
-    const result = await client.query(
-      `UPDATE waitpoints
-       SET status = 'timed_out'
-       WHERE status = 'waiting' AND timeout_at IS NOT NULL AND timeout_at <= NOW()
-       RETURNING id, job_id`,
-    );
-
-    // Move associated jobs back to 'pending'
-    for (const row of result.rows) {
-      if (row.job_id != null) {
-        await client.query(
-          `UPDATE job_queue
-           SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
-           WHERE id = $1 AND status = 'waiting'`,
-          [row.job_id],
-        );
-      }
-    }
+): Promise<WaitpointRecord | null> =>
+  new PostgresBackend(pool).getWaitpoint(tokenId);
 
-    await client.query('COMMIT');
-    const count = result.rowCount || 0;
-    if (count > 0) {
-      log(`Expired ${count} timed-out waitpoints`);
-    }
-    return count;
-  } catch (error) {
-    await client.query('ROLLBACK');
-    log(`Error expiring timed-out waitpoints: ${error}`);
-    throw error;
-  } finally {
-    client.release();
-  }
-};
+/** @deprecated Use backend.expireTimedOutWaitpoints() directly. Delegates to PostgresBackend. */
+export const expireTimedOutWaitpoints = async (pool: Pool): Promise<number> =>
+  new PostgresBackend(pool).expireTimedOutWaitpoints();

package/src/types.ts

@@ -485,6 +485,23 @@ export interface PostgresJobQueueConfig {
     user?: string;
     password?: string;
     ssl?: DatabaseSSLConfig;
+    /**
+     * Maximum number of clients in the pool (default: 10).
+     * Increase when running multiple processors in the same process.
+     */
+    max?: number;
+    /**
+     * Minimum number of idle clients in the pool (default: 0).
+     */
+    min?: number;
+    /**
+     * Milliseconds a client must sit idle before being closed (default: 10000).
+     */
+    idleTimeoutMillis?: number;
+    /**
+     * Milliseconds to wait for a connection before throwing (default: 0, no timeout).
+     */
+    connectionTimeoutMillis?: number;
   };
   verbose?: boolean;
 }
@@ -533,6 +550,90 @@ export type JobQueueConfigLegacy = PostgresJobQueueConfig;
 
 export type TagQueryMode = 'exact' | 'all' | 'any' | 'none';
 
+// ── Cron schedule types ──────────────────────────────────────────────
+
+/**
+ * Status of a cron schedule.
+ */
+export type CronScheduleStatus = 'active' | 'paused';
+
+/**
+ * Options for creating a recurring cron schedule.
+ * Each schedule defines a recurring job that is automatically enqueued
+ * when its cron expression matches.
+ */
+export interface CronScheduleOptions<
+  PayloadMap,
+  T extends JobType<PayloadMap>,
+> {
+  /** Unique human-readable name for the schedule. */
+  scheduleName: string;
+  /** Standard cron expression (5 fields, e.g. "0 * * * *"). */
+  cronExpression: string;
+  /** Job type from the PayloadMap. */
+  jobType: T;
+  /** Payload for each job instance. */
+  payload: PayloadMap[T];
+  /** Maximum retry attempts for each job instance (default: 3). */
+  maxAttempts?: number;
+  /** Priority for each job instance (default: 0). */
+  priority?: number;
+  /** Timeout in milliseconds for each job instance. */
+  timeoutMs?: number;
+  /** Whether to force-kill the job on timeout (default: false). */
+  forceKillOnTimeout?: boolean;
+  /** Tags for each job instance. */
+  tags?: string[];
+  /** IANA timezone string for cron evaluation (default: "UTC"). */
+  timezone?: string;
+  /**
+   * Whether to allow overlapping job instances (default: false).
+   * When false, a new job will not be enqueued if the previous instance
+   * is still pending, processing, or waiting.
+   */
+  allowOverlap?: boolean;
+}
+
+/**
+ * A persisted cron schedule record.
+ */
+export interface CronScheduleRecord {
+  id: number;
+  scheduleName: string;
+  cronExpression: string;
+  jobType: string;
+  payload: any;
+  maxAttempts: number;
+  priority: number;
+  timeoutMs: number | null;
+  forceKillOnTimeout: boolean;
+  tags: string[] | undefined;
+  timezone: string;
+  allowOverlap: boolean;
+  status: CronScheduleStatus;
+  lastEnqueuedAt: Date | null;
+  lastJobId: number | null;
+  nextRunAt: Date | null;
+  createdAt: Date;
+  updatedAt: Date;
+}
+
+/**
+ * Options for editing an existing cron schedule.
+ * All fields are optional; only provided fields are updated.
+ */
+export interface EditCronScheduleOptions {
+  cronExpression?: string;
+  payload?: any;
+  maxAttempts?: number;
+  priority?: number;
+  timeoutMs?: number | null;
+  forceKillOnTimeout?: boolean;
+  tags?: string[] | null;
+  timezone?: string;
+  allowOverlap?: boolean;
+}
+
 export interface JobQueue<PayloadMap> {
   /**
    * Add a job to the job queue.
@@ -606,12 +707,21 @@ export interface JobQueue<PayloadMap> {
   retryJob: (jobId: number) => Promise<void>;
   /**
    * Cleanup jobs that are older than the specified number of days.
+   * Deletes in batches for scale safety.
+   * @param daysToKeep - Number of days to retain completed jobs (default 30).
+   * @param batchSize - Number of rows to delete per batch (default 1000 for PostgreSQL, 200 for Redis).
    */
-  cleanupOldJobs: (daysToKeep?: number) => Promise<number>;
+  cleanupOldJobs: (daysToKeep?: number, batchSize?: number) => Promise<number>;
   /**
    * Cleanup job events that are older than the specified number of days.
+   * Deletes in batches for scale safety.
+   * @param daysToKeep - Number of days to retain events (default 30).
+   * @param batchSize - Number of rows to delete per batch (default 1000).
    */
-  cleanupOldJobEvents: (daysToKeep?: number) => Promise<number>;
+  cleanupOldJobEvents: (
+    daysToKeep?: number,
+    batchSize?: number,
+  ) => Promise<number>;
   /**
    * Cancel a job given its ID.
    * - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
@@ -695,8 +805,6 @@ export interface JobQueue<PayloadMap> {
    * Tokens can be completed externally to resume a waiting job.
    * Can be called outside of handlers (e.g., from an API route).
    *
-   * **PostgreSQL backend only.** Throws if the backend is Redis.
-   *
    * @param options - Optional token configuration (timeout, tags).
    * @returns A token object with `id`.
    */
@@ -706,8 +814,6 @@ export interface JobQueue<PayloadMap> {
    * Complete a waitpoint token, resuming the associated waiting job.
    * Can be called from anywhere (API routes, external services, etc.).
    *
-   * **PostgreSQL backend only.** Throws if the backend is Redis.
-   *
    * @param tokenId - The ID of the token to complete.
    * @param data - Optional data to pass to the waiting handler.
    */
@@ -716,8 +822,6 @@ export interface JobQueue<PayloadMap> {
   /**
    * Retrieve a waitpoint token by its ID.
    *
-   * **PostgreSQL backend only.** Throws if the backend is Redis.
-   *
    * @param tokenId - The ID of the token to retrieve.
    * @returns The token record, or null if not found.
    */
@@ -727,12 +831,75 @@ export interface JobQueue<PayloadMap> {
    * Expire timed-out waitpoint tokens and resume their associated jobs.
    * Call this periodically (e.g., alongside `reclaimStuckJobs`).
    *
-   * **PostgreSQL backend only.** Throws if the backend is Redis.
-   *
    * @returns The number of tokens that were expired.
    */
   expireTimedOutTokens: () => Promise<number>;
 
+  // ── Cron schedule operations ────────────────────────────────────────
+
+  /**
+   * Add a recurring cron schedule. The processor automatically enqueues
+   * due cron jobs before each batch, so no manual triggering is needed.
+   *
+   * @returns The ID of the created schedule.
+   * @throws If the cron expression is invalid or the schedule name is already taken.
+   */
+  addCronJob: <T extends JobType<PayloadMap>>(
+    options: CronScheduleOptions<PayloadMap, T>,
+  ) => Promise<number>;
+
+  /**
+   * Get a cron schedule by its ID.
+   */
+  getCronJob: (id: number) => Promise<CronScheduleRecord | null>;
+
+  /**
+   * Get a cron schedule by its unique name.
+   */
+  getCronJobByName: (name: string) => Promise<CronScheduleRecord | null>;
+
+  /**
+   * List all cron schedules, optionally filtered by status.
+   */
+  listCronJobs: (status?: CronScheduleStatus) => Promise<CronScheduleRecord[]>;
+
+  /**
+   * Remove a cron schedule by its ID. Does not cancel any already-enqueued jobs.
+   */
+  removeCronJob: (id: number) => Promise<void>;
+
+  /**
+   * Pause a cron schedule. Paused schedules are skipped by `enqueueDueCronJobs()`.
+   */
+  pauseCronJob: (id: number) => Promise<void>;
+
+  /**
+   * Resume a paused cron schedule.
+   */
+  resumeCronJob: (id: number) => Promise<void>;
+
+  /**
+   * Edit an existing cron schedule. Only provided fields are updated.
+   * If `cronExpression` or `timezone` changes, `nextRunAt` is recalculated.
+   */
+  editCronJob: (id: number, updates: EditCronScheduleOptions) => Promise<void>;
+
+  /**
+   * Check all active cron schedules and enqueue jobs for any whose
+   * `nextRunAt` has passed. When `allowOverlap` is false (the default),
+   * a new job is not enqueued if the previous instance is still
+   * pending, processing, or waiting.
+   *
+   * **Note:** The processor calls this automatically before each batch,
+   * so you typically do not need to call it yourself. It is exposed for
+   * manual use in tests or one-off scripts.
+   *
+   * @returns The number of jobs that were enqueued.
+   */
+  enqueueDueCronJobs: () => Promise<number>;
+
+  // ── Advanced access ───────────────────────────────────────────────────
+
   /**
    * Get the PostgreSQL database pool.
    * Throws if the backend is not PostgreSQL.