@nicnocquee/dataqueue 1.22.0 → 1.25.0

package/src/test-util.ts

@@ -65,3 +65,35 @@ export async function destroyTestDb(dbName: string) {
   await adminPool.query(`DROP DATABASE IF EXISTS ${dbName}`);
   await adminPool.end();
 }
+
+/**
+ * Create a Redis test setup with a unique prefix to isolate tests.
+ * Returns the prefix and a cleanup function.
+ */
+export function createRedisTestPrefix(): string {
+  return `test_${randomUUID().replace(/-/g, '').slice(0, 12)}:`;
+}
+
+/**
+ * Flush all keys with the given prefix from Redis.
+ */
+export async function cleanupRedisPrefix(
+  redisClient: any,
+  prefix: string,
+): Promise<void> {
+  // Use SCAN to find all keys with the prefix and delete them
+  let cursor = '0';
+  do {
+    const [nextCursor, keys] = await redisClient.scan(
+      cursor,
+      'MATCH',
+      `${prefix}*`,
+      'COUNT',
+      100,
+    );
+    cursor = nextCursor;
+    if (keys.length > 0) {
+      await redisClient.del(...keys);
+    }
+  } while (cursor !== '0');
+}

package/src/types.ts

@@ -1,5 +1,3 @@
-import { Pool } from 'pg';
-
 // Utility type for job type keys
 export type JobType<PayloadMap> = keyof PayloadMap & string;
 
@@ -13,12 +11,83 @@ export interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
    * Timeout for this job in milliseconds. If not set, uses the processor default or unlimited.
    */
   timeoutMs?: number;
+  /**
+   * If true, the job will be forcefully terminated (using Worker Threads) when timeout is reached.
+   * If false (default), the job will only receive an AbortSignal and must handle the abort gracefully.
+   *
+   * **⚠️ RUNTIME REQUIREMENTS**: This option requires **Node.js** and uses the `worker_threads` module.
+   * It will **not work** in Bun or other runtimes that don't support Node.js worker threads.
+   *
+   * **IMPORTANT**: When `forceKillOnTimeout` is true, the handler must be serializable. This means:
+   * - The handler should be a standalone function (not a closure over external variables)
+   * - It should not capture variables from outer scopes that reference external dependencies
+   * - It should not use 'this' context unless it's a bound method
+   * - All dependencies must be importable in the worker thread context
+   *
+   * **Examples of serializable handlers:**
+   * ```ts
+   * // ✅ Good - standalone function
+   * const handler = async (payload, signal) => {
+   *   await doSomething(payload);
+   * };
+   *
+   * // ✅ Good - function that imports dependencies
+   * const handler = async (payload, signal) => {
+   *   const { api } = await import('./api');
+   *   await api.call(payload);
+   * };
+   *
+   * // ❌ Bad - closure over external variable
+   * const db = getDatabase();
+   * const handler = async (payload, signal) => {
+   *   await db.query(payload); // 'db' is captured from closure
+   * };
+   *
+   * // ❌ Bad - uses 'this' context
+   * class MyHandler {
+   *   async handle(payload, signal) {
+   *     await this.doSomething(payload); // 'this' won't work
+   *   }
+   * }
+   * ```
+   *
+   * If your handler doesn't meet these requirements, use `forceKillOnTimeout: false` (default)
+   * and ensure your handler checks `signal.aborted` to exit gracefully.
+   *
+   * Note: forceKillOnTimeout requires timeoutMs to be set.
+   */
+  forceKillOnTimeout?: boolean;
   /**
    * Tags for this job. Used for grouping, searching, or batch operations.
    */
   tags?: string[];
+  /**
+   * Optional idempotency key. When provided, ensures that only one job exists for a given key.
+   * If a job with the same idempotency key already exists, `addJob` returns the existing job's ID
+   * instead of creating a duplicate.
+   *
+   * Useful for preventing duplicate jobs caused by retries, double-clicks, webhook replays,
+   * or serverless function re-invocations.
+   *
+   * The key is unique across the entire `job_queue` table regardless of job status.
+   * Once a key exists, it cannot be reused until the job is cleaned up (via `cleanupOldJobs`).
+   */
+  idempotencyKey?: string;
 }
 
+/**
+ * Options for editing a pending job.
+ * All fields are optional and only provided fields will be updated.
+ * Note: jobType cannot be changed.
+ * timeoutMs and tags can be set to null to clear them.
+ */
+export type EditJobOptions<PayloadMap, T extends JobType<PayloadMap>> = Partial<
+  Omit<JobOptions<PayloadMap, T>, 'jobType'>
+> & {
+  timeoutMs?: number | null;
+  tags?: string[] | null;
+};
+
 export enum JobEventType {
   Added = 'added',
   Processing = 'processing',
@@ -26,6 +95,9 @@ export enum JobEventType {
   Failed = 'failed',
   Cancelled = 'cancelled',
   Retried = 'retried',
+  Edited = 'edited',
+  Prolonged = 'prolonged',
+  Waiting = 'waiting',
 }
 
 export interface JobEvent {
@@ -47,7 +119,8 @@ export type JobStatus =
   | 'processing'
   | 'completed'
   | 'failed'
-  | 'cancelled';
+  | 'cancelled'
+  | 'waiting';
 
 export interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
   id: number;
@@ -69,6 +142,11 @@ export interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
    * Timeout for this job in milliseconds (null means no timeout).
    */
   timeoutMs?: number | null;
+  /**
+   * If true, the job will be forcefully terminated (using Worker Threads) when timeout is reached.
+   * If false (default), the job will only receive an AbortSignal and must handle the abort gracefully.
+   */
+  forceKillOnTimeout?: boolean | null;
   /**
    * The reason for the last failure, if any.
    */
@@ -97,11 +175,211 @@ export interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
    * Tags for this job. Used for grouping, searching, or batch operations.
    */
   tags?: string[];
+  /**
+   * The idempotency key for this job, if one was provided when the job was created.
+   */
+  idempotencyKey?: string | null;
+  /**
+   * The time the job is waiting until (for time-based waits).
+   */
+  waitUntil?: Date | null;
+  /**
+   * The waitpoint token ID the job is waiting for (for token-based waits).
+   */
+  waitTokenId?: string | null;
+  /**
+   * Step data for the job. Stores completed step results for replay on re-invocation.
+   */
+  stepData?: Record<string, any>;
+  /**
+   * Progress percentage for the job (0-100), or null if no progress has been reported.
+   * Updated by the handler via `ctx.setProgress(percent)`.
+   */
+  progress?: number | null;
+}
+
+/**
+ * Callback registered via `onTimeout`. Invoked when the timeout fires, before the AbortSignal is triggered.
+ * Return a number (ms) to extend the timeout, or return nothing to let the timeout proceed.
+ */
+export type OnTimeoutCallback = () => number | void | undefined;
+
+/**
+ * Context object passed to job handlers as the third argument.
+ * Provides mechanisms to extend the job's timeout while it's running,
+ * as well as step tracking and wait capabilities.
+ */
+export interface JobContext {
+  /**
+   * Proactively reset the timeout deadline.
+   * - If `ms` is provided, sets the deadline to `ms` milliseconds from now.
+   * - If omitted, resets the deadline to the original `timeoutMs` from now (heartbeat-style).
+   * - No-op if the job has no timeout set or if `forceKillOnTimeout` is true.
+   */
+  prolong: (ms?: number) => void;
+
+  /**
+   * Register a callback that is invoked when the timeout fires, **before** the AbortSignal is triggered.
+   * - If the callback returns a number > 0, the timeout is reset to that many ms from now.
+   * - If the callback returns `undefined`, `null`, `0`, or a negative number, the timeout proceeds normally.
+   * - The callback may be invoked multiple times if the job keeps extending.
+   * - Only one callback can be registered; subsequent calls replace the previous one.
+   * - No-op if the job has no timeout set or if `forceKillOnTimeout` is true.
+   */
+  onTimeout: (callback: OnTimeoutCallback) => void;
+
+  /**
+   * Execute a named step with memoization. If the step was already completed
+   * in a previous invocation (e.g., before a wait), the cached result is returned
+   * without re-executing the function.
+   *
+   * Step names must be unique within a handler and stable across re-invocations.
+   *
+   * @param stepName - A unique identifier for this step.
+   * @param fn - The function to execute. Its return value is cached.
+   * @returns The result of the step (from cache or fresh execution).
+   */
+  run: <T>(stepName: string, fn: () => Promise<T>) => Promise<T>;
+
+  /**
+   * Wait for a specified duration before continuing execution.
+   * The job will be paused and resumed after the duration elapses.
+   *
+   * When this is called, the handler throws a WaitSignal internally.
+   * The job is set to 'waiting' status and will be re-invoked after the
+   * specified duration. All steps completed via `ctx.run()` before this
+   * call will be replayed from cache on re-invocation.
+   *
+   * @param duration - The duration to wait (e.g., `{ hours: 1 }`, `{ days: 7 }`).
+   */
+  waitFor: (duration: WaitDuration) => Promise<void>;
+
+  /**
+   * Wait until a specific date/time before continuing execution.
+   * The job will be paused and resumed at (or after) the specified date.
+   *
+   * @param date - The date to wait until.
+   */
+  waitUntil: (date: Date) => Promise<void>;
+
+  /**
+   * Create a waitpoint token. The token can be completed externally
+   * (by calling `jobQueue.completeToken()`) to resume a waiting job.
+   *
+   * Tokens can be created inside handlers or outside (via `jobQueue.createToken()`).
+   *
+   * @param options - Optional token configuration (timeout, tags).
+   * @returns A token object with `id` that can be passed to `waitForToken()`.
+   */
+  createToken: (options?: CreateTokenOptions) => Promise<WaitToken>;
+
+  /**
+   * Wait for a waitpoint token to be completed by an external signal.
+   * The job will be paused until `jobQueue.completeToken(tokenId, data)` is called
+   * or the token times out.
+   *
+   * @param tokenId - The ID of the token to wait for.
+   * @returns A result object indicating success or timeout.
+   */
+  waitForToken: <T = any>(tokenId: string) => Promise<WaitTokenResult<T>>;
+
+  /**
+   * Report progress for this job (0-100).
+   * The value is persisted to the database and can be read by clients
+   * via `getJob()` or the React SDK's `useJob()` hook.
+   *
+   * @param percent - Progress percentage (0-100). Values are rounded to the nearest integer.
+   * @throws If percent is outside the 0-100 range.
+   */
+  setProgress: (percent: number) => Promise<void>;
+}
+
+/**
+ * Duration specification for `ctx.waitFor()`.
+ * At least one field must be provided. Fields are additive.
+ */
+export interface WaitDuration {
+  seconds?: number;
+  minutes?: number;
+  hours?: number;
+  days?: number;
+  weeks?: number;
+  months?: number;
+  years?: number;
+}
+
+/**
+ * Options for creating a waitpoint token.
+ */
+export interface CreateTokenOptions {
+  /**
+   * Maximum time to wait for the token to be completed.
+   * Accepts a duration string like '10m', '1h', '24h', '7d'.
+   * If not provided, the token has no timeout.
+   */
+  timeout?: string;
+  /**
+   * Tags to attach to the token for filtering.
+   */
+  tags?: string[];
+}
+
+/**
+ * A waitpoint token returned by `ctx.createToken()`.
+ */
+export interface WaitToken {
+  /** The unique token ID. */
+  id: string;
+}
+
+/**
+ * Result of `ctx.waitForToken()`.
+ */
+export type WaitTokenResult<T = any> =
+  | { ok: true; output: T }
+  | { ok: false; error: string };
+
+/**
+ * Internal signal thrown by wait methods to pause handler execution.
+ * This is not a real error -- the processor catches it and transitions the job to 'waiting' status.
+ */
+export class WaitSignal extends Error {
+  readonly isWaitSignal = true;
+
+  constructor(
+    public readonly type: 'duration' | 'date' | 'token',
+    public readonly waitUntil: Date | undefined,
+    public readonly tokenId: string | undefined,
+    public readonly stepData: Record<string, any>,
+  ) {
+    super('WaitSignal');
+    this.name = 'WaitSignal';
+  }
+}
+
+/**
+ * Status of a waitpoint token.
+ */
+export type WaitpointStatus = 'waiting' | 'completed' | 'timed_out';
+
+/**
+ * A waitpoint record from the database.
+ */
+export interface WaitpointRecord {
+  id: string;
+  jobId: number | null;
+  status: WaitpointStatus;
+  output: any;
+  timeoutAt: Date | null;
+  createdAt: Date;
+  completedAt: Date | null;
+  tags: string[] | null;
 }
 
 export type JobHandler<PayloadMap, T extends keyof PayloadMap> = (
   payload: PayloadMap[T],
   signal: AbortSignal,
+  ctx: JobContext,
 ) => Promise<void>;
 
 export type JobHandlers<PayloadMap> = {
@@ -149,8 +427,18 @@ export interface Processor {
   startInBackground: () => void;
   /**
    * Stop the job processor that runs in the background.
+   * Does not wait for in-flight jobs to complete.
    */
   stop: () => void;
+  /**
+   * Stop the job processor and wait for all in-flight jobs to complete.
+   * Useful for graceful shutdown (e.g., SIGTERM handling).
+   * No new batches will be started after calling this method.
+   *
+   * @param timeoutMs - Maximum time to wait for in-flight jobs (default: 30000ms).
+   *   If jobs don't complete within this time, the promise resolves anyway.
+   */
+  stopAndDrain: (timeoutMs?: number) => Promise<void>;
   /**
    * Check if the job processor is running.
    */
@@ -183,7 +471,12 @@ export interface DatabaseSSLConfig {
   rejectUnauthorized?: boolean;
 }
 
-export interface JobQueueConfig {
+/**
+ * Configuration for PostgreSQL backend (default).
+ * Backward-compatible: omitting `backend` defaults to 'postgres'.
+ */
+export interface PostgresJobQueueConfig {
+  backend?: 'postgres';
   databaseConfig: {
     connectionString?: string;
     host?: string;
@@ -196,6 +489,48 @@ export interface JobQueueConfig {
   verbose?: boolean;
 }
 
+/**
+ * TLS configuration for the Redis connection.
+ */
+export interface RedisTLSConfig {
+  ca?: string;
+  cert?: string;
+  key?: string;
+  rejectUnauthorized?: boolean;
+}
+
+/**
+ * Configuration for Redis backend.
+ */
+export interface RedisJobQueueConfig {
+  backend: 'redis';
+  redisConfig: {
+    /** Redis URL (e.g. redis://localhost:6379) */
+    url?: string;
+    host?: string;
+    port?: number;
+    password?: string;
+    /** Redis database number (default: 0) */
+    db?: number;
+    tls?: RedisTLSConfig;
+    /**
+     * Key prefix for all Redis keys (default: 'dq:').
+     * Useful to namespace multiple queues in the same Redis instance.
+     */
+    keyPrefix?: string;
+  };
+  verbose?: boolean;
+}
+
+/**
+ * Job queue configuration — discriminated union.
+ * If `backend` is omitted, PostgreSQL is used.
+ */
+export type JobQueueConfig = PostgresJobQueueConfig | RedisJobQueueConfig;
+
+/** @deprecated Use JobQueueConfig instead. Alias kept for backward compat. */
+export type JobQueueConfigLegacy = PostgresJobQueueConfig;
+
 export type TagQueryMode = 'exact' | 'all' | 'any' | 'none';
 
 export interface JobQueue<PayloadMap> {
@@ -245,16 +580,25 @@ export interface JobQueue<PayloadMap> {
     offset?: number,
   ) => Promise<JobRecord<PayloadMap, T>[]>;
   /**
-   * Get jobs by filters.
-  /**
-   * Get jobs by filters.
-   */
-  getJobs: <T extends JobType<PayloadMap>>(filters?: {
-    jobType?: string;
-    priority?: number;
-    runAt?: Date | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };
-    tags?: { values: string[]; mode?: TagQueryMode };
-  }) => Promise<JobRecord<PayloadMap, T>[]>;
+   * Get jobs by filters, with pagination support.
+   * - Use `cursor` for efficient keyset pagination (recommended for large datasets).
+   * - Use `limit` and `offset` for traditional pagination.
+   * - Do not combine `cursor` with `offset`.
+   */
+  getJobs: <T extends JobType<PayloadMap>>(
+    filters?: {
+      jobType?: string;
+      priority?: number;
+      runAt?:
+        | Date
+        | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };
+      tags?: { values: string[]; mode?: TagQueryMode };
+      /** Cursor for keyset pagination. Only return jobs with id < cursor. */
+      cursor?: number;
+    },
+    limit?: number,
+    offset?: number,
+  ) => Promise<JobRecord<PayloadMap, T>[]>;
   /**
    * Retry a job given its ID.
    * - This will set the job status back to 'pending', clear the locked_at and locked_by, and allow it to be picked up by other workers.
@@ -264,11 +608,52 @@ export interface JobQueue<PayloadMap> {
    * Cleanup jobs that are older than the specified number of days.
    */
   cleanupOldJobs: (daysToKeep?: number) => Promise<number>;
+  /**
+   * Cleanup job events that are older than the specified number of days.
+   */
+  cleanupOldJobEvents: (daysToKeep?: number) => Promise<number>;
   /**
    * Cancel a job given its ID.
    * - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
    */
   cancelJob: (jobId: number) => Promise<void>;
+  /**
+   * Edit a pending job given its ID.
+   * - Only works for jobs with status 'pending'. Silently fails for other statuses.
+   * - All fields in EditJobOptions are optional - only provided fields will be updated.
+   * - jobType cannot be changed.
+   * - Records an 'edited' event with the updated fields in metadata.
+   */
+  editJob: <T extends JobType<PayloadMap>>(
+    jobId: number,
+    updates: EditJobOptions<PayloadMap, T>,
+  ) => Promise<void>;
+  /**
+   * Edit all pending jobs that match the filters.
+   * - Only works for jobs with status 'pending'. Non-pending jobs are not affected.
+   * - All fields in EditJobOptions are optional - only provided fields will be updated.
+   * - jobType cannot be changed.
+   * - Records an 'edited' event with the updated fields in metadata for each affected job.
+   * - Returns the number of jobs that were edited.
+   * - The filters are:
+   *   - jobType: The job type to edit.
+   *   - priority: The priority of the job to edit.
+   *   - runAt: The time the job is scheduled to run at (now supports gt/gte/lt/lte/eq).
+   *   - tags: An object with 'values' (string[]) and 'mode' (TagQueryMode) for tag-based editing.
+   */
+  editAllPendingJobs: <T extends JobType<PayloadMap>>(
+    filters:
+      | {
+          jobType?: string;
+          priority?: number;
+          runAt?:
+            | Date
+            | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };
+          tags?: { values: string[]; mode?: TagQueryMode };
+        }
+      | undefined,
+    updates: EditJobOptions<PayloadMap, T>,
+  ) => Promise<number>;
   /**
    * Reclaim stuck jobs.
    * - If a process (e.g., API route or worker) crashes after marking a job as 'processing' but before completing it, the job can remain stuck in the 'processing' state indefinitely. This can happen if the process is killed or encounters an unhandled error after updating the job status but before marking it as 'completed' or 'failed'.
@@ -304,8 +689,58 @@ export interface JobQueue<PayloadMap> {
    * Get the job events for a job.
    */
   getJobEvents: (jobId: number) => Promise<JobEvent[]>;
+
+  /**
+   * Create a waitpoint token.
+   * Tokens can be completed externally to resume a waiting job.
+   * Can be called outside of handlers (e.g., from an API route).
+   *
+   * **PostgreSQL backend only.** Throws if the backend is Redis.
+   *
+   * @param options - Optional token configuration (timeout, tags).
+   * @returns A token object with `id`.
+   */
+  createToken: (options?: CreateTokenOptions) => Promise<WaitToken>;
+
+  /**
+   * Complete a waitpoint token, resuming the associated waiting job.
+   * Can be called from anywhere (API routes, external services, etc.).
+   *
+   * **PostgreSQL backend only.** Throws if the backend is Redis.
+   *
+   * @param tokenId - The ID of the token to complete.
+   * @param data - Optional data to pass to the waiting handler.
+   */
+  completeToken: (tokenId: string, data?: any) => Promise<void>;
+
+  /**
+   * Retrieve a waitpoint token by its ID.
+   *
+   * **PostgreSQL backend only.** Throws if the backend is Redis.
+   *
+   * @param tokenId - The ID of the token to retrieve.
+   * @returns The token record, or null if not found.
+   */
+  getToken: (tokenId: string) => Promise<WaitpointRecord | null>;
+
+  /**
+   * Expire timed-out waitpoint tokens and resume their associated jobs.
+   * Call this periodically (e.g., alongside `reclaimStuckJobs`).
+   *
+   * **PostgreSQL backend only.** Throws if the backend is Redis.
+   *
+   * @returns The number of tokens that were expired.
+   */
+  expireTimedOutTokens: () => Promise<number>;
+
+  /**
+   * Get the PostgreSQL database pool.
+   * Throws if the backend is not PostgreSQL.
+   */
+  getPool: () => import('pg').Pool;
   /**
-   * Get the database pool.
+   * Get the Redis client instance (ioredis).
+   * Throws if the backend is not Redis.
    */
-  getPool: () => Pool;
+  getRedisClient: () => unknown;
 }