@nicnocquee/dataqueue 1.24.0 → 1.25.0

package/src/test-util.ts

@@ -65,3 +65,35 @@ export async function destroyTestDb(dbName: string) {
   await adminPool.query(`DROP DATABASE IF EXISTS ${dbName}`);
   await adminPool.end();
 }
+
+/**
+ * Create a Redis test setup with a unique prefix to isolate tests.
+ * Returns the prefix and a cleanup function.
+ */
+export function createRedisTestPrefix(): string {
+  return `test_${randomUUID().replace(/-/g, '').slice(0, 12)}:`;
+}
+
+/**
+ * Flush all keys with the given prefix from Redis.
+ */
+export async function cleanupRedisPrefix(
+  redisClient: any,
+  prefix: string,
+): Promise<void> {
+  // Use SCAN to find all keys with the prefix and delete them
+  let cursor = '0';
+  do {
+    const [nextCursor, keys] = await redisClient.scan(
+      cursor,
+      'MATCH',
+      `${prefix}*`,
+      'COUNT',
+      100,
+    );
+    cursor = nextCursor;
+    if (keys.length > 0) {
+      await redisClient.del(...keys);
+    }
+  } while (cursor !== '0');
+}

package/src/types.ts

@@ -1,5 +1,3 @@
-import { Pool } from 'pg';
-
 // Utility type for job type keys
 export type JobType<PayloadMap> = keyof PayloadMap & string;
 
@@ -63,6 +61,18 @@ export interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
    * Tags for this job. Used for grouping, searching, or batch operations.
    */
   tags?: string[];
+  /**
+   * Optional idempotency key. When provided, ensures that only one job exists for a given key.
+   * If a job with the same idempotency key already exists, `addJob` returns the existing job's ID
+   * instead of creating a duplicate.
+   *
+   * Useful for preventing duplicate jobs caused by retries, double-clicks, webhook replays,
+   * or serverless function re-invocations.
+   *
+   * The key is unique across the entire `job_queue` table regardless of job status.
+   * Once a key exists, it cannot be reused until the job is cleaned up (via `cleanupOldJobs`).
+   */
+  idempotencyKey?: string;
 }
 
 /**
@@ -86,6 +96,8 @@ export enum JobEventType {
   Cancelled = 'cancelled',
   Retried = 'retried',
   Edited = 'edited',
+  Prolonged = 'prolonged',
+  Waiting = 'waiting',
 }
 
 export interface JobEvent {
@@ -107,7 +119,8 @@ export type JobStatus =
   | 'processing'
   | 'completed'
   | 'failed'
-  | 'cancelled';
+  | 'cancelled'
+  | 'waiting';
 
 export interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
   id: number;
@@ -162,11 +175,211 @@ export interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
    * Tags for this job. Used for grouping, searching, or batch operations.
    */
   tags?: string[];
+  /**
+   * The idempotency key for this job, if one was provided when the job was created.
+   */
+  idempotencyKey?: string | null;
+  /**
+   * The time the job is waiting until (for time-based waits).
+   */
+  waitUntil?: Date | null;
+  /**
+   * The waitpoint token ID the job is waiting for (for token-based waits).
+   */
+  waitTokenId?: string | null;
+  /**
+   * Step data for the job. Stores completed step results for replay on re-invocation.
+   */
+  stepData?: Record<string, any>;
+  /**
+   * Progress percentage for the job (0-100), or null if no progress has been reported.
+   * Updated by the handler via `ctx.setProgress(percent)`.
+   */
+  progress?: number | null;
+}
+
+/**
+ * Callback registered via `onTimeout`. Invoked when the timeout fires, before the AbortSignal is triggered.
+ * Return a number (ms) to extend the timeout, or return nothing to let the timeout proceed.
+ */
+export type OnTimeoutCallback = () => number | void | undefined;
+
+/**
+ * Context object passed to job handlers as the third argument.
+ * Provides mechanisms to extend the job's timeout while it's running,
+ * as well as step tracking and wait capabilities.
+ */
+export interface JobContext {
+  /**
+   * Proactively reset the timeout deadline.
+   * - If `ms` is provided, sets the deadline to `ms` milliseconds from now.
+   * - If omitted, resets the deadline to the original `timeoutMs` from now (heartbeat-style).
+   * - No-op if the job has no timeout set or if `forceKillOnTimeout` is true.
+   */
+  prolong: (ms?: number) => void;
+
+  /**
+   * Register a callback that is invoked when the timeout fires, **before** the AbortSignal is triggered.
+   * - If the callback returns a number > 0, the timeout is reset to that many ms from now.
+   * - If the callback returns `undefined`, `null`, `0`, or a negative number, the timeout proceeds normally.
+   * - The callback may be invoked multiple times if the job keeps extending.
+   * - Only one callback can be registered; subsequent calls replace the previous one.
+   * - No-op if the job has no timeout set or if `forceKillOnTimeout` is true.
+   */
+  onTimeout: (callback: OnTimeoutCallback) => void;
+
+  /**
+   * Execute a named step with memoization. If the step was already completed
+   * in a previous invocation (e.g., before a wait), the cached result is returned
+   * without re-executing the function.
+   *
+   * Step names must be unique within a handler and stable across re-invocations.
+   *
+   * @param stepName - A unique identifier for this step.
+   * @param fn - The function to execute. Its return value is cached.
+   * @returns The result of the step (from cache or fresh execution).
+   */
+  run: <T>(stepName: string, fn: () => Promise<T>) => Promise<T>;
+
+  /**
+   * Wait for a specified duration before continuing execution.
+   * The job will be paused and resumed after the duration elapses.
+   *
+   * When this is called, the handler throws a WaitSignal internally.
+   * The job is set to 'waiting' status and will be re-invoked after the
+   * specified duration. All steps completed via `ctx.run()` before this
+   * call will be replayed from cache on re-invocation.
+   *
+   * @param duration - The duration to wait (e.g., `{ hours: 1 }`, `{ days: 7 }`).
+   */
+  waitFor: (duration: WaitDuration) => Promise<void>;
+
+  /**
+   * Wait until a specific date/time before continuing execution.
+   * The job will be paused and resumed at (or after) the specified date.
+   *
+   * @param date - The date to wait until.
+   */
+  waitUntil: (date: Date) => Promise<void>;
+
+  /**
+   * Create a waitpoint token. The token can be completed externally
+   * (by calling `jobQueue.completeToken()`) to resume a waiting job.
+   *
+   * Tokens can be created inside handlers or outside (via `jobQueue.createToken()`).
+   *
+   * @param options - Optional token configuration (timeout, tags).
+   * @returns A token object with `id` that can be passed to `waitForToken()`.
+   */
+  createToken: (options?: CreateTokenOptions) => Promise<WaitToken>;
+
+  /**
+   * Wait for a waitpoint token to be completed by an external signal.
+   * The job will be paused until `jobQueue.completeToken(tokenId, data)` is called
+   * or the token times out.
+   *
+   * @param tokenId - The ID of the token to wait for.
+   * @returns A result object indicating success or timeout.
+   */
+  waitForToken: <T = any>(tokenId: string) => Promise<WaitTokenResult<T>>;
+
+  /**
+   * Report progress for this job (0-100).
+   * The value is persisted to the database and can be read by clients
+   * via `getJob()` or the React SDK's `useJob()` hook.
+   *
+   * @param percent - Progress percentage (0-100). Values are rounded to the nearest integer.
+   * @throws If percent is outside the 0-100 range.
+   */
+  setProgress: (percent: number) => Promise<void>;
+}
+
+/**
+ * Duration specification for `ctx.waitFor()`.
+ * At least one field must be provided. Fields are additive.
+ */
+export interface WaitDuration {
+  seconds?: number;
+  minutes?: number;
+  hours?: number;
+  days?: number;
+  weeks?: number;
+  months?: number;
+  years?: number;
+}
+
+/**
+ * Options for creating a waitpoint token.
+ */
+export interface CreateTokenOptions {
+  /**
+   * Maximum time to wait for the token to be completed.
+   * Accepts a duration string like '10m', '1h', '24h', '7d'.
+   * If not provided, the token has no timeout.
+   */
+  timeout?: string;
+  /**
+   * Tags to attach to the token for filtering.
+   */
+  tags?: string[];
+}
+
+/**
+ * A waitpoint token returned by `ctx.createToken()`.
+ */
+export interface WaitToken {
+  /** The unique token ID. */
+  id: string;
+}
+
+/**
+ * Result of `ctx.waitForToken()`.
+ */
+export type WaitTokenResult<T = any> =
+  | { ok: true; output: T }
+  | { ok: false; error: string };
+
+/**
+ * Internal signal thrown by wait methods to pause handler execution.
+ * This is not a real error -- the processor catches it and transitions the job to 'waiting' status.
+ */
+export class WaitSignal extends Error {
+  readonly isWaitSignal = true;
+
+  constructor(
+    public readonly type: 'duration' | 'date' | 'token',
+    public readonly waitUntil: Date | undefined,
+    public readonly tokenId: string | undefined,
+    public readonly stepData: Record<string, any>,
+  ) {
+    super('WaitSignal');
+    this.name = 'WaitSignal';
+  }
+}
+
+/**
+ * Status of a waitpoint token.
+ */
+export type WaitpointStatus = 'waiting' | 'completed' | 'timed_out';
+
+/**
+ * A waitpoint record from the database.
+ */
+export interface WaitpointRecord {
+  id: string;
+  jobId: number | null;
+  status: WaitpointStatus;
+  output: any;
+  timeoutAt: Date | null;
+  createdAt: Date;
+  completedAt: Date | null;
+  tags: string[] | null;
 }
 
 export type JobHandler<PayloadMap, T extends keyof PayloadMap> = (
   payload: PayloadMap[T],
   signal: AbortSignal,
+  ctx: JobContext,
 ) => Promise<void>;
 
 export type JobHandlers<PayloadMap> = {
@@ -214,8 +427,18 @@ export interface Processor {
   startInBackground: () => void;
   /**
    * Stop the job processor that runs in the background.
+   * Does not wait for in-flight jobs to complete.
    */
   stop: () => void;
+  /**
+   * Stop the job processor and wait for all in-flight jobs to complete.
+   * Useful for graceful shutdown (e.g., SIGTERM handling).
+   * No new batches will be started after calling this method.
+   *
+   * @param timeoutMs - Maximum time to wait for in-flight jobs (default: 30000ms).
+   *   If jobs don't complete within this time, the promise resolves anyway.
+   */
+  stopAndDrain: (timeoutMs?: number) => Promise<void>;
   /**
    * Check if the job processor is running.
    */
@@ -248,7 +471,12 @@ export interface DatabaseSSLConfig {
   rejectUnauthorized?: boolean;
 }
 
-export interface JobQueueConfig {
+/**
+ * Configuration for PostgreSQL backend (default).
+ * Backward-compatible: omitting `backend` defaults to 'postgres'.
+ */
+export interface PostgresJobQueueConfig {
+  backend?: 'postgres';
   databaseConfig: {
     connectionString?: string;
     host?: string;
@@ -261,6 +489,48 @@ export interface JobQueueConfig {
   verbose?: boolean;
 }
 
+/**
+ * TLS configuration for the Redis connection.
+ */
+export interface RedisTLSConfig {
+  ca?: string;
+  cert?: string;
+  key?: string;
+  rejectUnauthorized?: boolean;
+}
+
+/**
+ * Configuration for Redis backend.
+ */
+export interface RedisJobQueueConfig {
+  backend: 'redis';
+  redisConfig: {
+    /** Redis URL (e.g. redis://localhost:6379) */
+    url?: string;
+    host?: string;
+    port?: number;
+    password?: string;
+    /** Redis database number (default: 0) */
+    db?: number;
+    tls?: RedisTLSConfig;
+    /**
+     * Key prefix for all Redis keys (default: 'dq:').
+     * Useful to namespace multiple queues in the same Redis instance.
+     */
+    keyPrefix?: string;
+  };
+  verbose?: boolean;
+}
+
+/**
+ * Job queue configuration — discriminated union.
+ * If `backend` is omitted, PostgreSQL is used.
+ */
+export type JobQueueConfig = PostgresJobQueueConfig | RedisJobQueueConfig;
+
+/** @deprecated Use JobQueueConfig instead. Alias kept for backward compat. */
+export type JobQueueConfigLegacy = PostgresJobQueueConfig;
+
 export type TagQueryMode = 'exact' | 'all' | 'any' | 'none';
 
 export interface JobQueue<PayloadMap> {
@@ -310,16 +580,25 @@ export interface JobQueue<PayloadMap> {
     offset?: number,
   ) => Promise<JobRecord<PayloadMap, T>[]>;
   /**
-   * Get jobs by filters.
-  /**
-   * Get jobs by filters.
-   */
-  getJobs: <T extends JobType<PayloadMap>>(filters?: {
-    jobType?: string;
-    priority?: number;
-    runAt?: Date | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };
-    tags?: { values: string[]; mode?: TagQueryMode };
-  }) => Promise<JobRecord<PayloadMap, T>[]>;
+   * Get jobs by filters, with pagination support.
+   * - Use `cursor` for efficient keyset pagination (recommended for large datasets).
+   * - Use `limit` and `offset` for traditional pagination.
+   * - Do not combine `cursor` with `offset`.
+   */
+  getJobs: <T extends JobType<PayloadMap>>(
+    filters?: {
+      jobType?: string;
+      priority?: number;
+      runAt?:
+        | Date
+        | { gt?: Date; gte?: Date; lt?: Date; lte?: Date; eq?: Date };
+      tags?: { values: string[]; mode?: TagQueryMode };
+      /** Cursor for keyset pagination. Only return jobs with id < cursor. */
+      cursor?: number;
+    },
+    limit?: number,
+    offset?: number,
+  ) => Promise<JobRecord<PayloadMap, T>[]>;
   /**
    * Retry a job given its ID.
    * - This will set the job status back to 'pending', clear the locked_at and locked_by, and allow it to be picked up by other workers.
@@ -329,6 +608,10 @@ export interface JobQueue<PayloadMap> {
    * Cleanup jobs that are older than the specified number of days.
    */
   cleanupOldJobs: (daysToKeep?: number) => Promise<number>;
+  /**
+   * Cleanup job events that are older than the specified number of days.
+   */
+  cleanupOldJobEvents: (daysToKeep?: number) => Promise<number>;
   /**
    * Cancel a job given its ID.
    * - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
@@ -406,8 +689,58 @@ export interface JobQueue<PayloadMap> {
    * Get the job events for a job.
    */
   getJobEvents: (jobId: number) => Promise<JobEvent[]>;
+
+  /**
+   * Create a waitpoint token.
+   * Tokens can be completed externally to resume a waiting job.
+   * Can be called outside of handlers (e.g., from an API route).
+   *
+   * **PostgreSQL backend only.** Throws if the backend is Redis.
+   *
+   * @param options - Optional token configuration (timeout, tags).
+   * @returns A token object with `id`.
+   */
+  createToken: (options?: CreateTokenOptions) => Promise<WaitToken>;
+
+  /**
+   * Complete a waitpoint token, resuming the associated waiting job.
+   * Can be called from anywhere (API routes, external services, etc.).
+   *
+   * **PostgreSQL backend only.** Throws if the backend is Redis.
+   *
+   * @param tokenId - The ID of the token to complete.
+   * @param data - Optional data to pass to the waiting handler.
+   */
+  completeToken: (tokenId: string, data?: any) => Promise<void>;
+
+  /**
+   * Retrieve a waitpoint token by its ID.
+   *
+   * **PostgreSQL backend only.** Throws if the backend is Redis.
+   *
+   * @param tokenId - The ID of the token to retrieve.
+   * @returns The token record, or null if not found.
+   */
+  getToken: (tokenId: string) => Promise<WaitpointRecord | null>;
+
+  /**
+   * Expire timed-out waitpoint tokens and resume their associated jobs.
+   * Call this periodically (e.g., alongside `reclaimStuckJobs`).
+   *
+   * **PostgreSQL backend only.** Throws if the backend is Redis.
+   *
+   * @returns The number of tokens that were expired.
+   */
+  expireTimedOutTokens: () => Promise<number>;
+
+  /**
+   * Get the PostgreSQL database pool.
+   * Throws if the backend is not PostgreSQL.
+   */
+  getPool: () => import('pg').Pool;
   /**
-   * Get the database pool.
+   * Get the Redis client instance (ioredis).
+   * Throws if the backend is not Redis.
    */
-  getPool: () => Pool;
+  getRedisClient: () => unknown;
 }