@nicnocquee/dataqueue 1.24.0 → 1.25.0

package/dist/index.d.ts

@@ -1,3 +1,4 @@
+import * as pg from 'pg';
 import { Pool } from 'pg';
 
 type JobType<PayloadMap> = keyof PayloadMap & string;
@@ -61,6 +62,18 @@ interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
      * Tags for this job. Used for grouping, searching, or batch operations.
      */
     tags?: string[];
+    /**
+     * Optional idempotency key. When provided, ensures that only one job exists for a given key.
+     * If a job with the same idempotency key already exists, `addJob` returns the existing job's ID
+     * instead of creating a duplicate.
+     *
+     * Useful for preventing duplicate jobs caused by retries, double-clicks, webhook replays,
+     * or serverless function re-invocations.
+     *
+     * The key is unique across the entire `job_queue` table regardless of job status.
+     * Once a key exists, it cannot be reused until the job is cleaned up (via `cleanupOldJobs`).
+     */
+    idempotencyKey?: string;
 }
 /**
  * Options for editing a pending job.
@@ -79,7 +92,9 @@ declare enum JobEventType {
     Failed = "failed",
     Cancelled = "cancelled",
     Retried = "retried",
-    Edited = "edited"
+    Edited = "edited",
+    Prolonged = "prolonged",
+    Waiting = "waiting"
 }
 interface JobEvent {
     id: number;
@@ -93,7 +108,7 @@ declare enum FailureReason {
     HandlerError = "handler_error",
     NoHandler = "no_handler"
 }
-type JobStatus = 'pending' | 'processing' | 'completed' | 'failed' | 'cancelled';
+type JobStatus = 'pending' | 'processing' | 'completed' | 'failed' | 'cancelled' | 'waiting';
 interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
     id: number;
     jobType: T;
@@ -150,8 +165,190 @@ interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
      * Tags for this job. Used for grouping, searching, or batch operations.
      */
     tags?: string[];
+    /**
+     * The idempotency key for this job, if one was provided when the job was created.
+     */
+    idempotencyKey?: string | null;
+    /**
+     * The time the job is waiting until (for time-based waits).
+     */
+    waitUntil?: Date | null;
+    /**
+     * The waitpoint token ID the job is waiting for (for token-based waits).
+     */
+    waitTokenId?: string | null;
+    /**
+     * Step data for the job. Stores completed step results for replay on re-invocation.
+     */
+    stepData?: Record<string, any>;
+    /**
+     * Progress percentage for the job (0-100), or null if no progress has been reported.
+     * Updated by the handler via `ctx.setProgress(percent)`.
+     */
+    progress?: number | null;
 }
-type JobHandler<PayloadMap, T extends keyof PayloadMap> = (payload: PayloadMap[T], signal: AbortSignal) => Promise<void>;
+/**
+ * Callback registered via `onTimeout`. Invoked when the timeout fires, before the AbortSignal is triggered.
+ * Return a number (ms) to extend the timeout, or return nothing to let the timeout proceed.
+ */
+type OnTimeoutCallback = () => number | void | undefined;
+/**
+ * Context object passed to job handlers as the third argument.
+ * Provides mechanisms to extend the job's timeout while it's running,
+ * as well as step tracking and wait capabilities.
+ */
+interface JobContext {
+    /**
+     * Proactively reset the timeout deadline.
+     * - If `ms` is provided, sets the deadline to `ms` milliseconds from now.
+     * - If omitted, resets the deadline to the original `timeoutMs` from now (heartbeat-style).
+     * - No-op if the job has no timeout set or if `forceKillOnTimeout` is true.
+     */
+    prolong: (ms?: number) => void;
+    /**
+     * Register a callback that is invoked when the timeout fires, **before** the AbortSignal is triggered.
+     * - If the callback returns a number > 0, the timeout is reset to that many ms from now.
+     * - If the callback returns `undefined`, `null`, `0`, or a negative number, the timeout proceeds normally.
+     * - The callback may be invoked multiple times if the job keeps extending.
+     * - Only one callback can be registered; subsequent calls replace the previous one.
+     * - No-op if the job has no timeout set or if `forceKillOnTimeout` is true.
+     */
+    onTimeout: (callback: OnTimeoutCallback) => void;
+    /**
+     * Execute a named step with memoization. If the step was already completed
+     * in a previous invocation (e.g., before a wait), the cached result is returned
+     * without re-executing the function.
+     *
+     * Step names must be unique within a handler and stable across re-invocations.
+     *
+     * @param stepName - A unique identifier for this step.
+     * @param fn - The function to execute. Its return value is cached.
+     * @returns The result of the step (from cache or fresh execution).
+     */
+    run: <T>(stepName: string, fn: () => Promise<T>) => Promise<T>;
+    /**
+     * Wait for a specified duration before continuing execution.
+     * The job will be paused and resumed after the duration elapses.
+     *
+     * When this is called, the handler throws a WaitSignal internally.
+     * The job is set to 'waiting' status and will be re-invoked after the
+     * specified duration. All steps completed via `ctx.run()` before this
+     * call will be replayed from cache on re-invocation.
+     *
+     * @param duration - The duration to wait (e.g., `{ hours: 1 }`, `{ days: 7 }`).
+     */
+    waitFor: (duration: WaitDuration) => Promise<void>;
+    /**
+     * Wait until a specific date/time before continuing execution.
+     * The job will be paused and resumed at (or after) the specified date.
+     *
+     * @param date - The date to wait until.
+     */
+    waitUntil: (date: Date) => Promise<void>;
+    /**
+     * Create a waitpoint token. The token can be completed externally
+     * (by calling `jobQueue.completeToken()`) to resume a waiting job.
+     *
+     * Tokens can be created inside handlers or outside (via `jobQueue.createToken()`).
+     *
+     * @param options - Optional token configuration (timeout, tags).
+     * @returns A token object with `id` that can be passed to `waitForToken()`.
+     */
+    createToken: (options?: CreateTokenOptions) => Promise<WaitToken>;
+    /**
+     * Wait for a waitpoint token to be completed by an external signal.
+     * The job will be paused until `jobQueue.completeToken(tokenId, data)` is called
+     * or the token times out.
+     *
+     * @param tokenId - The ID of the token to wait for.
+     * @returns A result object indicating success or timeout.
+     */
+    waitForToken: <T = any>(tokenId: string) => Promise<WaitTokenResult<T>>;
+    /**
+     * Report progress for this job (0-100).
+     * The value is persisted to the database and can be read by clients
+     * via `getJob()` or the React SDK's `useJob()` hook.
+     *
+     * @param percent - Progress percentage (0-100). Values are rounded to the nearest integer.
+     * @throws If percent is outside the 0-100 range.
+     */
+    setProgress: (percent: number) => Promise<void>;
+}
+/**
+ * Duration specification for `ctx.waitFor()`.
+ * At least one field must be provided. Fields are additive.
+ */
+interface WaitDuration {
+    seconds?: number;
+    minutes?: number;
+    hours?: number;
+    days?: number;
+    weeks?: number;
+    months?: number;
+    years?: number;
+}
+/**
+ * Options for creating a waitpoint token.
+ */
+interface CreateTokenOptions {
+    /**
+     * Maximum time to wait for the token to be completed.
+     * Accepts a duration string like '10m', '1h', '24h', '7d'.
+     * If not provided, the token has no timeout.
+     */
+    timeout?: string;
+    /**
+     * Tags to attach to the token for filtering.
+     */
+    tags?: string[];
+}
+/**
+ * A waitpoint token returned by `ctx.createToken()`.
+ */
+interface WaitToken {
+    /** The unique token ID. */
+    id: string;
+}
+/**
+ * Result of `ctx.waitForToken()`.
+ */
+type WaitTokenResult<T = any> = {
+    ok: true;
+    output: T;
+} | {
+    ok: false;
+    error: string;
+};
+/**
+ * Internal signal thrown by wait methods to pause handler execution.
+ * This is not a real error -- the processor catches it and transitions the job to 'waiting' status.
+ */
+declare class WaitSignal extends Error {
+    readonly type: 'duration' | 'date' | 'token';
+    readonly waitUntil: Date | undefined;
+    readonly tokenId: string | undefined;
+    readonly stepData: Record<string, any>;
+    readonly isWaitSignal = true;
+    constructor(type: 'duration' | 'date' | 'token', waitUntil: Date | undefined, tokenId: string | undefined, stepData: Record<string, any>);
+}
+/**
+ * Status of a waitpoint token.
+ */
+type WaitpointStatus = 'waiting' | 'completed' | 'timed_out';
+/**
+ * A waitpoint record from the database.
+ */
+interface WaitpointRecord {
+    id: string;
+    jobId: number | null;
+    status: WaitpointStatus;
+    output: any;
+    timeoutAt: Date | null;
+    createdAt: Date;
+    completedAt: Date | null;
+    tags: string[] | null;
+}
+type JobHandler<PayloadMap, T extends keyof PayloadMap> = (payload: PayloadMap[T], signal: AbortSignal, ctx: JobContext) => Promise<void>;
 type JobHandlers<PayloadMap> = {
     [K in keyof PayloadMap]: JobHandler<PayloadMap, K>;
 };
@@ -195,8 +392,18 @@ interface Processor {
     startInBackground: () => void;
     /**
      * Stop the job processor that runs in the background.
+     * Does not wait for in-flight jobs to complete.
      */
     stop: () => void;
+    /**
+     * Stop the job processor and wait for all in-flight jobs to complete.
+     * Useful for graceful shutdown (e.g., SIGTERM handling).
+     * No new batches will be started after calling this method.
+     *
+     * @param timeoutMs - Maximum time to wait for in-flight jobs (default: 30000ms).
+     *   If jobs don't complete within this time, the promise resolves anyway.
+     */
+    stopAndDrain: (timeoutMs?: number) => Promise<void>;
     /**
      * Check if the job processor is running.
      */
@@ -227,7 +434,12 @@ interface DatabaseSSLConfig {
      */
     rejectUnauthorized?: boolean;
 }
-interface JobQueueConfig {
+/**
+ * Configuration for PostgreSQL backend (default).
+ * Backward-compatible: omitting `backend` defaults to 'postgres'.
+ */
+interface PostgresJobQueueConfig {
+    backend?: 'postgres';
     databaseConfig: {
         connectionString?: string;
         host?: string;
@@ -239,6 +451,44 @@ interface JobQueueConfig {
     };
     verbose?: boolean;
 }
+/**
+ * TLS configuration for the Redis connection.
+ */
+interface RedisTLSConfig {
+    ca?: string;
+    cert?: string;
+    key?: string;
+    rejectUnauthorized?: boolean;
+}
+/**
+ * Configuration for Redis backend.
+ */
+interface RedisJobQueueConfig {
+    backend: 'redis';
+    redisConfig: {
+        /** Redis URL (e.g. redis://localhost:6379) */
+        url?: string;
+        host?: string;
+        port?: number;
+        password?: string;
+        /** Redis database number (default: 0) */
+        db?: number;
+        tls?: RedisTLSConfig;
+        /**
+         * Key prefix for all Redis keys (default: 'dq:').
+         * Useful to namespace multiple queues in the same Redis instance.
+         */
+        keyPrefix?: string;
+    };
+    verbose?: boolean;
+}
+/**
+ * Job queue configuration — discriminated union.
+ * If `backend` is omitted, PostgreSQL is used.
+ */
+type JobQueueConfig = PostgresJobQueueConfig | RedisJobQueueConfig;
+/** @deprecated Use JobQueueConfig instead. Alias kept for backward compat. */
+type JobQueueConfigLegacy = PostgresJobQueueConfig;
 type TagQueryMode = 'exact' | 'all' | 'any' | 'none';
 interface JobQueue<PayloadMap> {
     /**
@@ -271,9 +521,10 @@ interface JobQueue<PayloadMap> {
      */
     getAllJobs: <T extends JobType<PayloadMap>>(limit?: number, offset?: number) => Promise<JobRecord<PayloadMap, T>[]>;
     /**
-     * Get jobs by filters.
-    /**
-     * Get jobs by filters.
+     * Get jobs by filters, with pagination support.
+     * - Use `cursor` for efficient keyset pagination (recommended for large datasets).
+     * - Use `limit` and `offset` for traditional pagination.
+     * - Do not combine `cursor` with `offset`.
      */
     getJobs: <T extends JobType<PayloadMap>>(filters?: {
         jobType?: string;
@@ -289,7 +540,9 @@ interface JobQueue<PayloadMap> {
             values: string[];
             mode?: TagQueryMode;
         };
-    }) => Promise<JobRecord<PayloadMap, T>[]>;
+        /** Cursor for keyset pagination. Only return jobs with id < cursor. */
+        cursor?: number;
+    }, limit?: number, offset?: number) => Promise<JobRecord<PayloadMap, T>[]>;
     /**
      * Retry a job given its ID.
      * - This will set the job status back to 'pending', clear the locked_at and locked_by, and allow it to be picked up by other workers.
@@ -299,6 +552,10 @@ interface JobQueue<PayloadMap> {
      * Cleanup jobs that are older than the specified number of days.
      */
     cleanupOldJobs: (daysToKeep?: number) => Promise<number>;
+    /**
+     * Cleanup job events that are older than the specified number of days.
+     */
+    cleanupOldJobEvents: (daysToKeep?: number) => Promise<number>;
     /**
      * Cancel a job given its ID.
      * - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
@@ -381,9 +638,178 @@ interface JobQueue<PayloadMap> {
      */
     getJobEvents: (jobId: number) => Promise<JobEvent[]>;
     /**
-     * Get the database pool.
+     * Create a waitpoint token.
+     * Tokens can be completed externally to resume a waiting job.
+     * Can be called outside of handlers (e.g., from an API route).
+     *
+     * **PostgreSQL backend only.** Throws if the backend is Redis.
+     *
+     * @param options - Optional token configuration (timeout, tags).
+     * @returns A token object with `id`.
+     */
+    createToken: (options?: CreateTokenOptions) => Promise<WaitToken>;
+    /**
+     * Complete a waitpoint token, resuming the associated waiting job.
+     * Can be called from anywhere (API routes, external services, etc.).
+     *
+     * **PostgreSQL backend only.** Throws if the backend is Redis.
+     *
+     * @param tokenId - The ID of the token to complete.
+     * @param data - Optional data to pass to the waiting handler.
+     */
+    completeToken: (tokenId: string, data?: any) => Promise<void>;
+    /**
+     * Retrieve a waitpoint token by its ID.
+     *
+     * **PostgreSQL backend only.** Throws if the backend is Redis.
+     *
+     * @param tokenId - The ID of the token to retrieve.
+     * @returns The token record, or null if not found.
+     */
+    getToken: (tokenId: string) => Promise<WaitpointRecord | null>;
+    /**
+     * Expire timed-out waitpoint tokens and resume their associated jobs.
+     * Call this periodically (e.g., alongside `reclaimStuckJobs`).
+     *
+     * **PostgreSQL backend only.** Throws if the backend is Redis.
+     *
+     * @returns The number of tokens that were expired.
+     */
+    expireTimedOutTokens: () => Promise<number>;
+    /**
+     * Get the PostgreSQL database pool.
+     * Throws if the backend is not PostgreSQL.
+     */
+    getPool: () => pg.Pool;
+    /**
+     * Get the Redis client instance (ioredis).
+     * Throws if the backend is not Redis.
+     */
+    getRedisClient: () => unknown;
+}
+
+/**
+ * Filter options used by getJobs, cancelAllUpcomingJobs, editAllPendingJobs
+ */
+interface JobFilters {
+    jobType?: string;
+    priority?: number;
+    runAt?: Date | {
+        gt?: Date;
+        gte?: Date;
+        lt?: Date;
+        lte?: Date;
+        eq?: Date;
+    };
+    tags?: {
+        values: string[];
+        mode?: TagQueryMode;
+    };
+    /**
+     * Cursor for keyset pagination. When provided, only return jobs with id < cursor.
+     * This is more efficient than OFFSET for large datasets.
+     * Cannot be used together with offset.
      */
-    getPool: () => Pool;
+    cursor?: number;
+}
+/**
+ * Fields that can be updated on a job
+ */
+interface JobUpdates {
+    payload?: any;
+    maxAttempts?: number;
+    priority?: number;
+    runAt?: Date | null;
+    timeoutMs?: number | null;
+    tags?: string[] | null;
+}
+/**
+ * Abstract backend interface that both PostgreSQL and Redis implement.
+ * All storage operations go through this interface so the processor
+ * and public API are backend-agnostic.
+ */
+interface QueueBackend {
+    /** Add a job and return its numeric ID. */
+    addJob<PayloadMap, T extends JobType<PayloadMap>>(job: JobOptions<PayloadMap, T>): Promise<number>;
+    /** Get a single job by ID, or null if not found. */
+    getJob<PayloadMap, T extends JobType<PayloadMap>>(id: number): Promise<JobRecord<PayloadMap, T> | null>;
+    /** Get jobs filtered by status, ordered by createdAt DESC. */
+    getJobsByStatus<PayloadMap, T extends JobType<PayloadMap>>(status: string, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
+    /** Get all jobs, ordered by createdAt DESC. */
+    getAllJobs<PayloadMap, T extends JobType<PayloadMap>>(limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
+    /** Get jobs matching arbitrary filters, ordered by createdAt DESC. */
+    getJobs<PayloadMap, T extends JobType<PayloadMap>>(filters?: JobFilters, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
+    /** Get jobs by tag(s) with query mode. */
+    getJobsByTags<PayloadMap, T extends JobType<PayloadMap>>(tags: string[], mode?: TagQueryMode, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
+    /**
+     * Atomically claim a batch of ready jobs for the given worker.
+     * Equivalent to SELECT … FOR UPDATE SKIP LOCKED in Postgres.
+     */
+    getNextBatch<PayloadMap, T extends JobType<PayloadMap>>(workerId: string, batchSize?: number, jobType?: string | string[]): Promise<JobRecord<PayloadMap, T>[]>;
+    /** Mark a job as completed. */
+    completeJob(jobId: number): Promise<void>;
+    /** Mark a job as failed with error info and schedule retry. */
+    failJob(jobId: number, error: Error, failureReason?: FailureReason): Promise<void>;
+    /** Update locked_at to keep the job alive (heartbeat). */
+    prolongJob(jobId: number): Promise<void>;
+    /** Retry a failed/cancelled job immediately. */
+    retryJob(jobId: number): Promise<void>;
+    /** Cancel a pending job. */
+    cancelJob(jobId: number): Promise<void>;
+    /** Cancel all pending jobs matching optional filters. Returns count. */
+    cancelAllUpcomingJobs(filters?: JobFilters): Promise<number>;
+    /** Edit a single pending job. */
+    editJob(jobId: number, updates: JobUpdates): Promise<void>;
+    /** Edit all pending jobs matching filters. Returns count. */
+    editAllPendingJobs(filters: JobFilters | undefined, updates: JobUpdates): Promise<number>;
+    /** Delete completed jobs older than N days. Returns count deleted. */
+    cleanupOldJobs(daysToKeep?: number): Promise<number>;
+    /** Delete job events older than N days. Returns count deleted. */
+    cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
+    /** Reclaim jobs stuck in 'processing' for too long. Returns count. */
+    reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
+    /** Update the progress percentage (0-100) for a job. */
+    updateProgress(jobId: number, progress: number): Promise<void>;
+    /** Record a job event. Should not throw. */
+    recordJobEvent(jobId: number, eventType: JobEventType, metadata?: any): Promise<void>;
+    /** Get all events for a job, ordered by createdAt ASC. */
+    getJobEvents(jobId: number): Promise<JobEvent[]>;
+    /** Set a pending reason for unpicked jobs of a given type. */
+    setPendingReasonForUnpickedJobs(reason: string, jobType?: string | string[]): Promise<void>;
+}
+
+declare class PostgresBackend implements QueueBackend {
+    private pool;
+    constructor(pool: Pool);
+    /** Expose the raw pool for advanced usage. */
+    getPool(): Pool;
+    recordJobEvent(jobId: number, eventType: JobEventType, metadata?: any): Promise<void>;
+    getJobEvents(jobId: number): Promise<JobEvent[]>;
+    addJob<PayloadMap, T extends JobType<PayloadMap>>({ jobType, payload, maxAttempts, priority, runAt, timeoutMs, forceKillOnTimeout, tags, idempotencyKey, }: JobOptions<PayloadMap, T>): Promise<number>;
+    getJob<PayloadMap, T extends JobType<PayloadMap>>(id: number): Promise<JobRecord<PayloadMap, T> | null>;
+    getJobsByStatus<PayloadMap, T extends JobType<PayloadMap>>(status: string, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
+    getAllJobs<PayloadMap, T extends JobType<PayloadMap>>(limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
+    getJobs<PayloadMap, T extends JobType<PayloadMap>>(filters?: JobFilters, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
+    getJobsByTags<PayloadMap, T extends JobType<PayloadMap>>(tags: string[], mode?: TagQueryMode, limit?: number, offset?: number): Promise<JobRecord<PayloadMap, T>[]>;
+    getNextBatch<PayloadMap, T extends JobType<PayloadMap>>(workerId: string, batchSize?: number, jobType?: string | string[]): Promise<JobRecord<PayloadMap, T>[]>;
+    completeJob(jobId: number): Promise<void>;
+    failJob(jobId: number, error: Error, failureReason?: FailureReason): Promise<void>;
+    prolongJob(jobId: number): Promise<void>;
+    updateProgress(jobId: number, progress: number): Promise<void>;
+    retryJob(jobId: number): Promise<void>;
+    cancelJob(jobId: number): Promise<void>;
+    cancelAllUpcomingJobs(filters?: JobFilters): Promise<number>;
+    editJob(jobId: number, updates: JobUpdates): Promise<void>;
+    editAllPendingJobs(filters: JobFilters | undefined, updates: JobUpdates): Promise<number>;
+    cleanupOldJobs(daysToKeep?: number): Promise<number>;
+    cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
+    reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
+    /**
+     * Batch-insert multiple job events in a single query.
+     * More efficient than individual recordJobEvent calls.
+     */
+    private recordJobEventsBatch;
+    setPendingReasonForUnpickedJobs(reason: string, jobType?: string | string[]): Promise<void>;
 }
 
 /**
@@ -438,8 +864,10 @@ declare function testHandlerSerialization<PayloadMap, T extends keyof PayloadMap
 }>;
 
 /**
- * Initialize the job queue system
+ * Initialize the job queue system.
+ *
+ * Defaults to PostgreSQL when `backend` is omitted.
  */
 declare const initJobQueue: <PayloadMap = any>(config: JobQueueConfig) => JobQueue<PayloadMap>;
 
-export { type DatabaseSSLConfig, type EditJobOptions, FailureReason, type JobEvent, JobEventType, type JobHandler, type JobHandlers, type JobOptions, type JobQueue, type JobQueueConfig, type JobRecord, type JobStatus, type JobType, type Processor, type ProcessorOptions, type TagQueryMode, initJobQueue, testHandlerSerialization, validateHandlerSerializable };
+export { type CreateTokenOptions, type DatabaseSSLConfig, type EditJobOptions, FailureReason, type JobContext, type JobEvent, JobEventType, type JobHandler, type JobHandlers, type JobOptions, type JobQueue, type JobQueueConfig, type JobQueueConfigLegacy, type JobRecord, type JobStatus, type JobType, type OnTimeoutCallback, PostgresBackend, type PostgresJobQueueConfig, type Processor, type ProcessorOptions, type QueueBackend, type RedisJobQueueConfig, type RedisTLSConfig, type TagQueryMode, type WaitDuration, WaitSignal, type WaitToken, type WaitTokenResult, type WaitpointRecord, type WaitpointStatus, initJobQueue, testHandlerSerialization, validateHandlerSerializable };