@nicnocquee/dataqueue 1.31.0 → 1.32.0

package/dist/index.d.cts

@@ -449,6 +449,23 @@ interface PostgresJobQueueConfig {
         user?: string;
         password?: string;
         ssl?: DatabaseSSLConfig;
+        /**
+         * Maximum number of clients in the pool (default: 10).
+         * Increase when running multiple processors in the same process.
+         */
+        max?: number;
+        /**
+         * Minimum number of idle clients in the pool (default: 0).
+         */
+        min?: number;
+        /**
+         * Milliseconds a client must sit idle before being closed (default: 10000).
+         */
+        idleTimeoutMillis?: number;
+        /**
+         * Milliseconds to wait for a connection before throwing (default: 0, no timeout).
+         */
+        connectionTimeoutMillis?: number;
     };
     verbose?: boolean;
 }
@@ -626,12 +643,18 @@ interface JobQueue<PayloadMap> {
     retryJob: (jobId: number) => Promise<void>;
     /**
      * Cleanup jobs that are older than the specified number of days.
+     * Deletes in batches for scale safety.
+     * @param daysToKeep - Number of days to retain completed jobs (default 30).
+     * @param batchSize - Number of rows to delete per batch (default 1000 for PostgreSQL, 200 for Redis).
      */
-    cleanupOldJobs: (daysToKeep?: number) => Promise<number>;
+    cleanupOldJobs: (daysToKeep?: number, batchSize?: number) => Promise<number>;
     /**
      * Cleanup job events that are older than the specified number of days.
+     * Deletes in batches for scale safety.
+     * @param daysToKeep - Number of days to retain events (default 30).
+     * @param batchSize - Number of rows to delete per batch (default 1000).
      */
-    cleanupOldJobEvents: (daysToKeep?: number) => Promise<number>;
+    cleanupOldJobEvents: (daysToKeep?: number, batchSize?: number) => Promise<number>;
     /**
      * Cancel a job given its ID.
      * - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
@@ -718,8 +741,6 @@ interface JobQueue<PayloadMap> {
      * Tokens can be completed externally to resume a waiting job.
      * Can be called outside of handlers (e.g., from an API route).
      *
-     * **PostgreSQL backend only.** Throws if the backend is Redis.
-     *
      * @param options - Optional token configuration (timeout, tags).
      * @returns A token object with `id`.
      */
@@ -728,8 +749,6 @@ interface JobQueue<PayloadMap> {
      * Complete a waitpoint token, resuming the associated waiting job.
      * Can be called from anywhere (API routes, external services, etc.).
      *
-     * **PostgreSQL backend only.** Throws if the backend is Redis.
-     *
      * @param tokenId - The ID of the token to complete.
      * @param data - Optional data to pass to the waiting handler.
      */
@@ -737,8 +756,6 @@ interface JobQueue<PayloadMap> {
     /**
      * Retrieve a waitpoint token by its ID.
      *
-     * **PostgreSQL backend only.** Throws if the backend is Redis.
-     *
      * @param tokenId - The ID of the token to retrieve.
      * @returns The token record, or null if not found.
      */
@@ -747,8 +764,6 @@ interface JobQueue<PayloadMap> {
      * Expire timed-out waitpoint tokens and resume their associated jobs.
      * Call this periodically (e.g., alongside `reclaimStuckJobs`).
      *
-     * **PostgreSQL backend only.** Throws if the backend is Redis.
-     *
      * @returns The number of tokens that were expired.
      */
     expireTimedOutTokens: () => Promise<number>;
@@ -906,10 +921,10 @@ interface QueueBackend {
     editJob(jobId: number, updates: JobUpdates): Promise<void>;
     /** Edit all pending jobs matching filters. Returns count. */
     editAllPendingJobs(filters: JobFilters | undefined, updates: JobUpdates): Promise<number>;
-    /** Delete completed jobs older than N days. Returns count deleted. */
-    cleanupOldJobs(daysToKeep?: number): Promise<number>;
-    /** Delete job events older than N days. Returns count deleted. */
-    cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
+    /** Delete completed jobs older than N days. Deletes in batches for scale safety. Returns count deleted. */
+    cleanupOldJobs(daysToKeep?: number, batchSize?: number): Promise<number>;
+    /** Delete job events older than N days. Deletes in batches for scale safety. Returns count deleted. */
+    cleanupOldJobEvents(daysToKeep?: number, batchSize?: number): Promise<number>;
     /** Reclaim jobs stuck in 'processing' for too long. Returns count. */
     reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
     /** Update the progress percentage (0-100) for a job. */
@@ -944,6 +959,58 @@ interface QueueBackend {
      * Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
      */
     updateCronScheduleAfterEnqueue(id: number, lastEnqueuedAt: Date, lastJobId: number, nextRunAt: Date | null): Promise<void>;
+    /**
+     * Transition a job from 'processing' to 'waiting' status.
+     * Persists step data so the handler can resume from where it left off.
+     *
+     * @param jobId - The job to pause.
+     * @param options - Wait configuration including optional waitUntil date, token ID, and step data.
+     */
+    waitJob(jobId: number, options: {
+        waitUntil?: Date;
+        waitTokenId?: string;
+        stepData: Record<string, any>;
+    }): Promise<void>;
+    /**
+     * Persist step data for a job. Called after each `ctx.run()` step completes
+     * to save intermediate progress. Best-effort: should not throw.
+     *
+     * @param jobId - The job to update.
+     * @param stepData - The step data to persist.
+     */
+    updateStepData(jobId: number, stepData: Record<string, any>): Promise<void>;
+    /**
+     * Create a waitpoint token that can pause a job until an external signal completes it.
+     *
+     * @param jobId - The job ID to associate with the token (null if created outside a handler).
+     * @param options - Optional timeout string (e.g. '10m', '1h') and tags.
+     * @returns The created waitpoint with its unique ID.
+     */
+    createWaitpoint(jobId: number | null, options?: CreateTokenOptions): Promise<{
+        id: string;
+    }>;
+    /**
+     * Complete a waitpoint token, optionally providing output data.
+     * Moves the associated job from 'waiting' back to 'pending' so it gets picked up.
+     *
+     * @param tokenId - The waitpoint token ID to complete.
+     * @param data - Optional data to pass to the waiting handler.
+     */
+    completeWaitpoint(tokenId: string, data?: any): Promise<void>;
+    /**
+     * Retrieve a waitpoint token by its ID.
+     *
+     * @param tokenId - The waitpoint token ID to look up.
+     * @returns The waitpoint record, or null if not found.
+     */
+    getWaitpoint(tokenId: string): Promise<WaitpointRecord | null>;
+    /**
+     * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
+     * Should be called periodically (e.g., alongside reclaimStuckJobs).
+     *
+     * @returns The number of tokens that were expired.
+     */
+    expireTimedOutWaitpoints(): Promise<number>;
     /** Set a pending reason for unpicked jobs of a given type. */
     setPendingReasonForUnpickedJobs(reason: string, jobType?: string | string[]): Promise<void>;
 }
@@ -971,8 +1038,26 @@ declare class PostgresBackend implements QueueBackend {
     cancelAllUpcomingJobs(filters?: JobFilters): Promise<number>;
     editJob(jobId: number, updates: JobUpdates): Promise<void>;
     editAllPendingJobs(filters: JobFilters | undefined, updates: JobUpdates): Promise<number>;
-    cleanupOldJobs(daysToKeep?: number): Promise<number>;
-    cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
+    /**
+     * Delete completed jobs older than the given number of days.
+     * Deletes in batches of 1000 to avoid long-running transactions
+     * and excessive WAL bloat at scale.
+     *
+     * @param daysToKeep - Number of days to retain completed jobs (default 30).
+     * @param batchSize - Number of rows to delete per batch (default 1000).
+     * @returns Total number of deleted jobs.
+     */
+    cleanupOldJobs(daysToKeep?: number, batchSize?: number): Promise<number>;
+    /**
+     * Delete job events older than the given number of days.
+     * Deletes in batches of 1000 to avoid long-running transactions
+     * and excessive WAL bloat at scale.
+     *
+     * @param daysToKeep - Number of days to retain events (default 30).
+     * @param batchSize - Number of rows to delete per batch (default 1000).
+     * @returns Total number of deleted events.
+     */
+    cleanupOldJobEvents(daysToKeep?: number, batchSize?: number): Promise<number>;
     reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
     /**
      * Batch-insert multiple job events in a single query.
@@ -1005,6 +1090,56 @@ declare class PostgresBackend implements QueueBackend {
      * Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
      */
     updateCronScheduleAfterEnqueue(id: number, lastEnqueuedAt: Date, lastJobId: number, nextRunAt: Date | null): Promise<void>;
+    /**
+     * Transition a job from 'processing' to 'waiting' status.
+     * Persists step data so the handler can resume from where it left off.
+     *
+     * @param jobId - The job to pause.
+     * @param options - Wait configuration including optional waitUntil date, token ID, and step data.
+     */
+    waitJob(jobId: number, options: {
+        waitUntil?: Date;
+        waitTokenId?: string;
+        stepData: Record<string, any>;
+    }): Promise<void>;
+    /**
+     * Persist step data for a job. Called after each ctx.run() step completes.
+     * Best-effort: does not throw to avoid killing the running handler.
+     *
+     * @param jobId - The job to update.
+     * @param stepData - The step data to persist.
+     */
+    updateStepData(jobId: number, stepData: Record<string, any>): Promise<void>;
+    /**
+     * Create a waitpoint token in the database.
+     *
+     * @param jobId - The job ID to associate with the token (null if created outside a handler).
+     * @param options - Optional timeout string (e.g. '10m', '1h') and tags.
+     * @returns The created waitpoint with its unique ID.
+     */
+    createWaitpoint(jobId: number | null, options?: CreateTokenOptions): Promise<{
+        id: string;
+    }>;
+    /**
+     * Complete a waitpoint token and move the associated job back to 'pending'.
+     *
+     * @param tokenId - The waitpoint token ID to complete.
+     * @param data - Optional data to pass to the waiting handler.
+     */
+    completeWaitpoint(tokenId: string, data?: any): Promise<void>;
+    /**
+     * Retrieve a waitpoint token by its ID.
+     *
+     * @param tokenId - The waitpoint token ID to look up.
+     * @returns The waitpoint record, or null if not found.
+     */
+    getWaitpoint(tokenId: string): Promise<WaitpointRecord | null>;
+    /**
+     * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
+     *
+     * @returns The number of tokens that were expired.
+     */
+    expireTimedOutWaitpoints(): Promise<number>;
     setPendingReasonForUnpickedJobs(reason: string, jobType?: string | string[]): Promise<void>;
 }
 

package/dist/index.d.ts

@@ -449,6 +449,23 @@ interface PostgresJobQueueConfig {
         user?: string;
         password?: string;
         ssl?: DatabaseSSLConfig;
+        /**
+         * Maximum number of clients in the pool (default: 10).
+         * Increase when running multiple processors in the same process.
+         */
+        max?: number;
+        /**
+         * Minimum number of idle clients in the pool (default: 0).
+         */
+        min?: number;
+        /**
+         * Milliseconds a client must sit idle before being closed (default: 10000).
+         */
+        idleTimeoutMillis?: number;
+        /**
+         * Milliseconds to wait for a connection before throwing (default: 0, no timeout).
+         */
+        connectionTimeoutMillis?: number;
     };
     verbose?: boolean;
 }
@@ -626,12 +643,18 @@ interface JobQueue<PayloadMap> {
     retryJob: (jobId: number) => Promise<void>;
     /**
      * Cleanup jobs that are older than the specified number of days.
+     * Deletes in batches for scale safety.
+     * @param daysToKeep - Number of days to retain completed jobs (default 30).
+     * @param batchSize - Number of rows to delete per batch (default 1000 for PostgreSQL, 200 for Redis).
      */
-    cleanupOldJobs: (daysToKeep?: number) => Promise<number>;
+    cleanupOldJobs: (daysToKeep?: number, batchSize?: number) => Promise<number>;
     /**
      * Cleanup job events that are older than the specified number of days.
+     * Deletes in batches for scale safety.
+     * @param daysToKeep - Number of days to retain events (default 30).
+     * @param batchSize - Number of rows to delete per batch (default 1000).
      */
-    cleanupOldJobEvents: (daysToKeep?: number) => Promise<number>;
+    cleanupOldJobEvents: (daysToKeep?: number, batchSize?: number) => Promise<number>;
     /**
      * Cancel a job given its ID.
      * - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
@@ -718,8 +741,6 @@ interface JobQueue<PayloadMap> {
      * Tokens can be completed externally to resume a waiting job.
      * Can be called outside of handlers (e.g., from an API route).
      *
-     * **PostgreSQL backend only.** Throws if the backend is Redis.
-     *
      * @param options - Optional token configuration (timeout, tags).
      * @returns A token object with `id`.
      */
@@ -728,8 +749,6 @@ interface JobQueue<PayloadMap> {
      * Complete a waitpoint token, resuming the associated waiting job.
      * Can be called from anywhere (API routes, external services, etc.).
      *
-     * **PostgreSQL backend only.** Throws if the backend is Redis.
-     *
      * @param tokenId - The ID of the token to complete.
      * @param data - Optional data to pass to the waiting handler.
      */
@@ -737,8 +756,6 @@ interface JobQueue<PayloadMap> {
     /**
      * Retrieve a waitpoint token by its ID.
      *
-     * **PostgreSQL backend only.** Throws if the backend is Redis.
-     *
      * @param tokenId - The ID of the token to retrieve.
      * @returns The token record, or null if not found.
      */
@@ -747,8 +764,6 @@ interface JobQueue<PayloadMap> {
      * Expire timed-out waitpoint tokens and resume their associated jobs.
      * Call this periodically (e.g., alongside `reclaimStuckJobs`).
      *
-     * **PostgreSQL backend only.** Throws if the backend is Redis.
-     *
      * @returns The number of tokens that were expired.
      */
     expireTimedOutTokens: () => Promise<number>;
@@ -906,10 +921,10 @@ interface QueueBackend {
     editJob(jobId: number, updates: JobUpdates): Promise<void>;
     /** Edit all pending jobs matching filters. Returns count. */
     editAllPendingJobs(filters: JobFilters | undefined, updates: JobUpdates): Promise<number>;
-    /** Delete completed jobs older than N days. Returns count deleted. */
-    cleanupOldJobs(daysToKeep?: number): Promise<number>;
-    /** Delete job events older than N days. Returns count deleted. */
-    cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
+    /** Delete completed jobs older than N days. Deletes in batches for scale safety. Returns count deleted. */
+    cleanupOldJobs(daysToKeep?: number, batchSize?: number): Promise<number>;
+    /** Delete job events older than N days. Deletes in batches for scale safety. Returns count deleted. */
+    cleanupOldJobEvents(daysToKeep?: number, batchSize?: number): Promise<number>;
     /** Reclaim jobs stuck in 'processing' for too long. Returns count. */
     reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
     /** Update the progress percentage (0-100) for a job. */
@@ -944,6 +959,58 @@ interface QueueBackend {
      * Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
      */
     updateCronScheduleAfterEnqueue(id: number, lastEnqueuedAt: Date, lastJobId: number, nextRunAt: Date | null): Promise<void>;
+    /**
+     * Transition a job from 'processing' to 'waiting' status.
+     * Persists step data so the handler can resume from where it left off.
+     *
+     * @param jobId - The job to pause.
+     * @param options - Wait configuration including optional waitUntil date, token ID, and step data.
+     */
+    waitJob(jobId: number, options: {
+        waitUntil?: Date;
+        waitTokenId?: string;
+        stepData: Record<string, any>;
+    }): Promise<void>;
+    /**
+     * Persist step data for a job. Called after each `ctx.run()` step completes
+     * to save intermediate progress. Best-effort: should not throw.
+     *
+     * @param jobId - The job to update.
+     * @param stepData - The step data to persist.
+     */
+    updateStepData(jobId: number, stepData: Record<string, any>): Promise<void>;
+    /**
+     * Create a waitpoint token that can pause a job until an external signal completes it.
+     *
+     * @param jobId - The job ID to associate with the token (null if created outside a handler).
+     * @param options - Optional timeout string (e.g. '10m', '1h') and tags.
+     * @returns The created waitpoint with its unique ID.
+     */
+    createWaitpoint(jobId: number | null, options?: CreateTokenOptions): Promise<{
+        id: string;
+    }>;
+    /**
+     * Complete a waitpoint token, optionally providing output data.
+     * Moves the associated job from 'waiting' back to 'pending' so it gets picked up.
+     *
+     * @param tokenId - The waitpoint token ID to complete.
+     * @param data - Optional data to pass to the waiting handler.
+     */
+    completeWaitpoint(tokenId: string, data?: any): Promise<void>;
+    /**
+     * Retrieve a waitpoint token by its ID.
+     *
+     * @param tokenId - The waitpoint token ID to look up.
+     * @returns The waitpoint record, or null if not found.
+     */
+    getWaitpoint(tokenId: string): Promise<WaitpointRecord | null>;
+    /**
+     * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
+     * Should be called periodically (e.g., alongside reclaimStuckJobs).
+     *
+     * @returns The number of tokens that were expired.
+     */
+    expireTimedOutWaitpoints(): Promise<number>;
     /** Set a pending reason for unpicked jobs of a given type. */
     setPendingReasonForUnpickedJobs(reason: string, jobType?: string | string[]): Promise<void>;
 }
@@ -971,8 +1038,26 @@ declare class PostgresBackend implements QueueBackend {
     cancelAllUpcomingJobs(filters?: JobFilters): Promise<number>;
     editJob(jobId: number, updates: JobUpdates): Promise<void>;
     editAllPendingJobs(filters: JobFilters | undefined, updates: JobUpdates): Promise<number>;
-    cleanupOldJobs(daysToKeep?: number): Promise<number>;
-    cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
+    /**
+     * Delete completed jobs older than the given number of days.
+     * Deletes in batches of 1000 to avoid long-running transactions
+     * and excessive WAL bloat at scale.
+     *
+     * @param daysToKeep - Number of days to retain completed jobs (default 30).
+     * @param batchSize - Number of rows to delete per batch (default 1000).
+     * @returns Total number of deleted jobs.
+     */
+    cleanupOldJobs(daysToKeep?: number, batchSize?: number): Promise<number>;
+    /**
+     * Delete job events older than the given number of days.
+     * Deletes in batches of 1000 to avoid long-running transactions
+     * and excessive WAL bloat at scale.
+     *
+     * @param daysToKeep - Number of days to retain events (default 30).
+     * @param batchSize - Number of rows to delete per batch (default 1000).
+     * @returns Total number of deleted events.
+     */
+    cleanupOldJobEvents(daysToKeep?: number, batchSize?: number): Promise<number>;
     reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
     /**
      * Batch-insert multiple job events in a single query.
@@ -1005,6 +1090,56 @@ declare class PostgresBackend implements QueueBackend {
      * Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
      */
     updateCronScheduleAfterEnqueue(id: number, lastEnqueuedAt: Date, lastJobId: number, nextRunAt: Date | null): Promise<void>;
+    /**
+     * Transition a job from 'processing' to 'waiting' status.
+     * Persists step data so the handler can resume from where it left off.
+     *
+     * @param jobId - The job to pause.
+     * @param options - Wait configuration including optional waitUntil date, token ID, and step data.
+     */
+    waitJob(jobId: number, options: {
+        waitUntil?: Date;
+        waitTokenId?: string;
+        stepData: Record<string, any>;
+    }): Promise<void>;
+    /**
+     * Persist step data for a job. Called after each ctx.run() step completes.
+     * Best-effort: does not throw to avoid killing the running handler.
+     *
+     * @param jobId - The job to update.
+     * @param stepData - The step data to persist.
+     */
+    updateStepData(jobId: number, stepData: Record<string, any>): Promise<void>;
+    /**
+     * Create a waitpoint token in the database.
+     *
+     * @param jobId - The job ID to associate with the token (null if created outside a handler).
+     * @param options - Optional timeout string (e.g. '10m', '1h') and tags.
+     * @returns The created waitpoint with its unique ID.
+     */
+    createWaitpoint(jobId: number | null, options?: CreateTokenOptions): Promise<{
+        id: string;
+    }>;
+    /**
+     * Complete a waitpoint token and move the associated job back to 'pending'.
+     *
+     * @param tokenId - The waitpoint token ID to complete.
+     * @param data - Optional data to pass to the waiting handler.
+     */
+    completeWaitpoint(tokenId: string, data?: any): Promise<void>;
+    /**
+     * Retrieve a waitpoint token by its ID.
+     *
+     * @param tokenId - The waitpoint token ID to look up.
+     * @returns The waitpoint record, or null if not found.
+     */
+    getWaitpoint(tokenId: string): Promise<WaitpointRecord | null>;
+    /**
+     * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
+     *
+     * @returns The number of tokens that were expired.
+     */
+    expireTimedOutWaitpoints(): Promise<number>;
     setPendingReasonForUnpickedJobs(reason: string, jobType?: string | string[]): Promise<void>;
 }