@nicnocquee/dataqueue 1.30.0 → 1.32.0

package/dist/index.d.cts

@@ -1,5 +1,6 @@
 import * as pg from 'pg';
 import { Pool } from 'pg';
+import { Cron } from 'croner';
 
 type JobType<PayloadMap> = keyof PayloadMap & string;
 interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
@@ -448,6 +449,23 @@ interface PostgresJobQueueConfig {
         user?: string;
         password?: string;
         ssl?: DatabaseSSLConfig;
+        /**
+         * Maximum number of clients in the pool (default: 10).
+         * Increase when running multiple processors in the same process.
+         */
+        max?: number;
+        /**
+         * Minimum number of idle clients in the pool (default: 0).
+         */
+        min?: number;
+        /**
+         * Milliseconds a client must sit idle before being closed (default: 10000).
+         */
+        idleTimeoutMillis?: number;
+        /**
+         * Milliseconds to wait for a connection before throwing (default: 0, no timeout).
+         */
+        connectionTimeoutMillis?: number;
     };
     verbose?: boolean;
 }
@@ -490,6 +508,81 @@ type JobQueueConfig = PostgresJobQueueConfig | RedisJobQueueConfig;
 /** @deprecated Use JobQueueConfig instead. Alias kept for backward compat. */
 type JobQueueConfigLegacy = PostgresJobQueueConfig;
 type TagQueryMode = 'exact' | 'all' | 'any' | 'none';
+/**
+ * Status of a cron schedule.
+ */
+type CronScheduleStatus = 'active' | 'paused';
+/**
+ * Options for creating a recurring cron schedule.
+ * Each schedule defines a recurring job that is automatically enqueued
+ * when its cron expression matches.
+ */
+interface CronScheduleOptions<PayloadMap, T extends JobType<PayloadMap>> {
+    /** Unique human-readable name for the schedule. */
+    scheduleName: string;
+    /** Standard cron expression (5 fields, e.g. "0 * * * *"). */
+    cronExpression: string;
+    /** Job type from the PayloadMap. */
+    jobType: T;
+    /** Payload for each job instance. */
+    payload: PayloadMap[T];
+    /** Maximum retry attempts for each job instance (default: 3). */
+    maxAttempts?: number;
+    /** Priority for each job instance (default: 0). */
+    priority?: number;
+    /** Timeout in milliseconds for each job instance. */
+    timeoutMs?: number;
+    /** Whether to force-kill the job on timeout (default: false). */
+    forceKillOnTimeout?: boolean;
+    /** Tags for each job instance. */
+    tags?: string[];
+    /** IANA timezone string for cron evaluation (default: "UTC"). */
+    timezone?: string;
+    /**
+     * Whether to allow overlapping job instances (default: false).
+     * When false, a new job will not be enqueued if the previous instance
+     * is still pending, processing, or waiting.
+     */
+    allowOverlap?: boolean;
+}
+/**
+ * A persisted cron schedule record.
+ */
+interface CronScheduleRecord {
+    id: number;
+    scheduleName: string;
+    cronExpression: string;
+    jobType: string;
+    payload: any;
+    maxAttempts: number;
+    priority: number;
+    timeoutMs: number | null;
+    forceKillOnTimeout: boolean;
+    tags: string[] | undefined;
+    timezone: string;
+    allowOverlap: boolean;
+    status: CronScheduleStatus;
+    lastEnqueuedAt: Date | null;
+    lastJobId: number | null;
+    nextRunAt: Date | null;
+    createdAt: Date;
+    updatedAt: Date;
+}
+/**
+ * Options for editing an existing cron schedule.
+ * All fields are optional; only provided fields are updated.
+ */
+interface EditCronScheduleOptions {
+    cronExpression?: string;
+    payload?: any;
+    maxAttempts?: number;
+    priority?: number;
+    timeoutMs?: number | null;
+    forceKillOnTimeout?: boolean;
+    tags?: string[] | null;
+    timezone?: string;
+    allowOverlap?: boolean;
+}
 interface JobQueue<PayloadMap> {
     /**
      * Add a job to the job queue.
@@ -550,12 +643,18 @@ interface JobQueue<PayloadMap> {
     retryJob: (jobId: number) => Promise<void>;
     /**
      * Cleanup jobs that are older than the specified number of days.
+     * Deletes in batches for scale safety.
+     * @param daysToKeep - Number of days to retain completed jobs (default 30).
+     * @param batchSize - Number of rows to delete per batch (default 1000 for PostgreSQL, 200 for Redis).
      */
-    cleanupOldJobs: (daysToKeep?: number) => Promise<number>;
+    cleanupOldJobs: (daysToKeep?: number, batchSize?: number) => Promise<number>;
     /**
      * Cleanup job events that are older than the specified number of days.
+     * Deletes in batches for scale safety.
+     * @param daysToKeep - Number of days to retain events (default 30).
+     * @param batchSize - Number of rows to delete per batch (default 1000).
      */
-    cleanupOldJobEvents: (daysToKeep?: number) => Promise<number>;
+    cleanupOldJobEvents: (daysToKeep?: number, batchSize?: number) => Promise<number>;
     /**
      * Cancel a job given its ID.
      * - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
@@ -642,8 +741,6 @@ interface JobQueue<PayloadMap> {
      * Tokens can be completed externally to resume a waiting job.
      * Can be called outside of handlers (e.g., from an API route).
      *
-     * **PostgreSQL backend only.** Throws if the backend is Redis.
-     *
      * @param options - Optional token configuration (timeout, tags).
      * @returns A token object with `id`.
      */
@@ -652,8 +749,6 @@ interface JobQueue<PayloadMap> {
      * Complete a waitpoint token, resuming the associated waiting job.
      * Can be called from anywhere (API routes, external services, etc.).
      *
-     * **PostgreSQL backend only.** Throws if the backend is Redis.
-     *
      * @param tokenId - The ID of the token to complete.
      * @param data - Optional data to pass to the waiting handler.
      */
@@ -661,8 +756,6 @@ interface JobQueue<PayloadMap> {
     /**
      * Retrieve a waitpoint token by its ID.
      *
-     * **PostgreSQL backend only.** Throws if the backend is Redis.
-     *
      * @param tokenId - The ID of the token to retrieve.
      * @returns The token record, or null if not found.
      */
@@ -671,11 +764,59 @@ interface JobQueue<PayloadMap> {
      * Expire timed-out waitpoint tokens and resume their associated jobs.
      * Call this periodically (e.g., alongside `reclaimStuckJobs`).
      *
-     * **PostgreSQL backend only.** Throws if the backend is Redis.
-     *
      * @returns The number of tokens that were expired.
      */
     expireTimedOutTokens: () => Promise<number>;
+    /**
+     * Add a recurring cron schedule. The processor automatically enqueues
+     * due cron jobs before each batch, so no manual triggering is needed.
+     *
+     * @returns The ID of the created schedule.
+     * @throws If the cron expression is invalid or the schedule name is already taken.
+     */
+    addCronJob: <T extends JobType<PayloadMap>>(options: CronScheduleOptions<PayloadMap, T>) => Promise<number>;
+    /**
+     * Get a cron schedule by its ID.
+     */
+    getCronJob: (id: number) => Promise<CronScheduleRecord | null>;
+    /**
+     * Get a cron schedule by its unique name.
+     */
+    getCronJobByName: (name: string) => Promise<CronScheduleRecord | null>;
+    /**
+     * List all cron schedules, optionally filtered by status.
+     */
+    listCronJobs: (status?: CronScheduleStatus) => Promise<CronScheduleRecord[]>;
+    /**
+     * Remove a cron schedule by its ID. Does not cancel any already-enqueued jobs.
+     */
+    removeCronJob: (id: number) => Promise<void>;
+    /**
+     * Pause a cron schedule. Paused schedules are skipped by `enqueueDueCronJobs()`.
+     */
+    pauseCronJob: (id: number) => Promise<void>;
+    /**
+     * Resume a paused cron schedule.
+     */
+    resumeCronJob: (id: number) => Promise<void>;
+    /**
+     * Edit an existing cron schedule. Only provided fields are updated.
+     * If `cronExpression` or `timezone` changes, `nextRunAt` is recalculated.
+     */
+    editCronJob: (id: number, updates: EditCronScheduleOptions) => Promise<void>;
+    /**
+     * Check all active cron schedules and enqueue jobs for any whose
+     * `nextRunAt` has passed. When `allowOverlap` is false (the default),
+     * a new job is not enqueued if the previous instance is still
+     * pending, processing, or waiting.
+     *
+     * **Note:** The processor calls this automatically before each batch,
+     * so you typically do not need to call it yourself. It is exposed for
+     * manual use in tests or one-off scripts.
+     *
+     * @returns The number of jobs that were enqueued.
+     */
+    enqueueDueCronJobs: () => Promise<number>;
     /**
      * Get the PostgreSQL database pool.
      * Throws if the backend is not PostgreSQL.
@@ -723,6 +864,24 @@ interface JobUpdates {
     timeoutMs?: number | null;
     tags?: string[] | null;
 }
+/**
+ * Input shape for creating a cron schedule in the backend.
+ * This is the backend-level version of CronScheduleOptions.
+ */
+interface CronScheduleInput {
+    scheduleName: string;
+    cronExpression: string;
+    jobType: string;
+    payload: any;
+    maxAttempts: number;
+    priority: number;
+    timeoutMs: number | null;
+    forceKillOnTimeout: boolean;
+    tags: string[] | undefined;
+    timezone: string;
+    allowOverlap: boolean;
+    nextRunAt: Date | null;
+}
 /**
  * Abstract backend interface that both PostgreSQL and Redis implement.
  * All storage operations go through this interface so the processor
@@ -762,10 +921,10 @@ interface QueueBackend {
     editJob(jobId: number, updates: JobUpdates): Promise<void>;
     /** Edit all pending jobs matching filters. Returns count. */
     editAllPendingJobs(filters: JobFilters | undefined, updates: JobUpdates): Promise<number>;
-    /** Delete completed jobs older than N days. Returns count deleted. */
-    cleanupOldJobs(daysToKeep?: number): Promise<number>;
-    /** Delete job events older than N days. Returns count deleted. */
-    cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
+    /** Delete completed jobs older than N days. Deletes in batches for scale safety. Returns count deleted. */
+    cleanupOldJobs(daysToKeep?: number, batchSize?: number): Promise<number>;
+    /** Delete job events older than N days. Deletes in batches for scale safety. Returns count deleted. */
+    cleanupOldJobEvents(daysToKeep?: number, batchSize?: number): Promise<number>;
     /** Reclaim jobs stuck in 'processing' for too long. Returns count. */
     reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
     /** Update the progress percentage (0-100) for a job. */
@@ -774,6 +933,84 @@ interface QueueBackend {
     recordJobEvent(jobId: number, eventType: JobEventType, metadata?: any): Promise<void>;
     /** Get all events for a job, ordered by createdAt ASC. */
     getJobEvents(jobId: number): Promise<JobEvent[]>;
+    /** Create a cron schedule and return its ID. */
+    addCronSchedule(input: CronScheduleInput): Promise<number>;
+    /** Get a cron schedule by ID, or null if not found. */
+    getCronSchedule(id: number): Promise<CronScheduleRecord | null>;
+    /** Get a cron schedule by its unique name, or null if not found. */
+    getCronScheduleByName(name: string): Promise<CronScheduleRecord | null>;
+    /** List cron schedules, optionally filtered by status. */
+    listCronSchedules(status?: CronScheduleStatus): Promise<CronScheduleRecord[]>;
+    /** Delete a cron schedule by ID. */
+    removeCronSchedule(id: number): Promise<void>;
+    /** Pause a cron schedule. */
+    pauseCronSchedule(id: number): Promise<void>;
+    /** Resume a cron schedule. */
+    resumeCronSchedule(id: number): Promise<void>;
+    /** Edit a cron schedule. */
+    editCronSchedule(id: number, updates: EditCronScheduleOptions, nextRunAt?: Date | null): Promise<void>;
+    /**
+     * Atomically fetch all active cron schedules whose nextRunAt <= now.
+     * In PostgreSQL this uses FOR UPDATE SKIP LOCKED to prevent duplicate enqueuing.
+     */
+    getDueCronSchedules(): Promise<CronScheduleRecord[]>;
+    /**
+     * Update a cron schedule after a job has been enqueued.
+     * Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
+     */
+    updateCronScheduleAfterEnqueue(id: number, lastEnqueuedAt: Date, lastJobId: number, nextRunAt: Date | null): Promise<void>;
+    /**
+     * Transition a job from 'processing' to 'waiting' status.
+     * Persists step data so the handler can resume from where it left off.
+     *
+     * @param jobId - The job to pause.
+     * @param options - Wait configuration including optional waitUntil date, token ID, and step data.
+     */
+    waitJob(jobId: number, options: {
+        waitUntil?: Date;
+        waitTokenId?: string;
+        stepData: Record<string, any>;
+    }): Promise<void>;
+    /**
+     * Persist step data for a job. Called after each `ctx.run()` step completes
+     * to save intermediate progress. Best-effort: should not throw.
+     *
+     * @param jobId - The job to update.
+     * @param stepData - The step data to persist.
+     */
+    updateStepData(jobId: number, stepData: Record<string, any>): Promise<void>;
+    /**
+     * Create a waitpoint token that can pause a job until an external signal completes it.
+     *
+     * @param jobId - The job ID to associate with the token (null if created outside a handler).
+     * @param options - Optional timeout string (e.g. '10m', '1h') and tags.
+     * @returns The created waitpoint with its unique ID.
+     */
+    createWaitpoint(jobId: number | null, options?: CreateTokenOptions): Promise<{
+        id: string;
+    }>;
+    /**
+     * Complete a waitpoint token, optionally providing output data.
+     * Moves the associated job from 'waiting' back to 'pending' so it gets picked up.
+     *
+     * @param tokenId - The waitpoint token ID to complete.
+     * @param data - Optional data to pass to the waiting handler.
+     */
+    completeWaitpoint(tokenId: string, data?: any): Promise<void>;
+    /**
+     * Retrieve a waitpoint token by its ID.
+     *
+     * @param tokenId - The waitpoint token ID to look up.
+     * @returns The waitpoint record, or null if not found.
+     */
+    getWaitpoint(tokenId: string): Promise<WaitpointRecord | null>;
+    /**
+     * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
+     * Should be called periodically (e.g., alongside reclaimStuckJobs).
+     *
+     * @returns The number of tokens that were expired.
+     */
+    expireTimedOutWaitpoints(): Promise<number>;
     /** Set a pending reason for unpicked jobs of a given type. */
     setPendingReasonForUnpickedJobs(reason: string, jobType?: string | string[]): Promise<void>;
 }
@@ -801,14 +1038,108 @@ declare class PostgresBackend implements QueueBackend {
     cancelAllUpcomingJobs(filters?: JobFilters): Promise<number>;
     editJob(jobId: number, updates: JobUpdates): Promise<void>;
     editAllPendingJobs(filters: JobFilters | undefined, updates: JobUpdates): Promise<number>;
-    cleanupOldJobs(daysToKeep?: number): Promise<number>;
-    cleanupOldJobEvents(daysToKeep?: number): Promise<number>;
+    /**
+     * Delete completed jobs older than the given number of days.
+     * Deletes in batches of 1000 to avoid long-running transactions
+     * and excessive WAL bloat at scale.
+     *
+     * @param daysToKeep - Number of days to retain completed jobs (default 30).
+     * @param batchSize - Number of rows to delete per batch (default 1000).
+     * @returns Total number of deleted jobs.
+     */
+    cleanupOldJobs(daysToKeep?: number, batchSize?: number): Promise<number>;
+    /**
+     * Delete job events older than the given number of days.
+     * Deletes in batches of 1000 to avoid long-running transactions
+     * and excessive WAL bloat at scale.
+     *
+     * @param daysToKeep - Number of days to retain events (default 30).
+     * @param batchSize - Number of rows to delete per batch (default 1000).
+     * @returns Total number of deleted events.
+     */
+    cleanupOldJobEvents(daysToKeep?: number, batchSize?: number): Promise<number>;
     reclaimStuckJobs(maxProcessingTimeMinutes?: number): Promise<number>;
     /**
      * Batch-insert multiple job events in a single query.
      * More efficient than individual recordJobEvent calls.
      */
     private recordJobEventsBatch;
+    /** Create a cron schedule and return its ID. */
+    addCronSchedule(input: CronScheduleInput): Promise<number>;
+    /** Get a cron schedule by ID. */
+    getCronSchedule(id: number): Promise<CronScheduleRecord | null>;
+    /** Get a cron schedule by its unique name. */
+    getCronScheduleByName(name: string): Promise<CronScheduleRecord | null>;
+    /** List cron schedules, optionally filtered by status. */
+    listCronSchedules(status?: CronScheduleStatus): Promise<CronScheduleRecord[]>;
+    /** Delete a cron schedule by ID. */
+    removeCronSchedule(id: number): Promise<void>;
+    /** Pause a cron schedule. */
+    pauseCronSchedule(id: number): Promise<void>;
+    /** Resume a paused cron schedule. */
+    resumeCronSchedule(id: number): Promise<void>;
+    /** Edit a cron schedule. */
+    editCronSchedule(id: number, updates: EditCronScheduleOptions, nextRunAt?: Date | null): Promise<void>;
+    /**
+     * Atomically fetch all active cron schedules whose nextRunAt <= NOW().
+     * Uses FOR UPDATE SKIP LOCKED to prevent duplicate enqueuing across workers.
+     */
+    getDueCronSchedules(): Promise<CronScheduleRecord[]>;
+    /**
+     * Update a cron schedule after a job has been enqueued.
+     * Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
+     */
+    updateCronScheduleAfterEnqueue(id: number, lastEnqueuedAt: Date, lastJobId: number, nextRunAt: Date | null): Promise<void>;
+    /**
+     * Transition a job from 'processing' to 'waiting' status.
+     * Persists step data so the handler can resume from where it left off.
+     *
+     * @param jobId - The job to pause.
+     * @param options - Wait configuration including optional waitUntil date, token ID, and step data.
+     */
+    waitJob(jobId: number, options: {
+        waitUntil?: Date;
+        waitTokenId?: string;
+        stepData: Record<string, any>;
+    }): Promise<void>;
+    /**
+     * Persist step data for a job. Called after each ctx.run() step completes.
+     * Best-effort: does not throw to avoid killing the running handler.
+     *
+     * @param jobId - The job to update.
+     * @param stepData - The step data to persist.
+     */
+    updateStepData(jobId: number, stepData: Record<string, any>): Promise<void>;
+    /**
+     * Create a waitpoint token in the database.
+     *
+     * @param jobId - The job ID to associate with the token (null if created outside a handler).
+     * @param options - Optional timeout string (e.g. '10m', '1h') and tags.
+     * @returns The created waitpoint with its unique ID.
+     */
+    createWaitpoint(jobId: number | null, options?: CreateTokenOptions): Promise<{
+        id: string;
+    }>;
+    /**
+     * Complete a waitpoint token and move the associated job back to 'pending'.
+     *
+     * @param tokenId - The waitpoint token ID to complete.
+     * @param data - Optional data to pass to the waiting handler.
+     */
+    completeWaitpoint(tokenId: string, data?: any): Promise<void>;
+    /**
+     * Retrieve a waitpoint token by its ID.
+     *
+     * @param tokenId - The waitpoint token ID to look up.
+     * @returns The waitpoint record, or null if not found.
+     */
+    getWaitpoint(tokenId: string): Promise<WaitpointRecord | null>;
+    /**
+     * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
+     *
+     * @returns The number of tokens that were expired.
+     */
+    expireTimedOutWaitpoints(): Promise<number>;
     setPendingReasonForUnpickedJobs(reason: string, jobType?: string | string[]): Promise<void>;
 }
 
@@ -863,6 +1194,25 @@ declare function testHandlerSerialization<PayloadMap, T extends keyof PayloadMap
     error?: string;
 }>;
 
+/**
+ * Calculate the next occurrence of a cron expression after a given date.
+ *
+ * @param cronExpression - A standard cron expression (5 fields, e.g. "0 * * * *").
+ * @param timezone - IANA timezone string (default: "UTC").
+ * @param after - The reference date to compute the next run from (default: now).
+ * @param CronImpl - Cron class for dependency injection (default: croner's Cron).
+ * @returns The next occurrence as a Date, or null if the expression will never fire again.
+ */
+declare function getNextCronOccurrence(cronExpression: string, timezone?: string, after?: Date, CronImpl?: typeof Cron): Date | null;
+/**
+ * Validate whether a string is a syntactically correct cron expression.
+ *
+ * @param cronExpression - The cron expression to validate.
+ * @param CronImpl - Cron class for dependency injection (default: croner's Cron).
+ * @returns True if the expression is valid, false otherwise.
+ */
+declare function validateCronExpression(cronExpression: string, CronImpl?: typeof Cron): boolean;
+
 /**
  * Initialize the job queue system.
  *
@@ -870,4 +1220,4 @@ declare function testHandlerSerialization<PayloadMap, T extends keyof PayloadMap
  */
 declare const initJobQueue: <PayloadMap = any>(config: JobQueueConfig) => JobQueue<PayloadMap>;
 
-export { type CreateTokenOptions, type DatabaseSSLConfig, type EditJobOptions, FailureReason, type JobContext, type JobEvent, JobEventType, type JobHandler, type JobHandlers, type JobOptions, type JobQueue, type JobQueueConfig, type JobQueueConfigLegacy, type JobRecord, type JobStatus, type JobType, type OnTimeoutCallback, PostgresBackend, type PostgresJobQueueConfig, type Processor, type ProcessorOptions, type QueueBackend, type RedisJobQueueConfig, type RedisTLSConfig, type TagQueryMode, type WaitDuration, WaitSignal, type WaitToken, type WaitTokenResult, type WaitpointRecord, type WaitpointStatus, initJobQueue, testHandlerSerialization, validateHandlerSerializable };
+export { type CreateTokenOptions, type CronScheduleInput, type CronScheduleOptions, type CronScheduleRecord, type CronScheduleStatus, type DatabaseSSLConfig, type EditCronScheduleOptions, type EditJobOptions, FailureReason, type JobContext, type JobEvent, JobEventType, type JobHandler, type JobHandlers, type JobOptions, type JobQueue, type JobQueueConfig, type JobQueueConfigLegacy, type JobRecord, type JobStatus, type JobType, type OnTimeoutCallback, PostgresBackend, type PostgresJobQueueConfig, type Processor, type ProcessorOptions, type QueueBackend, type RedisJobQueueConfig, type RedisTLSConfig, type TagQueryMode, type WaitDuration, WaitSignal, type WaitToken, type WaitTokenResult, type WaitpointRecord, type WaitpointStatus, getNextCronOccurrence, initJobQueue, testHandlerSerialization, validateCronExpression, validateHandlerSerializable };