@nicnocquee/dataqueue 1.25.0 → 1.26.0-beta.20260223202259

package/src/types.ts

@@ -1,6 +1,33 @@
 // Utility type for job type keys
 export type JobType<PayloadMap> = keyof PayloadMap & string;
 
+/**
+ * Abstract database client interface for transactional job creation.
+ * Compatible with `pg.Pool`, `pg.PoolClient`, `pg.Client`, or any object
+ * that exposes a `.query()` method matching the `pg` signature.
+ */
+export interface DatabaseClient {
+  query(
+    text: string,
+    values?: any[],
+  ): Promise<{ rows: any[]; rowCount: number | null }>;
+}
+
+/**
+ * Options for `addJob()` beyond the job itself.
+ * Use `db` to insert the job within an existing database transaction.
+ */
+export interface AddJobOptions {
+  /**
+   * An external database client (e.g., a `pg.PoolClient` inside a transaction).
+   * When provided, the INSERT runs on this client instead of the internal pool,
+   * so the job is part of the caller's transaction.
+   *
+   * **PostgreSQL only.** Throws if used with the Redis backend.
+   */
+  db?: DatabaseClient;
+}
+
 export interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
   jobType: T;
   payload: PayloadMap[T];
@@ -73,6 +100,26 @@ export interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
    * Once a key exists, it cannot be reused until the job is cleaned up (via `cleanupOldJobs`).
    */
   idempotencyKey?: string;
+  /**
+   * Base delay between retries in seconds. When `retryBackoff` is true (the default),
+   * this is the base for exponential backoff: `retryDelay * 2^attempts`.
+   * When `retryBackoff` is false, retries use this fixed delay.
+   * @default 60
+   */
+  retryDelay?: number;
+  /**
+   * Whether to use exponential backoff for retries. When true, delay doubles
+   * with each attempt and includes jitter to prevent thundering herd.
+   * When false, a fixed `retryDelay` is used between every retry.
+   * @default true
+   */
+  retryBackoff?: boolean;
+  /**
+   * Maximum delay between retries in seconds. Caps the exponential backoff
+   * so retries never wait longer than this value. Only meaningful when
+   * `retryBackoff` is true. No limit when omitted.
+   */
+  retryDelayMax?: number;
 }
 
 /**
@@ -196,6 +243,18 @@ export interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
    * Updated by the handler via `ctx.setProgress(percent)`.
    */
   progress?: number | null;
+  /**
+   * Base delay between retries in seconds, or null if using legacy default.
+   */
+  retryDelay?: number | null;
+  /**
+   * Whether exponential backoff is enabled for retries, or null if using legacy default.
+   */
+  retryBackoff?: boolean | null;
+  /**
+   * Maximum delay cap for retries in seconds, or null if no cap.
+   */
+  retryDelayMax?: number | null;
 }
 
 /**
@@ -452,6 +511,91 @@ export interface Processor {
   start: () => Promise<number>;
 }
 
+export interface SupervisorOptions {
+  /**
+   * How often the maintenance loop runs, in milliseconds.
+   * @default 60000 (1 minute)
+   */
+  intervalMs?: number;
+  /**
+   * Reclaim jobs stuck in `processing` longer than this many minutes.
+   * @default 10
+   */
+  stuckJobsTimeoutMinutes?: number;
+  /**
+   * Auto-delete completed jobs older than this many days. Set to 0 to disable.
+   * @default 30
+   */
+  cleanupJobsDaysToKeep?: number;
+  /**
+   * Auto-delete job events older than this many days. Set to 0 to disable.
+   * @default 30
+   */
+  cleanupEventsDaysToKeep?: number;
+  /**
+   * Batch size for cleanup deletions.
+   * @default 1000
+   */
+  cleanupBatchSize?: number;
+  /**
+   * Whether to reclaim stuck jobs each cycle.
+   * @default true
+   */
+  reclaimStuckJobs?: boolean;
+  /**
+   * Whether to expire timed-out waitpoint tokens each cycle.
+   * @default true
+   */
+  expireTimedOutTokens?: boolean;
+  /**
+   * Called when a maintenance task throws. One failure does not block other tasks.
+   * @default console.error
+   */
+  onError?: (error: Error) => void;
+  /** Enable verbose logging. */
+  verbose?: boolean;
+}
+
+export interface SupervisorRunResult {
+  /** Number of stuck jobs reclaimed back to pending. */
+  reclaimedJobs: number;
+  /** Number of old completed jobs deleted. */
+  cleanedUpJobs: number;
+  /** Number of old job events deleted. */
+  cleanedUpEvents: number;
+  /** Number of timed-out waitpoint tokens expired. */
+  expiredTokens: number;
+}
+
+export interface Supervisor {
+  /**
+   * Run all maintenance tasks once and return the results.
+   * Ideal for serverless or cron-triggered invocations.
+   */
+  start: () => Promise<SupervisorRunResult>;
+  /**
+   * Start the maintenance loop in the background.
+   * Runs every `intervalMs` milliseconds (default: 60 000).
+   * Call `stop()` or `stopAndDrain()` to halt the loop.
+   */
+  startInBackground: () => void;
+  /**
+   * Stop the background maintenance loop immediately.
+   * Does not wait for an in-flight maintenance run to complete.
+   */
+  stop: () => void;
+  /**
+   * Stop the background loop and wait for the current maintenance run
+   * (if any) to finish before resolving.
+   *
+   * @param timeoutMs - Maximum time to wait (default: 30 000 ms).
+   *   If the run does not finish within this time the promise resolves anyway.
+   */
+  stopAndDrain: (timeoutMs?: number) => Promise<void>;
+  /** Whether the background maintenance loop is currently running. */
+  isRunning: () => boolean;
+}
+
 export interface DatabaseSSLConfig {
   /**
    * CA certificate as PEM string or file path. If the value starts with 'file://', it will be loaded from file, otherwise treated as PEM string.
@@ -474,10 +618,13 @@ export interface DatabaseSSLConfig {
 /**
  * Configuration for PostgreSQL backend (default).
  * Backward-compatible: omitting `backend` defaults to 'postgres'.
+ *
+ * Provide either `databaseConfig` (the library creates a pool) or `pool`
+ * (bring your own `pg.Pool`). At least one must be set.
  */
 export interface PostgresJobQueueConfig {
   backend?: 'postgres';
-  databaseConfig: {
+  databaseConfig?: {
     connectionString?: string;
     host?: string;
     port?: number;
@@ -485,7 +632,29 @@ export interface PostgresJobQueueConfig {
     user?: string;
     password?: string;
     ssl?: DatabaseSSLConfig;
+    /**
+     * Maximum number of clients in the pool (default: 10).
+     * Increase when running multiple processors in the same process.
+     */
+    max?: number;
+    /**
+     * Minimum number of idle clients in the pool (default: 0).
+     */
+    min?: number;
+    /**
+     * Milliseconds a client must sit idle before being closed (default: 10000).
+     */
+    idleTimeoutMillis?: number;
+    /**
+     * Milliseconds to wait for a connection before throwing (default: 0, no timeout).
+     */
+    connectionTimeoutMillis?: number;
   };
+  /**
+   * Bring your own `pg.Pool` instance. When provided, `databaseConfig` is
+   * ignored and the library will not close the pool on shutdown.
+   */
+  pool?: import('pg').Pool;
   verbose?: boolean;
 }
 
@@ -501,10 +670,13 @@ export interface RedisTLSConfig {
 
 /**
  * Configuration for Redis backend.
+ *
+ * Provide either `redisConfig` (the library creates an ioredis client) or
+ * `client` (bring your own ioredis instance). At least one must be set.
  */
 export interface RedisJobQueueConfig {
   backend: 'redis';
-  redisConfig: {
+  redisConfig?: {
     /** Redis URL (e.g. redis://localhost:6379) */
     url?: string;
     host?: string;
@@ -519,6 +691,17 @@ export interface RedisJobQueueConfig {
      */
     keyPrefix?: string;
   };
+  /**
+   * Bring your own ioredis client instance. When provided, `redisConfig` is
+   * ignored and the library will not close the client on shutdown.
+   * Use `keyPrefix` to set the key namespace (default: 'dq:').
+   */
+  client?: unknown;
+  /**
+   * Key prefix when using an external `client`. Ignored when `redisConfig` is used
+   * (set `redisConfig.keyPrefix` instead). Default: 'dq:'.
+   */
+  keyPrefix?: string;
   verbose?: boolean;
 }
 
@@ -533,13 +716,176 @@ export type JobQueueConfigLegacy = PostgresJobQueueConfig;
 
 export type TagQueryMode = 'exact' | 'all' | 'any' | 'none';
 
+// ── Cron schedule types ──────────────────────────────────────────────
+
+/**
+ * Status of a cron schedule.
+ */
+export type CronScheduleStatus = 'active' | 'paused';
+
+/**
+ * Options for creating a recurring cron schedule.
+ * Each schedule defines a recurring job that is automatically enqueued
+ * when its cron expression matches.
+ */
+export interface CronScheduleOptions<
+  PayloadMap,
+  T extends JobType<PayloadMap>,
+> {
+  /** Unique human-readable name for the schedule. */
+  scheduleName: string;
+  /** Standard cron expression (5 fields, e.g. "0 * * * *"). */
+  cronExpression: string;
+  /** Job type from the PayloadMap. */
+  jobType: T;
+  /** Payload for each job instance. */
+  payload: PayloadMap[T];
+  /** Maximum retry attempts for each job instance (default: 3). */
+  maxAttempts?: number;
+  /** Priority for each job instance (default: 0). */
+  priority?: number;
+  /** Timeout in milliseconds for each job instance. */
+  timeoutMs?: number;
+  /** Whether to force-kill the job on timeout (default: false). */
+  forceKillOnTimeout?: boolean;
+  /** Tags for each job instance. */
+  tags?: string[];
+  /** IANA timezone string for cron evaluation (default: "UTC"). */
+  timezone?: string;
+  /**
+   * Whether to allow overlapping job instances (default: false).
+   * When false, a new job will not be enqueued if the previous instance
+   * is still pending, processing, or waiting.
+   */
+  allowOverlap?: boolean;
+  /** Base delay between retries in seconds for each job instance (default: 60). */
+  retryDelay?: number;
+  /** Whether to use exponential backoff for retries (default: true). */
+  retryBackoff?: boolean;
+  /** Maximum delay cap for retries in seconds. */
+  retryDelayMax?: number;
+}
+
+/**
+ * A persisted cron schedule record.
+ */
+export interface CronScheduleRecord {
+  id: number;
+  scheduleName: string;
+  cronExpression: string;
+  jobType: string;
+  payload: any;
+  maxAttempts: number;
+  priority: number;
+  timeoutMs: number | null;
+  forceKillOnTimeout: boolean;
+  tags: string[] | undefined;
+  timezone: string;
+  allowOverlap: boolean;
+  status: CronScheduleStatus;
+  lastEnqueuedAt: Date | null;
+  lastJobId: number | null;
+  nextRunAt: Date | null;
+  createdAt: Date;
+  updatedAt: Date;
+  retryDelay: number | null;
+  retryBackoff: boolean | null;
+  retryDelayMax: number | null;
+}
+
+/**
+ * Options for editing an existing cron schedule.
+ * All fields are optional; only provided fields are updated.
+ */
+export interface EditCronScheduleOptions {
+  cronExpression?: string;
+  payload?: any;
+  maxAttempts?: number;
+  priority?: number;
+  timeoutMs?: number | null;
+  forceKillOnTimeout?: boolean;
+  tags?: string[] | null;
+  timezone?: string;
+  allowOverlap?: boolean;
+  retryDelay?: number | null;
+  retryBackoff?: boolean | null;
+  retryDelayMax?: number | null;
+}
+
+// ── Event hooks ──────────────────────────────────────────────────────
+
+/**
+ * Payload types for each event emitted by the job queue.
+ */
+export interface QueueEventMap {
+  /** Fired after a job is successfully added to the queue. */
+  'job:added': { jobId: number; jobType: string };
+  /** Fired when a processor claims a job and begins executing its handler. */
+  'job:processing': { jobId: number; jobType: string };
+  /** Fired when a job handler completes successfully. */
+  'job:completed': { jobId: number; jobType: string };
+  /** Fired when a job handler fails. `willRetry` indicates whether the job will be retried. */
+  'job:failed': {
+    jobId: number;
+    jobType: string;
+    error: Error;
+    willRetry: boolean;
+  };
+  /** Fired after a job is cancelled via `cancelJob()`. */
+  'job:cancelled': { jobId: number };
+  /** Fired after a failed job is manually retried via `retryJob()`. */
+  'job:retried': { jobId: number };
+  /** Fired when a job enters the `waiting` state (via `ctx.waitFor`, `ctx.waitUntil`, or `ctx.waitForToken`). */
+  'job:waiting': { jobId: number; jobType: string };
+  /** Fired when a job reports progress via `ctx.setProgress()`. */
+  'job:progress': { jobId: number; progress: number };
+  /** Fired on internal errors from the processor or supervisor. */
+  error: Error;
+}
+
+/** Union of all event names supported by the job queue. */
+export type QueueEventName = keyof QueueEventMap;
+
+/**
+ * Callback type for `emit`. Used internally to pass the emitter
+ * from `initJobQueue` into the processor and supervisor.
+ */
+export type QueueEmitFn = <K extends QueueEventName>(
+  event: K,
+  data: QueueEventMap[K],
+) => void;
+
 export interface JobQueue<PayloadMap> {
   /**
    * Add a job to the job queue.
+   *
+   * @param job - The job to enqueue.
+   * @param options - Optional. Pass `{ db }` with an external database client
+   *   to insert the job within an existing transaction (PostgreSQL only).
    */
   addJob: <T extends JobType<PayloadMap>>(
     job: JobOptions<PayloadMap, T>,
+    options?: AddJobOptions,
   ) => Promise<number>;
+  /**
+   * Add multiple jobs to the queue in a single operation.
+   *
+   * More efficient than calling `addJob` in a loop because it batches the
+   * INSERT into a single database round-trip (PostgreSQL) or a single
+   * atomic Lua script (Redis).
+   *
+   * Returns an array of job IDs in the same order as the input array.
+   * Each job may independently have an `idempotencyKey`; duplicates
+   * resolve to the existing job's ID without creating a new row.
+   *
+   * @param jobs - Array of jobs to enqueue.
+   * @param options - Optional. Pass `{ db }` with an external database client
+   *   to insert the jobs within an existing transaction (PostgreSQL only).
+   */
+  addJobs: <T extends JobType<PayloadMap>>(
+    jobs: JobOptions<PayloadMap, T>[],
+    options?: AddJobOptions,
+  ) => Promise<number[]>;
   /**
    * Get a job by its ID.
    */
@@ -606,12 +952,21 @@ export interface JobQueue<PayloadMap> {
   retryJob: (jobId: number) => Promise<void>;
   /**
    * Cleanup jobs that are older than the specified number of days.
+   * Deletes in batches for scale safety.
+   * @param daysToKeep - Number of days to retain completed jobs (default 30).
+   * @param batchSize - Number of rows to delete per batch (default 1000 for PostgreSQL, 200 for Redis).
    */
-  cleanupOldJobs: (daysToKeep?: number) => Promise<number>;
+  cleanupOldJobs: (daysToKeep?: number, batchSize?: number) => Promise<number>;
   /**
    * Cleanup job events that are older than the specified number of days.
+   * Deletes in batches for scale safety.
+   * @param daysToKeep - Number of days to retain events (default 30).
+   * @param batchSize - Number of rows to delete per batch (default 1000).
    */
-  cleanupOldJobEvents: (daysToKeep?: number) => Promise<number>;
+  cleanupOldJobEvents: (
+    daysToKeep?: number,
+    batchSize?: number,
+  ) => Promise<number>;
   /**
    * Cancel a job given its ID.
    * - This will set the job status to 'cancelled' and clear the locked_at and locked_by.
@@ -685,6 +1040,13 @@ export interface JobQueue<PayloadMap> {
     options?: ProcessorOptions,
   ) => Processor;
 
+  /**
+   * Create a background supervisor that automatically reclaims stuck jobs,
+   * cleans up old completed jobs/events, and expires timed-out waitpoint
+   * tokens on a configurable interval.
+   */
+  createSupervisor: (options?: SupervisorOptions) => Supervisor;
+
   /**
    * Get the job events for a job.
    */
@@ -695,8 +1057,6 @@ export interface JobQueue<PayloadMap> {
    * Tokens can be completed externally to resume a waiting job.
    * Can be called outside of handlers (e.g., from an API route).
    *
-   * **PostgreSQL backend only.** Throws if the backend is Redis.
-   *
    * @param options - Optional token configuration (timeout, tags).
    * @returns A token object with `id`.
    */
@@ -706,8 +1066,6 @@ export interface JobQueue<PayloadMap> {
    * Complete a waitpoint token, resuming the associated waiting job.
    * Can be called from anywhere (API routes, external services, etc.).
    *
-   * **PostgreSQL backend only.** Throws if the backend is Redis.
-   *
    * @param tokenId - The ID of the token to complete.
    * @param data - Optional data to pass to the waiting handler.
    */
@@ -716,8 +1074,6 @@ export interface JobQueue<PayloadMap> {
   /**
    * Retrieve a waitpoint token by its ID.
    *
-   * **PostgreSQL backend only.** Throws if the backend is Redis.
-   *
    * @param tokenId - The ID of the token to retrieve.
    * @returns The token record, or null if not found.
    */
@@ -727,12 +1083,120 @@ export interface JobQueue<PayloadMap> {
    * Expire timed-out waitpoint tokens and resume their associated jobs.
    * Call this periodically (e.g., alongside `reclaimStuckJobs`).
    *
-   * **PostgreSQL backend only.** Throws if the backend is Redis.
-   *
    * @returns The number of tokens that were expired.
    */
   expireTimedOutTokens: () => Promise<number>;
 
+  // ── Cron schedule operations ────────────────────────────────────────
+
+  /**
+   * Add a recurring cron schedule. The processor automatically enqueues
+   * due cron jobs before each batch, so no manual triggering is needed.
+   *
+   * @returns The ID of the created schedule.
+   * @throws If the cron expression is invalid or the schedule name is already taken.
+   */
+  addCronJob: <T extends JobType<PayloadMap>>(
+    options: CronScheduleOptions<PayloadMap, T>,
+  ) => Promise<number>;
+
+  /**
+   * Get a cron schedule by its ID.
+   */
+  getCronJob: (id: number) => Promise<CronScheduleRecord | null>;
+
+  /**
+   * Get a cron schedule by its unique name.
+   */
+  getCronJobByName: (name: string) => Promise<CronScheduleRecord | null>;
+
+  /**
+   * List all cron schedules, optionally filtered by status.
+   */
+  listCronJobs: (status?: CronScheduleStatus) => Promise<CronScheduleRecord[]>;
+
+  /**
+   * Remove a cron schedule by its ID. Does not cancel any already-enqueued jobs.
+   */
+  removeCronJob: (id: number) => Promise<void>;
+
+  /**
+   * Pause a cron schedule. Paused schedules are skipped by `enqueueDueCronJobs()`.
+   */
+  pauseCronJob: (id: number) => Promise<void>;
+
+  /**
+   * Resume a paused cron schedule.
+   */
+  resumeCronJob: (id: number) => Promise<void>;
+
+  /**
+   * Edit an existing cron schedule. Only provided fields are updated.
+   * If `cronExpression` or `timezone` changes, `nextRunAt` is recalculated.
+   */
+  editCronJob: (id: number, updates: EditCronScheduleOptions) => Promise<void>;
+
+  /**
+   * Check all active cron schedules and enqueue jobs for any whose
+   * `nextRunAt` has passed. When `allowOverlap` is false (the default),
+   * a new job is not enqueued if the previous instance is still
+   * pending, processing, or waiting.
+   *
+   * **Note:** The processor calls this automatically before each batch,
+   * so you typically do not need to call it yourself. It is exposed for
+   * manual use in tests or one-off scripts.
+   *
+   * @returns The number of jobs that were enqueued.
+   */
+  enqueueDueCronJobs: () => Promise<number>;
+
+  // ── Event hooks ───────────────────────────────────────────────────────
+
+  /**
+   * Register a listener for a queue event. The listener is called every
+   * time the event fires. Works identically with both PostgreSQL and Redis.
+   *
+   * @param event - The event name (e.g. `'job:completed'`, `'error'`).
+   * @param listener - Callback receiving the event payload.
+   */
+  on: <K extends QueueEventName>(
+    event: K,
+    listener: (data: QueueEventMap[K]) => void,
+  ) => void;
+
+  /**
+   * Register a one-time listener. The listener is automatically removed
+   * after it fires once.
+   *
+   * @param event - The event name.
+   * @param listener - Callback receiving the event payload.
+   */
+  once: <K extends QueueEventName>(
+    event: K,
+    listener: (data: QueueEventMap[K]) => void,
+  ) => void;
+
+  /**
+   * Remove a previously registered listener.
+   *
+   * @param event - The event name.
+   * @param listener - The exact function reference passed to `on` or `once`.
+   */
+  off: <K extends QueueEventName>(
+    event: K,
+    listener: (data: QueueEventMap[K]) => void,
+  ) => void;
+
+  /**
+   * Remove all listeners for a specific event, or all listeners for
+   * all events when called without arguments.
+   *
+   * @param event - Optional event name. If omitted, removes everything.
+   */
+  removeAllListeners: (event?: QueueEventName) => void;
+
+  // ── Advanced access ───────────────────────────────────────────────────
+
   /**
    * Get the PostgreSQL database pool.
    * Throws if the backend is not PostgreSQL.

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) Nico Prananta 2024
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.