@nicnocquee/dataqueue 1.34.0 → 1.35.0-beta.20260224110011

package/src/types.ts

@@ -1,6 +1,47 @@
 // Utility type for job type keys
 export type JobType<PayloadMap> = keyof PayloadMap & string;
 
+/**
+ * Abstract database client interface for transactional job creation.
+ * Compatible with `pg.Pool`, `pg.PoolClient`, `pg.Client`, or any object
+ * that exposes a `.query()` method matching the `pg` signature.
+ */
+export interface DatabaseClient {
+  query(
+    text: string,
+    values?: any[],
+  ): Promise<{ rows: any[]; rowCount: number | null }>;
+}
+
+/**
+ * Options for `addJob()` beyond the job itself.
+ * Use `db` to insert the job within an existing database transaction.
+ */
+export interface AddJobOptions {
+  /**
+   * An external database client (e.g., a `pg.PoolClient` inside a transaction).
+   * When provided, the INSERT runs on this client instead of the internal pool,
+   * so the job is part of the caller's transaction.
+   *
+   * **PostgreSQL only.** Throws if used with the Redis backend.
+   */
+  db?: DatabaseClient;
+}
+
+/**
+ * Optional grouping metadata for a job.
+ * Use `id` to enforce global per-group concurrency limits when
+ * `ProcessorOptions.groupConcurrency` is set.
+ *
+ * `tier` is reserved for future tier-based policies.
+ */
+export interface JobGroup {
+  /** Stable group identifier (for example: tenant ID, user ID, organization ID). */
+  id: string;
+  /** Optional tier label reserved for future tier-based concurrency controls. */
+  tier?: string;
+}
+
 export interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
   jobType: T;
   payload: PayloadMap[T];
@@ -73,6 +114,32 @@ export interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
    * Once a key exists, it cannot be reused until the job is cleaned up (via `cleanupOldJobs`).
    */
   idempotencyKey?: string;
+  /**
+   * Base delay between retries in seconds. When `retryBackoff` is true (the default),
+   * this is the base for exponential backoff: `retryDelay * 2^attempts`.
+   * When `retryBackoff` is false, retries use this fixed delay.
+   * @default 60
+   */
+  retryDelay?: number;
+  /**
+   * Whether to use exponential backoff for retries. When true, delay doubles
+   * with each attempt and includes jitter to prevent thundering herd.
+   * When false, a fixed `retryDelay` is used between every retry.
+   * @default true
+   */
+  retryBackoff?: boolean;
+  /**
+   * Maximum delay between retries in seconds. Caps the exponential backoff
+   * so retries never wait longer than this value. Only meaningful when
+   * `retryBackoff` is true. No limit when omitted.
+   */
+  retryDelayMax?: number;
+  /**
+   * Optional group metadata for this job.
+   * When `ProcessorOptions.groupConcurrency` is configured, grouped jobs are
+   * globally limited by `group.id` across all workers/instances.
+   */
+  group?: JobGroup;
 }
 
 /**
@@ -196,6 +263,31 @@ export interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
    * Updated by the handler via `ctx.setProgress(percent)`.
    */
   progress?: number | null;
+  /**
+   * Handler output stored via `ctx.setOutput(data)` or by returning a value
+   * from the handler. `null` if no output has been stored.
+   */
+  output?: unknown;
+  /**
+   * Base delay between retries in seconds, or null if using legacy default.
+   */
+  retryDelay?: number | null;
+  /**
+   * Whether exponential backoff is enabled for retries, or null if using legacy default.
+   */
+  retryBackoff?: boolean | null;
+  /**
+   * Maximum delay cap for retries in seconds, or null if no cap.
+   */
+  retryDelayMax?: number | null;
+  /**
+   * Group identifier for this job, if provided at enqueue time.
+   */
+  groupId?: string | null;
+  /**
+   * Group tier for this job, if provided at enqueue time.
+   */
+  groupTier?: string | null;
 }
 
 /**
@@ -292,6 +384,17 @@ export interface JobContext {
    * @throws If percent is outside the 0-100 range.
    */
   setProgress: (percent: number) => Promise<void>;
+
+  /**
+   * Store an output/result for this job. The value is persisted to the database
+   * as JSONB and can be read by clients via `getJob()` or the React SDK's `useJob()` hook.
+   *
+   * Can be called multiple times — each call overwrites the previous value.
+   * If `setOutput()` is called, the handler's return value is ignored.
+   *
+   * @param data - Any JSON-serializable value to store as the job's output.
+   */
+  setOutput: (data: unknown) => Promise<void>;
 }
 
 /**
@@ -380,7 +483,7 @@ export type JobHandler<PayloadMap, T extends keyof PayloadMap> = (
   payload: PayloadMap[T],
   signal: AbortSignal,
   ctx: JobContext,
-) => Promise<void>;
+) => Promise<unknown>;
 
 export type JobHandlers<PayloadMap> = {
   [K in keyof PayloadMap]: JobHandler<PayloadMap, K>;
@@ -401,6 +504,13 @@ export interface ProcessorOptions {
    * - Set to a lower value to avoid resource exhaustion.
    */
   concurrency?: number;
+  /**
+   * Global per-group concurrency limit across all workers/instances.
+   * - Applies only to jobs with `group.id` set.
+   * - Jobs without a group are unaffected.
+   * - Disabled when omitted.
+   */
+  groupConcurrency?: number;
   /**
    * The interval in milliseconds to poll for new jobs.
    * - If not provided, the processor will process jobs every 5 seconds when startInBackground is called.
@@ -452,6 +562,91 @@ export interface Processor {
   start: () => Promise<number>;
 }
 
+export interface SupervisorOptions {
+  /**
+   * How often the maintenance loop runs, in milliseconds.
+   * @default 60000 (1 minute)
+   */
+  intervalMs?: number;
+  /**
+   * Reclaim jobs stuck in `processing` longer than this many minutes.
+   * @default 10
+   */
+  stuckJobsTimeoutMinutes?: number;
+  /**
+   * Auto-delete completed jobs older than this many days. Set to 0 to disable.
+   * @default 30
+   */
+  cleanupJobsDaysToKeep?: number;
+  /**
+   * Auto-delete job events older than this many days. Set to 0 to disable.
+   * @default 30
+   */
+  cleanupEventsDaysToKeep?: number;
+  /**
+   * Batch size for cleanup deletions.
+   * @default 1000
+   */
+  cleanupBatchSize?: number;
+  /**
+   * Whether to reclaim stuck jobs each cycle.
+   * @default true
+   */
+  reclaimStuckJobs?: boolean;
+  /**
+   * Whether to expire timed-out waitpoint tokens each cycle.
+   * @default true
+   */
+  expireTimedOutTokens?: boolean;
+  /**
+   * Called when a maintenance task throws. One failure does not block other tasks.
+   * @default console.error
+   */
+  onError?: (error: Error) => void;
+  /** Enable verbose logging. */
+  verbose?: boolean;
+}
+
+export interface SupervisorRunResult {
+  /** Number of stuck jobs reclaimed back to pending. */
+  reclaimedJobs: number;
+  /** Number of old completed jobs deleted. */
+  cleanedUpJobs: number;
+  /** Number of old job events deleted. */
+  cleanedUpEvents: number;
+  /** Number of timed-out waitpoint tokens expired. */
+  expiredTokens: number;
+}
+
+export interface Supervisor {
+  /**
+   * Run all maintenance tasks once and return the results.
+   * Ideal for serverless or cron-triggered invocations.
+   */
+  start: () => Promise<SupervisorRunResult>;
+  /**
+   * Start the maintenance loop in the background.
+   * Runs every `intervalMs` milliseconds (default: 60 000).
+   * Call `stop()` or `stopAndDrain()` to halt the loop.
+   */
+  startInBackground: () => void;
+  /**
+   * Stop the background maintenance loop immediately.
+   * Does not wait for an in-flight maintenance run to complete.
+   */
+  stop: () => void;
+  /**
+   * Stop the background loop and wait for the current maintenance run
+   * (if any) to finish before resolving.
+   *
+   * @param timeoutMs - Maximum time to wait (default: 30 000 ms).
+   *   If the run does not finish within this time the promise resolves anyway.
+   */
+  stopAndDrain: (timeoutMs?: number) => Promise<void>;
+  /** Whether the background maintenance loop is currently running. */
+  isRunning: () => boolean;
+}
+
 export interface DatabaseSSLConfig {
   /**
    * CA certificate as PEM string or file path. If the value starts with 'file://', it will be loaded from file, otherwise treated as PEM string.
@@ -474,10 +669,13 @@ export interface DatabaseSSLConfig {
 /**
  * Configuration for PostgreSQL backend (default).
  * Backward-compatible: omitting `backend` defaults to 'postgres'.
+ *
+ * Provide either `databaseConfig` (the library creates a pool) or `pool`
+ * (bring your own `pg.Pool`). At least one must be set.
  */
 export interface PostgresJobQueueConfig {
   backend?: 'postgres';
-  databaseConfig: {
+  databaseConfig?: {
     connectionString?: string;
     host?: string;
     port?: number;
@@ -503,6 +701,11 @@ export interface PostgresJobQueueConfig {
      */
     connectionTimeoutMillis?: number;
   };
+  /**
+   * Bring your own `pg.Pool` instance. When provided, `databaseConfig` is
+   * ignored and the library will not close the pool on shutdown.
+   */
+  pool?: import('pg').Pool;
   verbose?: boolean;
 }
 
@@ -518,10 +721,13 @@ export interface RedisTLSConfig {
 
 /**
  * Configuration for Redis backend.
+ *
+ * Provide either `redisConfig` (the library creates an ioredis client) or
+ * `client` (bring your own ioredis instance). At least one must be set.
  */
 export interface RedisJobQueueConfig {
   backend: 'redis';
-  redisConfig: {
+  redisConfig?: {
     /** Redis URL (e.g. redis://localhost:6379) */
     url?: string;
     host?: string;
@@ -536,6 +742,17 @@ export interface RedisJobQueueConfig {
      */
     keyPrefix?: string;
   };
+  /**
+   * Bring your own ioredis client instance. When provided, `redisConfig` is
+   * ignored and the library will not close the client on shutdown.
+   * Use `keyPrefix` to set the key namespace (default: 'dq:').
+   */
+  client?: unknown;
+  /**
+   * Key prefix when using an external `client`. Ignored when `redisConfig` is used
+   * (set `redisConfig.keyPrefix` instead). Default: 'dq:'.
+   */
+  keyPrefix?: string;
   verbose?: boolean;
 }
 
@@ -592,6 +809,12 @@ export interface CronScheduleOptions<
    * is still pending, processing, or waiting.
    */
   allowOverlap?: boolean;
+  /** Base delay between retries in seconds for each job instance (default: 60). */
+  retryDelay?: number;
+  /** Whether to use exponential backoff for retries (default: true). */
+  retryBackoff?: boolean;
+  /** Maximum delay cap for retries in seconds. */
+  retryDelayMax?: number;
 }
 
 /**
@@ -616,6 +839,9 @@ export interface CronScheduleRecord {
   nextRunAt: Date | null;
   createdAt: Date;
   updatedAt: Date;
+  retryDelay: number | null;
+  retryBackoff: boolean | null;
+  retryDelayMax: number | null;
 }
 
 /**
@@ -632,15 +858,87 @@ export interface EditCronScheduleOptions {
   tags?: string[] | null;
   timezone?: string;
   allowOverlap?: boolean;
+  retryDelay?: number | null;
+  retryBackoff?: boolean | null;
+  retryDelayMax?: number | null;
+}
+
+// ── Event hooks ──────────────────────────────────────────────────────
+
+/**
+ * Payload types for each event emitted by the job queue.
+ */
+export interface QueueEventMap {
+  /** Fired after a job is successfully added to the queue. */
+  'job:added': { jobId: number; jobType: string };
+  /** Fired when a processor claims a job and begins executing its handler. */
+  'job:processing': { jobId: number; jobType: string };
+  /** Fired when a job handler completes successfully. */
+  'job:completed': { jobId: number; jobType: string };
+  /** Fired when a job handler fails. `willRetry` indicates whether the job will be retried. */
+  'job:failed': {
+    jobId: number;
+    jobType: string;
+    error: Error;
+    willRetry: boolean;
+  };
+  /** Fired after a job is cancelled via `cancelJob()`. */
+  'job:cancelled': { jobId: number };
+  /** Fired after a failed job is manually retried via `retryJob()`. */
+  'job:retried': { jobId: number };
+  /** Fired when a job enters the `waiting` state (via `ctx.waitFor`, `ctx.waitUntil`, or `ctx.waitForToken`). */
+  'job:waiting': { jobId: number; jobType: string };
+  /** Fired when a job reports progress via `ctx.setProgress()`. */
+  'job:progress': { jobId: number; progress: number };
+  /** Fired when a job stores output via `ctx.setOutput()`. */
+  'job:output': { jobId: number; output: unknown };
+  /** Fired on internal errors from the processor or supervisor. */
+  error: Error;
 }
 
+/** Union of all event names supported by the job queue. */
+export type QueueEventName = keyof QueueEventMap;
+
+/**
+ * Callback type for `emit`. Used internally to pass the emitter
+ * from `initJobQueue` into the processor and supervisor.
+ */
+export type QueueEmitFn = <K extends QueueEventName>(
+  event: K,
+  data: QueueEventMap[K],
+) => void;
+
 export interface JobQueue<PayloadMap> {
   /**
    * Add a job to the job queue.
+   *
+   * @param job - The job to enqueue.
+   * @param options - Optional. Pass `{ db }` with an external database client
+   *   to insert the job within an existing transaction (PostgreSQL only).
    */
   addJob: <T extends JobType<PayloadMap>>(
     job: JobOptions<PayloadMap, T>,
+    options?: AddJobOptions,
   ) => Promise<number>;
+  /**
+   * Add multiple jobs to the queue in a single operation.
+   *
+   * More efficient than calling `addJob` in a loop because it batches the
+   * INSERT into a single database round-trip (PostgreSQL) or a single
+   * atomic Lua script (Redis).
+   *
+   * Returns an array of job IDs in the same order as the input array.
+   * Each job may independently have an `idempotencyKey`; duplicates
+   * resolve to the existing job's ID without creating a new row.
+   *
+   * @param jobs - Array of jobs to enqueue.
+   * @param options - Optional. Pass `{ db }` with an external database client
+   *   to insert the jobs within an existing transaction (PostgreSQL only).
+   */
+  addJobs: <T extends JobType<PayloadMap>>(
+    jobs: JobOptions<PayloadMap, T>[],
+    options?: AddJobOptions,
+  ) => Promise<number[]>;
   /**
    * Get a job by its ID.
    */
@@ -795,6 +1093,13 @@ export interface JobQueue<PayloadMap> {
     options?: ProcessorOptions,
   ) => Processor;
 
+  /**
+   * Create a background supervisor that automatically reclaims stuck jobs,
+   * cleans up old completed jobs/events, and expires timed-out waitpoint
+   * tokens on a configurable interval.
+   */
+  createSupervisor: (options?: SupervisorOptions) => Supervisor;
+
   /**
    * Get the job events for a job.
    */
@@ -898,6 +1203,51 @@ export interface JobQueue<PayloadMap> {
    */
   enqueueDueCronJobs: () => Promise<number>;
 
+  // ── Event hooks ───────────────────────────────────────────────────────
+
+  /**
+   * Register a listener for a queue event. The listener is called every
+   * time the event fires. Works identically with both PostgreSQL and Redis.
+   *
+   * @param event - The event name (e.g. `'job:completed'`, `'error'`).
+   * @param listener - Callback receiving the event payload.
+   */
+  on: <K extends QueueEventName>(
+    event: K,
+    listener: (data: QueueEventMap[K]) => void,
+  ) => void;
+
+  /**
+   * Register a one-time listener. The listener is automatically removed
+   * after it fires once.
+   *
+   * @param event - The event name.
+   * @param listener - Callback receiving the event payload.
+   */
+  once: <K extends QueueEventName>(
+    event: K,
+    listener: (data: QueueEventMap[K]) => void,
+  ) => void;
+
+  /**
+   * Remove a previously registered listener.
+   *
+   * @param event - The event name.
+   * @param listener - The exact function reference passed to `on` or `once`.
+   */
+  off: <K extends QueueEventName>(
+    event: K,
+    listener: (data: QueueEventMap[K]) => void,
+  ) => void;
+
+  /**
+   * Remove all listeners for a specific event, or all listeners for
+   * all events when called without arguments.
+   *
+   * @param event - Optional event name. If omitted, removes everything.
+   */
+  removeAllListeners: (event?: QueueEventName) => void;
+
   // ── Advanced access ───────────────────────────────────────────────────
 
   /**