@nicnocquee/dataqueue 1.34.0 → 1.35.0-beta.20260224110011

package/src/index.ts

@@ -1,9 +1,13 @@
+import { EventEmitter } from 'node:events';
 import { createProcessor } from './processor.js';
+import { createSupervisor } from './supervisor.js';
 import {
   JobQueueConfig,
   JobQueue,
   JobOptions,
+  AddJobOptions,
   ProcessorOptions,
+  SupervisorOptions,
   JobHandlers,
   JobType,
   PostgresJobQueueConfig,
@@ -11,6 +15,9 @@ import {
   CronScheduleOptions,
   CronScheduleStatus,
   EditCronScheduleOptions,
+  QueueEventMap,
+  QueueEventName,
+  QueueEmitFn,
 } from './types.js';
 import { QueueBackend, CronScheduleInput } from './backend.js';
 import { setLogContext } from './log-context.js';
@@ -23,6 +30,8 @@ import { getNextCronOccurrence, validateCronExpression } from './cron.js';
  * Initialize the job queue system.
  *
  * Defaults to PostgreSQL when `backend` is omitted.
+ * For PostgreSQL, provide either `databaseConfig` or `pool` (bring your own).
+ * For Redis, provide either `redisConfig` or `client` (bring your own).
  */
 export const initJobQueue = <PayloadMap = any>(
   config: JobQueueConfig,
@@ -34,15 +43,39 @@ export const initJobQueue = <PayloadMap = any>(
 
   if (backendType === 'postgres') {
     const pgConfig = config as PostgresJobQueueConfig;
-    const pool = createPool(pgConfig.databaseConfig);
-    backend = new PostgresBackend(pool);
+    if (pgConfig.pool) {
+      backend = new PostgresBackend(pgConfig.pool);
+    } else if (pgConfig.databaseConfig) {
+      const pool = createPool(pgConfig.databaseConfig);
+      backend = new PostgresBackend(pool);
+    } else {
+      throw new Error(
+        'PostgreSQL backend requires either "databaseConfig" or "pool" to be provided.',
+      );
+    }
   } else if (backendType === 'redis') {
-    const redisConfig = (config as RedisJobQueueConfig).redisConfig;
-    backend = new RedisBackend(redisConfig);
+    const redisConfig = config as RedisJobQueueConfig;
+    if (redisConfig.client) {
+      backend = new RedisBackend(
+        redisConfig.client as any,
+        redisConfig.keyPrefix,
+      );
+    } else if (redisConfig.redisConfig) {
+      backend = new RedisBackend(redisConfig.redisConfig);
+    } else {
+      throw new Error(
+        'Redis backend requires either "redisConfig" or "client" to be provided.',
+      );
+    }
   } else {
     throw new Error(`Unknown backend: ${backendType}`);
   }
 
+  const emitter = new EventEmitter();
+  const emit: QueueEmitFn = (event, data) => {
+    emitter.emit(event, data);
+  };
+
   /**
    * Enqueue due cron jobs. Shared by the public API and the processor hook.
    */
@@ -84,6 +117,9 @@ export const initJobQueue = <PayloadMap = any>(
         timeoutMs: schedule.timeoutMs ?? undefined,
         forceKillOnTimeout: schedule.forceKillOnTimeout,
         tags: schedule.tags,
+        retryDelay: schedule.retryDelay ?? undefined,
+        retryBackoff: schedule.retryBackoff ?? undefined,
+        retryDelayMax: schedule.retryDelayMax ?? undefined,
       });
 
       // Advance to next occurrence
@@ -107,8 +143,21 @@ export const initJobQueue = <PayloadMap = any>(
   return {
     // Job queue operations
     addJob: withLogContext(
-      (job: JobOptions<PayloadMap, any>) =>
-        backend.addJob<PayloadMap, any>(job),
+      async (job: JobOptions<PayloadMap, any>, options?: AddJobOptions) => {
+        const jobId = await backend.addJob<PayloadMap, any>(job, options);
+        emit('job:added', { jobId, jobType: job.jobType });
+        return jobId;
+      },
+      config.verbose ?? false,
+    ),
+    addJobs: withLogContext(
+      async (jobs: JobOptions<PayloadMap, any>[], options?: AddJobOptions) => {
+        const jobIds = await backend.addJobs<PayloadMap, any>(jobs, options);
+        for (let i = 0; i < jobIds.length; i++) {
+          emit('job:added', { jobId: jobIds[i], jobType: jobs[i].jobType });
+        }
+        return jobIds;
+      },
       config.verbose ?? false,
     ),
     getJob: withLogContext(
@@ -140,15 +189,18 @@ export const initJobQueue = <PayloadMap = any>(
       ) => backend.getJobs<PayloadMap, any>(filters, limit, offset),
       config.verbose ?? false,
     ),
-    retryJob: (jobId: number) => backend.retryJob(jobId),
+    retryJob: async (jobId: number) => {
+      await backend.retryJob(jobId);
+      emit('job:retried', { jobId });
+    },
     cleanupOldJobs: (daysToKeep?: number, batchSize?: number) =>
       backend.cleanupOldJobs(daysToKeep, batchSize),
     cleanupOldJobEvents: (daysToKeep?: number, batchSize?: number) =>
       backend.cleanupOldJobEvents(daysToKeep, batchSize),
-    cancelJob: withLogContext(
-      (jobId: number) => backend.cancelJob(jobId),
-      config.verbose ?? false,
-    ),
+    cancelJob: withLogContext(async (jobId: number) => {
+      await backend.cancelJob(jobId);
+      emit('job:cancelled', { jobId });
+    }, config.verbose ?? false),
     editJob: withLogContext(
       <T extends JobType<PayloadMap>>(
         jobId: number,
@@ -206,9 +258,19 @@ export const initJobQueue = <PayloadMap = any>(
       handlers: JobHandlers<PayloadMap>,
       options?: ProcessorOptions,
     ) =>
-      createProcessor<PayloadMap>(backend, handlers, options, async () => {
-        await enqueueDueCronJobsImpl();
-      }),
+      createProcessor<PayloadMap>(
+        backend,
+        handlers,
+        options,
+        async () => {
+          await enqueueDueCronJobsImpl();
+        },
+        emit,
+      ),
+
+    // Background supervisor — automated maintenance
+    createSupervisor: (options?: SupervisorOptions) =>
+      createSupervisor(backend, options, emit),
 
     // Job events
     getJobEvents: withLogContext(
@@ -262,6 +324,9 @@ export const initJobQueue = <PayloadMap = any>(
           timezone: options.timezone ?? 'UTC',
           allowOverlap: options.allowOverlap ?? false,
           nextRunAt,
+          retryDelay: options.retryDelay ?? null,
+          retryBackoff: options.retryBackoff ?? null,
+          retryDelayMax: options.retryDelayMax ?? null,
         };
         return backend.addCronSchedule(input);
       },
@@ -320,6 +385,33 @@ export const initJobQueue = <PayloadMap = any>(
       config.verbose ?? false,
     ),
 
+    // Event hooks
+    on: <K extends QueueEventName>(
+      event: K,
+      listener: (data: QueueEventMap[K]) => void,
+    ) => {
+      emitter.on(event, listener as (...args: any[]) => void);
+    },
+    once: <K extends QueueEventName>(
+      event: K,
+      listener: (data: QueueEventMap[K]) => void,
+    ) => {
+      emitter.once(event, listener as (...args: any[]) => void);
+    },
+    off: <K extends QueueEventName>(
+      event: K,
+      listener: (data: QueueEventMap[K]) => void,
+    ) => {
+      emitter.off(event, listener as (...args: any[]) => void);
+    },
+    removeAllListeners: (event?: QueueEventName) => {
+      if (event) {
+        emitter.removeAllListeners(event);
+      } else {
+        emitter.removeAllListeners();
+      }
+    },
+
     // Advanced access
     getPool: () => {
       if (!(backend instanceof PostgresBackend)) {

package/src/processor.test.ts

@@ -436,6 +436,24 @@ describe('concurrency option', () => {
     await processor.start();
     expect(maxParallel).toBe(1);
   });
+
+  it('should throw when groupConcurrency is not a positive integer', async () => {
+    const handlers = { test: vi.fn(async () => {}) };
+    expect(() =>
+      createProcessor(backend, handlers, {
+        groupConcurrency: 0,
+      }),
+    ).toThrow(
+      'Processor option "groupConcurrency" must be a positive integer when provided.',
+    );
+    expect(() =>
+      createProcessor(backend, handlers, {
+        groupConcurrency: 1.5,
+      }),
+    ).toThrow(
+      'Processor option "groupConcurrency" must be a positive integer when provided.',
+    );
+  });
 });
 
 describe('per-job timeout', () => {

package/src/processor.ts

@@ -12,6 +12,7 @@ import {
   WaitSignal,
   WaitDuration,
   WaitTokenResult,
+  QueueEmitFn,
 } from './types.js';
 import { QueueBackend } from './backend.js';
 import { log, setLogContext } from './log-context.js';
@@ -90,7 +91,7 @@ async function runHandlerInWorker<
   payload: PayloadMap[T],
   timeoutMs: number,
   jobType: string,
-): Promise<void> {
+): Promise<unknown> {
   // Validate handler can be serialized before attempting to run in worker
   validateHandlerSerializable(handler, jobType);
 
@@ -155,9 +156,9 @@ async function runHandlerInWorker<
           }
           
           handlerFn(payload, signal)
-            .then(() => {
+            .then((result) => {
               clearTimeout(timeoutId);
-              parentPort.postMessage({ type: 'success' });
+              parentPort.postMessage({ type: 'success', output: result });
             })
             .catch((error) => {
               clearTimeout(timeoutId);
@@ -195,26 +196,29 @@ async function runHandlerInWorker<
 
     let resolved = false;
 
-    worker.on('message', (message: { type: string; error?: any }) => {
-      if (resolved) return;
-      resolved = true;
+    worker.on(
+      'message',
+      (message: { type: string; error?: any; output?: unknown }) => {
+        if (resolved) return;
+        resolved = true;
 
-      if (message.type === 'success') {
-        resolve();
-      } else if (message.type === 'timeout') {
-        const timeoutError = new Error(
-          `Job timed out after ${timeoutMs} ms and was forcefully terminated`,
-        );
-        // @ts-ignore
-        timeoutError.failureReason = FailureReason.Timeout;
-        reject(timeoutError);
-      } else if (message.type === 'error') {
-        const error = new Error(message.error.message);
-        error.stack = message.error.stack;
-        error.name = message.error.name;
-        reject(error);
-      }
-    });
+        if (message.type === 'success') {
+          resolve(message.output);
+        } else if (message.type === 'timeout') {
+          const timeoutError = new Error(
+            `Job timed out after ${timeoutMs} ms and was forcefully terminated`,
+          );
+          // @ts-ignore
+          timeoutError.failureReason = FailureReason.Timeout;
+          reject(timeoutError);
+        } else if (message.type === 'error') {
+          const error = new Error(message.error.message);
+          error.stack = message.error.stack;
+          error.name = message.error.name;
+          reject(error);
+        }
+      },
+    );
 
     worker.on('error', (error) => {
       if (resolved) return;
@@ -318,6 +322,9 @@ function createNoOpContext(
         throw new Error('Progress must be between 0 and 100');
       await backend.updateProgress(jobId, Math.round(percent));
     },
+    setOutput: async (data: unknown) => {
+      await backend.updateOutput(jobId, data);
+    },
   };
 }
 
@@ -495,13 +502,21 @@ function buildWaitContext(
         throw new Error('Progress must be between 0 and 100');
       await backend.updateProgress(jobId, Math.round(percent));
     },
+    setOutput: async (data: unknown) => {
+      await backend.updateOutput(jobId, data);
+    },
   };
 
   return ctx;
 }
 
 /**
- * Process a single job using the provided handler map
+ * Process a single job using the provided handler map.
+ *
+ * @param backend - The queue backend.
+ * @param job - The job record to process.
+ * @param jobHandlers - Map of job type to handler function.
+ * @param emit - Optional callback to emit lifecycle events to the queue's EventEmitter.
  */
 export async function processJobWithHandlers<
   PayloadMap,
@@ -510,6 +525,7 @@ export async function processJobWithHandlers<
   backend: QueueBackend,
   job: JobRecord<PayloadMap, T>,
   jobHandlers: JobHandlers<PayloadMap>,
+  emit?: QueueEmitFn,
 ): Promise<void> {
   const handler = jobHandlers[job.jobType];
 
@@ -518,11 +534,16 @@ export async function processJobWithHandlers<
       `No handler registered for job type: ${job.jobType}`,
       job.jobType,
     );
-    await backend.failJob(
-      job.id,
-      new Error(`No handler registered for job type: ${job.jobType}`),
-      FailureReason.NoHandler,
+    const noHandlerError = new Error(
+      `No handler registered for job type: ${job.jobType}`,
     );
+    await backend.failJob(job.id, noHandlerError, FailureReason.NoHandler);
+    emit?.('job:failed', {
+      jobId: job.id,
+      jobType: job.jobType,
+      error: noHandlerError,
+      willRetry: false,
+    });
     return;
   }
 
@@ -544,11 +565,18 @@ export async function processJobWithHandlers<
   const forceKillOnTimeout = job.forceKillOnTimeout ?? false;
   let timeoutId: NodeJS.Timeout | undefined;
   const controller = new AbortController();
+  let setOutputCalled = false;
+  let handlerReturnValue: unknown;
   try {
     // If forceKillOnTimeout is true, run handler in a worker thread
-    // Note: wait features are not available in forceKillOnTimeout mode
+    // Note: wait features and setOutput are not available in forceKillOnTimeout mode
     if (forceKillOnTimeout && timeoutMs && timeoutMs > 0) {
-      await runHandlerInWorker(handler, job.payload, timeoutMs, job.jobType);
+      handlerReturnValue = await runHandlerInWorker(
+        handler,
+        job.payload,
+        timeoutMs,
+        job.jobType,
+      );
     } else {
       // Build the JobContext for prolong/onTimeout support
       let onTimeoutCallback: OnTimeoutCallback | undefined;
@@ -623,6 +651,26 @@ export async function processJobWithHandlers<
       // Build context: full wait support for all backends
       const ctx = buildWaitContext(backend, job.id, stepData, baseCtx);
 
+      // Wrap setProgress to also emit the event
+      if (emit) {
+        const originalSetProgress = ctx.setProgress;
+        ctx.setProgress = async (percent: number) => {
+          await originalSetProgress(percent);
+          emit('job:progress', {
+            jobId: job.id,
+            progress: Math.round(percent),
+          });
+        };
+      }
+
+      // Wrap setOutput to track calls and emit the event
+      const originalSetOutput = ctx.setOutput;
+      ctx.setOutput = async (data: unknown) => {
+        setOutputCalled = true;
+        await originalSetOutput(data);
+        emit?.('job:output', { jobId: job.id, output: data });
+      };
+
       // If forceKillOnTimeout was set but timeoutMs was missing, warn
       if (forceKillOnTimeout && !hasTimeout) {
         log(
@@ -633,7 +681,7 @@ export async function processJobWithHandlers<
       const jobPromise = handler(job.payload, controller.signal, ctx);
 
       if (hasTimeout) {
-        await Promise.race([
+        handlerReturnValue = await Promise.race([
           jobPromise,
           new Promise<never>((_, reject) => {
             timeoutReject = reject;
@@ -641,13 +689,22 @@ export async function processJobWithHandlers<
           }),
         ]);
       } else {
-        await jobPromise;
+        handlerReturnValue = await jobPromise;
       }
     }
     if (timeoutId) clearTimeout(timeoutId);
 
+    // Determine the output to persist on completion.
+    // If setOutput() was called, the value is already in the DB -- pass undefined
+    // so completeJob preserves it. Otherwise, use the handler's return value.
+    const completionOutput =
+      setOutputCalled || handlerReturnValue === undefined
+        ? undefined
+        : handlerReturnValue;
+
     // Job completed successfully -- complete via backend
-    await backend.completeJob(job.id);
+    await backend.completeJob(job.id, completionOutput);
+    emit?.('job:completed', { jobId: job.id, jobType: job.jobType });
   } catch (error) {
     if (timeoutId) clearTimeout(timeoutId);
 
@@ -661,6 +718,7 @@ export async function processJobWithHandlers<
         waitTokenId: error.tokenId,
         stepData: error.stepData,
       });
+      emit?.('job:waiting', { jobId: job.id, jobType: job.jobType });
       return;
     }
 
@@ -676,16 +734,29 @@ export async function processJobWithHandlers<
     ) {
       failureReason = FailureReason.Timeout;
     }
-    await backend.failJob(
-      job.id,
-      error instanceof Error ? error : new Error(String(error)),
-      failureReason,
-    );
+    const failError = error instanceof Error ? error : new Error(String(error));
+    await backend.failJob(job.id, failError, failureReason);
+    emit?.('job:failed', {
+      jobId: job.id,
+      jobType: job.jobType,
+      error: failError,
+      willRetry: job.attempts + 1 < job.maxAttempts,
+    });
   }
 }
 
 /**
- * Process a batch of jobs using the provided handler map and concurrency limit
+ * Process a batch of jobs using the provided handler map and concurrency limit.
+ *
+ * @param backend - The queue backend.
+ * @param workerId - Identifier for the worker claiming jobs.
+ * @param batchSize - Maximum jobs to claim per batch.
+ * @param jobType - Optional job type filter.
+ * @param jobHandlers - Map of job type to handler function.
+ * @param concurrency - Max parallel jobs within the batch.
+ * @param groupConcurrency - Optional global per-group concurrency limit.
+ * @param onError - Legacy error callback.
+ * @param emit - Optional callback to emit lifecycle events.
  */
 export async function processBatchWithHandlers<PayloadMap>(
   backend: QueueBackend,
@@ -694,17 +765,29 @@ export async function processBatchWithHandlers<PayloadMap>(
   jobType: string | string[] | undefined,
   jobHandlers: JobHandlers<PayloadMap>,
   concurrency?: number,
+  groupConcurrency?: number,
   onError?: (error: Error) => void,
+  emit?: QueueEmitFn,
 ): Promise<number> {
   const jobs = await backend.getNextBatch<PayloadMap, JobType<PayloadMap>>(
     workerId,
     batchSize,
     jobType,
+    groupConcurrency,
   );
+
+  // Emit job:processing for each claimed job
+  if (emit) {
+    for (const job of jobs) {
+      emit('job:processing', { jobId: job.id, jobType: job.jobType });
+    }
+  }
+
   if (!concurrency || concurrency >= jobs.length) {
-    // Default: all in parallel
     await Promise.all(
-      jobs.map((job) => processJobWithHandlers(backend, job, jobHandlers)),
+      jobs.map((job) =>
+        processJobWithHandlers(backend, job, jobHandlers, emit),
+      ),
     );
     return jobs.length;
   }
@@ -718,7 +801,7 @@ export async function processBatchWithHandlers<PayloadMap>(
       while (running < concurrency && idx < jobs.length) {
         const job = jobs[idx++];
         running++;
-        processJobWithHandlers(backend, job, jobHandlers)
+        processJobWithHandlers(backend, job, jobHandlers, emit)
           .then(() => {
             running--;
             finished++;
@@ -740,17 +823,20 @@ export async function processBatchWithHandlers<PayloadMap>(
 
 /**
  * Start a job processor that continuously processes jobs.
+ *
  * @param backend - The queue backend.
  * @param handlers - The job handlers for this processor instance.
  * @param options - The processor options. Leave pollInterval empty to run only once. Use jobType to filter jobs by type.
  * @param onBeforeBatch - Optional callback invoked before each batch. Used internally to enqueue due cron jobs.
- * @returns {Processor} The processor instance.
+ * @param emit - Optional callback to emit lifecycle events to the queue's EventEmitter.
+ * @returns The processor instance.
  */
 export const createProcessor = <PayloadMap = any>(
   backend: QueueBackend,
   handlers: JobHandlers<PayloadMap>,
   options: ProcessorOptions = {},
   onBeforeBatch?: () => Promise<void>,
+  emit?: QueueEmitFn,
 ): Processor => {
   const {
     workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
@@ -759,8 +845,18 @@ export const createProcessor = <PayloadMap = any>(
     onError = (error: Error) => console.error('Job processor error:', error),
     jobType,
     concurrency = 3,
+    groupConcurrency,
   } = options;
 
+  if (
+    groupConcurrency !== undefined &&
+    (!Number.isInteger(groupConcurrency) || groupConcurrency <= 0)
+  ) {
+    throw new Error(
+      'Processor option "groupConcurrency" must be a positive integer when provided.',
+    );
+  }
+
   let running = false;
   let intervalId: NodeJS.Timeout | null = null;
   let currentBatchPromise: Promise<number> | null = null;
@@ -776,13 +872,12 @@ export const createProcessor = <PayloadMap = any>(
         await onBeforeBatch();
       } catch (hookError) {
         log(`onBeforeBatch hook error: ${hookError}`);
+        const err =
+          hookError instanceof Error ? hookError : new Error(String(hookError));
         if (onError) {
-          onError(
-            hookError instanceof Error
-              ? hookError
-              : new Error(String(hookError)),
-          );
+          onError(err);
         }
+        emit?.('error', err);
       }
     }
 
@@ -798,12 +893,15 @@ export const createProcessor = <PayloadMap = any>(
         jobType,
         handlers,
         concurrency,
+        groupConcurrency,
         onError,
+        emit,
       );
-      // Only process one batch in start; do not schedule next batch here
       return processed;
     } catch (error) {
-      onError(error instanceof Error ? error : new Error(String(error)));
+      const err = error instanceof Error ? error : new Error(String(error));
+      onError(err);
+      emit?.('error', err);
     }
     return 0;
   };