@nicnocquee/dataqueue 1.30.0 → 1.32.0

package/src/index.ts

@@ -1,9 +1,3 @@
-import {
-  createWaitpoint,
-  completeWaitpoint,
-  getWaitpoint,
-  expireTimedOutWaitpoints,
-} from './queue.js';
 import { createProcessor } from './processor.js';
 import {
   JobQueueConfig,
@@ -14,12 +8,16 @@ import {
   JobType,
   PostgresJobQueueConfig,
   RedisJobQueueConfig,
+  CronScheduleOptions,
+  CronScheduleStatus,
+  EditCronScheduleOptions,
 } from './types.js';
-import { QueueBackend } from './backend.js';
+import { QueueBackend, CronScheduleInput } from './backend.js';
 import { setLogContext } from './log-context.js';
 import { createPool } from './db-util.js';
 import { PostgresBackend } from './backends/postgres.js';
 import { RedisBackend } from './backends/redis.js';
+import { getNextCronOccurrence, validateCronExpression } from './cron.js';
 
 /**
  * Initialize the job queue system.
@@ -33,27 +31,76 @@ export const initJobQueue = <PayloadMap = any>(
   setLogContext(config.verbose ?? false);
 
   let backend: QueueBackend;
-  let pool: import('pg').Pool | undefined;
 
   if (backendType === 'postgres') {
     const pgConfig = config as PostgresJobQueueConfig;
-    pool = createPool(pgConfig.databaseConfig);
+    const pool = createPool(pgConfig.databaseConfig);
     backend = new PostgresBackend(pool);
   } else if (backendType === 'redis') {
     const redisConfig = (config as RedisJobQueueConfig).redisConfig;
-    // RedisBackend constructor will throw if ioredis is not installed
     backend = new RedisBackend(redisConfig);
   } else {
     throw new Error(`Unknown backend: ${backendType}`);
   }
 
-  const requirePool = () => {
-    if (!pool) {
-      throw new Error(
-        'Wait/Token features require the PostgreSQL backend. Configure with backend: "postgres" to use these features.',
+  /**
+   * Enqueue due cron jobs. Shared by the public API and the processor hook.
+   */
+  const enqueueDueCronJobsImpl = async (): Promise<number> => {
+    const dueSchedules = await backend.getDueCronSchedules();
+    let count = 0;
+
+    for (const schedule of dueSchedules) {
+      // Overlap check: skip if allowOverlap is false and last job is still active
+      if (!schedule.allowOverlap && schedule.lastJobId !== null) {
+        const lastJob = await backend.getJob(schedule.lastJobId);
+        if (
+          lastJob &&
+          (lastJob.status === 'pending' ||
+            lastJob.status === 'processing' ||
+            lastJob.status === 'waiting')
+        ) {
+          // Still active — advance nextRunAt but don't enqueue
+          const nextRunAt = getNextCronOccurrence(
+            schedule.cronExpression,
+            schedule.timezone,
+          );
+          await backend.updateCronScheduleAfterEnqueue(
+            schedule.id,
+            new Date(),
+            schedule.lastJobId,
+            nextRunAt,
+          );
+          continue;
+        }
+      }
+
+      // Enqueue a new job instance
+      const jobId = await backend.addJob<any, any>({
+        jobType: schedule.jobType,
+        payload: schedule.payload,
+        maxAttempts: schedule.maxAttempts,
+        priority: schedule.priority,
+        timeoutMs: schedule.timeoutMs ?? undefined,
+        forceKillOnTimeout: schedule.forceKillOnTimeout,
+        tags: schedule.tags,
+      });
+
+      // Advance to next occurrence
+      const nextRunAt = getNextCronOccurrence(
+        schedule.cronExpression,
+        schedule.timezone,
+      );
+      await backend.updateCronScheduleAfterEnqueue(
+        schedule.id,
+        new Date(),
+        jobId,
+        nextRunAt,
       );
+      count++;
     }
-    return pool;
+
+    return count;
   };
 
   // Return the job queue API
@@ -94,9 +141,10 @@ export const initJobQueue = <PayloadMap = any>(
       config.verbose ?? false,
     ),
     retryJob: (jobId: number) => backend.retryJob(jobId),
-    cleanupOldJobs: (daysToKeep?: number) => backend.cleanupOldJobs(daysToKeep),
-    cleanupOldJobEvents: (daysToKeep?: number) =>
-      backend.cleanupOldJobEvents(daysToKeep),
+    cleanupOldJobs: (daysToKeep?: number, batchSize?: number) =>
+      backend.cleanupOldJobs(daysToKeep, batchSize),
+    cleanupOldJobEvents: (daysToKeep?: number, batchSize?: number) =>
+      backend.cleanupOldJobEvents(daysToKeep, batchSize),
     cancelJob: withLogContext(
       (jobId: number) => backend.cancelJob(jobId),
       config.verbose ?? false,
@@ -153,11 +201,14 @@ export const initJobQueue = <PayloadMap = any>(
       config.verbose ?? false,
     ),
 
-    // Job processing
+    // Job processing — automatically enqueues due cron jobs before each batch
     createProcessor: (
       handlers: JobHandlers<PayloadMap>,
       options?: ProcessorOptions,
-    ) => createProcessor<PayloadMap>(backend, handlers, options),
+    ) =>
+      createProcessor<PayloadMap>(backend, handlers, options, async () => {
+        await enqueueDueCronJobsImpl();
+      }),
 
     // Job events
     getJobEvents: withLogContext(
@@ -165,34 +216,118 @@ export const initJobQueue = <PayloadMap = any>(
       config.verbose ?? false,
     ),
 
-    // Wait / Token support (PostgreSQL-only for now)
+    // Wait / Token support (works with all backends)
     createToken: withLogContext(
       (options?: import('./types.js').CreateTokenOptions) =>
-        createWaitpoint(requirePool(), null, options),
+        backend.createWaitpoint(null, options),
       config.verbose ?? false,
     ),
     completeToken: withLogContext(
-      (tokenId: string, data?: any) =>
-        completeWaitpoint(requirePool(), tokenId, data),
+      (tokenId: string, data?: any) => backend.completeWaitpoint(tokenId, data),
       config.verbose ?? false,
     ),
     getToken: withLogContext(
-      (tokenId: string) => getWaitpoint(requirePool(), tokenId),
+      (tokenId: string) => backend.getWaitpoint(tokenId),
       config.verbose ?? false,
     ),
     expireTimedOutTokens: withLogContext(
-      () => expireTimedOutWaitpoints(requirePool()),
+      () => backend.expireTimedOutWaitpoints(),
+      config.verbose ?? false,
+    ),
+
+    // Cron schedule operations
+    addCronJob: withLogContext(
+      <T extends JobType<PayloadMap>>(
+        options: CronScheduleOptions<PayloadMap, T>,
+      ) => {
+        if (!validateCronExpression(options.cronExpression)) {
+          return Promise.reject(
+            new Error(`Invalid cron expression: "${options.cronExpression}"`),
+          );
+        }
+        const nextRunAt = getNextCronOccurrence(
+          options.cronExpression,
+          options.timezone ?? 'UTC',
+        );
+        const input: CronScheduleInput = {
+          scheduleName: options.scheduleName,
+          cronExpression: options.cronExpression,
+          jobType: options.jobType as string,
+          payload: options.payload,
+          maxAttempts: options.maxAttempts ?? 3,
+          priority: options.priority ?? 0,
+          timeoutMs: options.timeoutMs ?? null,
+          forceKillOnTimeout: options.forceKillOnTimeout ?? false,
+          tags: options.tags,
+          timezone: options.timezone ?? 'UTC',
+          allowOverlap: options.allowOverlap ?? false,
+          nextRunAt,
+        };
+        return backend.addCronSchedule(input);
+      },
+      config.verbose ?? false,
+    ),
+    getCronJob: withLogContext(
+      (id: number) => backend.getCronSchedule(id),
+      config.verbose ?? false,
+    ),
+    getCronJobByName: withLogContext(
+      (name: string) => backend.getCronScheduleByName(name),
+      config.verbose ?? false,
+    ),
+    listCronJobs: withLogContext(
+      (status?: CronScheduleStatus) => backend.listCronSchedules(status),
+      config.verbose ?? false,
+    ),
+    removeCronJob: withLogContext(
+      (id: number) => backend.removeCronSchedule(id),
+      config.verbose ?? false,
+    ),
+    pauseCronJob: withLogContext(
+      (id: number) => backend.pauseCronSchedule(id),
+      config.verbose ?? false,
+    ),
+    resumeCronJob: withLogContext(
+      (id: number) => backend.resumeCronSchedule(id),
+      config.verbose ?? false,
+    ),
+    editCronJob: withLogContext(
+      async (id: number, updates: EditCronScheduleOptions) => {
+        if (
+          updates.cronExpression !== undefined &&
+          !validateCronExpression(updates.cronExpression)
+        ) {
+          throw new Error(
+            `Invalid cron expression: "${updates.cronExpression}"`,
+          );
+        }
+        let nextRunAt: Date | null | undefined;
+        if (
+          updates.cronExpression !== undefined ||
+          updates.timezone !== undefined
+        ) {
+          const existing = await backend.getCronSchedule(id);
+          const expr = updates.cronExpression ?? existing?.cronExpression ?? '';
+          const tz = updates.timezone ?? existing?.timezone ?? 'UTC';
+          nextRunAt = getNextCronOccurrence(expr, tz);
+        }
+        await backend.editCronSchedule(id, updates, nextRunAt);
+      },
+      config.verbose ?? false,
+    ),
+    enqueueDueCronJobs: withLogContext(
+      () => enqueueDueCronJobsImpl(),
       config.verbose ?? false,
     ),
 
     // Advanced access
     getPool: () => {
-      if (backendType !== 'postgres') {
+      if (!(backend instanceof PostgresBackend)) {
         throw new Error(
           'getPool() is only available with the PostgreSQL backend.',
         );
       }
-      return (backend as PostgresBackend).getPool();
+      return backend.getPool();
     },
     getRedisClient: () => {
       if (backendType !== 'redis') {
@@ -213,9 +348,10 @@ const withLogContext =
   };
 
 export * from './types.js';
-export { QueueBackend } from './backend.js';
+export { QueueBackend, CronScheduleInput } from './backend.js';
 export { PostgresBackend } from './backends/postgres.js';
 export {
   validateHandlerSerializable,
   testHandlerSerialization,
 } from './handler-validation.js';
+export { getNextCronOccurrence, validateCronExpression } from './cron.js';

package/src/processor.ts

@@ -1,5 +1,4 @@
 import { Worker } from 'worker_threads';
-import { Pool } from 'pg';
 import {
   JobRecord,
   ProcessorOptions,
@@ -15,69 +14,8 @@ import {
   WaitTokenResult,
 } from './types.js';
 import { QueueBackend } from './backend.js';
-import { PostgresBackend } from './backends/postgres.js';
-import {
-  waitJob,
-  updateStepData,
-  createWaitpoint,
-  getWaitpoint,
-} from './queue.js';
 import { log, setLogContext } from './log-context.js';
 
-/**
- * Try to extract the underlying pg Pool from a QueueBackend.
- * Returns null for non-PostgreSQL backends.
- */
-function tryExtractPool(backend: QueueBackend): Pool | null {
-  if (backend instanceof PostgresBackend) {
-    return backend.getPool();
-  }
-  return null;
-}
-
-/**
- * Build a JobContext without wait support (for non-PostgreSQL backends).
- * prolong/onTimeout work normally; wait-related methods throw helpful errors.
- */
-function buildBasicContext(
-  backend: QueueBackend,
-  jobId: number,
-  baseCtx: {
-    prolong: JobContext['prolong'];
-    onTimeout: JobContext['onTimeout'];
-  },
-): JobContext {
-  const waitError = () =>
-    new Error(
-      'Wait features (waitFor, waitUntil, createToken, waitForToken, ctx.run) are currently only supported with the PostgreSQL backend.',
-    );
-  return {
-    prolong: baseCtx.prolong,
-    onTimeout: baseCtx.onTimeout,
-    run: async <T>(_stepName: string, fn: () => Promise<T>): Promise<T> => {
-      // Without PostgreSQL, just execute the function directly (no persistence)
-      return fn();
-    },
-    waitFor: async () => {
-      throw waitError();
-    },
-    waitUntil: async () => {
-      throw waitError();
-    },
-    createToken: async () => {
-      throw waitError();
-    },
-    waitForToken: async () => {
-      throw waitError();
-    },
-    setProgress: async (percent: number) => {
-      if (percent < 0 || percent > 100)
-        throw new Error('Progress must be between 0 and 100');
-      await backend.updateProgress(jobId, Math.round(percent));
-    },
-  };
-}
-
 /**
  * Validates that a handler can be serialized for worker thread execution.
  * Throws an error with helpful message if serialization fails.
@@ -388,7 +326,7 @@ function createNoOpContext(
  * Marks pending waits as completed and fetches token outputs.
  */
 async function resolveCompletedWaits(
-  pool: Pool,
+  backend: QueueBackend,
   stepData: Record<string, any>,
 ): Promise<void> {
   for (const key of Object.keys(stepData)) {
@@ -401,7 +339,7 @@ async function resolveCompletedWaits(
       stepData[key] = { ...entry, completed: true };
     } else if (entry.type === 'token' && entry.tokenId) {
       // Token-based wait -- fetch the waitpoint result
-      const wp = await getWaitpoint(pool, entry.tokenId);
+      const wp = await backend.getWaitpoint(entry.tokenId);
       if (wp && wp.status === 'completed') {
         stepData[key] = {
           ...entry,
@@ -422,10 +360,10 @@ async function resolveCompletedWaits(
 
 /**
  * Build the extended JobContext with step tracking and wait support.
+ * Works with any QueueBackend (Postgres or Redis).
  */
 function buildWaitContext(
   backend: QueueBackend,
-  pool: Pool,
   jobId: number,
   stepData: Record<string, any>,
   baseCtx: {
@@ -455,7 +393,7 @@ function buildWaitContext(
 
       // Persist step result
       stepData[stepName] = { __completed: true, result };
-      await updateStepData(pool, jobId, stepData);
+      await backend.updateStepData(jobId, stepData);
 
       return result;
     },
@@ -498,7 +436,7 @@ function buildWaitContext(
     },
 
     createToken: async (options?) => {
-      const token = await createWaitpoint(pool, jobId, options);
+      const token = await backend.createWaitpoint(jobId, options);
       return token;
     },
 
@@ -517,7 +455,7 @@ function buildWaitContext(
       }
 
       // Check if the token is already completed (e.g., completed while job was still processing)
-      const wp = await getWaitpoint(pool, tokenId);
+      const wp = await backend.getWaitpoint(tokenId);
       if (wp && wp.status === 'completed') {
         const result: WaitTokenResult<T> = {
           ok: true,
@@ -529,7 +467,7 @@ function buildWaitContext(
           completed: true,
           result,
         };
-        await updateStepData(pool, jobId, stepData);
+        await backend.updateStepData(jobId, stepData);
         return result;
       }
       if (wp && wp.status === 'timed_out') {
@@ -543,7 +481,7 @@ function buildWaitContext(
           completed: true,
           result,
         };
-        await updateStepData(pool, jobId, stepData);
+        await backend.updateStepData(jobId, stepData);
         return result;
       }
 
@@ -591,17 +529,14 @@ export async function processJobWithHandlers<
   // Load step data (may contain completed steps from previous invocations)
   const stepData: Record<string, any> = { ...(job.stepData || {}) };
 
-  // Try to get pool for wait features (PostgreSQL-only)
-  const pool = tryExtractPool(backend);
-
   // If resuming from a wait, resolve any pending wait entries
   const hasStepHistory = Object.keys(stepData).some((k) =>
     k.startsWith('__wait_'),
   );
-  if (hasStepHistory && pool) {
-    await resolveCompletedWaits(pool, stepData);
+  if (hasStepHistory) {
+    await resolveCompletedWaits(backend, stepData);
     // Persist the resolved step data
-    await updateStepData(pool, job.id, stepData);
+    await backend.updateStepData(job.id, stepData);
   }
 
   // Per-job timeout logic
@@ -685,10 +620,8 @@ export async function processJobWithHandlers<
             },
           };
 
-      // Build context: full wait support for PostgreSQL, basic for others
-      const ctx = pool
-        ? buildWaitContext(backend, pool, job.id, stepData, baseCtx)
-        : buildBasicContext(backend, job.id, baseCtx);
+      // Build context: full wait support for all backends
+      const ctx = buildWaitContext(backend, job.id, stepData, baseCtx);
 
       // If forceKillOnTimeout was set but timeoutMs was missing, warn
       if (forceKillOnTimeout && !hasTimeout) {
@@ -720,22 +653,10 @@ export async function processJobWithHandlers<
 
     // Check if this is a WaitSignal (not a real error)
     if (error instanceof WaitSignal) {
-      if (!pool) {
-        // Wait signals should never happen with non-PostgreSQL backends
-        // since the context methods throw, but guard just in case
-        await backend.failJob(
-          job.id,
-          new Error(
-            'WaitSignal received but wait features require the PostgreSQL backend.',
-          ),
-          FailureReason.HandlerError,
-        );
-        return;
-      }
       log(
         `Job ${job.id} entering wait: type=${error.type}, waitUntil=${error.waitUntil?.toISOString() ?? 'none'}, tokenId=${error.tokenId ?? 'none'}`,
       );
-      await waitJob(pool, job.id, {
+      await backend.waitJob(job.id, {
         waitUntil: error.waitUntil,
         waitTokenId: error.tokenId,
         stepData: error.stepData,
@@ -818,16 +739,18 @@ export async function processBatchWithHandlers<PayloadMap>(
 }
 
 /**
- * Start a job processor that continuously processes jobs
- * @param backend - The queue backend
- * @param handlers - The job handlers for this processor instance
+ * Start a job processor that continuously processes jobs.
+ * @param backend - The queue backend.
+ * @param handlers - The job handlers for this processor instance.
  * @param options - The processor options. Leave pollInterval empty to run only once. Use jobType to filter jobs by type.
- * @returns {Processor} The processor instance
+ * @param onBeforeBatch - Optional callback invoked before each batch. Used internally to enqueue due cron jobs.
+ * @returns {Processor} The processor instance.
  */
 export const createProcessor = <PayloadMap = any>(
   backend: QueueBackend,
   handlers: JobHandlers<PayloadMap>,
   options: ProcessorOptions = {},
+  onBeforeBatch?: () => Promise<void>,
 ): Processor => {
   const {
     workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
@@ -847,6 +770,22 @@ export const createProcessor = <PayloadMap = any>(
   const processJobs = async (): Promise<number> => {
     if (!running) return 0;
 
+    // Run pre-batch hook (e.g. enqueue due cron jobs) before processing
+    if (onBeforeBatch) {
+      try {
+        await onBeforeBatch();
+      } catch (hookError) {
+        log(`onBeforeBatch hook error: ${hookError}`);
+        if (onError) {
+          onError(
+            hookError instanceof Error
+              ? hookError
+              : new Error(String(hookError)),
+          );
+        }
+      }
+    }
+
     log(
       `Processing jobs with workerId: ${workerId}${jobType ? ` and jobType: ${Array.isArray(jobType) ? jobType.join(',') : jobType}` : ''}`,
     );

package/src/queue.test.ts

@@ -141,6 +141,35 @@ describe('queue integration', () => {
     expect(job).toBeNull();
   });
 
+  it('should cleanup old completed jobs in batches', async () => {
+    // Add and complete 5 jobs
+    const ids: number[] = [];
+    for (let i = 0; i < 5; i++) {
+      const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(
+        pool,
+        {
+          jobType: 'email',
+          payload: { to: `batch-${i}@example.com` },
+        },
+      );
+      await queue.getNextBatch(pool, 'worker-batch-cleanup', 1);
+      await queue.completeJob(pool, jobId);
+      ids.push(jobId);
+    }
+    // Manually backdate all 5
+    await pool.query(
+      `UPDATE job_queue SET updated_at = NOW() - INTERVAL '31 days' WHERE id = ANY($1::int[])`,
+      [ids],
+    );
+    // Cleanup with batchSize=2 so it takes multiple iterations
+    const deleted = await queue.cleanupOldJobs(pool, 30, 2);
+    expect(deleted).toBe(5);
+    for (const id of ids) {
+      const job = await queue.getJob(pool, id);
+      expect(job).toBeNull();
+    }
+  });
+
   it('should cancel a scheduled job', async () => {
     const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
       jobType: 'email',