@nicnocquee/dataqueue 1.16.0

package/src/processor.test.ts

@@ -0,0 +1,478 @@
+import { Pool } from 'pg';
+import { describe, expect, it, beforeEach, afterEach, vi } from 'vitest';
+import {
+  createProcessor,
+  processBatchWithHandlers,
+  processJobWithHandlers,
+} from './processor.js';
+import * as queue from './queue.js';
+import { createTestDbAndPool, destroyTestDb } from './test-util.js';
+import { FailureReason, JobHandler } from './types.js';
+
+// Define the payload map for test jobs
+interface TestPayloadMap {
+  test: { foo: string };
+  fail: {};
+  missing: {};
+  batch: { i: number };
+  proc: { x: number };
+  typeA: { n: number };
+  typeB: { n: number };
+  typeC: { n: number };
+}
+
+// Integration tests for processor
+
+describe('processor integration', () => {
+  let pool: Pool;
+  let dbName: string;
+
+  beforeEach(async () => {
+    const setup = await createTestDbAndPool();
+    pool = setup.pool;
+    dbName = setup.dbName;
+  });
+
+  afterEach(async () => {
+    await pool.end();
+    await destroyTestDb(dbName);
+  });
+
+  it('should process a job with a registered handler', async () => {
+    const handler = vi.fn(async () => {});
+    const handlers = {
+      test: handler,
+      fail: vi.fn(async () => {}),
+      missing: vi.fn(async () => {}),
+      batch: vi.fn(async () => {}),
+      proc: vi.fn(async () => {}),
+      typeA: vi.fn(async () => {}),
+      typeB: vi.fn(async () => {}),
+      typeC: vi.fn(async () => {}),
+    };
+    const jobId = await queue.addJob<TestPayloadMap, 'test'>(pool, {
+      job_type: 'test',
+      payload: { foo: 'bar' },
+    });
+    const job = await queue.getJob<TestPayloadMap, 'test'>(pool, jobId);
+    expect(job).not.toBeNull();
+    await processJobWithHandlers(pool, job!, handlers);
+    expect(handler).toHaveBeenCalledWith(
+      { foo: 'bar' },
+      expect.any(AbortSignal),
+    );
+    const completed = await queue.getJob(pool, jobId);
+    expect(completed?.status).toBe('completed');
+  });
+
+  it('should mark job as failed if handler throws', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('fail!');
+    });
+    const handlers = {
+      test: vi.fn(async () => {}),
+      fail: handler,
+      missing: vi.fn(async () => {}),
+      batch: vi.fn(async () => {}),
+      proc: vi.fn(async () => {}),
+      typeA: vi.fn(async () => {}),
+      typeB: vi.fn(async () => {}),
+      typeC: vi.fn(async () => {}),
+    };
+    const jobId = await queue.addJob<TestPayloadMap, 'fail'>(pool, {
+      job_type: 'fail',
+      payload: {},
+    });
+    const job = await queue.getJob<TestPayloadMap, 'fail'>(pool, jobId);
+    expect(job).not.toBeNull();
+    await processJobWithHandlers(pool, job!, handlers);
+    const failed = await queue.getJob(pool, jobId);
+    expect(failed?.status).toBe('failed');
+    expect(failed?.error_history?.[0]?.message).toBe('fail!');
+    expect(failed?.failure_reason).toBe('handler_error');
+  });
+
+  it('should mark job as failed if no handler registered', async () => {
+    const handler = vi.fn(async () => {
+      throw new Error('No handler registered');
+    });
+    const handlers = {
+      test: vi.fn(async () => {}),
+      fail: handler,
+      batch: vi.fn(async () => {}),
+      proc: vi.fn(async () => {}),
+      typeA: vi.fn(async () => {}),
+      typeB: vi.fn(async () => {}),
+      typeC: vi.fn(async () => {}),
+    };
+    const jobId = await queue.addJob<TestPayloadMap, 'missing'>(pool, {
+      job_type: 'missing',
+      payload: {},
+    });
+    const job = await queue.getJob<TestPayloadMap, 'missing'>(pool, jobId);
+    expect(job).not.toBeNull();
+    // @ts-expect-error - test handler is missing
+    await processJobWithHandlers(pool, job!, handlers);
+    const failed = await queue.getJob(pool, jobId);
+    expect(failed?.status).toBe('failed');
+    expect(failed?.error_history?.[0]?.message).toContain(
+      'No handler registered',
+    );
+    expect(failed?.failure_reason).toBe('no_handler');
+  });
+
+  it('should process a batch of jobs', async () => {
+    const handler = vi.fn(async () => {});
+    const handlers = {
+      test: vi.fn(async () => {}),
+      fail: vi.fn(async () => {}),
+      missing: vi.fn(async () => {}),
+      batch: handler,
+      proc: vi.fn(async () => {}),
+      typeA: vi.fn(async () => {}),
+      typeB: vi.fn(async () => {}),
+      typeC: vi.fn(async () => {}),
+    };
+    const ids = await Promise.all([
+      queue.addJob<TestPayloadMap, 'batch'>(pool, {
+        job_type: 'batch',
+        payload: { i: 1 },
+      }),
+      queue.addJob<TestPayloadMap, 'batch'>(pool, {
+        job_type: 'batch',
+        payload: { i: 2 },
+      }),
+    ]);
+    const processed = await processBatchWithHandlers(
+      pool,
+      'worker-batch',
+      2,
+      undefined,
+      handlers,
+    );
+    expect(processed).toBe(2);
+    const jobs = await queue.getJobsByStatus<TestPayloadMap, 'batch'>(
+      pool,
+      'completed',
+    );
+    expect(jobs.length).toBeGreaterThanOrEqual(2);
+    expect(handler).toHaveBeenCalledTimes(2);
+  });
+
+  it('should start and stop the processor', async () => {
+    const handler = vi.fn(async () => {});
+    const handlers = {
+      test: vi.fn(async () => {}),
+      fail: vi.fn(async () => {}),
+      missing: vi.fn(async () => {}),
+      batch: vi.fn(async () => {}),
+      proc: handler,
+      typeA: vi.fn(async () => {}),
+      typeB: vi.fn(async () => {}),
+      typeC: vi.fn(async () => {}),
+    };
+    await queue.addJob<TestPayloadMap, 'proc'>(pool, {
+      job_type: 'proc',
+      payload: { x: 1 },
+    });
+    const processor = createProcessor(pool, handlers, { pollInterval: 200 });
+    processor.start();
+    // Wait for job to be processed
+    await new Promise((r) => setTimeout(r, 500));
+    processor.stop();
+    expect(processor.isRunning()).toBe(false);
+    const jobs = await queue.getJobsByStatus(pool, 'completed');
+    expect(jobs.some((j) => j.job_type === 'proc')).toBe(true);
+  });
+
+  it('should process only jobs of a specific job type with processBatch', async () => {
+    const handlerA = vi.fn(async () => {});
+    const handlerB = vi.fn(async () => {});
+    const handlers = {
+      test: vi.fn(async () => {}),
+      fail: vi.fn(async () => {}),
+      missing: vi.fn(async () => {}),
+      batch: vi.fn(async () => {}),
+      proc: vi.fn(async () => {}),
+      typeA: handlerA,
+      typeB: handlerB,
+      typeC: vi.fn(async () => {}),
+    };
+    const idA1 = await queue.addJob<TestPayloadMap, 'typeA'>(pool, {
+      job_type: 'typeA',
+      payload: { n: 1 },
+    });
+    const idA2 = await queue.addJob<TestPayloadMap, 'typeA'>(pool, {
+      job_type: 'typeA',
+      payload: { n: 2 },
+    });
+    const idB1 = await queue.addJob<TestPayloadMap, 'typeB'>(pool, {
+      job_type: 'typeB',
+      payload: { n: 3 },
+    });
+    // Only process typeA
+    const processed = await processBatchWithHandlers(
+      pool,
+      'worker-typeA',
+      10,
+      'typeA',
+      handlers,
+    );
+    expect(processed).toBe(2);
+    expect(handlerA).toHaveBeenCalledTimes(2);
+    expect(handlerB).not.toHaveBeenCalled();
+    const jobsA = await queue.getJobsByStatus<TestPayloadMap, 'typeA'>(
+      pool,
+      'completed',
+    );
+    expect(jobsA.some((j) => j.id === idA1)).toBe(true);
+    expect(jobsA.some((j) => j.id === idA2)).toBe(true);
+    const jobB = await queue.getJob<TestPayloadMap, 'typeB'>(pool, idB1);
+    expect(jobB?.status).not.toBe('completed');
+  });
+
+  it('should process only jobs of specific job types (array) with processBatch', async () => {
+    const handlerA = vi.fn(async () => {});
+    const handlerB = vi.fn(async () => {});
+    const handlerC = vi.fn(async () => {});
+    const handlers = {
+      test: vi.fn(async () => {}),
+      fail: vi.fn(async () => {}),
+      missing: vi.fn(async () => {}),
+      batch: vi.fn(async () => {}),
+      proc: vi.fn(async () => {}),
+      typeA: handlerA,
+      typeB: handlerB,
+      typeC: handlerC,
+    };
+    const idA = await queue.addJob<TestPayloadMap, 'typeA'>(pool, {
+      job_type: 'typeA',
+      payload: { n: 1 },
+    });
+    const idB = await queue.addJob<TestPayloadMap, 'typeB'>(pool, {
+      job_type: 'typeB',
+      payload: { n: 2 },
+    });
+    const idC = await queue.addJob<TestPayloadMap, 'typeC'>(pool, {
+      job_type: 'typeC',
+      payload: { n: 3 },
+    });
+    // Only process typeA and typeC
+    const processed = await processBatchWithHandlers(
+      pool,
+      'worker-multi',
+      10,
+      ['typeA', 'typeC'],
+      handlers,
+    );
+    expect(processed).toBe(2);
+    expect(handlerA).toHaveBeenCalledTimes(1);
+    expect(handlerB).not.toHaveBeenCalled();
+    expect(handlerC).toHaveBeenCalledTimes(1);
+    const jobs = await queue.getJobsByStatus<TestPayloadMap, 'typeA' | 'typeC'>(
+      pool,
+      'completed',
+    );
+    expect(jobs.some((j) => j.id === idA)).toBe(true);
+    expect(jobs.some((j) => j.id === idC)).toBe(true);
+    const jobB = await queue.getJob<TestPayloadMap, 'typeB'>(pool, idB);
+    expect(jobB?.status).not.toBe('completed');
+  });
+
+  it('should process only jobs of a specific job type with createProcessor', async () => {
+    const handlerA = vi.fn(async () => {});
+    const handlerB = vi.fn(async () => {});
+    const handlers = {
+      test: vi.fn(async () => {}),
+      fail: vi.fn(async () => {}),
+      missing: vi.fn(async () => {}),
+      batch: vi.fn(async () => {}),
+      proc: vi.fn(async () => {}),
+      typeA: handlerA,
+      typeB: handlerB,
+      typeC: vi.fn(async () => {}),
+    };
+    const idA = await queue.addJob<TestPayloadMap, 'typeA'>(pool, {
+      job_type: 'typeA',
+      payload: { n: 1 },
+    });
+    const idB = await queue.addJob<TestPayloadMap, 'typeB'>(pool, {
+      job_type: 'typeB',
+      payload: { n: 2 },
+    });
+    const processor = createProcessor(pool, handlers, {
+      pollInterval: 100,
+      jobType: 'typeA',
+    });
+    processor.start();
+    await new Promise((r) => setTimeout(r, 300));
+    processor.stop();
+    expect(processor.isRunning()).toBe(false);
+    expect(handlerA).toHaveBeenCalledTimes(1);
+    expect(handlerB).not.toHaveBeenCalled();
+    const jobA = await queue.getJob<TestPayloadMap, 'typeA'>(pool, idA);
+    const jobB = await queue.getJob<TestPayloadMap, 'typeB'>(pool, idB);
+    expect(jobA?.status).toBe('completed');
+    expect(jobB?.status).not.toBe('completed');
+  });
+});
+
+describe('concurrency option', () => {
+  let pool: Pool;
+  let dbName: string;
+
+  beforeEach(async () => {
+    const setup = await createTestDbAndPool();
+    pool = setup.pool;
+    dbName = setup.dbName;
+  });
+
+  afterEach(async () => {
+    await pool.end();
+    await destroyTestDb(dbName);
+  });
+
+  async function addJobs(n: number) {
+    for (let i = 0; i < n; i++) {
+      await queue.addJob<{ test: {} }, 'test'>(pool, {
+        job_type: 'test',
+        payload: {},
+      });
+    }
+  }
+
+  it('should not process more than default concurrency (3) jobs in parallel', async () => {
+    let running = 0;
+    let maxParallel = 0;
+    const handler = async () => {
+      running++;
+      maxParallel = Math.max(maxParallel, running);
+      await new Promise((r) => setTimeout(r, 30));
+      running--;
+    };
+    const handlers = { test: handler };
+    await addJobs(10);
+    const processor = createProcessor(pool, handlers, { batchSize: 10 });
+    await processor.start();
+    expect(maxParallel).toBeLessThanOrEqual(3);
+  });
+
+  it('should not process more than custom concurrency jobs in parallel', async () => {
+    let running = 0;
+    let maxParallel = 0;
+    const handler = async () => {
+      running++;
+      maxParallel = Math.max(maxParallel, running);
+      await new Promise((r) => setTimeout(r, 30));
+      running--;
+    };
+    const handlers = { test: handler };
+    await addJobs(10);
+    const processor = createProcessor(pool, handlers, {
+      batchSize: 10,
+      concurrency: 2,
+    });
+    await processor.start();
+    expect(maxParallel).toBeLessThanOrEqual(2);
+  });
+
+  it('should not process more than batchSize jobs in parallel if concurrency > batchSize', async () => {
+    let running = 0;
+    let maxParallel = 0;
+    const handler = async () => {
+      running++;
+      maxParallel = Math.max(maxParallel, running);
+      await new Promise((r) => setTimeout(r, 30));
+      running--;
+    };
+    const handlers = { test: handler };
+    await addJobs(2);
+    const processor = createProcessor(pool, handlers, {
+      batchSize: 2,
+      concurrency: 5,
+    });
+    await processor.start();
+    expect(maxParallel).toBeLessThanOrEqual(2);
+  });
+
+  it('should process jobs sequentially if concurrency is 1', async () => {
+    let running = 0;
+    let maxParallel = 0;
+    const handler = async () => {
+      running++;
+      maxParallel = Math.max(maxParallel, running);
+      await new Promise((r) => setTimeout(r, 30));
+      running--;
+    };
+    const handlers = { test: handler };
+    await addJobs(5);
+    const processor = createProcessor(pool, handlers, {
+      batchSize: 5,
+      concurrency: 1,
+    });
+    await processor.start();
+    expect(maxParallel).toBe(1);
+  });
+});
+
+describe('per-job timeout', () => {
+  let pool: Pool;
+  let dbName: string;
+
+  beforeEach(async () => {
+    const setup = await createTestDbAndPool();
+    pool = setup.pool;
+    dbName = setup.dbName;
+  });
+
+  afterEach(async () => {
+    await pool.end();
+    await destroyTestDb(dbName);
+  });
+
+  it('should fail the job if handler exceeds timeoutMs', async () => {
+    const handler = vi.fn(async (_payload, signal) => {
+      await new Promise((resolve, reject) => {
+        const t = setTimeout(resolve, 200);
+        signal.addEventListener('abort', () => {
+          clearTimeout(t);
+          reject(new Error('aborted'));
+        });
+      });
+    });
+    const handlers: { test: JobHandler<{ test: {} }, 'test'> } = {
+      test: handler,
+    };
+    const jobId = await queue.addJob<{ test: {} }, 'test'>(pool, {
+      job_type: 'test',
+      payload: {},
+      timeoutMs: 50, // 50ms
+    });
+    const job = await queue.getJob<{ test: {} }, 'test'>(pool, jobId);
+    expect(job).not.toBeNull();
+    await processJobWithHandlers(pool, job!, handlers);
+    const failed = await queue.getJob(pool, jobId);
+    expect(failed?.status).toBe('failed');
+    expect(failed?.error_history?.[0]?.message).toContain('timed out');
+    expect(failed?.failure_reason).toBe(FailureReason.Timeout);
+  });
+
+  it('should complete the job if handler finishes before timeoutMs', async () => {
+    const handler = vi.fn(async (_payload, _signal) => {
+      await new Promise((r) => setTimeout(r, 20));
+    });
+    const handlers: { test: JobHandler<{ test: {} }, 'test'> } = {
+      test: handler,
+    };
+    const jobId = await queue.addJob<{ test: {} }, 'test'>(pool, {
+      job_type: 'test',
+      payload: {},
+      timeoutMs: 200, // 200ms
+    });
+    const job = await queue.getJob<{ test: {} }, 'test'>(pool, jobId);
+    expect(job).not.toBeNull();
+    await processJobWithHandlers(pool, job!, handlers);
+    const completed = await queue.getJob(pool, jobId);
+    expect(completed?.status).toBe('completed');
+  });
+});

package/src/processor.ts

@@ -0,0 +1,242 @@
+import { Pool } from 'pg';
+import {
+  JobRecord,
+  ProcessorOptions,
+  Processor,
+  JobHandler,
+  JobType,
+  FailureReason,
+  JobHandlers,
+} from './types.js';
+import {
+  getNextBatch,
+  completeJob,
+  failJob,
+  setPendingReasonForUnpickedJobs,
+} from './queue.js';
+import { log, setLogContext } from './log-context.js';
+
+/**
+ * Process a single job using the provided handler map
+ */
+export async function processJobWithHandlers<
+  PayloadMap,
+  T extends keyof PayloadMap & string,
+>(
+  pool: Pool,
+  job: JobRecord<PayloadMap, T>,
+  jobHandlers: JobHandlers<PayloadMap>,
+): Promise<void> {
+  const handler = jobHandlers[job.job_type];
+
+  if (!handler) {
+    await setPendingReasonForUnpickedJobs(
+      pool,
+      `No handler registered for job type: ${job.job_type}`,
+      job.job_type,
+    );
+    await failJob(
+      pool,
+      job.id,
+      new Error(`No handler registered for job type: ${job.job_type}`),
+      FailureReason.NoHandler,
+    );
+    return;
+  }
+
+  // Per-job timeout logic
+  const timeoutMs = job.timeout_ms ?? undefined;
+  let timeoutId: NodeJS.Timeout | undefined;
+  const controller = new AbortController();
+  try {
+    const jobPromise = handler(job.payload, controller.signal);
+    if (timeoutMs && timeoutMs > 0) {
+      await Promise.race([
+        jobPromise,
+        new Promise((_, reject) => {
+          timeoutId = setTimeout(() => {
+            controller.abort();
+            const timeoutError = new Error(
+              `Job timed out after ${timeoutMs} ms`,
+            );
+            // @ts-ignore
+            timeoutError.failureReason = FailureReason.Timeout;
+            reject(timeoutError);
+          }, timeoutMs);
+        }),
+      ]);
+    } else {
+      await jobPromise;
+    }
+    if (timeoutId) clearTimeout(timeoutId);
+    await completeJob(pool, job.id);
+  } catch (error) {
+    if (timeoutId) clearTimeout(timeoutId);
+    console.error(`Error processing job ${job.id}:`, error);
+    let failureReason = FailureReason.HandlerError;
+    if (
+      error &&
+      typeof error === 'object' &&
+      'failureReason' in error &&
+      (error as any).failureReason === FailureReason.Timeout
+    ) {
+      failureReason = FailureReason.Timeout;
+    }
+    await failJob(
+      pool,
+      job.id,
+      error instanceof Error ? error : new Error(String(error)),
+      failureReason,
+    );
+  }
+}
+
+/**
+ * Process a batch of jobs using the provided handler map and concurrency limit
+ */
+export async function processBatchWithHandlers<PayloadMap>(
+  pool: Pool,
+  workerId: string,
+  batchSize: number,
+  jobType: string | string[] | undefined,
+  jobHandlers: JobHandlers<PayloadMap>,
+  concurrency?: number,
+): Promise<number> {
+  const jobs = await getNextBatch<PayloadMap, JobType<PayloadMap>>(
+    pool,
+    workerId,
+    batchSize,
+    jobType,
+  );
+  if (!concurrency || concurrency >= jobs.length) {
+    // Default: all in parallel
+    await Promise.all(
+      jobs.map((job) => processJobWithHandlers(pool, job, jobHandlers)),
+    );
+    return jobs.length;
+  }
+  // Concurrency-limited pool
+  let idx = 0;
+  let running = 0;
+  let finished = 0;
+  return new Promise((resolve, reject) => {
+    const next = () => {
+      if (finished === jobs.length) return resolve(jobs.length);
+      while (running < concurrency && idx < jobs.length) {
+        const job = jobs[idx++];
+        running++;
+        processJobWithHandlers(pool, job, jobHandlers)
+          .then(() => {
+            running--;
+            finished++;
+            next();
+          })
+          .catch((err) => {
+            running--;
+            finished++;
+            next();
+          });
+      }
+    };
+    next();
+  });
+}
+
+/**
+ * Start a job processor that continuously processes jobs
+ * @param pool - The database pool
+ * @param handlers - The job handlers for this processor instance
+ * @param options - The processor options. Leave pollInterval empty to run only once. Use jobType to filter jobs by type.
+ * @returns {Processor} The processor instance
+ */
+export const createProcessor = <PayloadMap = any>(
+  pool: Pool,
+  handlers: JobHandlers<PayloadMap>,
+  options: ProcessorOptions = {},
+): Processor => {
+  const {
+    workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
+    batchSize = 10,
+    pollInterval = 5000,
+    onError = (error: Error) => console.error('Job processor error:', error),
+    jobType,
+    concurrency = 3,
+  } = options;
+
+  let running = false;
+  let intervalId: NodeJS.Timeout | null = null;
+
+  setLogContext(options.verbose ?? false);
+
+  const processJobs = async (): Promise<number> => {
+    if (!running) return 0;
+
+    log(
+      `Processing jobs with workerId: ${workerId}${jobType ? ` and jobType: ${Array.isArray(jobType) ? jobType.join(',') : jobType}` : ''}`,
+    );
+
+    try {
+      const processed = await processBatchWithHandlers(
+        pool,
+        workerId,
+        batchSize,
+        jobType,
+        handlers,
+        concurrency,
+      );
+      // Only process one batch in start; do not schedule next batch here
+      return processed;
+    } catch (error) {
+      onError(error instanceof Error ? error : new Error(String(error)));
+    }
+    return 0;
+  };
+
+  return {
+    /**
+     * Start the job processor in the background.
+     * - This will run periodically (every pollInterval milliseconds or 5 seconds if not provided) and process jobs as they become available.
+     * - You have to call the stop method to stop the processor.
+     */
+    startInBackground: () => {
+      if (running) return;
+
+      log(`Starting job processor with workerId: ${workerId}`);
+      running = true;
+      // Background: process batches repeatedly if needed
+      const processBatches = async () => {
+        if (!running) return;
+        const processed = await processJobs();
+        if (processed === batchSize && running) {
+          setImmediate(processBatches);
+        }
+      };
+      processBatches(); // Process immediately on start
+      intervalId = setInterval(processJobs, pollInterval);
+    },
+    /**
+     * Stop the job processor that runs in the background
+     */
+    stop: () => {
+      log(`Stopping job processor with workerId: ${workerId}`);
+      running = false;
+      if (intervalId) {
+        clearInterval(intervalId);
+        intervalId = null;
+      }
+    },
+    /**
+     * Start the job processor synchronously.
+     * - This will process all jobs immediately and then stop.
+     * - The pollInterval is ignored.
+     */
+    start: async () => {
+      log(`Starting job processor with workerId: ${workerId}`);
+      running = true;
+      const processed = await processJobs();
+      running = false;
+      return processed;
+    },
+    isRunning: () => running,
+  };
+};