@nicnocquee/dataqueue 1.34.0 → 1.35.0-beta.20260224110011

package/src/queue.test.ts

@@ -79,6 +79,47 @@ describe('queue integration', () => {
     expect(job?.status).toBe('completed');
   });
 
+  it('should store output when completing a job', async () => {
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'output@example.com' },
+    });
+    await queue.getNextBatch(pool, 'worker-output', 1);
+    await queue.completeJob(pool, jobId, {
+      url: 'https://example.com/report.pdf',
+    });
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.status).toBe('completed');
+    expect(job?.output).toEqual({ url: 'https://example.com/report.pdf' });
+  });
+
+  it('should have null output when completing without output', async () => {
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'no-output@example.com' },
+    });
+    await queue.getNextBatch(pool, 'worker-no-output', 1);
+    await queue.completeJob(pool, jobId);
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.status).toBe('completed');
+    expect(job?.output).toBeNull();
+  });
+
+  it('should preserve output set via updateOutput when completing without output arg', async () => {
+    const { PostgresBackend } = await import('./backends/postgres.js');
+    const backend = new PostgresBackend(pool);
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'pre-output@example.com' },
+    });
+    await queue.getNextBatch(pool, 'worker-pre-output', 1);
+    await backend.updateOutput(jobId, { interim: true });
+    await queue.completeJob(pool, jobId);
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.status).toBe('completed');
+    expect(job?.output).toEqual({ interim: true });
+  });
+
   it('should get the next batch of jobs to process', async () => {
     // Add jobs (do not set runAt, use DB default)
     const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
@@ -1958,4 +1999,547 @@ describe('getJobs', () => {
     const batch3 = await queue.getNextBatch(pool, 'worker-1', 1);
     expect(batch3.length).toBe(0);
   });
+
+  // ── Configurable retry strategy tests ────────────────────────────────
+
+  it('uses legacy backoff when no retry config is set', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'legacy@example.com' },
+      maxAttempts: 3,
+    });
+
+    // Act
+    await queue.getNextBatch(pool, 'worker-1', 1);
+    await queue.failJob(pool, jobId, new Error('fail'));
+
+    // Assert — legacy formula: 2^1 * 60s = 120s from now
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.nextAttemptAt).not.toBeNull();
+    const delaySec =
+      (job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
+    expect(delaySec).toBeGreaterThanOrEqual(115);
+    expect(delaySec).toBeLessThanOrEqual(125);
+  });
+
+  it('uses fixed delay when retryBackoff is false', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'fixed@example.com' },
+      maxAttempts: 3,
+      retryDelay: 10,
+      retryBackoff: false,
+    });
+
+    // Act
+    await queue.getNextBatch(pool, 'worker-1', 1);
+    await queue.failJob(pool, jobId, new Error('fail'));
+
+    // Assert — fixed 10s delay
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.nextAttemptAt).not.toBeNull();
+    expect(job?.retryDelay).toBe(10);
+    expect(job?.retryBackoff).toBe(false);
+    const delaySec =
+      (job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
+    expect(delaySec).toBeGreaterThanOrEqual(9);
+    expect(delaySec).toBeLessThanOrEqual(11);
+  });
+
+  it('uses exponential backoff with custom retryDelay', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'expo@example.com' },
+      maxAttempts: 3,
+      retryDelay: 5,
+      retryBackoff: true,
+    });
+
+    // Act — attempt 1
+    await queue.getNextBatch(pool, 'worker-1', 1);
+    await queue.failJob(pool, jobId, new Error('fail'));
+
+    // Assert — exponential: 5 * 2^1 = 10s, with jitter [5, 10]
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.nextAttemptAt).not.toBeNull();
+    const delaySec =
+      (job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
+    expect(delaySec).toBeGreaterThanOrEqual(4);
+    expect(delaySec).toBeLessThanOrEqual(11);
+  });
+
+  it('caps exponential backoff with retryDelayMax', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'capped@example.com' },
+      maxAttempts: 5,
+      retryDelay: 100,
+      retryBackoff: true,
+      retryDelayMax: 30,
+    });
+
+    // Act — attempt 1
+    await queue.getNextBatch(pool, 'worker-1', 1);
+    await queue.failJob(pool, jobId, new Error('fail'));
+
+    // Assert — 100 * 2^1 = 200s but capped at 30s, with jitter [15, 30]
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.nextAttemptAt).not.toBeNull();
+    expect(job?.retryDelayMax).toBe(30);
+    const delaySec =
+      (job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
+    expect(delaySec).toBeGreaterThanOrEqual(14);
+    expect(delaySec).toBeLessThanOrEqual(31);
+  });
+
+  it('stores retry config on job record', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'config@example.com' },
+      retryDelay: 30,
+      retryBackoff: false,
+      retryDelayMax: 120,
+    });
+
+    // Act
+    const job = await queue.getJob(pool, jobId);
+
+    // Assert
+    expect(job?.retryDelay).toBe(30);
+    expect(job?.retryBackoff).toBe(false);
+    expect(job?.retryDelayMax).toBe(120);
+  });
+
+  it('returns null retry config for jobs without it', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'noconfig@example.com' },
+    });
+
+    // Act
+    const job = await queue.getJob(pool, jobId);
+
+    // Assert
+    expect(job?.retryDelay).toBeNull();
+    expect(job?.retryBackoff).toBeNull();
+    expect(job?.retryDelayMax).toBeNull();
+  });
+
+  it('allows editing retry config via editJob', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'edit@example.com' },
+    });
+
+    // Act
+    await queue.editJob(pool, jobId, {
+      retryDelay: 15,
+      retryBackoff: false,
+      retryDelayMax: 60,
+    });
+
+    // Assert
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.retryDelay).toBe(15);
+    expect(job?.retryBackoff).toBe(false);
+    expect(job?.retryDelayMax).toBe(60);
+  });
+});
+
+describe('queue.addJob with db option (BYOC)', () => {
+  let pool: Pool;
+  let dbName: string;
+
+  beforeEach(async () => {
+    const setup = await createTestDbAndPool();
+    pool = setup.pool;
+    dbName = setup.dbName;
+  });
+
+  afterEach(async () => {
+    await pool.end();
+    await destroyTestDb(dbName);
+  });
+
+  it('rolls back the job when the transaction is rolled back', async () => {
+    // Setup
+    const client = await pool.connect();
+    await client.query('BEGIN');
+
+    // Act
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      { jobType: 'email', payload: { to: 'rollback@example.com' } },
+      { db: client },
+    );
+    await client.query('ROLLBACK');
+    client.release();
+
+    // Assert
+    const job = await queue.getJob(pool, jobId);
+    expect(job).toBeNull();
+  });
+
+  it('persists the job when the transaction is committed', async () => {
+    // Setup
+    const client = await pool.connect();
+    await client.query('BEGIN');
+
+    // Act
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      { jobType: 'email', payload: { to: 'commit@example.com' } },
+      { db: client },
+    );
+    await client.query('COMMIT');
+    client.release();
+
+    // Assert
+    const job = await queue.getJob(pool, jobId);
+    expect(job).not.toBeNull();
+    expect(job?.payload).toEqual({ to: 'commit@example.com' });
+  });
+});
+
+describe('addJobs batch insert', () => {
+  let pool: Pool;
+  let dbName: string;
+
+  beforeEach(async () => {
+    const setup = await createTestDbAndPool();
+    pool = setup.pool;
+    dbName = setup.dbName;
+  });
+
+  afterEach(async () => {
+    await pool.end();
+    await destroyTestDb(dbName);
+  });
+
+  it('inserts multiple jobs and returns IDs in order', async () => {
+    // Act
+    const ids = await queue.addJobs<
+      { email: { to: string }; report: { id: string } },
+      'email' | 'report'
+    >(pool, [
+      { jobType: 'email', payload: { to: 'a@test.com' } },
+      { jobType: 'report', payload: { id: 'r1' } },
+      { jobType: 'email', payload: { to: 'b@test.com' } },
+    ]);
+
+    // Assert
+    expect(ids).toHaveLength(3);
+    expect(ids[0]).toBeLessThan(ids[1]);
+    expect(ids[1]).toBeLessThan(ids[2]);
+
+    const job0 = await queue.getJob(pool, ids[0]);
+    expect(job0?.jobType).toBe('email');
+    expect(job0?.payload).toEqual({ to: 'a@test.com' });
+
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1?.jobType).toBe('report');
+    expect(job1?.payload).toEqual({ id: 'r1' });
+
+    const job2 = await queue.getJob(pool, ids[2]);
+    expect(job2?.jobType).toBe('email');
+    expect(job2?.payload).toEqual({ to: 'b@test.com' });
+  });
+
+  it('returns empty array for empty input', async () => {
+    // Act
+    const ids = await queue.addJobs(pool, []);
+
+    // Assert
+    expect(ids).toEqual([]);
+  });
+
+  it('respects priority and runAt per job', async () => {
+    // Setup
+    const futureDate = new Date(Date.now() + 60_000);
+
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(pool, [
+      { jobType: 'task', payload: { n: 1 }, priority: 5 },
+      { jobType: 'task', payload: { n: 2 }, priority: 10, runAt: futureDate },
+    ]);
+
+    // Assert
+    const job0 = await queue.getJob(pool, ids[0]);
+    expect(job0?.priority).toBe(5);
+
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1?.priority).toBe(10);
+    expect(job1?.runAt.getTime()).toBeCloseTo(futureDate.getTime(), -3);
+  });
+
+  it('handles idempotency keys for new jobs', async () => {
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(pool, [
+      { jobType: 'task', payload: { n: 1 }, idempotencyKey: 'key-a' },
+      { jobType: 'task', payload: { n: 2 }, idempotencyKey: 'key-b' },
+    ]);
+
+    // Assert
+    expect(ids).toHaveLength(2);
+    expect(ids[0]).not.toBe(ids[1]);
+
+    const job0 = await queue.getJob(pool, ids[0]);
+    expect(job0?.idempotencyKey).toBe('key-a');
+
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1?.idempotencyKey).toBe('key-b');
+  });
+
+  it('returns existing IDs for conflicting idempotency keys', async () => {
+    // Setup — insert a job first
+    const existingId = await queue.addJob<{ task: { n: number } }, 'task'>(
+      pool,
+      { jobType: 'task', payload: { n: 0 }, idempotencyKey: 'dup-key' },
+    );
+
+    // Act — batch includes a duplicate key
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(pool, [
+      { jobType: 'task', payload: { n: 1 } },
+      { jobType: 'task', payload: { n: 2 }, idempotencyKey: 'dup-key' },
+      { jobType: 'task', payload: { n: 3 } },
+    ]);
+
+    // Assert
+    expect(ids).toHaveLength(3);
+    expect(ids[1]).toBe(existingId);
+    expect(ids[0]).not.toBe(existingId);
+    expect(ids[2]).not.toBe(existingId);
+  });
+
+  it('handles mix of keyed and non-keyed jobs', async () => {
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(pool, [
+      { jobType: 'task', payload: { n: 1 } },
+      { jobType: 'task', payload: { n: 2 }, idempotencyKey: 'mix-1' },
+      { jobType: 'task', payload: { n: 3 } },
+      { jobType: 'task', payload: { n: 4 }, idempotencyKey: 'mix-2' },
+      { jobType: 'task', payload: { n: 5 } },
+    ]);
+
+    // Assert
+    expect(ids).toHaveLength(5);
+    const uniqueIds = new Set(ids);
+    expect(uniqueIds.size).toBe(5);
+
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1?.idempotencyKey).toBe('mix-1');
+
+    const job3 = await queue.getJob(pool, ids[3]);
+    expect(job3?.idempotencyKey).toBe('mix-2');
+  });
+
+  it('records added events only for newly inserted jobs', async () => {
+    // Setup — pre-insert a job with a known key
+    const existingId = await queue.addJob<{ task: { n: number } }, 'task'>(
+      pool,
+      { jobType: 'task', payload: { n: 0 }, idempotencyKey: 'evt-key' },
+    );
+
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(pool, [
+      { jobType: 'task', payload: { n: 1 } },
+      { jobType: 'task', payload: { n: 2 }, idempotencyKey: 'evt-key' },
+    ]);
+
+    // Assert — the new job should have an event from addJobs
+    const events0 = await queue.getJobEvents(pool, ids[0]);
+    const addedEvents0 = events0.filter(
+      (e: JobEvent) => e.eventType === JobEventType.Added,
+    );
+    expect(addedEvents0).toHaveLength(1);
+
+    // The duplicate should only have the original event from addJob, not a second from addJobs
+    const eventsExisting = await queue.getJobEvents(pool, existingId);
+    const addedEventsExisting = eventsExisting.filter(
+      (e: JobEvent) => e.eventType === JobEventType.Added,
+    );
+    expect(addedEventsExisting).toHaveLength(1);
+  });
+
+  it('stores tags correctly per job', async () => {
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(pool, [
+      { jobType: 'task', payload: { n: 1 }, tags: ['urgent', 'billing'] },
+      { jobType: 'task', payload: { n: 2 }, tags: ['low-priority'] },
+      { jobType: 'task', payload: { n: 3 } },
+    ]);
+
+    // Assert
+    const job0 = await queue.getJob(pool, ids[0]);
+    expect(job0?.tags).toEqual(['urgent', 'billing']);
+
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1?.tags).toEqual(['low-priority']);
+
+    const job2 = await queue.getJob(pool, ids[2]);
+    expect(job2?.tags).toBeNull();
+  });
+
+  it('works with transactional db option — commit', async () => {
+    // Setup
+    const client = await pool.connect();
+    await client.query('BEGIN');
+
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(
+      pool,
+      [
+        { jobType: 'task', payload: { n: 1 } },
+        { jobType: 'task', payload: { n: 2 } },
+      ],
+      { db: client },
+    );
+    await client.query('COMMIT');
+    client.release();
+
+    // Assert
+    expect(ids).toHaveLength(2);
+    const job0 = await queue.getJob(pool, ids[0]);
+    expect(job0).not.toBeNull();
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1).not.toBeNull();
+  });
+
+  it('works with transactional db option — rollback', async () => {
+    // Setup
+    const client = await pool.connect();
+    await client.query('BEGIN');
+
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(
+      pool,
+      [
+        { jobType: 'task', payload: { n: 1 } },
+        { jobType: 'task', payload: { n: 2 } },
+      ],
+      { db: client },
+    );
+    await client.query('ROLLBACK');
+    client.release();
+
+    // Assert — jobs should not exist after rollback
+    const job0 = await queue.getJob(pool, ids[0]);
+    expect(job0).toBeNull();
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1).toBeNull();
+  });
+});
+
+describe('group-based concurrency limits (Postgres)', () => {
+  let pool: Pool;
+  let dbName: string;
+
+  beforeEach(async () => {
+    const setup = await createTestDbAndPool();
+    pool = setup.pool;
+    dbName = setup.dbName;
+  });
+
+  afterEach(async () => {
+    await pool.end();
+    await destroyTestDb(dbName);
+  });
+
+  it('stores and returns group metadata on jobs', async () => {
+    const id = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'grouped@example.com' },
+      group: { id: 'tenant-a', tier: 'gold' },
+    });
+
+    const job = await queue.getJob(pool, id);
+    expect(job?.groupId).toBe('tenant-a');
+    expect(job?.groupTier).toBe('gold');
+  });
+
+  it('enforces global per-group limits across workers', async () => {
+    await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'a-1@example.com' },
+      group: { id: 'tenant-a' },
+    });
+    await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'a-2@example.com' },
+      group: { id: 'tenant-a' },
+    });
+    await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'b-1@example.com' },
+      group: { id: 'tenant-b' },
+    });
+
+    const firstBatch = await queue.getNextBatch(
+      pool,
+      'worker-1',
+      10,
+      undefined,
+      1,
+    );
+    expect(firstBatch).toHaveLength(2);
+    expect(new Set(firstBatch.map((job) => job.groupId))).toEqual(
+      new Set(['tenant-a', 'tenant-b']),
+    );
+
+    const blockedBatch = await queue.getNextBatch(
+      pool,
+      'worker-2',
+      10,
+      undefined,
+      1,
+    );
+    expect(blockedBatch).toHaveLength(0);
+
+    const groupAJob = firstBatch.find((job) => job.groupId === 'tenant-a');
+    expect(groupAJob).toBeDefined();
+    await queue.completeJob(pool, groupAJob!.id);
+
+    const resumedBatch = await queue.getNextBatch(
+      pool,
+      'worker-3',
+      10,
+      undefined,
+      1,
+    );
+    expect(resumedBatch).toHaveLength(1);
+    expect(resumedBatch[0]?.groupId).toBe('tenant-a');
+  });
+
+  it('keeps ungrouped jobs unaffected by groupConcurrency', async () => {
+    await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'ungrouped-1@example.com' },
+    });
+    await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'ungrouped-2@example.com' },
+    });
+    await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'grouped-1@example.com' },
+      group: { id: 'tenant-c' },
+    });
+    await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'grouped-2@example.com' },
+      group: { id: 'tenant-c' },
+    });
+
+    const batch = await queue.getNextBatch(pool, 'worker-u', 10, undefined, 1);
+    expect(batch).toHaveLength(3);
+    expect(batch.filter((job) => job.groupId === 'tenant-c')).toHaveLength(1);
+    expect(batch.filter((job) => !job.groupId)).toHaveLength(2);
+  });
 });

package/src/queue.ts

@@ -16,6 +16,7 @@ import {
   JobEventType,
   TagQueryMode,
   WaitpointRecord,
+  AddJobOptions,
 } from './types.js';
 import { PostgresBackend } from './backends/postgres.js';
 
@@ -34,7 +35,14 @@ export const recordJobEvent = async (
 export const addJob = async <PayloadMap, T extends keyof PayloadMap & string>(
   pool: Pool,
   job: JobOptions<PayloadMap, T>,
-): Promise<number> => new PostgresBackend(pool).addJob(job);
+  options?: AddJobOptions,
+): Promise<number> => new PostgresBackend(pool).addJob(job, options);
+
+export const addJobs = async <PayloadMap, T extends keyof PayloadMap & string>(
+  pool: Pool,
+  jobs: JobOptions<PayloadMap, T>[],
+  options?: AddJobOptions,
+): Promise<number[]> => new PostgresBackend(pool).addJobs(jobs, options);
 
 export const getJob = async <PayloadMap, T extends keyof PayloadMap & string>(
   pool: Pool,
@@ -65,15 +73,20 @@ export const getNextBatch = async <
   workerId: string,
   batchSize = 10,
   jobType?: string | string[],
+  groupConcurrency?: number,
 ): Promise<JobRecord<PayloadMap, T>[]> =>
   new PostgresBackend(pool).getNextBatch<PayloadMap, T>(
     workerId,
     batchSize,
     jobType,
+    groupConcurrency,
   );
 
-export const completeJob = async (pool: Pool, jobId: number): Promise<void> =>
-  new PostgresBackend(pool).completeJob(jobId);
+export const completeJob = async (
+  pool: Pool,
+  jobId: number,
+  output?: unknown,
+): Promise<void> => new PostgresBackend(pool).completeJob(jobId, output);
 
 export const prolongJob = async (pool: Pool, jobId: number): Promise<void> =>
   new PostgresBackend(pool).prolongJob(jobId);
@@ -109,6 +122,9 @@ export const editJob = async <PayloadMap, T extends keyof PayloadMap & string>(
     runAt?: Date | null;
     timeoutMs?: number | null;
     tags?: string[] | null;
+    retryDelay?: number | null;
+    retryBackoff?: boolean | null;
+    retryDelayMax?: number | null;
   },
 ): Promise<void> => new PostgresBackend(pool).editJob(jobId, updates);
 
@@ -134,6 +150,9 @@ export const editAllPendingJobs = async <
     runAt?: Date | null;
     timeoutMs?: number;
     tags?: string[];
+    retryDelay?: number | null;
+    retryBackoff?: boolean | null;
+    retryDelayMax?: number | null;
   },
 ): Promise<number> =>
   new PostgresBackend(pool).editAllPendingJobs(filters, updates);