@nicnocquee/dataqueue 1.25.0 → 1.26.0-beta.20260223195940

package/src/queue.test.ts

@@ -141,6 +141,35 @@ describe('queue integration', () => {
     expect(job).toBeNull();
   });
 
+  it('should cleanup old completed jobs in batches', async () => {
+    // Add and complete 5 jobs
+    const ids: number[] = [];
+    for (let i = 0; i < 5; i++) {
+      const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(
+        pool,
+        {
+          jobType: 'email',
+          payload: { to: `batch-${i}@example.com` },
+        },
+      );
+      await queue.getNextBatch(pool, 'worker-batch-cleanup', 1);
+      await queue.completeJob(pool, jobId);
+      ids.push(jobId);
+    }
+    // Manually backdate all 5
+    await pool.query(
+      `UPDATE job_queue SET updated_at = NOW() - INTERVAL '31 days' WHERE id = ANY($1::int[])`,
+      [ids],
+    );
+    // Cleanup with batchSize=2 so it takes multiple iterations
+    const deleted = await queue.cleanupOldJobs(pool, 30, 2);
+    expect(deleted).toBe(5);
+    for (const id of ids) {
+      const job = await queue.getJob(pool, id);
+      expect(job).toBeNull();
+    }
+  });
+
   it('should cancel a scheduled job', async () => {
     const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
       jobType: 'email',
@@ -1929,4 +1958,440 @@ describe('getJobs', () => {
     const batch3 = await queue.getNextBatch(pool, 'worker-1', 1);
     expect(batch3.length).toBe(0);
   });
+
+  // ── Configurable retry strategy tests ────────────────────────────────
+
+  it('uses legacy backoff when no retry config is set', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'legacy@example.com' },
+      maxAttempts: 3,
+    });
+
+    // Act
+    await queue.getNextBatch(pool, 'worker-1', 1);
+    await queue.failJob(pool, jobId, new Error('fail'));
+
+    // Assert — legacy formula: 2^1 * 60s = 120s from now
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.nextAttemptAt).not.toBeNull();
+    const delaySec =
+      (job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
+    expect(delaySec).toBeGreaterThanOrEqual(115);
+    expect(delaySec).toBeLessThanOrEqual(125);
+  });
+
+  it('uses fixed delay when retryBackoff is false', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'fixed@example.com' },
+      maxAttempts: 3,
+      retryDelay: 10,
+      retryBackoff: false,
+    });
+
+    // Act
+    await queue.getNextBatch(pool, 'worker-1', 1);
+    await queue.failJob(pool, jobId, new Error('fail'));
+
+    // Assert — fixed 10s delay
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.nextAttemptAt).not.toBeNull();
+    expect(job?.retryDelay).toBe(10);
+    expect(job?.retryBackoff).toBe(false);
+    const delaySec =
+      (job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
+    expect(delaySec).toBeGreaterThanOrEqual(9);
+    expect(delaySec).toBeLessThanOrEqual(11);
+  });
+
+  it('uses exponential backoff with custom retryDelay', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'expo@example.com' },
+      maxAttempts: 3,
+      retryDelay: 5,
+      retryBackoff: true,
+    });
+
+    // Act — attempt 1
+    await queue.getNextBatch(pool, 'worker-1', 1);
+    await queue.failJob(pool, jobId, new Error('fail'));
+
+    // Assert — exponential: 5 * 2^1 = 10s, with jitter [5, 10]
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.nextAttemptAt).not.toBeNull();
+    const delaySec =
+      (job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
+    expect(delaySec).toBeGreaterThanOrEqual(4);
+    expect(delaySec).toBeLessThanOrEqual(11);
+  });
+
+  it('caps exponential backoff with retryDelayMax', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'capped@example.com' },
+      maxAttempts: 5,
+      retryDelay: 100,
+      retryBackoff: true,
+      retryDelayMax: 30,
+    });
+
+    // Act — attempt 1
+    await queue.getNextBatch(pool, 'worker-1', 1);
+    await queue.failJob(pool, jobId, new Error('fail'));
+
+    // Assert — 100 * 2^1 = 200s but capped at 30s, with jitter [15, 30]
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.nextAttemptAt).not.toBeNull();
+    expect(job?.retryDelayMax).toBe(30);
+    const delaySec =
+      (job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
+    expect(delaySec).toBeGreaterThanOrEqual(14);
+    expect(delaySec).toBeLessThanOrEqual(31);
+  });
+
+  it('stores retry config on job record', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'config@example.com' },
+      retryDelay: 30,
+      retryBackoff: false,
+      retryDelayMax: 120,
+    });
+
+    // Act
+    const job = await queue.getJob(pool, jobId);
+
+    // Assert
+    expect(job?.retryDelay).toBe(30);
+    expect(job?.retryBackoff).toBe(false);
+    expect(job?.retryDelayMax).toBe(120);
+  });
+
+  it('returns null retry config for jobs without it', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'noconfig@example.com' },
+    });
+
+    // Act
+    const job = await queue.getJob(pool, jobId);
+
+    // Assert
+    expect(job?.retryDelay).toBeNull();
+    expect(job?.retryBackoff).toBeNull();
+    expect(job?.retryDelayMax).toBeNull();
+  });
+
+  it('allows editing retry config via editJob', async () => {
+    // Setup
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'edit@example.com' },
+    });
+
+    // Act
+    await queue.editJob(pool, jobId, {
+      retryDelay: 15,
+      retryBackoff: false,
+      retryDelayMax: 60,
+    });
+
+    // Assert
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.retryDelay).toBe(15);
+    expect(job?.retryBackoff).toBe(false);
+    expect(job?.retryDelayMax).toBe(60);
+  });
+});
+
+describe('queue.addJob with db option (BYOC)', () => {
+  let pool: Pool;
+  let dbName: string;
+
+  beforeEach(async () => {
+    const setup = await createTestDbAndPool();
+    pool = setup.pool;
+    dbName = setup.dbName;
+  });
+
+  afterEach(async () => {
+    await pool.end();
+    await destroyTestDb(dbName);
+  });
+
+  it('rolls back the job when the transaction is rolled back', async () => {
+    // Setup
+    const client = await pool.connect();
+    await client.query('BEGIN');
+
+    // Act
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      { jobType: 'email', payload: { to: 'rollback@example.com' } },
+      { db: client },
+    );
+    await client.query('ROLLBACK');
+    client.release();
+
+    // Assert
+    const job = await queue.getJob(pool, jobId);
+    expect(job).toBeNull();
+  });
+
+  it('persists the job when the transaction is committed', async () => {
+    // Setup
+    const client = await pool.connect();
+    await client.query('BEGIN');
+
+    // Act
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      { jobType: 'email', payload: { to: 'commit@example.com' } },
+      { db: client },
+    );
+    await client.query('COMMIT');
+    client.release();
+
+    // Assert
+    const job = await queue.getJob(pool, jobId);
+    expect(job).not.toBeNull();
+    expect(job?.payload).toEqual({ to: 'commit@example.com' });
+  });
+});
+
+describe('addJobs batch insert', () => {
+  let pool: Pool;
+  let dbName: string;
+
+  beforeEach(async () => {
+    const setup = await createTestDbAndPool();
+    pool = setup.pool;
+    dbName = setup.dbName;
+  });
+
+  afterEach(async () => {
+    await pool.end();
+    await destroyTestDb(dbName);
+  });
+
+  it('inserts multiple jobs and returns IDs in order', async () => {
+    // Act
+    const ids = await queue.addJobs<
+      { email: { to: string }; report: { id: string } },
+      'email' | 'report'
+    >(pool, [
+      { jobType: 'email', payload: { to: 'a@test.com' } },
+      { jobType: 'report', payload: { id: 'r1' } },
+      { jobType: 'email', payload: { to: 'b@test.com' } },
+    ]);
+
+    // Assert
+    expect(ids).toHaveLength(3);
+    expect(ids[0]).toBeLessThan(ids[1]);
+    expect(ids[1]).toBeLessThan(ids[2]);
+
+    const job0 = await queue.getJob(pool, ids[0]);
+    expect(job0?.jobType).toBe('email');
+    expect(job0?.payload).toEqual({ to: 'a@test.com' });
+
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1?.jobType).toBe('report');
+    expect(job1?.payload).toEqual({ id: 'r1' });
+
+    const job2 = await queue.getJob(pool, ids[2]);
+    expect(job2?.jobType).toBe('email');
+    expect(job2?.payload).toEqual({ to: 'b@test.com' });
+  });
+
+  it('returns empty array for empty input', async () => {
+    // Act
+    const ids = await queue.addJobs(pool, []);
+
+    // Assert
+    expect(ids).toEqual([]);
+  });
+
+  it('respects priority and runAt per job', async () => {
+    // Setup
+    const futureDate = new Date(Date.now() + 60_000);
+
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(pool, [
+      { jobType: 'task', payload: { n: 1 }, priority: 5 },
+      { jobType: 'task', payload: { n: 2 }, priority: 10, runAt: futureDate },
+    ]);
+
+    // Assert
+    const job0 = await queue.getJob(pool, ids[0]);
+    expect(job0?.priority).toBe(5);
+
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1?.priority).toBe(10);
+    expect(job1?.runAt.getTime()).toBeCloseTo(futureDate.getTime(), -3);
+  });
+
+  it('handles idempotency keys for new jobs', async () => {
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(pool, [
+      { jobType: 'task', payload: { n: 1 }, idempotencyKey: 'key-a' },
+      { jobType: 'task', payload: { n: 2 }, idempotencyKey: 'key-b' },
+    ]);
+
+    // Assert
+    expect(ids).toHaveLength(2);
+    expect(ids[0]).not.toBe(ids[1]);
+
+    const job0 = await queue.getJob(pool, ids[0]);
+    expect(job0?.idempotencyKey).toBe('key-a');
+
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1?.idempotencyKey).toBe('key-b');
+  });
+
+  it('returns existing IDs for conflicting idempotency keys', async () => {
+    // Setup — insert a job first
+    const existingId = await queue.addJob<{ task: { n: number } }, 'task'>(
+      pool,
+      { jobType: 'task', payload: { n: 0 }, idempotencyKey: 'dup-key' },
+    );
+
+    // Act — batch includes a duplicate key
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(pool, [
+      { jobType: 'task', payload: { n: 1 } },
+      { jobType: 'task', payload: { n: 2 }, idempotencyKey: 'dup-key' },
+      { jobType: 'task', payload: { n: 3 } },
+    ]);
+
+    // Assert
+    expect(ids).toHaveLength(3);
+    expect(ids[1]).toBe(existingId);
+    expect(ids[0]).not.toBe(existingId);
+    expect(ids[2]).not.toBe(existingId);
+  });
+
+  it('handles mix of keyed and non-keyed jobs', async () => {
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(pool, [
+      { jobType: 'task', payload: { n: 1 } },
+      { jobType: 'task', payload: { n: 2 }, idempotencyKey: 'mix-1' },
+      { jobType: 'task', payload: { n: 3 } },
+      { jobType: 'task', payload: { n: 4 }, idempotencyKey: 'mix-2' },
+      { jobType: 'task', payload: { n: 5 } },
+    ]);
+
+    // Assert
+    expect(ids).toHaveLength(5);
+    const uniqueIds = new Set(ids);
+    expect(uniqueIds.size).toBe(5);
+
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1?.idempotencyKey).toBe('mix-1');
+
+    const job3 = await queue.getJob(pool, ids[3]);
+    expect(job3?.idempotencyKey).toBe('mix-2');
+  });
+
+  it('records added events only for newly inserted jobs', async () => {
+    // Setup — pre-insert a job with a known key
+    const existingId = await queue.addJob<{ task: { n: number } }, 'task'>(
+      pool,
+      { jobType: 'task', payload: { n: 0 }, idempotencyKey: 'evt-key' },
+    );
+
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(pool, [
+      { jobType: 'task', payload: { n: 1 } },
+      { jobType: 'task', payload: { n: 2 }, idempotencyKey: 'evt-key' },
+    ]);
+
+    // Assert — the new job should have an event from addJobs
+    const events0 = await queue.getJobEvents(pool, ids[0]);
+    const addedEvents0 = events0.filter(
+      (e: JobEvent) => e.eventType === JobEventType.Added,
+    );
+    expect(addedEvents0).toHaveLength(1);
+
+    // The duplicate should only have the original event from addJob, not a second from addJobs
+    const eventsExisting = await queue.getJobEvents(pool, existingId);
+    const addedEventsExisting = eventsExisting.filter(
+      (e: JobEvent) => e.eventType === JobEventType.Added,
+    );
+    expect(addedEventsExisting).toHaveLength(1);
+  });
+
+  it('stores tags correctly per job', async () => {
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(pool, [
+      { jobType: 'task', payload: { n: 1 }, tags: ['urgent', 'billing'] },
+      { jobType: 'task', payload: { n: 2 }, tags: ['low-priority'] },
+      { jobType: 'task', payload: { n: 3 } },
+    ]);
+
+    // Assert
+    const job0 = await queue.getJob(pool, ids[0]);
+    expect(job0?.tags).toEqual(['urgent', 'billing']);
+
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1?.tags).toEqual(['low-priority']);
+
+    const job2 = await queue.getJob(pool, ids[2]);
+    expect(job2?.tags).toBeNull();
+  });
+
+  it('works with transactional db option — commit', async () => {
+    // Setup
+    const client = await pool.connect();
+    await client.query('BEGIN');
+
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(
+      pool,
+      [
+        { jobType: 'task', payload: { n: 1 } },
+        { jobType: 'task', payload: { n: 2 } },
+      ],
+      { db: client },
+    );
+    await client.query('COMMIT');
+    client.release();
+
+    // Assert
+    expect(ids).toHaveLength(2);
+    const job0 = await queue.getJob(pool, ids[0]);
+    expect(job0).not.toBeNull();
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1).not.toBeNull();
+  });
+
+  it('works with transactional db option — rollback', async () => {
+    // Setup
+    const client = await pool.connect();
+    await client.query('BEGIN');
+
+    // Act
+    const ids = await queue.addJobs<{ task: { n: number } }, 'task'>(
+      pool,
+      [
+        { jobType: 'task', payload: { n: 1 } },
+        { jobType: 'task', payload: { n: 2 } },
+      ],
+      { db: client },
+    );
+    await client.query('ROLLBACK');
+    client.release();
+
+    // Assert — jobs should not exist after rollback
+    const job0 = await queue.getJob(pool, ids[0]);
+    expect(job0).toBeNull();
+    const job1 = await queue.getJob(pool, ids[1]);
+    expect(job1).toBeNull();
+  });
 });