@nicnocquee/dataqueue 1.24.0 → 1.25.0

package/src/queue.test.ts

@@ -72,6 +72,8 @@ describe('queue integration', () => {
       jobType: 'email',
       payload: { to: 'done@example.com' },
     });
+    // Claim the job first (sets status to 'processing')
+    await queue.getNextBatch(pool, 'worker-complete', 1);
     await queue.completeJob(pool, jobId);
     const job = await queue.getJob(pool, jobId);
     expect(job?.status).toBe('completed');
@@ -124,6 +126,8 @@ describe('queue integration', () => {
       jobType: 'email',
       payload: { to: 'cleanup@example.com' },
     });
+    // Claim then complete the job
+    await queue.getNextBatch(pool, 'worker-cleanup', 1);
     await queue.completeJob(pool, jobId);
     // Manually update updated_at to be old
     await pool.query(
@@ -154,6 +158,7 @@ describe('queue integration', () => {
         payload: { to: 'done@example.com' },
       },
     );
+    await queue.getNextBatch(pool, 'worker-cancel-done', 1);
     await queue.completeJob(pool, jobId2);
     await queue.cancelJob(pool, jobId2);
     const completedJob = await queue.getJob(pool, jobId2);
@@ -229,6 +234,7 @@ describe('queue integration', () => {
       jobType: 'email',
       payload: { to: 'original@example.com' },
     });
+    await queue.getNextBatch(pool, 'worker-edit-noop', 1);
     await queue.completeJob(pool, jobId);
 
     // Try to edit a completed job - should silently fail
@@ -367,7 +373,7 @@ describe('queue integration', () => {
         priority: 0,
       },
     );
-    // Add a completed job
+    // Add a completed job (set via SQL since this test is about edit behavior)
     const jobId4 = await queue.addJob<{ email: { to: string } }, 'email'>(
       pool,
       {
@@ -375,7 +381,10 @@ describe('queue integration', () => {
         payload: { to: 'done@example.com' },
       },
     );
-    await queue.completeJob(pool, jobId4);
+    await pool.query(
+      `UPDATE job_queue SET status = 'completed' WHERE id = $1`,
+      [jobId4],
+    );
 
     // Edit all pending jobs
     const editedCount = await queue.editAllPendingJobs<
@@ -615,7 +624,11 @@ describe('queue integration', () => {
       payload: { to: 'completed@example.com' },
       priority: 0,
     });
-    await queue.completeJob(pool, completedJobId);
+    // Set to completed via SQL (bypassing status check since we're testing edit behavior)
+    await pool.query(
+      `UPDATE job_queue SET status = 'completed' WHERE id = $1`,
+      [completedJobId],
+    );
 
     // Edit all pending jobs
     const editedCount = await queue.editAllPendingJobs<
@@ -705,7 +718,7 @@ describe('queue integration', () => {
         payload: { to: 'cancelall3@example.com' },
       },
     );
-    // Add a completed job
+    // Add a completed job (set via SQL since this test is about cancel behavior)
     const jobId4 = await queue.addJob<{ email: { to: string } }, 'email'>(
       pool,
       {
@@ -713,7 +726,10 @@ describe('queue integration', () => {
         payload: { to: 'done@example.com' },
       },
     );
-    await queue.completeJob(pool, jobId4);
+    await pool.query(
+      `UPDATE job_queue SET status = 'completed' WHERE id = $1`,
+      [jobId4],
+    );
 
     // Cancel all upcoming jobs
     const cancelledCount = await queue.cancelAllUpcomingJobs(pool);
@@ -809,8 +825,12 @@ describe('queue integration', () => {
       jobType: 'email',
       payload: { to: 'failhistory@example.com' },
     });
-    // Fail the job twice with different errors
+    // Claim and fail the job (first error)
+    await queue.getNextBatch(pool, 'worker-fail-1', 1);
     await queue.failJob(pool, jobId, new Error('first error'));
+    // Retry, claim again, and fail again (second error)
+    await queue.retryJob(pool, jobId);
+    await queue.getNextBatch(pool, 'worker-fail-2', 1);
     await queue.failJob(pool, jobId, new Error('second error'));
     const job = await queue.getJob(pool, jobId);
     expect(job?.status).toBe('failed');
@@ -843,6 +863,50 @@ describe('queue integration', () => {
     expect(job?.lockedAt).toBeNull();
     expect(job?.lockedBy).toBeNull();
   });
+
+  it('should not reclaim a job whose timeoutMs exceeds the reclaim threshold', async () => {
+    // Add a job with a 30-minute timeout
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'long-timeout@example.com' },
+      timeoutMs: 30 * 60 * 1000, // 30 minutes
+    });
+    // Simulate: processing for 15 minutes (exceeds 10-min global threshold but within 30-min job timeout)
+    await pool.query(
+      `UPDATE job_queue SET status = 'processing', locked_at = NOW() - INTERVAL '15 minutes' WHERE id = $1`,
+      [jobId],
+    );
+    let job = await queue.getJob(pool, jobId);
+    expect(job?.status).toBe('processing');
+
+    // Reclaim with 10-minute global threshold — should NOT reclaim this job
+    const reclaimed = await queue.reclaimStuckJobs(pool, 10);
+    expect(reclaimed).toBe(0);
+    job = await queue.getJob(pool, jobId);
+    expect(job?.status).toBe('processing');
+  });
+
+  it('should reclaim a job whose timeoutMs has also been exceeded', async () => {
+    // Add a job with a 20-minute timeout
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'expired-timeout@example.com' },
+      timeoutMs: 20 * 60 * 1000, // 20 minutes
+    });
+    // Simulate: processing for 25 minutes (exceeds both 10-min threshold and 20-min job timeout)
+    await pool.query(
+      `UPDATE job_queue SET status = 'processing', locked_at = NOW() - INTERVAL '25 minutes' WHERE id = $1`,
+      [jobId],
+    );
+    let job = await queue.getJob(pool, jobId);
+    expect(job?.status).toBe('processing');
+
+    // Reclaim with 10-minute global threshold — should reclaim since 25 min > 20 min timeout
+    const reclaimed = await queue.reclaimStuckJobs(pool, 10);
+    expect(reclaimed).toBeGreaterThanOrEqual(1);
+    job = await queue.getJob(pool, jobId);
+    expect(job?.status).toBe('pending');
+  });
 });
 
 describe('job event tracking', () => {
@@ -1718,4 +1782,151 @@ describe('getJobs', () => {
     expect(jobs.map((j) => j.id)).toContain(id1);
     expect(jobs.map((j) => j.id)).not.toContain(id2);
   });
+
+  // --- Idempotency tests ---
+
+  it('should store and return idempotencyKey when provided', async () => {
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'test@example.com' },
+      idempotencyKey: 'unique-key-1',
+    });
+    const job = await queue.getJob(pool, jobId);
+    expect(job).not.toBeNull();
+    expect(job?.idempotencyKey).toBe('unique-key-1');
+  });
+
+  it('should return the same job ID when adding a job with a duplicate idempotencyKey', async () => {
+    const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'first@example.com' },
+        idempotencyKey: 'dedup-key',
+      },
+    );
+    const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'second@example.com' },
+        idempotencyKey: 'dedup-key',
+      },
+    );
+    expect(jobId1).toBe(jobId2);
+
+    // The original job's payload should be preserved (not updated)
+    const job = await queue.getJob(pool, jobId1);
+    expect(job?.payload).toEqual({ to: 'first@example.com' });
+  });
+
+  it('should create separate jobs when no idempotencyKey is provided', async () => {
+    const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'a@example.com' },
+      },
+    );
+    const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'a@example.com' },
+      },
+    );
+    expect(jobId1).not.toBe(jobId2);
+  });
+
+  it('should create separate jobs when different idempotencyKeys are provided', async () => {
+    const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'same@example.com' },
+        idempotencyKey: 'key-a',
+      },
+    );
+    const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'same@example.com' },
+        idempotencyKey: 'key-b',
+      },
+    );
+    expect(jobId1).not.toBe(jobId2);
+  });
+
+  it('should only record the added event once for duplicate idempotencyKey', async () => {
+    const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'once@example.com' },
+        idempotencyKey: 'event-dedup-key',
+      },
+    );
+    // Add again with same key
+    await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'twice@example.com' },
+      idempotencyKey: 'event-dedup-key',
+    });
+
+    const events = await queue.getJobEvents(pool, jobId1);
+    const addedEvents = events.filter(
+      (e: JobEvent) => e.eventType === JobEventType.Added,
+    );
+    expect(addedEvents.length).toBe(1);
+  });
+
+  it('should return null idempotencyKey for jobs created without one', async () => {
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'nokey@example.com' },
+    });
+    const job = await queue.getJob(pool, jobId);
+    expect(job).not.toBeNull();
+    expect(job?.idempotencyKey).toBeNull();
+  });
+
+  it('should permanently fail a job when max attempts are exhausted', async () => {
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'exhaust@example.com' },
+      maxAttempts: 2,
+    });
+
+    // Claim the job (attempt 1)
+    const batch1 = await queue.getNextBatch(pool, 'worker-1', 1);
+    expect(batch1.length).toBe(1);
+    expect(batch1[0].attempts).toBe(1);
+
+    // Fail it
+    await queue.failJob(pool, jobId, new Error('attempt 1 failed'));
+    let job = await queue.getJob(pool, jobId);
+    expect(job?.status).toBe('failed');
+    expect(job?.nextAttemptAt).not.toBeNull(); // Should have a retry scheduled
+
+    // Wait a moment so next_attempt_at <= NOW() and claim again (attempt 2)
+    await pool.query(
+      `UPDATE job_queue SET next_attempt_at = NOW() WHERE id = $1`,
+      [jobId],
+    );
+    const batch2 = await queue.getNextBatch(pool, 'worker-1', 1);
+    expect(batch2.length).toBe(1);
+    expect(batch2[0].attempts).toBe(2);
+
+    // Fail it again — now attempts === maxAttempts
+    await queue.failJob(pool, jobId, new Error('attempt 2 failed'));
+    job = await queue.getJob(pool, jobId);
+    expect(job?.status).toBe('failed');
+    expect(job?.nextAttemptAt).toBeNull(); // No more retries
+    expect(job?.errorHistory?.length).toBe(2);
+
+    // Should NOT be picked up again
+    const batch3 = await queue.getNextBatch(pool, 'worker-1', 1);
+    expect(batch3.length).toBe(0);
+  });
 });