@nicnocquee/dataqueue 1.33.0 → 1.35.0-beta.20260224075710

package/src/backends/redis.test.ts

@@ -580,6 +580,167 @@ describe('Redis backend integration', () => {
     const job = await jobQueue.getJob(jobId);
     expect(job?.status).toBe('pending');
   });
+
+  // ── Configurable retry strategy tests ────────────────────────────────
+
+  it('stores retry config on a job', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'retry-config@example.com' },
+      retryDelay: 30,
+      retryBackoff: false,
+      retryDelayMax: 120,
+    });
+
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.retryDelay).toBe(30);
+    expect(job?.retryBackoff).toBe(false);
+    expect(job?.retryDelayMax).toBe(120);
+  });
+
+  it('returns null retry config for jobs without it', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'no-retry-config@example.com' },
+    });
+
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.retryDelay).toBeNull();
+    expect(job?.retryBackoff).toBeNull();
+    expect(job?.retryDelayMax).toBeNull();
+  });
+
+  it('uses legacy backoff when no retry config is set', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'legacy-retry@example.com' },
+      maxAttempts: 3,
+    });
+
+    const handler = vi.fn(async () => {
+      throw new Error('fail');
+    });
+    const processor = jobQueue.createProcessor({
+      email: handler,
+      sms: vi.fn(async () => {}),
+      test: vi.fn(async () => {}),
+    });
+    await processor.start();
+
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.status).toBe('failed');
+    expect(job?.nextAttemptAt).not.toBeNull();
+    const delayMs =
+      job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime();
+    // Legacy: 2^1 * 60s = 120s = 120000ms
+    expect(delayMs).toBeGreaterThanOrEqual(115000);
+    expect(delayMs).toBeLessThanOrEqual(125000);
+  });
+
+  it('uses fixed delay when retryBackoff is false', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'fixed-retry@example.com' },
+      maxAttempts: 3,
+      retryDelay: 10,
+      retryBackoff: false,
+    });
+
+    const handler = vi.fn(async () => {
+      throw new Error('fail');
+    });
+    const processor = jobQueue.createProcessor({
+      email: handler,
+      sms: vi.fn(async () => {}),
+      test: vi.fn(async () => {}),
+    });
+    await processor.start();
+
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.status).toBe('failed');
+    expect(job?.nextAttemptAt).not.toBeNull();
+    const delaySec =
+      (job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
+    expect(delaySec).toBeGreaterThanOrEqual(9);
+    expect(delaySec).toBeLessThanOrEqual(11);
+  });
+
+  it('uses exponential backoff with custom retryDelay', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'expo-retry@example.com' },
+      maxAttempts: 3,
+      retryDelay: 5,
+      retryBackoff: true,
+    });
+
+    const handler = vi.fn(async () => {
+      throw new Error('fail');
+    });
+    const processor = jobQueue.createProcessor({
+      email: handler,
+      sms: vi.fn(async () => {}),
+      test: vi.fn(async () => {}),
+    });
+    await processor.start();
+
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.status).toBe('failed');
+    expect(job?.nextAttemptAt).not.toBeNull();
+    // 5 * 2^1 = 10s, with jitter [5, 10]
+    const delaySec =
+      (job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
+    expect(delaySec).toBeGreaterThanOrEqual(4);
+    expect(delaySec).toBeLessThanOrEqual(11);
+  });
+
+  it('caps exponential backoff with retryDelayMax', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'capped-retry@example.com' },
+      maxAttempts: 5,
+      retryDelay: 100,
+      retryBackoff: true,
+      retryDelayMax: 30,
+    });
+
+    const handler = vi.fn(async () => {
+      throw new Error('fail');
+    });
+    const processor = jobQueue.createProcessor({
+      email: handler,
+      sms: vi.fn(async () => {}),
+      test: vi.fn(async () => {}),
+    });
+    await processor.start();
+
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.status).toBe('failed');
+    expect(job?.nextAttemptAt).not.toBeNull();
+    // 100 * 2^1 = 200 capped to 30, with jitter [15, 30]
+    const delaySec =
+      (job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
+    expect(delaySec).toBeGreaterThanOrEqual(14);
+    expect(delaySec).toBeLessThanOrEqual(31);
+  });
+
+  it('allows editing retry config via editJob', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'edit-retry@example.com' },
+    });
+
+    await jobQueue.editJob(jobId, {
+      retryDelay: 15,
+      retryBackoff: false,
+      retryDelayMax: 60,
+    });
+
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.retryDelay).toBe(15);
+    expect(job?.retryBackoff).toBe(false);
+    expect(job?.retryDelayMax).toBe(60);
+  });
 });
 
 describe('Redis cron schedules integration', () => {
@@ -1105,6 +1266,113 @@ describe('Redis parity features', () => {
     expect(job?.waitTokenId).toBeNull();
   });
 
+  // ── Job output ─────────────────────────────────────────────────────
+
+  it('stores output from ctx.setOutput() and retrieves via getJob', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'output@test.com' },
+    });
+
+    const processor = jobQueue.createProcessor({
+      email: vi.fn(async (_payload, _signal, ctx) => {
+        await ctx.setOutput({ reportUrl: 'https://example.com/report.pdf' });
+      }),
+      sms: vi.fn(async () => {}),
+      test: vi.fn(async () => {}),
+    });
+    await processor.start();
+
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.status).toBe('completed');
+    expect(job?.output).toEqual({
+      reportUrl: 'https://example.com/report.pdf',
+    });
+  });
+
+  it('stores handler return value as output when setOutput is not called', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'return@test.com' },
+    });
+
+    const processor = jobQueue.createProcessor({
+      email: vi.fn(async () => {
+        return { processed: true, count: 42 };
+      }),
+      sms: vi.fn(async () => {}),
+      test: vi.fn(async () => {}),
+    });
+    await processor.start();
+
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.status).toBe('completed');
+    expect(job?.output).toEqual({ processed: true, count: 42 });
+  });
+
+  it('setOutput takes precedence over handler return value', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'precedence@test.com' },
+    });
+
+    const processor = jobQueue.createProcessor({
+      email: vi.fn(async (_payload, _signal, ctx) => {
+        await ctx.setOutput({ fromSetOutput: true });
+        return { fromReturn: true };
+      }),
+      sms: vi.fn(async () => {}),
+      test: vi.fn(async () => {}),
+    });
+    await processor.start();
+
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.status).toBe('completed');
+    expect(job?.output).toEqual({ fromSetOutput: true });
+  });
+
+  it('output is null for jobs that do not set output (backward compat)', async () => {
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'no-output@test.com' },
+    });
+
+    const processor = jobQueue.createProcessor({
+      email: vi.fn(async () => {}),
+      sms: vi.fn(async () => {}),
+      test: vi.fn(async () => {}),
+    });
+    await processor.start();
+
+    const job = await jobQueue.getJob(jobId);
+    expect(job?.status).toBe('completed');
+    expect(job?.output).toBeNull();
+  });
+
+  it('stores scalar output values (string, number, array)', async () => {
+    const jobId1 = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'string-output@test.com' },
+    });
+    const jobId2 = await jobQueue.addJob({
+      jobType: 'sms',
+      payload: { to: '+123' },
+    });
+
+    const processor = jobQueue.createProcessor({
+      email: vi.fn(async () => 'simple string'),
+      sms: vi.fn(async () => 42),
+      test: vi.fn(async () => {}),
+    });
+    await processor.start();
+
+    const job1 = await jobQueue.getJob(jobId1);
+    expect(job1?.output).toBe('simple string');
+
+    const job2 = await jobQueue.getJob(jobId2);
+    expect(job2?.output).toBe(42);
+  });
+
   // ── cleanupOldJobEvents ─────────────────────────────────────────────
 
   it('cleanupOldJobEvents removes old events', async () => {
@@ -1512,3 +1780,403 @@ describe('Redis parity features', () => {
     expect(invocationCount).toBe(2);
   });
 });
+
+// ── BYOC (Bring Your Own Connection) tests for Redis ────────────────────
+
+describe('Redis BYOC: init with external client', () => {
+  let prefix: string;
+  let externalClient: any;
+  let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
+
+  beforeEach(async () => {
+    prefix = createRedisTestPrefix();
+    const { default: IORedis } = await import('ioredis');
+    externalClient = new (IORedis as any)(REDIS_URL);
+    jobQueue = initJobQueue<TestPayloadMap>({
+      backend: 'redis',
+      client: externalClient,
+      keyPrefix: prefix,
+    });
+  });
+
+  afterEach(async () => {
+    await cleanupRedisPrefix(externalClient, prefix);
+    await externalClient.quit();
+  });
+
+  it('uses the provided client for addJob and getJob', async () => {
+    // Act
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'byoc-redis@example.com' },
+    });
+
+    // Assert
+    const job = await jobQueue.getJob(jobId);
+    expect(job).not.toBeNull();
+    expect(job?.jobType).toBe('email');
+    expect(job?.payload).toEqual({ to: 'byoc-redis@example.com' });
+  });
+
+  it('returns the same client instance from getRedisClient()', () => {
+    // Act
+    const returned = jobQueue.getRedisClient();
+
+    // Assert
+    expect(returned).toBe(externalClient);
+  });
+});
+
+describe('Redis BYOC: addJob with db option throws', () => {
+  let prefix: string;
+  let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
+  let redisClient: any;
+
+  beforeEach(async () => {
+    prefix = createRedisTestPrefix();
+    jobQueue = initJobQueue<TestPayloadMap>({
+      backend: 'redis',
+      redisConfig: { url: REDIS_URL, keyPrefix: prefix },
+    });
+    redisClient = jobQueue.getRedisClient();
+  });
+
+  afterEach(async () => {
+    await cleanupRedisPrefix(redisClient, prefix);
+    await redisClient.quit();
+  });
+
+  it('throws a clear error when db option is provided', async () => {
+    // Setup — fake db client
+    const fakeDb = { query: async () => ({ rows: [], rowCount: 0 }) };
+
+    // Act & Assert
+    await expect(
+      jobQueue.addJob(
+        { jobType: 'email', payload: { to: 'fail@example.com' } },
+        { db: fakeDb },
+      ),
+    ).rejects.toThrow('The db option is not supported with the Redis backend.');
+  });
+});
+
+describe('Redis addJobs batch insert', () => {
+  let prefix: string;
+  let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
+  let redisClient: any;
+
+  beforeEach(async () => {
+    prefix = createRedisTestPrefix();
+    jobQueue = initJobQueue<TestPayloadMap>({
+      backend: 'redis',
+      redisConfig: { url: REDIS_URL, keyPrefix: prefix },
+    });
+    redisClient = jobQueue.getRedisClient();
+  });
+
+  afterEach(async () => {
+    await cleanupRedisPrefix(redisClient, prefix);
+    await redisClient.quit();
+  });
+
+  it('inserts multiple jobs and returns IDs in order', async () => {
+    // Act
+    const ids = await jobQueue.addJobs([
+      { jobType: 'email', payload: { to: 'a@test.com' } },
+      { jobType: 'sms', payload: { to: '+1234' } },
+      { jobType: 'email', payload: { to: 'b@test.com' } },
+    ]);
+
+    // Assert
+    expect(ids).toHaveLength(3);
+
+    const job0 = await jobQueue.getJob(ids[0]);
+    expect(job0?.jobType).toBe('email');
+    expect(job0?.payload).toEqual({ to: 'a@test.com' });
+
+    const job1 = await jobQueue.getJob(ids[1]);
+    expect(job1?.jobType).toBe('sms');
+    expect(job1?.payload).toEqual({ to: '+1234' });
+
+    const job2 = await jobQueue.getJob(ids[2]);
+    expect(job2?.jobType).toBe('email');
+    expect(job2?.payload).toEqual({ to: 'b@test.com' });
+  });
+
+  it('returns empty array for empty input', async () => {
+    // Act
+    const ids = await jobQueue.addJobs([]);
+
+    // Assert
+    expect(ids).toEqual([]);
+  });
+
+  it('handles idempotency keys for new jobs', async () => {
+    // Act
+    const ids = await jobQueue.addJobs([
+      {
+        jobType: 'email',
+        payload: { to: 'a@test.com' },
+        idempotencyKey: 'r-key-a',
+      },
+      {
+        jobType: 'email',
+        payload: { to: 'b@test.com' },
+        idempotencyKey: 'r-key-b',
+      },
+    ]);
+
+    // Assert
+    expect(ids).toHaveLength(2);
+    expect(ids[0]).not.toBe(ids[1]);
+
+    const job0 = await jobQueue.getJob(ids[0]);
+    expect(job0?.idempotencyKey).toBe('r-key-a');
+  });
+
+  it('returns existing IDs for conflicting idempotency keys', async () => {
+    // Setup
+    const existingId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'existing@test.com' },
+      idempotencyKey: 'r-dup',
+    });
+
+    // Act
+    const ids = await jobQueue.addJobs([
+      { jobType: 'email', payload: { to: 'new@test.com' } },
+      {
+        jobType: 'email',
+        payload: { to: 'dup@test.com' },
+        idempotencyKey: 'r-dup',
+      },
+    ]);
+
+    // Assert
+    expect(ids).toHaveLength(2);
+    expect(ids[1]).toBe(existingId);
+    expect(ids[0]).not.toBe(existingId);
+  });
+
+  it('records added events for each inserted job', async () => {
+    // Act
+    const ids = await jobQueue.addJobs([
+      { jobType: 'email', payload: { to: 'a@test.com' } },
+      { jobType: 'sms', payload: { to: '+999' } },
+    ]);
+
+    // Assert
+    const events0 = await jobQueue.getJobEvents(ids[0]);
+    expect(events0.filter((e) => e.eventType === 'added')).toHaveLength(1);
+
+    const events1 = await jobQueue.getJobEvents(ids[1]);
+    expect(events1.filter((e) => e.eventType === 'added')).toHaveLength(1);
+  });
+
+  it('throws when db option is used with addJobs', async () => {
+    // Setup
+    const fakeDb = { query: async () => ({ rows: [], rowCount: 0 }) };
+
+    // Act & Assert
+    await expect(
+      jobQueue.addJobs(
+        [{ jobType: 'email', payload: { to: 'fail@test.com' } }],
+        { db: fakeDb },
+      ),
+    ).rejects.toThrow('The db option is not supported with the Redis backend.');
+  });
+
+  it('stores tags and priority correctly per job', async () => {
+    // Act
+    const ids = await jobQueue.addJobs([
+      {
+        jobType: 'email',
+        payload: { to: 'a@test.com' },
+        tags: ['urgent'],
+        priority: 10,
+      },
+      { jobType: 'sms', payload: { to: '+1' }, priority: 5 },
+      { jobType: 'email', payload: { to: 'c@test.com' }, tags: ['low'] },
+    ]);
+
+    // Assert
+    const job0 = await jobQueue.getJob(ids[0]);
+    expect(job0?.tags).toEqual(['urgent']);
+    expect(job0?.priority).toBe(10);
+
+    const job1 = await jobQueue.getJob(ids[1]);
+    expect(job1?.priority).toBe(5);
+
+    const job2 = await jobQueue.getJob(ids[2]);
+    expect(job2?.tags).toEqual(['low']);
+  });
+});
+
+describe('Redis event hooks', () => {
+  let prefix: string;
+  let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
+  let redisClient: any;
+
+  beforeEach(async () => {
+    prefix = createRedisTestPrefix();
+    jobQueue = initJobQueue<TestPayloadMap>({
+      backend: 'redis',
+      redisConfig: { url: REDIS_URL, keyPrefix: prefix },
+    });
+    redisClient = jobQueue.getRedisClient();
+  });
+
+  afterEach(async () => {
+    jobQueue.removeAllListeners();
+    await cleanupRedisPrefix(redisClient, prefix);
+    await redisClient.quit();
+  });
+
+  it('emits job:added on addJob', async () => {
+    const listener = vi.fn();
+    jobQueue.on('job:added', listener);
+
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'test@example.com' },
+    });
+
+    expect(listener).toHaveBeenCalledTimes(1);
+    expect(listener).toHaveBeenCalledWith({ jobId, jobType: 'email' });
+  });
+
+  it('emits job:added for each job in addJobs', async () => {
+    const listener = vi.fn();
+    jobQueue.on('job:added', listener);
+
+    const ids = await jobQueue.addJobs([
+      { jobType: 'email', payload: { to: 'a@test.com' } },
+      { jobType: 'sms', payload: { to: '+1234' } },
+    ]);
+
+    expect(listener).toHaveBeenCalledTimes(2);
+    expect(listener).toHaveBeenCalledWith({ jobId: ids[0], jobType: 'email' });
+    expect(listener).toHaveBeenCalledWith({ jobId: ids[1], jobType: 'sms' });
+  });
+
+  it('emits job:cancelled on cancelJob', async () => {
+    const listener = vi.fn();
+    jobQueue.on('job:cancelled', listener);
+
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'test@example.com' },
+    });
+    await jobQueue.cancelJob(jobId);
+
+    expect(listener).toHaveBeenCalledTimes(1);
+    expect(listener).toHaveBeenCalledWith({ jobId });
+  });
+
+  it('emits job:retried on retryJob', async () => {
+    const listener = vi.fn();
+    jobQueue.on('job:retried', listener);
+
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'test@example.com' },
+    });
+
+    const processor = jobQueue.createProcessor({
+      email: vi.fn(async () => {
+        throw new Error('fail');
+      }),
+      sms: vi.fn(async () => {}),
+      test: vi.fn(async () => {}),
+    });
+    await processor.start();
+
+    await jobQueue.retryJob(jobId);
+
+    expect(listener).toHaveBeenCalledTimes(1);
+    expect(listener).toHaveBeenCalledWith({ jobId });
+  });
+
+  it('emits job:processing and job:completed on successful processing', async () => {
+    const processingListener = vi.fn();
+    const completedListener = vi.fn();
+    jobQueue.on('job:processing', processingListener);
+    jobQueue.on('job:completed', completedListener);
+
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'test@example.com' },
+    });
+
+    const processor = jobQueue.createProcessor({
+      email: vi.fn(async () => {}),
+      sms: vi.fn(async () => {}),
+      test: vi.fn(async () => {}),
+    });
+    await processor.start();
+
+    expect(processingListener).toHaveBeenCalledTimes(1);
+    expect(processingListener).toHaveBeenCalledWith({
+      jobId,
+      jobType: 'email',
+    });
+    expect(completedListener).toHaveBeenCalledTimes(1);
+    expect(completedListener).toHaveBeenCalledWith({
+      jobId,
+      jobType: 'email',
+    });
+  });
+
+  it('emits job:failed with willRetry flag', async () => {
+    const listener = vi.fn();
+    jobQueue.on('job:failed', listener);
+
+    const jobId = await jobQueue.addJob({
+      jobType: 'email',
+      payload: { to: 'test@example.com' },
+      maxAttempts: 1,
+    });
+
+    const processor = jobQueue.createProcessor({
+      email: vi.fn(async () => {
+        throw new Error('boom');
+      }),
+      sms: vi.fn(async () => {}),
+      test: vi.fn(async () => {}),
+    });
+    await processor.start();
+
+    expect(listener).toHaveBeenCalledTimes(1);
+    expect(listener).toHaveBeenCalledWith(
+      expect.objectContaining({
+        jobId,
+        jobType: 'email',
+        willRetry: false,
+        error: expect.any(Error),
+      }),
+    );
+  });
+
+  it('once fires only once then auto-unsubscribes', async () => {
+    const listener = vi.fn();
+    jobQueue.once('job:added', listener);
+
+    await jobQueue.addJob({ jobType: 'email', payload: { to: 'a@test.com' } });
+    await jobQueue.addJob({ jobType: 'sms', payload: { to: '+1234' } });
+
+    expect(listener).toHaveBeenCalledTimes(1);
+  });
+
+  it('off removes a listener', async () => {
+    const listener = vi.fn();
+    jobQueue.on('job:added', listener);
+
+    await jobQueue.addJob({ jobType: 'email', payload: { to: 'a@test.com' } });
+    expect(listener).toHaveBeenCalledTimes(1);
+
+    jobQueue.off('job:added', listener);
+
+    await jobQueue.addJob({ jobType: 'sms', payload: { to: '+1234' } });
+    expect(listener).toHaveBeenCalledTimes(1);
+  });
+});