@nicnocquee/dataqueue 1.34.0 → 1.35.0-beta.20260224110011
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ai/docs-content.json +27 -15
- package/ai/rules/advanced.md +78 -1
- package/ai/rules/basic.md +73 -3
- package/ai/rules/react-dashboard.md +5 -1
- package/ai/skills/dataqueue-advanced/SKILL.md +181 -0
- package/ai/skills/dataqueue-core/SKILL.md +109 -3
- package/ai/skills/dataqueue-react/SKILL.md +19 -7
- package/dist/index.cjs +1168 -173
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +394 -13
- package/dist/index.d.ts +394 -13
- package/dist/index.js +1168 -173
- package/dist/index.js.map +1 -1
- package/migrations/1781200000005_add_retry_config_to_job_queue.sql +17 -0
- package/migrations/1781200000006_add_output_to_job_queue.sql +3 -0
- package/migrations/1781200000007_add_group_fields_to_job_queue.sql +16 -0
- package/package.json +1 -1
- package/src/backend.ts +37 -3
- package/src/backends/postgres.ts +458 -76
- package/src/backends/redis-scripts.ts +273 -37
- package/src/backends/redis.test.ts +753 -0
- package/src/backends/redis.ts +253 -15
- package/src/db-util.ts +1 -1
- package/src/index.test.ts +811 -12
- package/src/index.ts +106 -14
- package/src/processor.test.ts +18 -0
- package/src/processor.ts +147 -49
- package/src/queue.test.ts +584 -0
- package/src/queue.ts +22 -3
- package/src/supervisor.test.ts +340 -0
- package/src/supervisor.ts +177 -0
- package/src/types.ts +353 -3
|
@@ -580,6 +580,167 @@ describe('Redis backend integration', () => {
|
|
|
580
580
|
const job = await jobQueue.getJob(jobId);
|
|
581
581
|
expect(job?.status).toBe('pending');
|
|
582
582
|
});
|
|
583
|
+
|
|
584
|
+
// ── Configurable retry strategy tests ────────────────────────────────
|
|
585
|
+
|
|
586
|
+
it('stores retry config on a job', async () => {
|
|
587
|
+
const jobId = await jobQueue.addJob({
|
|
588
|
+
jobType: 'email',
|
|
589
|
+
payload: { to: 'retry-config@example.com' },
|
|
590
|
+
retryDelay: 30,
|
|
591
|
+
retryBackoff: false,
|
|
592
|
+
retryDelayMax: 120,
|
|
593
|
+
});
|
|
594
|
+
|
|
595
|
+
const job = await jobQueue.getJob(jobId);
|
|
596
|
+
expect(job?.retryDelay).toBe(30);
|
|
597
|
+
expect(job?.retryBackoff).toBe(false);
|
|
598
|
+
expect(job?.retryDelayMax).toBe(120);
|
|
599
|
+
});
|
|
600
|
+
|
|
601
|
+
it('returns null retry config for jobs without it', async () => {
|
|
602
|
+
const jobId = await jobQueue.addJob({
|
|
603
|
+
jobType: 'email',
|
|
604
|
+
payload: { to: 'no-retry-config@example.com' },
|
|
605
|
+
});
|
|
606
|
+
|
|
607
|
+
const job = await jobQueue.getJob(jobId);
|
|
608
|
+
expect(job?.retryDelay).toBeNull();
|
|
609
|
+
expect(job?.retryBackoff).toBeNull();
|
|
610
|
+
expect(job?.retryDelayMax).toBeNull();
|
|
611
|
+
});
|
|
612
|
+
|
|
613
|
+
it('uses legacy backoff when no retry config is set', async () => {
|
|
614
|
+
const jobId = await jobQueue.addJob({
|
|
615
|
+
jobType: 'email',
|
|
616
|
+
payload: { to: 'legacy-retry@example.com' },
|
|
617
|
+
maxAttempts: 3,
|
|
618
|
+
});
|
|
619
|
+
|
|
620
|
+
const handler = vi.fn(async () => {
|
|
621
|
+
throw new Error('fail');
|
|
622
|
+
});
|
|
623
|
+
const processor = jobQueue.createProcessor({
|
|
624
|
+
email: handler,
|
|
625
|
+
sms: vi.fn(async () => {}),
|
|
626
|
+
test: vi.fn(async () => {}),
|
|
627
|
+
});
|
|
628
|
+
await processor.start();
|
|
629
|
+
|
|
630
|
+
const job = await jobQueue.getJob(jobId);
|
|
631
|
+
expect(job?.status).toBe('failed');
|
|
632
|
+
expect(job?.nextAttemptAt).not.toBeNull();
|
|
633
|
+
const delayMs =
|
|
634
|
+
job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime();
|
|
635
|
+
// Legacy: 2^1 * 60s = 120s = 120000ms
|
|
636
|
+
expect(delayMs).toBeGreaterThanOrEqual(115000);
|
|
637
|
+
expect(delayMs).toBeLessThanOrEqual(125000);
|
|
638
|
+
});
|
|
639
|
+
|
|
640
|
+
it('uses fixed delay when retryBackoff is false', async () => {
|
|
641
|
+
const jobId = await jobQueue.addJob({
|
|
642
|
+
jobType: 'email',
|
|
643
|
+
payload: { to: 'fixed-retry@example.com' },
|
|
644
|
+
maxAttempts: 3,
|
|
645
|
+
retryDelay: 10,
|
|
646
|
+
retryBackoff: false,
|
|
647
|
+
});
|
|
648
|
+
|
|
649
|
+
const handler = vi.fn(async () => {
|
|
650
|
+
throw new Error('fail');
|
|
651
|
+
});
|
|
652
|
+
const processor = jobQueue.createProcessor({
|
|
653
|
+
email: handler,
|
|
654
|
+
sms: vi.fn(async () => {}),
|
|
655
|
+
test: vi.fn(async () => {}),
|
|
656
|
+
});
|
|
657
|
+
await processor.start();
|
|
658
|
+
|
|
659
|
+
const job = await jobQueue.getJob(jobId);
|
|
660
|
+
expect(job?.status).toBe('failed');
|
|
661
|
+
expect(job?.nextAttemptAt).not.toBeNull();
|
|
662
|
+
const delaySec =
|
|
663
|
+
(job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
|
|
664
|
+
expect(delaySec).toBeGreaterThanOrEqual(9);
|
|
665
|
+
expect(delaySec).toBeLessThanOrEqual(11);
|
|
666
|
+
});
|
|
667
|
+
|
|
668
|
+
it('uses exponential backoff with custom retryDelay', async () => {
|
|
669
|
+
const jobId = await jobQueue.addJob({
|
|
670
|
+
jobType: 'email',
|
|
671
|
+
payload: { to: 'expo-retry@example.com' },
|
|
672
|
+
maxAttempts: 3,
|
|
673
|
+
retryDelay: 5,
|
|
674
|
+
retryBackoff: true,
|
|
675
|
+
});
|
|
676
|
+
|
|
677
|
+
const handler = vi.fn(async () => {
|
|
678
|
+
throw new Error('fail');
|
|
679
|
+
});
|
|
680
|
+
const processor = jobQueue.createProcessor({
|
|
681
|
+
email: handler,
|
|
682
|
+
sms: vi.fn(async () => {}),
|
|
683
|
+
test: vi.fn(async () => {}),
|
|
684
|
+
});
|
|
685
|
+
await processor.start();
|
|
686
|
+
|
|
687
|
+
const job = await jobQueue.getJob(jobId);
|
|
688
|
+
expect(job?.status).toBe('failed');
|
|
689
|
+
expect(job?.nextAttemptAt).not.toBeNull();
|
|
690
|
+
// 5 * 2^1 = 10s, with jitter [5, 10]
|
|
691
|
+
const delaySec =
|
|
692
|
+
(job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
|
|
693
|
+
expect(delaySec).toBeGreaterThanOrEqual(4);
|
|
694
|
+
expect(delaySec).toBeLessThanOrEqual(11);
|
|
695
|
+
});
|
|
696
|
+
|
|
697
|
+
it('caps exponential backoff with retryDelayMax', async () => {
|
|
698
|
+
const jobId = await jobQueue.addJob({
|
|
699
|
+
jobType: 'email',
|
|
700
|
+
payload: { to: 'capped-retry@example.com' },
|
|
701
|
+
maxAttempts: 5,
|
|
702
|
+
retryDelay: 100,
|
|
703
|
+
retryBackoff: true,
|
|
704
|
+
retryDelayMax: 30,
|
|
705
|
+
});
|
|
706
|
+
|
|
707
|
+
const handler = vi.fn(async () => {
|
|
708
|
+
throw new Error('fail');
|
|
709
|
+
});
|
|
710
|
+
const processor = jobQueue.createProcessor({
|
|
711
|
+
email: handler,
|
|
712
|
+
sms: vi.fn(async () => {}),
|
|
713
|
+
test: vi.fn(async () => {}),
|
|
714
|
+
});
|
|
715
|
+
await processor.start();
|
|
716
|
+
|
|
717
|
+
const job = await jobQueue.getJob(jobId);
|
|
718
|
+
expect(job?.status).toBe('failed');
|
|
719
|
+
expect(job?.nextAttemptAt).not.toBeNull();
|
|
720
|
+
// 100 * 2^1 = 200 capped to 30, with jitter [15, 30]
|
|
721
|
+
const delaySec =
|
|
722
|
+
(job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
|
|
723
|
+
expect(delaySec).toBeGreaterThanOrEqual(14);
|
|
724
|
+
expect(delaySec).toBeLessThanOrEqual(31);
|
|
725
|
+
});
|
|
726
|
+
|
|
727
|
+
it('allows editing retry config via editJob', async () => {
|
|
728
|
+
const jobId = await jobQueue.addJob({
|
|
729
|
+
jobType: 'email',
|
|
730
|
+
payload: { to: 'edit-retry@example.com' },
|
|
731
|
+
});
|
|
732
|
+
|
|
733
|
+
await jobQueue.editJob(jobId, {
|
|
734
|
+
retryDelay: 15,
|
|
735
|
+
retryBackoff: false,
|
|
736
|
+
retryDelayMax: 60,
|
|
737
|
+
});
|
|
738
|
+
|
|
739
|
+
const job = await jobQueue.getJob(jobId);
|
|
740
|
+
expect(job?.retryDelay).toBe(15);
|
|
741
|
+
expect(job?.retryBackoff).toBe(false);
|
|
742
|
+
expect(job?.retryDelayMax).toBe(60);
|
|
743
|
+
});
|
|
583
744
|
});
|
|
584
745
|
|
|
585
746
|
describe('Redis cron schedules integration', () => {
|
|
@@ -1105,6 +1266,113 @@ describe('Redis parity features', () => {
|
|
|
1105
1266
|
expect(job?.waitTokenId).toBeNull();
|
|
1106
1267
|
});
|
|
1107
1268
|
|
|
1269
|
+
// ── Job output ─────────────────────────────────────────────────────
|
|
1270
|
+
|
|
1271
|
+
it('stores output from ctx.setOutput() and retrieves via getJob', async () => {
|
|
1272
|
+
const jobId = await jobQueue.addJob({
|
|
1273
|
+
jobType: 'email',
|
|
1274
|
+
payload: { to: 'output@test.com' },
|
|
1275
|
+
});
|
|
1276
|
+
|
|
1277
|
+
const processor = jobQueue.createProcessor({
|
|
1278
|
+
email: vi.fn(async (_payload, _signal, ctx) => {
|
|
1279
|
+
await ctx.setOutput({ reportUrl: 'https://example.com/report.pdf' });
|
|
1280
|
+
}),
|
|
1281
|
+
sms: vi.fn(async () => {}),
|
|
1282
|
+
test: vi.fn(async () => {}),
|
|
1283
|
+
});
|
|
1284
|
+
await processor.start();
|
|
1285
|
+
|
|
1286
|
+
const job = await jobQueue.getJob(jobId);
|
|
1287
|
+
expect(job?.status).toBe('completed');
|
|
1288
|
+
expect(job?.output).toEqual({
|
|
1289
|
+
reportUrl: 'https://example.com/report.pdf',
|
|
1290
|
+
});
|
|
1291
|
+
});
|
|
1292
|
+
|
|
1293
|
+
it('stores handler return value as output when setOutput is not called', async () => {
|
|
1294
|
+
const jobId = await jobQueue.addJob({
|
|
1295
|
+
jobType: 'email',
|
|
1296
|
+
payload: { to: 'return@test.com' },
|
|
1297
|
+
});
|
|
1298
|
+
|
|
1299
|
+
const processor = jobQueue.createProcessor({
|
|
1300
|
+
email: vi.fn(async () => {
|
|
1301
|
+
return { processed: true, count: 42 };
|
|
1302
|
+
}),
|
|
1303
|
+
sms: vi.fn(async () => {}),
|
|
1304
|
+
test: vi.fn(async () => {}),
|
|
1305
|
+
});
|
|
1306
|
+
await processor.start();
|
|
1307
|
+
|
|
1308
|
+
const job = await jobQueue.getJob(jobId);
|
|
1309
|
+
expect(job?.status).toBe('completed');
|
|
1310
|
+
expect(job?.output).toEqual({ processed: true, count: 42 });
|
|
1311
|
+
});
|
|
1312
|
+
|
|
1313
|
+
it('setOutput takes precedence over handler return value', async () => {
|
|
1314
|
+
const jobId = await jobQueue.addJob({
|
|
1315
|
+
jobType: 'email',
|
|
1316
|
+
payload: { to: 'precedence@test.com' },
|
|
1317
|
+
});
|
|
1318
|
+
|
|
1319
|
+
const processor = jobQueue.createProcessor({
|
|
1320
|
+
email: vi.fn(async (_payload, _signal, ctx) => {
|
|
1321
|
+
await ctx.setOutput({ fromSetOutput: true });
|
|
1322
|
+
return { fromReturn: true };
|
|
1323
|
+
}),
|
|
1324
|
+
sms: vi.fn(async () => {}),
|
|
1325
|
+
test: vi.fn(async () => {}),
|
|
1326
|
+
});
|
|
1327
|
+
await processor.start();
|
|
1328
|
+
|
|
1329
|
+
const job = await jobQueue.getJob(jobId);
|
|
1330
|
+
expect(job?.status).toBe('completed');
|
|
1331
|
+
expect(job?.output).toEqual({ fromSetOutput: true });
|
|
1332
|
+
});
|
|
1333
|
+
|
|
1334
|
+
it('output is null for jobs that do not set output (backward compat)', async () => {
|
|
1335
|
+
const jobId = await jobQueue.addJob({
|
|
1336
|
+
jobType: 'email',
|
|
1337
|
+
payload: { to: 'no-output@test.com' },
|
|
1338
|
+
});
|
|
1339
|
+
|
|
1340
|
+
const processor = jobQueue.createProcessor({
|
|
1341
|
+
email: vi.fn(async () => {}),
|
|
1342
|
+
sms: vi.fn(async () => {}),
|
|
1343
|
+
test: vi.fn(async () => {}),
|
|
1344
|
+
});
|
|
1345
|
+
await processor.start();
|
|
1346
|
+
|
|
1347
|
+
const job = await jobQueue.getJob(jobId);
|
|
1348
|
+
expect(job?.status).toBe('completed');
|
|
1349
|
+
expect(job?.output).toBeNull();
|
|
1350
|
+
});
|
|
1351
|
+
|
|
1352
|
+
it('stores scalar output values (string, number, array)', async () => {
|
|
1353
|
+
const jobId1 = await jobQueue.addJob({
|
|
1354
|
+
jobType: 'email',
|
|
1355
|
+
payload: { to: 'string-output@test.com' },
|
|
1356
|
+
});
|
|
1357
|
+
const jobId2 = await jobQueue.addJob({
|
|
1358
|
+
jobType: 'sms',
|
|
1359
|
+
payload: { to: '+123' },
|
|
1360
|
+
});
|
|
1361
|
+
|
|
1362
|
+
const processor = jobQueue.createProcessor({
|
|
1363
|
+
email: vi.fn(async () => 'simple string'),
|
|
1364
|
+
sms: vi.fn(async () => 42),
|
|
1365
|
+
test: vi.fn(async () => {}),
|
|
1366
|
+
});
|
|
1367
|
+
await processor.start();
|
|
1368
|
+
|
|
1369
|
+
const job1 = await jobQueue.getJob(jobId1);
|
|
1370
|
+
expect(job1?.output).toBe('simple string');
|
|
1371
|
+
|
|
1372
|
+
const job2 = await jobQueue.getJob(jobId2);
|
|
1373
|
+
expect(job2?.output).toBe(42);
|
|
1374
|
+
});
|
|
1375
|
+
|
|
1108
1376
|
// ── cleanupOldJobEvents ─────────────────────────────────────────────
|
|
1109
1377
|
|
|
1110
1378
|
it('cleanupOldJobEvents removes old events', async () => {
|
|
@@ -1512,3 +1780,488 @@ describe('Redis parity features', () => {
|
|
|
1512
1780
|
expect(invocationCount).toBe(2);
|
|
1513
1781
|
});
|
|
1514
1782
|
});
|
|
1783
|
+
|
|
1784
|
+
// ── BYOC (Bring Your Own Connection) tests for Redis ────────────────────
|
|
1785
|
+
|
|
1786
|
+
describe('Redis BYOC: init with external client', () => {
|
|
1787
|
+
let prefix: string;
|
|
1788
|
+
let externalClient: any;
|
|
1789
|
+
let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
|
|
1790
|
+
|
|
1791
|
+
beforeEach(async () => {
|
|
1792
|
+
prefix = createRedisTestPrefix();
|
|
1793
|
+
const { default: IORedis } = await import('ioredis');
|
|
1794
|
+
externalClient = new (IORedis as any)(REDIS_URL);
|
|
1795
|
+
jobQueue = initJobQueue<TestPayloadMap>({
|
|
1796
|
+
backend: 'redis',
|
|
1797
|
+
client: externalClient,
|
|
1798
|
+
keyPrefix: prefix,
|
|
1799
|
+
});
|
|
1800
|
+
});
|
|
1801
|
+
|
|
1802
|
+
afterEach(async () => {
|
|
1803
|
+
await cleanupRedisPrefix(externalClient, prefix);
|
|
1804
|
+
await externalClient.quit();
|
|
1805
|
+
});
|
|
1806
|
+
|
|
1807
|
+
it('uses the provided client for addJob and getJob', async () => {
|
|
1808
|
+
// Act
|
|
1809
|
+
const jobId = await jobQueue.addJob({
|
|
1810
|
+
jobType: 'email',
|
|
1811
|
+
payload: { to: 'byoc-redis@example.com' },
|
|
1812
|
+
});
|
|
1813
|
+
|
|
1814
|
+
// Assert
|
|
1815
|
+
const job = await jobQueue.getJob(jobId);
|
|
1816
|
+
expect(job).not.toBeNull();
|
|
1817
|
+
expect(job?.jobType).toBe('email');
|
|
1818
|
+
expect(job?.payload).toEqual({ to: 'byoc-redis@example.com' });
|
|
1819
|
+
});
|
|
1820
|
+
|
|
1821
|
+
it('returns the same client instance from getRedisClient()', () => {
|
|
1822
|
+
// Act
|
|
1823
|
+
const returned = jobQueue.getRedisClient();
|
|
1824
|
+
|
|
1825
|
+
// Assert
|
|
1826
|
+
expect(returned).toBe(externalClient);
|
|
1827
|
+
});
|
|
1828
|
+
});
|
|
1829
|
+
|
|
1830
|
+
describe('Redis BYOC: addJob with db option throws', () => {
|
|
1831
|
+
let prefix: string;
|
|
1832
|
+
let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
|
|
1833
|
+
let redisClient: any;
|
|
1834
|
+
|
|
1835
|
+
beforeEach(async () => {
|
|
1836
|
+
prefix = createRedisTestPrefix();
|
|
1837
|
+
jobQueue = initJobQueue<TestPayloadMap>({
|
|
1838
|
+
backend: 'redis',
|
|
1839
|
+
redisConfig: { url: REDIS_URL, keyPrefix: prefix },
|
|
1840
|
+
});
|
|
1841
|
+
redisClient = jobQueue.getRedisClient();
|
|
1842
|
+
});
|
|
1843
|
+
|
|
1844
|
+
afterEach(async () => {
|
|
1845
|
+
await cleanupRedisPrefix(redisClient, prefix);
|
|
1846
|
+
await redisClient.quit();
|
|
1847
|
+
});
|
|
1848
|
+
|
|
1849
|
+
it('throws a clear error when db option is provided', async () => {
|
|
1850
|
+
// Setup — fake db client
|
|
1851
|
+
const fakeDb = { query: async () => ({ rows: [], rowCount: 0 }) };
|
|
1852
|
+
|
|
1853
|
+
// Act & Assert
|
|
1854
|
+
await expect(
|
|
1855
|
+
jobQueue.addJob(
|
|
1856
|
+
{ jobType: 'email', payload: { to: 'fail@example.com' } },
|
|
1857
|
+
{ db: fakeDb },
|
|
1858
|
+
),
|
|
1859
|
+
).rejects.toThrow('The db option is not supported with the Redis backend.');
|
|
1860
|
+
});
|
|
1861
|
+
});
|
|
1862
|
+
|
|
1863
|
+
describe('Redis addJobs batch insert', () => {
|
|
1864
|
+
let prefix: string;
|
|
1865
|
+
let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
|
|
1866
|
+
let redisClient: any;
|
|
1867
|
+
|
|
1868
|
+
beforeEach(async () => {
|
|
1869
|
+
prefix = createRedisTestPrefix();
|
|
1870
|
+
jobQueue = initJobQueue<TestPayloadMap>({
|
|
1871
|
+
backend: 'redis',
|
|
1872
|
+
redisConfig: { url: REDIS_URL, keyPrefix: prefix },
|
|
1873
|
+
});
|
|
1874
|
+
redisClient = jobQueue.getRedisClient();
|
|
1875
|
+
});
|
|
1876
|
+
|
|
1877
|
+
afterEach(async () => {
|
|
1878
|
+
await cleanupRedisPrefix(redisClient, prefix);
|
|
1879
|
+
await redisClient.quit();
|
|
1880
|
+
});
|
|
1881
|
+
|
|
1882
|
+
it('inserts multiple jobs and returns IDs in order', async () => {
|
|
1883
|
+
// Act
|
|
1884
|
+
const ids = await jobQueue.addJobs([
|
|
1885
|
+
{ jobType: 'email', payload: { to: 'a@test.com' } },
|
|
1886
|
+
{ jobType: 'sms', payload: { to: '+1234' } },
|
|
1887
|
+
{ jobType: 'email', payload: { to: 'b@test.com' } },
|
|
1888
|
+
]);
|
|
1889
|
+
|
|
1890
|
+
// Assert
|
|
1891
|
+
expect(ids).toHaveLength(3);
|
|
1892
|
+
|
|
1893
|
+
const job0 = await jobQueue.getJob(ids[0]);
|
|
1894
|
+
expect(job0?.jobType).toBe('email');
|
|
1895
|
+
expect(job0?.payload).toEqual({ to: 'a@test.com' });
|
|
1896
|
+
|
|
1897
|
+
const job1 = await jobQueue.getJob(ids[1]);
|
|
1898
|
+
expect(job1?.jobType).toBe('sms');
|
|
1899
|
+
expect(job1?.payload).toEqual({ to: '+1234' });
|
|
1900
|
+
|
|
1901
|
+
const job2 = await jobQueue.getJob(ids[2]);
|
|
1902
|
+
expect(job2?.jobType).toBe('email');
|
|
1903
|
+
expect(job2?.payload).toEqual({ to: 'b@test.com' });
|
|
1904
|
+
});
|
|
1905
|
+
|
|
1906
|
+
it('returns empty array for empty input', async () => {
|
|
1907
|
+
// Act
|
|
1908
|
+
const ids = await jobQueue.addJobs([]);
|
|
1909
|
+
|
|
1910
|
+
// Assert
|
|
1911
|
+
expect(ids).toEqual([]);
|
|
1912
|
+
});
|
|
1913
|
+
|
|
1914
|
+
it('handles idempotency keys for new jobs', async () => {
|
|
1915
|
+
// Act
|
|
1916
|
+
const ids = await jobQueue.addJobs([
|
|
1917
|
+
{
|
|
1918
|
+
jobType: 'email',
|
|
1919
|
+
payload: { to: 'a@test.com' },
|
|
1920
|
+
idempotencyKey: 'r-key-a',
|
|
1921
|
+
},
|
|
1922
|
+
{
|
|
1923
|
+
jobType: 'email',
|
|
1924
|
+
payload: { to: 'b@test.com' },
|
|
1925
|
+
idempotencyKey: 'r-key-b',
|
|
1926
|
+
},
|
|
1927
|
+
]);
|
|
1928
|
+
|
|
1929
|
+
// Assert
|
|
1930
|
+
expect(ids).toHaveLength(2);
|
|
1931
|
+
expect(ids[0]).not.toBe(ids[1]);
|
|
1932
|
+
|
|
1933
|
+
const job0 = await jobQueue.getJob(ids[0]);
|
|
1934
|
+
expect(job0?.idempotencyKey).toBe('r-key-a');
|
|
1935
|
+
});
|
|
1936
|
+
|
|
1937
|
+
it('returns existing IDs for conflicting idempotency keys', async () => {
|
|
1938
|
+
// Setup
|
|
1939
|
+
const existingId = await jobQueue.addJob({
|
|
1940
|
+
jobType: 'email',
|
|
1941
|
+
payload: { to: 'existing@test.com' },
|
|
1942
|
+
idempotencyKey: 'r-dup',
|
|
1943
|
+
});
|
|
1944
|
+
|
|
1945
|
+
// Act
|
|
1946
|
+
const ids = await jobQueue.addJobs([
|
|
1947
|
+
{ jobType: 'email', payload: { to: 'new@test.com' } },
|
|
1948
|
+
{
|
|
1949
|
+
jobType: 'email',
|
|
1950
|
+
payload: { to: 'dup@test.com' },
|
|
1951
|
+
idempotencyKey: 'r-dup',
|
|
1952
|
+
},
|
|
1953
|
+
]);
|
|
1954
|
+
|
|
1955
|
+
// Assert
|
|
1956
|
+
expect(ids).toHaveLength(2);
|
|
1957
|
+
expect(ids[1]).toBe(existingId);
|
|
1958
|
+
expect(ids[0]).not.toBe(existingId);
|
|
1959
|
+
});
|
|
1960
|
+
|
|
1961
|
+
it('records added events for each inserted job', async () => {
|
|
1962
|
+
// Act
|
|
1963
|
+
const ids = await jobQueue.addJobs([
|
|
1964
|
+
{ jobType: 'email', payload: { to: 'a@test.com' } },
|
|
1965
|
+
{ jobType: 'sms', payload: { to: '+999' } },
|
|
1966
|
+
]);
|
|
1967
|
+
|
|
1968
|
+
// Assert
|
|
1969
|
+
const events0 = await jobQueue.getJobEvents(ids[0]);
|
|
1970
|
+
expect(events0.filter((e) => e.eventType === 'added')).toHaveLength(1);
|
|
1971
|
+
|
|
1972
|
+
const events1 = await jobQueue.getJobEvents(ids[1]);
|
|
1973
|
+
expect(events1.filter((e) => e.eventType === 'added')).toHaveLength(1);
|
|
1974
|
+
});
|
|
1975
|
+
|
|
1976
|
+
it('throws when db option is used with addJobs', async () => {
|
|
1977
|
+
// Setup
|
|
1978
|
+
const fakeDb = { query: async () => ({ rows: [], rowCount: 0 }) };
|
|
1979
|
+
|
|
1980
|
+
// Act & Assert
|
|
1981
|
+
await expect(
|
|
1982
|
+
jobQueue.addJobs(
|
|
1983
|
+
[{ jobType: 'email', payload: { to: 'fail@test.com' } }],
|
|
1984
|
+
{ db: fakeDb },
|
|
1985
|
+
),
|
|
1986
|
+
).rejects.toThrow('The db option is not supported with the Redis backend.');
|
|
1987
|
+
});
|
|
1988
|
+
|
|
1989
|
+
it('stores tags and priority correctly per job', async () => {
|
|
1990
|
+
// Act
|
|
1991
|
+
const ids = await jobQueue.addJobs([
|
|
1992
|
+
{
|
|
1993
|
+
jobType: 'email',
|
|
1994
|
+
payload: { to: 'a@test.com' },
|
|
1995
|
+
tags: ['urgent'],
|
|
1996
|
+
priority: 10,
|
|
1997
|
+
},
|
|
1998
|
+
{ jobType: 'sms', payload: { to: '+1' }, priority: 5 },
|
|
1999
|
+
{ jobType: 'email', payload: { to: 'c@test.com' }, tags: ['low'] },
|
|
2000
|
+
]);
|
|
2001
|
+
|
|
2002
|
+
// Assert
|
|
2003
|
+
const job0 = await jobQueue.getJob(ids[0]);
|
|
2004
|
+
expect(job0?.tags).toEqual(['urgent']);
|
|
2005
|
+
expect(job0?.priority).toBe(10);
|
|
2006
|
+
|
|
2007
|
+
const job1 = await jobQueue.getJob(ids[1]);
|
|
2008
|
+
expect(job1?.priority).toBe(5);
|
|
2009
|
+
|
|
2010
|
+
const job2 = await jobQueue.getJob(ids[2]);
|
|
2011
|
+
expect(job2?.tags).toEqual(['low']);
|
|
2012
|
+
});
|
|
2013
|
+
});
|
|
2014
|
+
|
|
2015
|
+
describe('Redis event hooks', () => {
|
|
2016
|
+
let prefix: string;
|
|
2017
|
+
let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
|
|
2018
|
+
let redisClient: any;
|
|
2019
|
+
|
|
2020
|
+
beforeEach(async () => {
|
|
2021
|
+
prefix = createRedisTestPrefix();
|
|
2022
|
+
jobQueue = initJobQueue<TestPayloadMap>({
|
|
2023
|
+
backend: 'redis',
|
|
2024
|
+
redisConfig: { url: REDIS_URL, keyPrefix: prefix },
|
|
2025
|
+
});
|
|
2026
|
+
redisClient = jobQueue.getRedisClient();
|
|
2027
|
+
});
|
|
2028
|
+
|
|
2029
|
+
afterEach(async () => {
|
|
2030
|
+
jobQueue.removeAllListeners();
|
|
2031
|
+
await cleanupRedisPrefix(redisClient, prefix);
|
|
2032
|
+
await redisClient.quit();
|
|
2033
|
+
});
|
|
2034
|
+
|
|
2035
|
+
it('emits job:added on addJob', async () => {
|
|
2036
|
+
const listener = vi.fn();
|
|
2037
|
+
jobQueue.on('job:added', listener);
|
|
2038
|
+
|
|
2039
|
+
const jobId = await jobQueue.addJob({
|
|
2040
|
+
jobType: 'email',
|
|
2041
|
+
payload: { to: 'test@example.com' },
|
|
2042
|
+
});
|
|
2043
|
+
|
|
2044
|
+
expect(listener).toHaveBeenCalledTimes(1);
|
|
2045
|
+
expect(listener).toHaveBeenCalledWith({ jobId, jobType: 'email' });
|
|
2046
|
+
});
|
|
2047
|
+
|
|
2048
|
+
it('emits job:added for each job in addJobs', async () => {
|
|
2049
|
+
const listener = vi.fn();
|
|
2050
|
+
jobQueue.on('job:added', listener);
|
|
2051
|
+
|
|
2052
|
+
const ids = await jobQueue.addJobs([
|
|
2053
|
+
{ jobType: 'email', payload: { to: 'a@test.com' } },
|
|
2054
|
+
{ jobType: 'sms', payload: { to: '+1234' } },
|
|
2055
|
+
]);
|
|
2056
|
+
|
|
2057
|
+
expect(listener).toHaveBeenCalledTimes(2);
|
|
2058
|
+
expect(listener).toHaveBeenCalledWith({ jobId: ids[0], jobType: 'email' });
|
|
2059
|
+
expect(listener).toHaveBeenCalledWith({ jobId: ids[1], jobType: 'sms' });
|
|
2060
|
+
});
|
|
2061
|
+
|
|
2062
|
+
it('emits job:cancelled on cancelJob', async () => {
|
|
2063
|
+
const listener = vi.fn();
|
|
2064
|
+
jobQueue.on('job:cancelled', listener);
|
|
2065
|
+
|
|
2066
|
+
const jobId = await jobQueue.addJob({
|
|
2067
|
+
jobType: 'email',
|
|
2068
|
+
payload: { to: 'test@example.com' },
|
|
2069
|
+
});
|
|
2070
|
+
await jobQueue.cancelJob(jobId);
|
|
2071
|
+
|
|
2072
|
+
expect(listener).toHaveBeenCalledTimes(1);
|
|
2073
|
+
expect(listener).toHaveBeenCalledWith({ jobId });
|
|
2074
|
+
});
|
|
2075
|
+
|
|
2076
|
+
it('emits job:retried on retryJob', async () => {
|
|
2077
|
+
const listener = vi.fn();
|
|
2078
|
+
jobQueue.on('job:retried', listener);
|
|
2079
|
+
|
|
2080
|
+
const jobId = await jobQueue.addJob({
|
|
2081
|
+
jobType: 'email',
|
|
2082
|
+
payload: { to: 'test@example.com' },
|
|
2083
|
+
});
|
|
2084
|
+
|
|
2085
|
+
const processor = jobQueue.createProcessor({
|
|
2086
|
+
email: vi.fn(async () => {
|
|
2087
|
+
throw new Error('fail');
|
|
2088
|
+
}),
|
|
2089
|
+
sms: vi.fn(async () => {}),
|
|
2090
|
+
test: vi.fn(async () => {}),
|
|
2091
|
+
});
|
|
2092
|
+
await processor.start();
|
|
2093
|
+
|
|
2094
|
+
await jobQueue.retryJob(jobId);
|
|
2095
|
+
|
|
2096
|
+
expect(listener).toHaveBeenCalledTimes(1);
|
|
2097
|
+
expect(listener).toHaveBeenCalledWith({ jobId });
|
|
2098
|
+
});
|
|
2099
|
+
|
|
2100
|
+
it('emits job:processing and job:completed on successful processing', async () => {
|
|
2101
|
+
const processingListener = vi.fn();
|
|
2102
|
+
const completedListener = vi.fn();
|
|
2103
|
+
jobQueue.on('job:processing', processingListener);
|
|
2104
|
+
jobQueue.on('job:completed', completedListener);
|
|
2105
|
+
|
|
2106
|
+
const jobId = await jobQueue.addJob({
|
|
2107
|
+
jobType: 'email',
|
|
2108
|
+
payload: { to: 'test@example.com' },
|
|
2109
|
+
});
|
|
2110
|
+
|
|
2111
|
+
const processor = jobQueue.createProcessor({
|
|
2112
|
+
email: vi.fn(async () => {}),
|
|
2113
|
+
sms: vi.fn(async () => {}),
|
|
2114
|
+
test: vi.fn(async () => {}),
|
|
2115
|
+
});
|
|
2116
|
+
await processor.start();
|
|
2117
|
+
|
|
2118
|
+
expect(processingListener).toHaveBeenCalledTimes(1);
|
|
2119
|
+
expect(processingListener).toHaveBeenCalledWith({
|
|
2120
|
+
jobId,
|
|
2121
|
+
jobType: 'email',
|
|
2122
|
+
});
|
|
2123
|
+
expect(completedListener).toHaveBeenCalledTimes(1);
|
|
2124
|
+
expect(completedListener).toHaveBeenCalledWith({
|
|
2125
|
+
jobId,
|
|
2126
|
+
jobType: 'email',
|
|
2127
|
+
});
|
|
2128
|
+
});
|
|
2129
|
+
|
|
2130
|
+
it('emits job:failed with willRetry flag', async () => {
|
|
2131
|
+
const listener = vi.fn();
|
|
2132
|
+
jobQueue.on('job:failed', listener);
|
|
2133
|
+
|
|
2134
|
+
const jobId = await jobQueue.addJob({
|
|
2135
|
+
jobType: 'email',
|
|
2136
|
+
payload: { to: 'test@example.com' },
|
|
2137
|
+
maxAttempts: 1,
|
|
2138
|
+
});
|
|
2139
|
+
|
|
2140
|
+
const processor = jobQueue.createProcessor({
|
|
2141
|
+
email: vi.fn(async () => {
|
|
2142
|
+
throw new Error('boom');
|
|
2143
|
+
}),
|
|
2144
|
+
sms: vi.fn(async () => {}),
|
|
2145
|
+
test: vi.fn(async () => {}),
|
|
2146
|
+
});
|
|
2147
|
+
await processor.start();
|
|
2148
|
+
|
|
2149
|
+
expect(listener).toHaveBeenCalledTimes(1);
|
|
2150
|
+
expect(listener).toHaveBeenCalledWith(
|
|
2151
|
+
expect.objectContaining({
|
|
2152
|
+
jobId,
|
|
2153
|
+
jobType: 'email',
|
|
2154
|
+
willRetry: false,
|
|
2155
|
+
error: expect.any(Error),
|
|
2156
|
+
}),
|
|
2157
|
+
);
|
|
2158
|
+
});
|
|
2159
|
+
|
|
2160
|
+
it('once fires only once then auto-unsubscribes', async () => {
|
|
2161
|
+
const listener = vi.fn();
|
|
2162
|
+
jobQueue.once('job:added', listener);
|
|
2163
|
+
|
|
2164
|
+
await jobQueue.addJob({ jobType: 'email', payload: { to: 'a@test.com' } });
|
|
2165
|
+
await jobQueue.addJob({ jobType: 'sms', payload: { to: '+1234' } });
|
|
2166
|
+
|
|
2167
|
+
expect(listener).toHaveBeenCalledTimes(1);
|
|
2168
|
+
});
|
|
2169
|
+
|
|
2170
|
+
it('off removes a listener', async () => {
|
|
2171
|
+
const listener = vi.fn();
|
|
2172
|
+
jobQueue.on('job:added', listener);
|
|
2173
|
+
|
|
2174
|
+
await jobQueue.addJob({ jobType: 'email', payload: { to: 'a@test.com' } });
|
|
2175
|
+
expect(listener).toHaveBeenCalledTimes(1);
|
|
2176
|
+
|
|
2177
|
+
jobQueue.off('job:added', listener);
|
|
2178
|
+
|
|
2179
|
+
await jobQueue.addJob({ jobType: 'sms', payload: { to: '+1234' } });
|
|
2180
|
+
expect(listener).toHaveBeenCalledTimes(1);
|
|
2181
|
+
});
|
|
2182
|
+
});
|
|
2183
|
+
|
|
2184
|
+
describe('Redis group-based concurrency limits', () => {
|
|
2185
|
+
let prefix: string;
|
|
2186
|
+
let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
|
|
2187
|
+
let redisClient: any;
|
|
2188
|
+
|
|
2189
|
+
beforeEach(async () => {
|
|
2190
|
+
prefix = createRedisTestPrefix();
|
|
2191
|
+
const config: RedisJobQueueConfig = {
|
|
2192
|
+
backend: 'redis',
|
|
2193
|
+
redisConfig: {
|
|
2194
|
+
url: REDIS_URL,
|
|
2195
|
+
keyPrefix: prefix,
|
|
2196
|
+
},
|
|
2197
|
+
};
|
|
2198
|
+
jobQueue = initJobQueue<TestPayloadMap>(config);
|
|
2199
|
+
redisClient = jobQueue.getRedisClient();
|
|
2200
|
+
});
|
|
2201
|
+
|
|
2202
|
+
afterEach(async () => {
|
|
2203
|
+
await cleanupRedisPrefix(redisClient, prefix);
|
|
2204
|
+
await redisClient.quit();
|
|
2205
|
+
});
|
|
2206
|
+
|
|
2207
|
+
it('stores group metadata for Redis jobs', async () => {
|
|
2208
|
+
const jobId = await jobQueue.addJob({
|
|
2209
|
+
jobType: 'test',
|
|
2210
|
+
payload: { foo: 'grouped' },
|
|
2211
|
+
group: { id: 'tenant-r1', tier: 'silver' },
|
|
2212
|
+
});
|
|
2213
|
+
|
|
2214
|
+
const job = await jobQueue.getJob(jobId);
|
|
2215
|
+
expect(job?.groupId).toBe('tenant-r1');
|
|
2216
|
+
expect(job?.groupTier).toBe('silver');
|
|
2217
|
+
});
|
|
2218
|
+
|
|
2219
|
+
it('enforces global grouped limits across processor instances', async () => {
|
|
2220
|
+
await jobQueue.addJob({
|
|
2221
|
+
jobType: 'test',
|
|
2222
|
+
payload: { foo: 'job-1' },
|
|
2223
|
+
group: { id: 'tenant-r2' },
|
|
2224
|
+
});
|
|
2225
|
+
await jobQueue.addJob({
|
|
2226
|
+
jobType: 'test',
|
|
2227
|
+
payload: { foo: 'job-2' },
|
|
2228
|
+
group: { id: 'tenant-r2' },
|
|
2229
|
+
});
|
|
2230
|
+
|
|
2231
|
+
let started = 0;
|
|
2232
|
+
let release!: () => void;
|
|
2233
|
+
const gate = new Promise<void>((resolve) => {
|
|
2234
|
+
release = resolve;
|
|
2235
|
+
});
|
|
2236
|
+
|
|
2237
|
+
const handler = vi.fn(async () => {
|
|
2238
|
+
started += 1;
|
|
2239
|
+
await gate;
|
|
2240
|
+
});
|
|
2241
|
+
|
|
2242
|
+
const processorA = jobQueue.createProcessor(
|
|
2243
|
+
{ email: vi.fn(), sms: vi.fn(), test: handler },
|
|
2244
|
+
{ batchSize: 2, concurrency: 2, groupConcurrency: 1 },
|
|
2245
|
+
);
|
|
2246
|
+
const processorB = jobQueue.createProcessor(
|
|
2247
|
+
{ email: vi.fn(), sms: vi.fn(), test: handler },
|
|
2248
|
+
{ batchSize: 2, concurrency: 2, groupConcurrency: 1 },
|
|
2249
|
+
);
|
|
2250
|
+
|
|
2251
|
+
const runA = processorA.start();
|
|
2252
|
+
await new Promise((resolve) => setTimeout(resolve, 40));
|
|
2253
|
+
const processedByB = await processorB.start();
|
|
2254
|
+
|
|
2255
|
+
expect(processedByB).toBe(0);
|
|
2256
|
+
expect(started).toBe(1);
|
|
2257
|
+
|
|
2258
|
+
release();
|
|
2259
|
+
await runA;
|
|
2260
|
+
|
|
2261
|
+
const pendingAfterA = await jobQueue.getJobsByStatus('pending');
|
|
2262
|
+
expect(pendingAfterA).toHaveLength(1);
|
|
2263
|
+
|
|
2264
|
+
const processedByBSecondRun = await processorB.start();
|
|
2265
|
+
expect(processedByBSecondRun).toBe(1);
|
|
2266
|
+
});
|
|
2267
|
+
});
|