@nicnocquee/dataqueue 1.25.0 → 1.26.0-beta.20260223202259

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/ai/build-docs-content.ts +96 -0
  2. package/ai/build-llms-full.ts +42 -0
  3. package/ai/docs-content.json +284 -0
  4. package/ai/rules/advanced.md +150 -0
  5. package/ai/rules/basic.md +159 -0
  6. package/ai/rules/react-dashboard.md +83 -0
  7. package/ai/skills/dataqueue-advanced/SKILL.md +370 -0
  8. package/ai/skills/dataqueue-core/SKILL.md +234 -0
  9. package/ai/skills/dataqueue-react/SKILL.md +189 -0
  10. package/dist/cli.cjs +1149 -14
  11. package/dist/cli.cjs.map +1 -1
  12. package/dist/cli.d.cts +66 -1
  13. package/dist/cli.d.ts +66 -1
  14. package/dist/cli.js +1146 -13
  15. package/dist/cli.js.map +1 -1
  16. package/dist/index.cjs +3236 -1237
  17. package/dist/index.cjs.map +1 -1
  18. package/dist/index.d.cts +697 -23
  19. package/dist/index.d.ts +697 -23
  20. package/dist/index.js +3235 -1238
  21. package/dist/index.js.map +1 -1
  22. package/dist/mcp-server.cjs +186 -0
  23. package/dist/mcp-server.cjs.map +1 -0
  24. package/dist/mcp-server.d.cts +32 -0
  25. package/dist/mcp-server.d.ts +32 -0
  26. package/dist/mcp-server.js +175 -0
  27. package/dist/mcp-server.js.map +1 -0
  28. package/migrations/1781200000004_create_cron_schedules_table.sql +33 -0
  29. package/migrations/1781200000005_add_retry_config_to_job_queue.sql +17 -0
  30. package/package.json +24 -21
  31. package/src/backend.ts +170 -5
  32. package/src/backends/postgres.ts +992 -63
  33. package/src/backends/redis-scripts.ts +358 -26
  34. package/src/backends/redis.test.ts +1532 -0
  35. package/src/backends/redis.ts +993 -35
  36. package/src/cli.test.ts +82 -6
  37. package/src/cli.ts +73 -10
  38. package/src/cron.test.ts +126 -0
  39. package/src/cron.ts +40 -0
  40. package/src/db-util.ts +1 -1
  41. package/src/index.test.ts +1034 -11
  42. package/src/index.ts +267 -39
  43. package/src/init-command.test.ts +449 -0
  44. package/src/init-command.ts +709 -0
  45. package/src/install-mcp-command.test.ts +216 -0
  46. package/src/install-mcp-command.ts +185 -0
  47. package/src/install-rules-command.test.ts +218 -0
  48. package/src/install-rules-command.ts +233 -0
  49. package/src/install-skills-command.test.ts +176 -0
  50. package/src/install-skills-command.ts +124 -0
  51. package/src/mcp-server.test.ts +162 -0
  52. package/src/mcp-server.ts +231 -0
  53. package/src/processor.ts +104 -113
  54. package/src/queue.test.ts +465 -0
  55. package/src/queue.ts +34 -252
  56. package/src/supervisor.test.ts +340 -0
  57. package/src/supervisor.ts +177 -0
  58. package/src/types.ts +476 -12
  59. package/LICENSE +0 -21
package/src/index.test.ts CHANGED
@@ -2,6 +2,7 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
2
2
  import { initJobQueue, JobQueueConfig } from './index.js';
3
3
  import { createTestDbAndPool, destroyTestDb } from './test-util.js';
4
4
  import { Pool } from 'pg';
5
+ import type { CronScheduleRecord, AddJobOptions } from './types.js';
5
6
 
6
7
  // Integration tests for index.ts
7
8
 
@@ -77,9 +78,7 @@ describe('index integration', () => {
77
78
  },
78
79
  { pollInterval: 100 },
79
80
  );
80
- processor.start();
81
- await new Promise((r) => setTimeout(r, 300));
82
- processor.stop();
81
+ await processor.start();
83
82
  const job = await jobQueue.getJob(jobId);
84
83
  expect(handler).toHaveBeenCalledWith(
85
84
  { foo: 'bar' },
@@ -431,9 +430,7 @@ describe('index integration', () => {
431
430
  },
432
431
  { pollInterval: 100 },
433
432
  );
434
- processor.start();
435
- await new Promise((r) => setTimeout(r, 300));
436
- processor.stop();
433
+ await processor.start();
437
434
 
438
435
  expect(handler).toHaveBeenCalledWith(
439
436
  { foo: 'updated@example.com' },
@@ -458,9 +455,7 @@ describe('index integration', () => {
458
455
  },
459
456
  { pollInterval: 100 },
460
457
  );
461
- processor.start();
462
- await new Promise((r) => setTimeout(r, 300));
463
- processor.stop();
458
+ await processor.start();
464
459
 
465
460
  const originalJob = await jobQueue.getJob(jobId1);
466
461
  expect(originalJob?.status).toBe('completed');
@@ -493,7 +488,7 @@ describe('index integration', () => {
493
488
  jobType: 'email',
494
489
  payload: { to: 'processing@example.com' },
495
490
  });
496
- processor2.start();
491
+ const startPromise = processor2.start();
497
492
  // Wait a bit for job to be picked up
498
493
  await new Promise((r) => setTimeout(r, 150));
499
494
  // Job should be processing now
@@ -509,7 +504,7 @@ describe('index integration', () => {
509
504
  expect(job2?.payload).toEqual({ to: 'processing@example.com' });
510
505
  }
511
506
  }
512
- processor2.stop();
507
+ await startPromise;
513
508
  });
514
509
 
515
510
  it('should record edited event when editing via JobQueue API', async () => {
@@ -531,4 +526,1032 @@ describe('index integration', () => {
531
526
  priority: 10,
532
527
  });
533
528
  });
529
+
530
+ // ── Configurable retry strategy integration tests ────────────────────
531
+
532
+ it('should store and return retry config through public API', async () => {
533
+ const jobId = await jobQueue.addJob({
534
+ jobType: 'email',
535
+ payload: { to: 'retry-api@example.com' },
536
+ retryDelay: 20,
537
+ retryBackoff: true,
538
+ retryDelayMax: 300,
539
+ });
540
+
541
+ const job = await jobQueue.getJob(jobId);
542
+ expect(job?.retryDelay).toBe(20);
543
+ expect(job?.retryBackoff).toBe(true);
544
+ expect(job?.retryDelayMax).toBe(300);
545
+ });
546
+
547
+ it('should use fixed delay on failure through public API', async () => {
548
+ const jobId = await jobQueue.addJob({
549
+ jobType: 'email',
550
+ payload: { to: 'fixed-api@example.com' },
551
+ maxAttempts: 3,
552
+ retryDelay: 10,
553
+ retryBackoff: false,
554
+ });
555
+
556
+ const handler = vi.fn(async () => {
557
+ throw new Error('fail');
558
+ });
559
+ const processor = jobQueue.createProcessor({
560
+ email: handler,
561
+ sms: vi.fn(async () => {}),
562
+ test: vi.fn(async () => {}),
563
+ });
564
+ await processor.start();
565
+
566
+ const job = await jobQueue.getJob(jobId);
567
+ expect(job?.status).toBe('failed');
568
+ expect(job?.nextAttemptAt).not.toBeNull();
569
+ const delaySec =
570
+ (job!.nextAttemptAt!.getTime() - job!.lastFailedAt!.getTime()) / 1000;
571
+ expect(delaySec).toBeGreaterThanOrEqual(9);
572
+ expect(delaySec).toBeLessThanOrEqual(11);
573
+ });
574
+ });
575
+
576
+ describe('cron schedules integration', () => {
577
+ let pool: Pool;
578
+ let dbName: string;
579
+ let testDbUrl: string;
580
+ let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
581
+
582
+ beforeEach(async () => {
583
+ const setup = await createTestDbAndPool();
584
+ pool = setup.pool;
585
+ dbName = setup.dbName;
586
+ testDbUrl = setup.testDbUrl;
587
+ const config: JobQueueConfig = {
588
+ databaseConfig: {
589
+ connectionString: testDbUrl,
590
+ },
591
+ };
592
+ jobQueue = initJobQueue<TestPayloadMap>(config);
593
+ });
594
+
595
+ afterEach(async () => {
596
+ vi.restoreAllMocks();
597
+ jobQueue.getPool().end();
598
+ await pool.end();
599
+ await destroyTestDb(dbName);
600
+ });
601
+
602
+ it('creates a cron schedule and retrieves it by ID', async () => {
603
+ // Act
604
+ const id = await jobQueue.addCronJob({
605
+ scheduleName: 'every-5-min-email',
606
+ cronExpression: '*/5 * * * *',
607
+ jobType: 'email',
608
+ payload: { to: 'cron@example.com' },
609
+ });
610
+
611
+ // Assert
612
+ const schedule = await jobQueue.getCronJob(id);
613
+ expect(schedule).not.toBeNull();
614
+ expect(schedule!.scheduleName).toBe('every-5-min-email');
615
+ expect(schedule!.cronExpression).toBe('*/5 * * * *');
616
+ expect(schedule!.jobType).toBe('email');
617
+ expect(schedule!.payload).toEqual({ to: 'cron@example.com' });
618
+ expect(schedule!.status).toBe('active');
619
+ expect(schedule!.allowOverlap).toBe(false);
620
+ expect(schedule!.timezone).toBe('UTC');
621
+ expect(schedule!.nextRunAt).toBeInstanceOf(Date);
622
+ });
623
+
624
+ it('retrieves a cron schedule by name', async () => {
625
+ // Setup
626
+ await jobQueue.addCronJob({
627
+ scheduleName: 'my-schedule',
628
+ cronExpression: '0 * * * *',
629
+ jobType: 'email',
630
+ payload: { to: 'test@example.com' },
631
+ });
632
+
633
+ // Act
634
+ const schedule = await jobQueue.getCronJobByName('my-schedule');
635
+
636
+ // Assert
637
+ expect(schedule).not.toBeNull();
638
+ expect(schedule!.scheduleName).toBe('my-schedule');
639
+ });
640
+
641
+ it('returns null for nonexistent schedule', async () => {
642
+ // Act
643
+ const byId = await jobQueue.getCronJob(99999);
644
+ const byName = await jobQueue.getCronJobByName('nonexistent');
645
+
646
+ // Assert
647
+ expect(byId).toBeNull();
648
+ expect(byName).toBeNull();
649
+ });
650
+
651
+ it('rejects duplicate schedule names', async () => {
652
+ // Setup
653
+ await jobQueue.addCronJob({
654
+ scheduleName: 'unique-name',
655
+ cronExpression: '* * * * *',
656
+ jobType: 'email',
657
+ payload: { to: 'a@example.com' },
658
+ });
659
+
660
+ // Act & Assert
661
+ await expect(
662
+ jobQueue.addCronJob({
663
+ scheduleName: 'unique-name',
664
+ cronExpression: '*/5 * * * *',
665
+ jobType: 'sms',
666
+ payload: { to: 'b@example.com' },
667
+ }),
668
+ ).rejects.toThrow();
669
+ });
670
+
671
+ it('rejects invalid cron expressions', async () => {
672
+ // Act & Assert
673
+ await expect(
674
+ jobQueue.addCronJob({
675
+ scheduleName: 'bad-cron',
676
+ cronExpression: 'not a cron',
677
+ jobType: 'email',
678
+ payload: { to: 'a@example.com' },
679
+ }),
680
+ ).rejects.toThrow('Invalid cron expression');
681
+ });
682
+
683
+ it('lists active and paused schedules', async () => {
684
+ // Setup
685
+ const id1 = await jobQueue.addCronJob({
686
+ scheduleName: 'schedule-1',
687
+ cronExpression: '* * * * *',
688
+ jobType: 'email',
689
+ payload: { to: 'a@example.com' },
690
+ });
691
+ await jobQueue.addCronJob({
692
+ scheduleName: 'schedule-2',
693
+ cronExpression: '*/5 * * * *',
694
+ jobType: 'sms',
695
+ payload: { to: 'b@example.com' },
696
+ });
697
+ await jobQueue.pauseCronJob(id1);
698
+
699
+ // Act
700
+ const all = await jobQueue.listCronJobs();
701
+ const active = await jobQueue.listCronJobs('active');
702
+ const paused = await jobQueue.listCronJobs('paused');
703
+
704
+ // Assert
705
+ expect(all).toHaveLength(2);
706
+ expect(active).toHaveLength(1);
707
+ expect(active[0].scheduleName).toBe('schedule-2');
708
+ expect(paused).toHaveLength(1);
709
+ expect(paused[0].scheduleName).toBe('schedule-1');
710
+ });
711
+
712
+ it('pauses and resumes a schedule', async () => {
713
+ // Setup
714
+ const id = await jobQueue.addCronJob({
715
+ scheduleName: 'pausable',
716
+ cronExpression: '* * * * *',
717
+ jobType: 'email',
718
+ payload: { to: 'a@example.com' },
719
+ });
720
+
721
+ // Act — pause
722
+ await jobQueue.pauseCronJob(id);
723
+ const paused = await jobQueue.getCronJob(id);
724
+
725
+ // Assert
726
+ expect(paused!.status).toBe('paused');
727
+
728
+ // Act — resume
729
+ await jobQueue.resumeCronJob(id);
730
+ const resumed = await jobQueue.getCronJob(id);
731
+
732
+ // Assert
733
+ expect(resumed!.status).toBe('active');
734
+ });
735
+
736
+ it('edits a schedule and recalculates nextRunAt when expression changes', async () => {
737
+ // Setup
738
+ const id = await jobQueue.addCronJob({
739
+ scheduleName: 'editable',
740
+ cronExpression: '* * * * *',
741
+ jobType: 'email',
742
+ payload: { to: 'old@example.com' },
743
+ });
744
+ const before = await jobQueue.getCronJob(id);
745
+
746
+ // Act
747
+ await jobQueue.editCronJob(id, {
748
+ cronExpression: '0 0 * * *',
749
+ payload: { to: 'new@example.com' },
750
+ });
751
+
752
+ // Assert
753
+ const after = await jobQueue.getCronJob(id);
754
+ expect(after!.cronExpression).toBe('0 0 * * *');
755
+ expect(after!.payload).toEqual({ to: 'new@example.com' });
756
+ expect(after!.nextRunAt!.getTime()).not.toBe(before!.nextRunAt!.getTime());
757
+ });
758
+
759
+ it('removes a schedule', async () => {
760
+ // Setup
761
+ const id = await jobQueue.addCronJob({
762
+ scheduleName: 'removable',
763
+ cronExpression: '* * * * *',
764
+ jobType: 'email',
765
+ payload: { to: 'a@example.com' },
766
+ });
767
+
768
+ // Act
769
+ await jobQueue.removeCronJob(id);
770
+
771
+ // Assert
772
+ const removed = await jobQueue.getCronJob(id);
773
+ expect(removed).toBeNull();
774
+ });
775
+
776
+ it('enqueueDueCronJobs enqueues a job when nextRunAt is due', async () => {
777
+ // Setup — insert a schedule with nextRunAt in the past
778
+ const id = await jobQueue.addCronJob({
779
+ scheduleName: 'due-now',
780
+ cronExpression: '* * * * *',
781
+ jobType: 'email',
782
+ payload: { to: 'due@example.com' },
783
+ });
784
+ // Force nextRunAt to be in the past
785
+ await pool.query(
786
+ `UPDATE cron_schedules SET next_run_at = NOW() - INTERVAL '1 minute' WHERE id = $1`,
787
+ [id],
788
+ );
789
+
790
+ // Act
791
+ const count = await jobQueue.enqueueDueCronJobs();
792
+
793
+ // Assert
794
+ expect(count).toBe(1);
795
+ const jobs = await jobQueue.getJobsByStatus('pending');
796
+ const cronJob = jobs.find(
797
+ (j) =>
798
+ j.jobType === 'email' && (j.payload as any).to === 'due@example.com',
799
+ );
800
+ expect(cronJob).toBeDefined();
801
+ });
802
+
803
+ it('enqueueDueCronJobs advances nextRunAt and sets lastJobId', async () => {
804
+ // Setup
805
+ const id = await jobQueue.addCronJob({
806
+ scheduleName: 'advance-test',
807
+ cronExpression: '* * * * *',
808
+ jobType: 'email',
809
+ payload: { to: 'advance@example.com' },
810
+ });
811
+ await pool.query(
812
+ `UPDATE cron_schedules SET next_run_at = NOW() - INTERVAL '1 minute' WHERE id = $1`,
813
+ [id],
814
+ );
815
+
816
+ // Act
817
+ await jobQueue.enqueueDueCronJobs();
818
+
819
+ // Assert
820
+ const schedule = await jobQueue.getCronJob(id);
821
+ expect(schedule!.lastJobId).not.toBeNull();
822
+ expect(schedule!.lastEnqueuedAt).toBeInstanceOf(Date);
823
+ expect(schedule!.nextRunAt).toBeInstanceOf(Date);
824
+ expect(schedule!.nextRunAt!.getTime()).toBeGreaterThan(Date.now() - 5000);
825
+ });
826
+
827
+ it('enqueueDueCronJobs skips paused schedules', async () => {
828
+ // Setup
829
+ const id = await jobQueue.addCronJob({
830
+ scheduleName: 'paused-skip',
831
+ cronExpression: '* * * * *',
832
+ jobType: 'email',
833
+ payload: { to: 'paused@example.com' },
834
+ });
835
+ await pool.query(
836
+ `UPDATE cron_schedules SET next_run_at = NOW() - INTERVAL '1 minute' WHERE id = $1`,
837
+ [id],
838
+ );
839
+ await jobQueue.pauseCronJob(id);
840
+
841
+ // Act
842
+ const count = await jobQueue.enqueueDueCronJobs();
843
+
844
+ // Assert
845
+ expect(count).toBe(0);
846
+ });
847
+
848
+ it('enqueueDueCronJobs skips schedules not yet due', async () => {
849
+ // Setup — nextRunAt is calculated to the future by addCronJob
850
+ await jobQueue.addCronJob({
851
+ scheduleName: 'future-schedule',
852
+ cronExpression: '0 0 1 1 *',
853
+ jobType: 'email',
854
+ payload: { to: 'future@example.com' },
855
+ });
856
+
857
+ // Act
858
+ const count = await jobQueue.enqueueDueCronJobs();
859
+
860
+ // Assert
861
+ expect(count).toBe(0);
862
+ });
863
+
864
+ it('enqueueDueCronJobs skips when allowOverlap=false and last job is still active', async () => {
865
+ // Setup
866
+ const id = await jobQueue.addCronJob({
867
+ scheduleName: 'no-overlap',
868
+ cronExpression: '* * * * *',
869
+ jobType: 'email',
870
+ payload: { to: 'overlap@example.com' },
871
+ allowOverlap: false,
872
+ });
873
+ await pool.query(
874
+ `UPDATE cron_schedules SET next_run_at = NOW() - INTERVAL '1 minute' WHERE id = $1`,
875
+ [id],
876
+ );
877
+
878
+ // First enqueue should succeed
879
+ const count1 = await jobQueue.enqueueDueCronJobs();
880
+ expect(count1).toBe(1);
881
+
882
+ // Set nextRunAt to past again (simulating next tick)
883
+ await pool.query(
884
+ `UPDATE cron_schedules SET next_run_at = NOW() - INTERVAL '1 minute' WHERE id = $1`,
885
+ [id],
886
+ );
887
+
888
+ // Act — second enqueue should be skipped because previous job is still pending
889
+ const count2 = await jobQueue.enqueueDueCronJobs();
890
+
891
+ // Assert
892
+ expect(count2).toBe(0);
893
+ });
894
+
895
+ it('enqueueDueCronJobs enqueues when allowOverlap=true even if last job is still active', async () => {
896
+ // Setup
897
+ const id = await jobQueue.addCronJob({
898
+ scheduleName: 'with-overlap',
899
+ cronExpression: '* * * * *',
900
+ jobType: 'email',
901
+ payload: { to: 'overlap@example.com' },
902
+ allowOverlap: true,
903
+ });
904
+ await pool.query(
905
+ `UPDATE cron_schedules SET next_run_at = NOW() - INTERVAL '1 minute' WHERE id = $1`,
906
+ [id],
907
+ );
908
+
909
+ // First enqueue
910
+ const count1 = await jobQueue.enqueueDueCronJobs();
911
+ expect(count1).toBe(1);
912
+
913
+ // Set nextRunAt to past again
914
+ await pool.query(
915
+ `UPDATE cron_schedules SET next_run_at = NOW() - INTERVAL '1 minute' WHERE id = $1`,
916
+ [id],
917
+ );
918
+
919
+ // Act — second enqueue should succeed because allowOverlap=true
920
+ const count2 = await jobQueue.enqueueDueCronJobs();
921
+
922
+ // Assert
923
+ expect(count2).toBe(1);
924
+
925
+ // Verify there are two pending jobs
926
+ const jobs = await jobQueue.getJobsByStatus('pending');
927
+ const cronJobs = jobs.filter(
928
+ (j) =>
929
+ j.jobType === 'email' &&
930
+ (j.payload as any).to === 'overlap@example.com',
931
+ );
932
+ expect(cronJobs).toHaveLength(2);
933
+ });
934
+
935
+ it('should propagate retry config from cron schedule to enqueued jobs', async () => {
936
+ const cronId = await jobQueue.addCronJob({
937
+ scheduleName: 'retry-cron',
938
+ cronExpression: '* * * * *',
939
+ jobType: 'email',
940
+ payload: { to: 'cron-retry@example.com' },
941
+ retryDelay: 15,
942
+ retryBackoff: false,
943
+ retryDelayMax: 90,
944
+ });
945
+
946
+ // Force next_run_at to the past
947
+ await pool.query(
948
+ `UPDATE cron_schedules SET next_run_at = NOW() - INTERVAL '1 minute' WHERE id = $1`,
949
+ [cronId],
950
+ );
951
+
952
+ const count = await jobQueue.enqueueDueCronJobs();
953
+ expect(count).toBe(1);
954
+
955
+ const jobs = await jobQueue.getJobsByStatus('pending');
956
+ const cronJob = jobs.find(
957
+ (j) => (j.payload as any).to === 'cron-retry@example.com',
958
+ );
959
+ expect(cronJob).toBeDefined();
960
+ expect(cronJob?.retryDelay).toBe(15);
961
+ expect(cronJob?.retryBackoff).toBe(false);
962
+ expect(cronJob?.retryDelayMax).toBe(90);
963
+ });
964
+
965
+ it('should store retry config on cron schedule record', async () => {
966
+ const cronId = await jobQueue.addCronJob({
967
+ scheduleName: 'retry-cron-record',
968
+ cronExpression: '0 */2 * * *',
969
+ jobType: 'email',
970
+ payload: { to: 'cron-record@example.com' },
971
+ retryDelay: 30,
972
+ retryBackoff: true,
973
+ retryDelayMax: 600,
974
+ });
975
+
976
+ const schedule = await jobQueue.getCronJob(cronId);
977
+ expect(schedule?.retryDelay).toBe(30);
978
+ expect(schedule?.retryBackoff).toBe(true);
979
+ expect(schedule?.retryDelayMax).toBe(600);
980
+ });
981
+ });
982
+
983
+ // ── BYOC (Bring Your Own Connection) tests ──────────────────────────────
984
+
985
+ describe('BYOC: init with external pool', () => {
986
+ let pool: Pool;
987
+ let dbName: string;
988
+ let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
989
+
990
+ beforeEach(async () => {
991
+ const setup = await createTestDbAndPool();
992
+ pool = setup.pool;
993
+ dbName = setup.dbName;
994
+ jobQueue = initJobQueue<TestPayloadMap>({ pool });
995
+ });
996
+
997
+ afterEach(async () => {
998
+ await pool.end();
999
+ await destroyTestDb(dbName);
1000
+ });
1001
+
1002
+ it('uses the provided pool for addJob and getJob', async () => {
1003
+ // Act
1004
+ const jobId = await jobQueue.addJob({
1005
+ jobType: 'email',
1006
+ payload: { to: 'byoc@example.com' },
1007
+ });
1008
+
1009
+ // Assert
1010
+ const job = await jobQueue.getJob(jobId);
1011
+ expect(job).not.toBeNull();
1012
+ expect(job?.jobType).toBe('email');
1013
+ expect(job?.payload).toEqual({ to: 'byoc@example.com' });
1014
+ });
1015
+
1016
+ it('returns the same pool instance from getPool()', () => {
1017
+ // Act
1018
+ const returnedPool = jobQueue.getPool();
1019
+
1020
+ // Assert
1021
+ expect(returnedPool).toBe(pool);
1022
+ });
1023
+ });
1024
+
1025
+ describe('BYOC: transactional addJob with db option', () => {
1026
+ let pool: Pool;
1027
+ let dbName: string;
1028
+ let testDbUrl: string;
1029
+ let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
1030
+
1031
+ beforeEach(async () => {
1032
+ const setup = await createTestDbAndPool();
1033
+ pool = setup.pool;
1034
+ dbName = setup.dbName;
1035
+ testDbUrl = setup.testDbUrl;
1036
+ jobQueue = initJobQueue<TestPayloadMap>({
1037
+ databaseConfig: { connectionString: testDbUrl },
1038
+ });
1039
+ });
1040
+
1041
+ afterEach(async () => {
1042
+ jobQueue.getPool().end();
1043
+ await pool.end();
1044
+ await destroyTestDb(dbName);
1045
+ });
1046
+
1047
+ it('rolls back the job when the transaction is rolled back', async () => {
1048
+ // Setup
1049
+ const client = await pool.connect();
1050
+ await client.query('BEGIN');
1051
+
1052
+ // Act
1053
+ const jobId = await jobQueue.addJob(
1054
+ { jobType: 'email', payload: { to: 'rollback@example.com' } },
1055
+ { db: client },
1056
+ );
1057
+ await client.query('ROLLBACK');
1058
+ client.release();
1059
+
1060
+ // Assert — job should not exist after rollback
1061
+ const job = await jobQueue.getJob(jobId);
1062
+ expect(job).toBeNull();
1063
+ });
1064
+
1065
+ it('persists the job and event when the transaction is committed', async () => {
1066
+ // Setup
1067
+ const client = await pool.connect();
1068
+ await client.query('BEGIN');
1069
+
1070
+ // Act
1071
+ const jobId = await jobQueue.addJob(
1072
+ { jobType: 'email', payload: { to: 'commit@example.com' } },
1073
+ { db: client },
1074
+ );
1075
+ await client.query('COMMIT');
1076
+ client.release();
1077
+
1078
+ // Assert — job exists
1079
+ const job = await jobQueue.getJob(jobId);
1080
+ expect(job).not.toBeNull();
1081
+ expect(job?.payload).toEqual({ to: 'commit@example.com' });
1082
+
1083
+ // Assert — event was recorded in the same transaction
1084
+ const events = await jobQueue.getJobEvents(jobId);
1085
+ expect(events.length).toBeGreaterThanOrEqual(1);
1086
+ expect(events[0].eventType).toBe('added');
1087
+ });
1088
+
1089
+ it('job is visible within the transaction before commit', async () => {
1090
+ // Setup
1091
+ const client = await pool.connect();
1092
+ await client.query('BEGIN');
1093
+
1094
+ // Act
1095
+ const jobId = await jobQueue.addJob(
1096
+ { jobType: 'sms', payload: { to: 'in-tx@example.com' } },
1097
+ { db: client },
1098
+ );
1099
+
1100
+ // Assert — visible within the transaction
1101
+ const res = await client.query('SELECT id FROM job_queue WHERE id = $1', [
1102
+ jobId,
1103
+ ]);
1104
+ expect(res.rows).toHaveLength(1);
1105
+
1106
+ await client.query('ROLLBACK');
1107
+ client.release();
1108
+ });
1109
+ });
1110
+
1111
+ describe('addJobs batch insert', () => {
1112
+ let pool: Pool;
1113
+ let dbName: string;
1114
+ let testDbUrl: string;
1115
+ let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
1116
+
1117
+ beforeEach(async () => {
1118
+ const setup = await createTestDbAndPool();
1119
+ pool = setup.pool;
1120
+ dbName = setup.dbName;
1121
+ testDbUrl = setup.testDbUrl;
1122
+ const config: JobQueueConfig = {
1123
+ databaseConfig: {
1124
+ connectionString: testDbUrl,
1125
+ },
1126
+ };
1127
+ jobQueue = initJobQueue<TestPayloadMap>(config);
1128
+ });
1129
+
1130
+ afterEach(async () => {
1131
+ jobQueue.getPool().end();
1132
+ await pool.end();
1133
+ await destroyTestDb(dbName);
1134
+ });
1135
+
1136
+ it('inserts multiple jobs and returns IDs in order', async () => {
1137
+ // Act
1138
+ const ids = await jobQueue.addJobs([
1139
+ { jobType: 'email', payload: { to: 'a@test.com' } },
1140
+ { jobType: 'sms', payload: { to: '+1234' } },
1141
+ { jobType: 'email', payload: { to: 'b@test.com' } },
1142
+ ]);
1143
+
1144
+ // Assert
1145
+ expect(ids).toHaveLength(3);
1146
+
1147
+ const job0 = await jobQueue.getJob(ids[0]);
1148
+ expect(job0?.jobType).toBe('email');
1149
+ expect(job0?.payload).toEqual({ to: 'a@test.com' });
1150
+
1151
+ const job1 = await jobQueue.getJob(ids[1]);
1152
+ expect(job1?.jobType).toBe('sms');
1153
+
1154
+ const job2 = await jobQueue.getJob(ids[2]);
1155
+ expect(job2?.jobType).toBe('email');
1156
+ expect(job2?.payload).toEqual({ to: 'b@test.com' });
1157
+ });
1158
+
1159
+ it('returns empty array for empty input', async () => {
1160
+ // Act
1161
+ const ids = await jobQueue.addJobs([]);
1162
+
1163
+ // Assert
1164
+ expect(ids).toEqual([]);
1165
+ });
1166
+
1167
+ it('handles idempotency keys correctly', async () => {
1168
+ // Setup
1169
+ const existingId = await jobQueue.addJob({
1170
+ jobType: 'email',
1171
+ payload: { to: 'existing@test.com' },
1172
+ idempotencyKey: 'batch-dup',
1173
+ });
1174
+
1175
+ // Act
1176
+ const ids = await jobQueue.addJobs([
1177
+ { jobType: 'email', payload: { to: 'new@test.com' } },
1178
+ {
1179
+ jobType: 'email',
1180
+ payload: { to: 'dup@test.com' },
1181
+ idempotencyKey: 'batch-dup',
1182
+ },
1183
+ ]);
1184
+
1185
+ // Assert
1186
+ expect(ids).toHaveLength(2);
1187
+ expect(ids[1]).toBe(existingId);
1188
+ expect(ids[0]).not.toBe(existingId);
1189
+ });
1190
+ });
1191
+
1192
+ describe('BYOC: validation errors', () => {
1193
+ it('throws when neither databaseConfig nor pool is provided for postgres', () => {
1194
+ // Act & Assert
1195
+ expect(() =>
1196
+ initJobQueue<TestPayloadMap>({ backend: 'postgres' } as any),
1197
+ ).toThrow(
1198
+ 'PostgreSQL backend requires either "databaseConfig" or "pool" to be provided.',
1199
+ );
1200
+ });
1201
+
1202
+ it('throws when neither redisConfig nor client is provided for redis', () => {
1203
+ // Act & Assert
1204
+ expect(() =>
1205
+ initJobQueue<TestPayloadMap>({ backend: 'redis' } as any),
1206
+ ).toThrow(
1207
+ 'Redis backend requires either "redisConfig" or "client" to be provided.',
1208
+ );
1209
+ });
1210
+ });
1211
+
1212
+ describe('event hooks', () => {
1213
+ let pool: Pool;
1214
+ let dbName: string;
1215
+ let testDbUrl: string;
1216
+ let jobQueue: ReturnType<typeof initJobQueue<TestPayloadMap>>;
1217
+
1218
+ beforeEach(async () => {
1219
+ const setup = await createTestDbAndPool();
1220
+ pool = setup.pool;
1221
+ dbName = setup.dbName;
1222
+ testDbUrl = setup.testDbUrl;
1223
+ const config: JobQueueConfig = {
1224
+ databaseConfig: {
1225
+ connectionString: testDbUrl,
1226
+ },
1227
+ };
1228
+ jobQueue = initJobQueue<TestPayloadMap>(config);
1229
+ });
1230
+
1231
+ afterEach(async () => {
1232
+ jobQueue.removeAllListeners();
1233
+ jobQueue.getPool().end();
1234
+ await pool.end();
1235
+ await destroyTestDb(dbName);
1236
+ });
1237
+
1238
+ it('emits job:added on addJob', async () => {
1239
+ const listener = vi.fn();
1240
+ jobQueue.on('job:added', listener);
1241
+
1242
+ const jobId = await jobQueue.addJob({
1243
+ jobType: 'email',
1244
+ payload: { to: 'test@example.com' },
1245
+ });
1246
+
1247
+ expect(listener).toHaveBeenCalledTimes(1);
1248
+ expect(listener).toHaveBeenCalledWith({ jobId, jobType: 'email' });
1249
+ });
1250
+
1251
+ it('emits job:added for each job in addJobs', async () => {
1252
+ const listener = vi.fn();
1253
+ jobQueue.on('job:added', listener);
1254
+
1255
+ const ids = await jobQueue.addJobs([
1256
+ { jobType: 'email', payload: { to: 'a@test.com' } },
1257
+ { jobType: 'sms', payload: { to: '+1234' } },
1258
+ ]);
1259
+
1260
+ expect(listener).toHaveBeenCalledTimes(2);
1261
+ expect(listener).toHaveBeenCalledWith({ jobId: ids[0], jobType: 'email' });
1262
+ expect(listener).toHaveBeenCalledWith({ jobId: ids[1], jobType: 'sms' });
1263
+ });
1264
+
1265
+ it('emits job:cancelled on cancelJob', async () => {
1266
+ const listener = vi.fn();
1267
+ jobQueue.on('job:cancelled', listener);
1268
+
1269
+ const jobId = await jobQueue.addJob({
1270
+ jobType: 'email',
1271
+ payload: { to: 'test@example.com' },
1272
+ });
1273
+ await jobQueue.cancelJob(jobId);
1274
+
1275
+ expect(listener).toHaveBeenCalledTimes(1);
1276
+ expect(listener).toHaveBeenCalledWith({ jobId });
1277
+ });
1278
+
1279
+ it('emits job:retried on retryJob', async () => {
1280
+ const listener = vi.fn();
1281
+ jobQueue.on('job:retried', listener);
1282
+
1283
+ const jobId = await jobQueue.addJob({
1284
+ jobType: 'email',
1285
+ payload: { to: 'test@example.com' },
1286
+ });
1287
+
1288
+ const handler = vi.fn(async () => {
1289
+ throw new Error('fail');
1290
+ });
1291
+ const processor = jobQueue.createProcessor({
1292
+ email: handler,
1293
+ sms: vi.fn(async () => {}),
1294
+ test: vi.fn(async () => {}),
1295
+ });
1296
+ await processor.start();
1297
+
1298
+ await jobQueue.retryJob(jobId);
1299
+
1300
+ expect(listener).toHaveBeenCalledTimes(1);
1301
+ expect(listener).toHaveBeenCalledWith({ jobId });
1302
+ });
1303
+
1304
+ it('emits job:processing and job:completed on successful processing', async () => {
1305
+ const processingListener = vi.fn();
1306
+ const completedListener = vi.fn();
1307
+ jobQueue.on('job:processing', processingListener);
1308
+ jobQueue.on('job:completed', completedListener);
1309
+
1310
+ const jobId = await jobQueue.addJob({
1311
+ jobType: 'email',
1312
+ payload: { to: 'test@example.com' },
1313
+ });
1314
+
1315
+ const processor = jobQueue.createProcessor({
1316
+ email: vi.fn(async () => {}),
1317
+ sms: vi.fn(async () => {}),
1318
+ test: vi.fn(async () => {}),
1319
+ });
1320
+ await processor.start();
1321
+
1322
+ expect(processingListener).toHaveBeenCalledTimes(1);
1323
+ expect(processingListener).toHaveBeenCalledWith({
1324
+ jobId,
1325
+ jobType: 'email',
1326
+ });
1327
+ expect(completedListener).toHaveBeenCalledTimes(1);
1328
+ expect(completedListener).toHaveBeenCalledWith({
1329
+ jobId,
1330
+ jobType: 'email',
1331
+ });
1332
+ });
1333
+
1334
+ it('emits job:failed with willRetry true when attempts remain', async () => {
1335
+ const listener = vi.fn();
1336
+ jobQueue.on('job:failed', listener);
1337
+
1338
+ const jobId = await jobQueue.addJob({
1339
+ jobType: 'email',
1340
+ payload: { to: 'test@example.com' },
1341
+ maxAttempts: 3,
1342
+ });
1343
+
1344
+ const processor = jobQueue.createProcessor({
1345
+ email: vi.fn(async () => {
1346
+ throw new Error('boom');
1347
+ }),
1348
+ sms: vi.fn(async () => {}),
1349
+ test: vi.fn(async () => {}),
1350
+ });
1351
+ await processor.start();
1352
+
1353
+ expect(listener).toHaveBeenCalledTimes(1);
1354
+ expect(listener).toHaveBeenCalledWith(
1355
+ expect.objectContaining({
1356
+ jobId,
1357
+ jobType: 'email',
1358
+ willRetry: true,
1359
+ error: expect.any(Error),
1360
+ }),
1361
+ );
1362
+ });
1363
+
1364
+ it('emits job:failed with willRetry false when no attempts remain', async () => {
1365
+ const listener = vi.fn();
1366
+ jobQueue.on('job:failed', listener);
1367
+
1368
+ const jobId = await jobQueue.addJob({
1369
+ jobType: 'email',
1370
+ payload: { to: 'test@example.com' },
1371
+ maxAttempts: 1,
1372
+ });
1373
+
1374
+ const processor = jobQueue.createProcessor({
1375
+ email: vi.fn(async () => {
1376
+ throw new Error('boom');
1377
+ }),
1378
+ sms: vi.fn(async () => {}),
1379
+ test: vi.fn(async () => {}),
1380
+ });
1381
+ await processor.start();
1382
+
1383
+ expect(listener).toHaveBeenCalledTimes(1);
1384
+ expect(listener).toHaveBeenCalledWith(
1385
+ expect.objectContaining({
1386
+ jobId,
1387
+ jobType: 'email',
1388
+ willRetry: false,
1389
+ }),
1390
+ );
1391
+ });
1392
+
1393
+ it('emits job:waiting when handler calls ctx.waitFor', async () => {
1394
+ const listener = vi.fn();
1395
+ jobQueue.on('job:waiting', listener);
1396
+
1397
+ const jobId = await jobQueue.addJob({
1398
+ jobType: 'email',
1399
+ payload: { to: 'test@example.com' },
1400
+ });
1401
+
1402
+ const processor = jobQueue.createProcessor({
1403
+ email: vi.fn(async (_payload, _signal, ctx) => {
1404
+ await ctx.waitFor({ hours: 1 });
1405
+ }),
1406
+ sms: vi.fn(async () => {}),
1407
+ test: vi.fn(async () => {}),
1408
+ });
1409
+ await processor.start();
1410
+
1411
+ expect(listener).toHaveBeenCalledTimes(1);
1412
+ expect(listener).toHaveBeenCalledWith({ jobId, jobType: 'email' });
1413
+ });
1414
+
1415
+ it('emits job:progress when handler calls ctx.setProgress', async () => {
1416
+ const listener = vi.fn();
1417
+ jobQueue.on('job:progress', listener);
1418
+
1419
+ const jobId = await jobQueue.addJob({
1420
+ jobType: 'email',
1421
+ payload: { to: 'test@example.com' },
1422
+ });
1423
+
1424
+ const processor = jobQueue.createProcessor({
1425
+ email: vi.fn(async (_payload, _signal, ctx) => {
1426
+ await ctx.setProgress(50);
1427
+ await ctx.setProgress(100);
1428
+ }),
1429
+ sms: vi.fn(async () => {}),
1430
+ test: vi.fn(async () => {}),
1431
+ });
1432
+ await processor.start();
1433
+
1434
+ expect(listener).toHaveBeenCalledTimes(2);
1435
+ expect(listener).toHaveBeenCalledWith({ jobId, progress: 50 });
1436
+ expect(listener).toHaveBeenCalledWith({ jobId, progress: 100 });
1437
+ });
1438
+
1439
+ it('once fires only once then auto-unsubscribes', async () => {
1440
+ const listener = vi.fn();
1441
+ jobQueue.once('job:added', listener);
1442
+
1443
+ await jobQueue.addJob({
1444
+ jobType: 'email',
1445
+ payload: { to: 'a@test.com' },
1446
+ });
1447
+ await jobQueue.addJob({
1448
+ jobType: 'sms',
1449
+ payload: { to: '+1234' },
1450
+ });
1451
+
1452
+ expect(listener).toHaveBeenCalledTimes(1);
1453
+ });
1454
+
1455
+ it('off removes a listener', async () => {
1456
+ const listener = vi.fn();
1457
+ jobQueue.on('job:added', listener);
1458
+
1459
+ await jobQueue.addJob({
1460
+ jobType: 'email',
1461
+ payload: { to: 'a@test.com' },
1462
+ });
1463
+ expect(listener).toHaveBeenCalledTimes(1);
1464
+
1465
+ jobQueue.off('job:added', listener);
1466
+
1467
+ await jobQueue.addJob({
1468
+ jobType: 'sms',
1469
+ payload: { to: '+1234' },
1470
+ });
1471
+ expect(listener).toHaveBeenCalledTimes(1);
1472
+ });
1473
+
1474
+ it('removeAllListeners clears all listeners for a specific event', async () => {
1475
+ const listener1 = vi.fn();
1476
+ const listener2 = vi.fn();
1477
+ const otherListener = vi.fn();
1478
+ jobQueue.on('job:added', listener1);
1479
+ jobQueue.on('job:added', listener2);
1480
+ jobQueue.on('job:cancelled', otherListener);
1481
+
1482
+ jobQueue.removeAllListeners('job:added');
1483
+
1484
+ await jobQueue.addJob({
1485
+ jobType: 'email',
1486
+ payload: { to: 'a@test.com' },
1487
+ });
1488
+ const jobId = await jobQueue.addJob({
1489
+ jobType: 'sms',
1490
+ payload: { to: '+1234' },
1491
+ });
1492
+ await jobQueue.cancelJob(jobId);
1493
+
1494
+ expect(listener1).not.toHaveBeenCalled();
1495
+ expect(listener2).not.toHaveBeenCalled();
1496
+ expect(otherListener).toHaveBeenCalledTimes(1);
1497
+ });
1498
+
1499
+ it('removeAllListeners with no args clears everything', async () => {
1500
+ const addedListener = vi.fn();
1501
+ const cancelledListener = vi.fn();
1502
+ jobQueue.on('job:added', addedListener);
1503
+ jobQueue.on('job:cancelled', cancelledListener);
1504
+
1505
+ jobQueue.removeAllListeners();
1506
+
1507
+ const jobId = await jobQueue.addJob({
1508
+ jobType: 'email',
1509
+ payload: { to: 'a@test.com' },
1510
+ });
1511
+ await jobQueue.cancelJob(jobId);
1512
+
1513
+ expect(addedListener).not.toHaveBeenCalled();
1514
+ expect(cancelledListener).not.toHaveBeenCalled();
1515
+ });
1516
+
1517
+ it('onError callback still works alongside error event', async () => {
1518
+ const onErrorSpy = vi.fn();
1519
+ const errorListener = vi.fn();
1520
+ jobQueue.on('error', errorListener);
1521
+
1522
+ await jobQueue.addJob({
1523
+ jobType: 'email',
1524
+ payload: { to: 'test@example.com' },
1525
+ maxAttempts: 1,
1526
+ });
1527
+
1528
+ const processor = jobQueue.createProcessor(
1529
+ {
1530
+ email: vi.fn(async () => {
1531
+ throw new Error('boom');
1532
+ }),
1533
+ sms: vi.fn(async () => {}),
1534
+ test: vi.fn(async () => {}),
1535
+ },
1536
+ { onError: onErrorSpy },
1537
+ );
1538
+ await processor.start();
1539
+
1540
+ // job:failed fires for individual job failures; error fires for
1541
+ // batch-level errors caught in processBatchWithHandlers. In this case
1542
+ // the job failure is handled inside processJobWithHandlers and doesn't
1543
+ // propagate to the batch-level error handler. So we verify that
1544
+ // onError still works as configured and job:failed events fire.
1545
+ const failedListener = vi.fn();
1546
+ jobQueue.on('job:failed', failedListener);
1547
+
1548
+ await jobQueue.addJob({
1549
+ jobType: 'email',
1550
+ payload: { to: 'test2@example.com' },
1551
+ maxAttempts: 1,
1552
+ });
1553
+ await processor.start();
1554
+
1555
+ expect(failedListener).toHaveBeenCalledTimes(1);
1556
+ });
534
1557
  });