bullmq 5.15.0 → 5.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/dist/cjs/classes/child-pool.js +2 -2
  2. package/dist/cjs/classes/child-pool.js.map +1 -1
  3. package/dist/cjs/classes/job-scheduler.js +140 -0
  4. package/dist/cjs/classes/job-scheduler.js.map +1 -0
  5. package/dist/cjs/classes/job.js +6 -2
  6. package/dist/cjs/classes/job.js.map +1 -1
  7. package/dist/cjs/classes/queue-events.js.map +1 -1
  8. package/dist/cjs/classes/queue-getters.js +10 -0
  9. package/dist/cjs/classes/queue-getters.js.map +1 -1
  10. package/dist/cjs/classes/queue-keys.js +1 -1
  11. package/dist/cjs/classes/queue-keys.js.map +1 -1
  12. package/dist/cjs/classes/queue.js +74 -0
  13. package/dist/cjs/classes/queue.js.map +1 -1
  14. package/dist/cjs/classes/repeat.js +9 -10
  15. package/dist/cjs/classes/repeat.js.map +1 -1
  16. package/dist/cjs/classes/sandbox.js +57 -47
  17. package/dist/cjs/classes/sandbox.js.map +1 -1
  18. package/dist/cjs/classes/scripts.js +46 -40
  19. package/dist/cjs/classes/scripts.js.map +1 -1
  20. package/dist/cjs/classes/worker.js +23 -4
  21. package/dist/cjs/classes/worker.js.map +1 -1
  22. package/dist/cjs/commands/addDelayedJob-6.lua +7 -7
  23. package/dist/cjs/commands/addJobScheduler-2.lua +75 -0
  24. package/dist/cjs/commands/addParentJob-4.lua +7 -7
  25. package/dist/cjs/commands/addPrioritizedJob-8.lua +7 -7
  26. package/dist/cjs/commands/addStandardJob-8.lua +7 -7
  27. package/dist/cjs/commands/{cleanJobsInSet-2.lua → cleanJobsInSet-3.lua} +6 -5
  28. package/dist/cjs/commands/drain-5.lua +41 -0
  29. package/dist/cjs/commands/includes/cleanSet.lua +50 -29
  30. package/dist/cjs/commands/includes/deduplicateJob.lua +25 -0
  31. package/dist/cjs/commands/includes/removeZSetJobs.lua +13 -1
  32. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +2 -4
  33. package/dist/cjs/commands/removeJob-2.lua +17 -9
  34. package/dist/cjs/commands/removeJobScheduler-3.lua +43 -0
  35. package/dist/cjs/enums/error-code.js +1 -0
  36. package/dist/cjs/enums/error-code.js.map +1 -1
  37. package/dist/cjs/scripts/addDelayedJob-6.js +18 -16
  38. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  39. package/dist/cjs/scripts/addJobScheduler-2.js +223 -0
  40. package/dist/cjs/scripts/addJobScheduler-2.js.map +1 -0
  41. package/dist/cjs/scripts/addParentJob-4.js +18 -16
  42. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  43. package/dist/cjs/scripts/addPrioritizedJob-8.js +18 -16
  44. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
  45. package/dist/cjs/scripts/addStandardJob-8.js +18 -16
  46. package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
  47. package/dist/cjs/scripts/{cleanJobsInSet-2.js → cleanJobsInSet-3.js} +57 -35
  48. package/dist/cjs/scripts/{cleanJobsInSet-2.js.map → cleanJobsInSet-3.js.map} +1 -1
  49. package/dist/cjs/scripts/{drain-4.js → drain-5.js} +28 -7
  50. package/dist/cjs/scripts/drain-5.js.map +1 -0
  51. package/dist/cjs/scripts/index.js +4 -2
  52. package/dist/cjs/scripts/index.js.map +1 -1
  53. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +2 -3
  54. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  55. package/dist/cjs/scripts/obliterate-2.js +11 -1
  56. package/dist/cjs/scripts/obliterate-2.js.map +1 -1
  57. package/dist/cjs/scripts/removeJob-2.js +16 -9
  58. package/dist/cjs/scripts/removeJob-2.js.map +1 -1
  59. package/dist/cjs/scripts/removeJobScheduler-3.js +49 -0
  60. package/dist/cjs/scripts/removeJobScheduler-3.js.map +1 -0
  61. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  62. package/dist/cjs/utils.js +25 -1
  63. package/dist/cjs/utils.js.map +1 -1
  64. package/dist/esm/classes/child-pool.d.ts +1 -1
  65. package/dist/esm/classes/child-pool.js +2 -2
  66. package/dist/esm/classes/child-pool.js.map +1 -1
  67. package/dist/esm/classes/job-scheduler.d.ts +30 -0
  68. package/dist/esm/classes/job-scheduler.js +135 -0
  69. package/dist/esm/classes/job-scheduler.js.map +1 -0
  70. package/dist/esm/classes/job.d.ts +6 -1
  71. package/dist/esm/classes/job.js +7 -3
  72. package/dist/esm/classes/job.js.map +1 -1
  73. package/dist/esm/classes/queue-events.d.ts +10 -0
  74. package/dist/esm/classes/queue-events.js.map +1 -1
  75. package/dist/esm/classes/queue-getters.d.ts +7 -0
  76. package/dist/esm/classes/queue-getters.js +10 -0
  77. package/dist/esm/classes/queue-getters.js.map +1 -1
  78. package/dist/esm/classes/queue-keys.js +1 -1
  79. package/dist/esm/classes/queue-keys.js.map +1 -1
  80. package/dist/esm/classes/queue.d.ts +54 -0
  81. package/dist/esm/classes/queue.js +74 -0
  82. package/dist/esm/classes/queue.js.map +1 -1
  83. package/dist/esm/classes/repeat.js +9 -10
  84. package/dist/esm/classes/repeat.js.map +1 -1
  85. package/dist/esm/classes/sandbox.js +57 -47
  86. package/dist/esm/classes/sandbox.js.map +1 -1
  87. package/dist/esm/classes/scripts.d.ts +3 -7
  88. package/dist/esm/classes/scripts.js +47 -41
  89. package/dist/esm/classes/scripts.js.map +1 -1
  90. package/dist/esm/classes/worker.d.ts +3 -0
  91. package/dist/esm/classes/worker.js +23 -4
  92. package/dist/esm/classes/worker.js.map +1 -1
  93. package/dist/esm/commands/addDelayedJob-6.lua +7 -7
  94. package/dist/esm/commands/addJobScheduler-2.lua +75 -0
  95. package/dist/esm/commands/addParentJob-4.lua +7 -7
  96. package/dist/esm/commands/addPrioritizedJob-8.lua +7 -7
  97. package/dist/esm/commands/addStandardJob-8.lua +7 -7
  98. package/dist/esm/commands/{cleanJobsInSet-2.lua → cleanJobsInSet-3.lua} +6 -5
  99. package/dist/esm/commands/drain-5.lua +41 -0
  100. package/dist/esm/commands/includes/cleanSet.lua +50 -29
  101. package/dist/esm/commands/includes/deduplicateJob.lua +25 -0
  102. package/dist/esm/commands/includes/removeZSetJobs.lua +13 -1
  103. package/dist/esm/commands/moveStalledJobsToWait-9.lua +2 -4
  104. package/dist/esm/commands/removeJob-2.lua +17 -9
  105. package/dist/esm/commands/removeJobScheduler-3.lua +43 -0
  106. package/dist/esm/enums/error-code.d.ts +2 -1
  107. package/dist/esm/enums/error-code.js +1 -0
  108. package/dist/esm/enums/error-code.js.map +1 -1
  109. package/dist/esm/interfaces/job-json.d.ts +1 -0
  110. package/dist/esm/scripts/addDelayedJob-6.js +18 -16
  111. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  112. package/dist/esm/scripts/addJobScheduler-2.d.ts +5 -0
  113. package/dist/esm/scripts/addJobScheduler-2.js +220 -0
  114. package/dist/esm/scripts/addJobScheduler-2.js.map +1 -0
  115. package/dist/esm/scripts/addParentJob-4.js +18 -16
  116. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  117. package/dist/esm/scripts/addPrioritizedJob-8.js +18 -16
  118. package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
  119. package/dist/esm/scripts/addStandardJob-8.js +18 -16
  120. package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
  121. package/dist/esm/scripts/{cleanJobsInSet-2.js → cleanJobsInSet-3.js} +57 -35
  122. package/dist/esm/scripts/{cleanJobsInSet-2.js.map → cleanJobsInSet-3.js.map} +1 -1
  123. package/dist/esm/scripts/{drain-4.js → drain-5.js} +28 -7
  124. package/dist/esm/scripts/drain-5.js.map +1 -0
  125. package/dist/esm/scripts/index.d.ts +4 -2
  126. package/dist/esm/scripts/index.js +4 -2
  127. package/dist/esm/scripts/index.js.map +1 -1
  128. package/dist/esm/scripts/moveStalledJobsToWait-9.js +2 -3
  129. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  130. package/dist/esm/scripts/obliterate-2.js +11 -1
  131. package/dist/esm/scripts/obliterate-2.js.map +1 -1
  132. package/dist/esm/scripts/removeJob-2.js +16 -9
  133. package/dist/esm/scripts/removeJob-2.js.map +1 -1
  134. package/dist/esm/scripts/removeJobScheduler-3.d.ts +5 -0
  135. package/dist/esm/scripts/removeJobScheduler-3.js +46 -0
  136. package/dist/esm/scripts/removeJobScheduler-3.js.map +1 -0
  137. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  138. package/dist/esm/types/job-options.d.ts +5 -0
  139. package/dist/esm/utils.d.ts +7 -0
  140. package/dist/esm/utils.js +23 -0
  141. package/dist/esm/utils.js.map +1 -1
  142. package/package.json +1 -1
  143. package/dist/cjs/commands/drain-4.lua +0 -26
  144. package/dist/cjs/commands/includes/debounceJob.lua +0 -23
  145. package/dist/cjs/scripts/drain-4.js.map +0 -1
  146. package/dist/esm/commands/drain-4.lua +0 -26
  147. package/dist/esm/commands/includes/debounceJob.lua +0 -23
  148. package/dist/esm/scripts/drain-4.js.map +0 -1
  149. /package/dist/esm/scripts/{cleanJobsInSet-2.d.ts → cleanJobsInSet-3.d.ts} +0 -0
  150. /package/dist/esm/scripts/{drain-4.d.ts → drain-5.d.ts} +0 -0
@@ -0,0 +1,75 @@
1
+ --[[
2
+ Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
3
+
4
+ Input:
5
+ KEYS[1] 'repeat' key
6
+ KEYS[2] 'delayed' key
7
+
8
+ ARGV[1] next milliseconds
9
+ ARGV[2] msgpacked options
10
+ [1] name
11
+ [2] tz?
12
+ [3] patten?
13
+ [4] endDate?
14
+ [5] every?
15
+ ARGV[3] jobs scheduler id
16
+ ARGV[4] prefix key
17
+
18
+ Output:
19
+ repeatableKey - OK
20
+ ]]
21
+ local rcall = redis.call
22
+ local repeatKey = KEYS[1]
23
+ local delayedKey = KEYS[2]
24
+
25
+ local nextMillis = ARGV[1]
26
+ local jobSchedulerId = ARGV[3]
27
+ local prefixKey = ARGV[4]
28
+
29
+ -- Includes
30
+ --- @include "includes/removeJob"
31
+
32
+ local function storeRepeatableJob(repeatKey, nextMillis, rawOpts)
33
+ rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
34
+ local opts = cmsgpack.unpack(rawOpts)
35
+
36
+ local optionalValues = {}
37
+ if opts['tz'] then
38
+ table.insert(optionalValues, "tz")
39
+ table.insert(optionalValues, opts['tz'])
40
+ end
41
+
42
+ if opts['pattern'] then
43
+ table.insert(optionalValues, "pattern")
44
+ table.insert(optionalValues, opts['pattern'])
45
+ end
46
+
47
+ if opts['endDate'] then
48
+ table.insert(optionalValues, "endDate")
49
+ table.insert(optionalValues, opts['endDate'])
50
+ end
51
+
52
+ if opts['every'] then
53
+ table.insert(optionalValues, "every")
54
+ table.insert(optionalValues, opts['every'])
55
+ end
56
+
57
+ rcall("HMSET", repeatKey .. ":" .. jobSchedulerId, "name", opts['name'],
58
+ unpack(optionalValues))
59
+ end
60
+
61
+ -- If we are overriding a repeatable job we must delete the delayed job for
62
+ -- the next iteration.
63
+ local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
64
+ if prevMillis ~= false then
65
+ local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
66
+ local nextDelayedJobId = repeatKey .. ":" .. jobSchedulerId .. ":" .. nextMillis
67
+
68
+ if rcall("ZSCORE", delayedKey, delayedJobId) ~= false
69
+ and rcall("EXISTS", nextDelayedJobId) ~= 1 then
70
+ removeJob(delayedJobId, true, prefixKey, true --[[remove debounce key]])
71
+ rcall("ZREM", delayedKey, delayedJobId)
72
+ end
73
+ end
74
+
75
+ return storeRepeatableJob(repeatKey, nextMillis, ARGV[2])
@@ -20,7 +20,7 @@
20
20
  [7] parent dependencies key.
21
21
  [8] parent? {id, queueKey}
22
22
  [9] repeat job key
23
- [10] debounce key
23
+ [10] deduplication key
24
24
 
25
25
  ARGV[2] Json stringified job data
26
26
  ARGV[3] msgpacked options
@@ -47,11 +47,11 @@ local opts = cmsgpack.unpack(ARGV[3])
47
47
  local parentKey = args[5]
48
48
  local parent = args[8]
49
49
  local repeatJobKey = args[9]
50
- local debounceKey = args[10]
50
+ local deduplicationKey = args[10]
51
51
  local parentData
52
52
 
53
53
  -- Includes
54
- --- @include "includes/debounceJob"
54
+ --- @include "includes/deduplicateJob"
55
55
  --- @include "includes/getOrSetMaxEvents"
56
56
  --- @include "includes/handleDuplicatedJob"
57
57
  --- @include "includes/storeJob"
@@ -81,10 +81,10 @@ else
81
81
  end
82
82
  end
83
83
 
84
- local debouncedJobId = debounceJob(args[1], opts['de'],
85
- jobId, debounceKey, eventsKey, maxEvents)
86
- if debouncedJobId then
87
- return debouncedJobId
84
+ local deduplicationJobId = deduplicateJob(args[1], opts['de'],
85
+ jobId, deduplicationKey, eventsKey, maxEvents)
86
+ if deduplicationJobId then
87
+ return deduplicationJobId
88
88
  end
89
89
 
90
90
  -- Store the job.
@@ -24,7 +24,7 @@
24
24
  [7] parent dependencies key.
25
25
  [8] parent? {id, queueKey}
26
26
  [9] repeat job key
27
- [10] debounce key
27
+ [10] deduplication key
28
28
 
29
29
  ARGV[2] Json stringified job data
30
30
  ARGV[3] msgpacked options
@@ -54,12 +54,12 @@ local opts = cmsgpack.unpack(ARGV[3])
54
54
  local parentKey = args[5]
55
55
  local parent = args[8]
56
56
  local repeatJobKey = args[9]
57
- local debounceKey = args[10]
57
+ local deduplicationKey = args[10]
58
58
  local parentData
59
59
 
60
60
  -- Includes
61
61
  --- @include "includes/addJobWithPriority"
62
- --- @include "includes/debounceJob"
62
+ --- @include "includes/deduplicateJob"
63
63
  --- @include "includes/storeJob"
64
64
  --- @include "includes/getOrSetMaxEvents"
65
65
  --- @include "includes/handleDuplicatedJob"
@@ -90,10 +90,10 @@ else
90
90
  end
91
91
  end
92
92
 
93
- local debouncedJobId = debounceJob(args[1], opts['de'],
94
- jobId, debounceKey, eventsKey, maxEvents)
95
- if debouncedJobId then
96
- return debouncedJobId
93
+ local deduplicationJobId = deduplicateJob(args[1], opts['de'],
94
+ jobId, deduplicationKey, eventsKey, maxEvents)
95
+ if deduplicationJobId then
96
+ return deduplicationJobId
97
97
  end
98
98
 
99
99
  -- Store the job.
@@ -34,7 +34,7 @@
34
34
  [7] parent dependencies key.
35
35
  [8] parent? {id, queueKey}
36
36
  [9] repeat job key
37
- [10] debounce key
37
+ [10] deduplication key
38
38
 
39
39
  ARGV[2] Json stringified job data
40
40
  ARGV[3] msgpacked options
@@ -57,12 +57,12 @@ local opts = cmsgpack.unpack(ARGV[3])
57
57
  local parentKey = args[5]
58
58
  local parent = args[8]
59
59
  local repeatJobKey = args[9]
60
- local debounceKey = args[10]
60
+ local deduplicationKey = args[10]
61
61
  local parentData
62
62
 
63
63
  -- Includes
64
64
  --- @include "includes/addJobInTargetList"
65
- --- @include "includes/debounceJob"
65
+ --- @include "includes/deduplicateJob"
66
66
  --- @include "includes/getOrSetMaxEvents"
67
67
  --- @include "includes/getTargetQueueList"
68
68
  --- @include "includes/handleDuplicatedJob"
@@ -94,10 +94,10 @@ else
94
94
  end
95
95
  end
96
96
 
97
- local debouncedJobId = debounceJob(args[1], opts['de'],
98
- jobId, debounceKey, eventsKey, maxEvents)
99
- if debouncedJobId then
100
- return debouncedJobId
97
+ local deduplicationJobId = deduplicateJob(args[1], opts['de'],
98
+ jobId, deduplicationKey, eventsKey, maxEvents)
99
+ if deduplicationJobId then
100
+ return deduplicationJobId
101
101
  end
102
102
 
103
103
  -- Store the job.
@@ -4,6 +4,7 @@
4
4
  Input:
5
5
  KEYS[1] set key,
6
6
  KEYS[2] events stream key
7
+ KEYS[3] job schedulers key
7
8
 
8
9
  ARGV[1] jobKey prefix
9
10
  ARGV[2] timestamp
@@ -32,21 +33,21 @@ end
32
33
 
33
34
  local result
34
35
  if ARGV[4] == "active" then
35
- result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
36
+ result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false --[[ hasFinished ]])
36
37
  elseif ARGV[4] == "delayed" then
37
38
  rangeEnd = "+inf"
38
39
  result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
39
- {"processedOn", "timestamp"}, false)
40
+ {"processedOn", "timestamp"}, false --[[ hasFinished ]], KEYS[3])
40
41
  elseif ARGV[4] == "prioritized" then
41
42
  rangeEnd = "+inf"
42
43
  result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
43
- {"timestamp"}, false)
44
+ {"timestamp"}, false --[[ hasFinished ]])
44
45
  elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
45
- result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
46
+ result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true --[[ hasFinished ]])
46
47
  else
47
48
  rangeEnd = ARGV[2]
48
49
  result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
49
- {"finishedOn"}, true)
50
+ {"finishedOn"}, true --[[ hasFinished ]])
50
51
  end
51
52
 
52
53
  rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
@@ -0,0 +1,41 @@
1
+ --[[
2
+ Drains the queue, removes all jobs that are waiting
3
+ or delayed, but not active, completed or failed
4
+
5
+ Input:
6
+ KEYS[1] 'wait',
7
+ KEYS[2] 'paused'
8
+ KEYS[3] 'delayed'
9
+ KEYS[4] 'prioritized'
10
+ KEYS[5] 'jobschedulers' (repeat)
11
+
12
+ ARGV[1] queue key prefix
13
+ ]]
14
+ local rcall = redis.call
15
+ local queueBaseKey = ARGV[1]
16
+
17
+ --- @include "includes/removeListJobs"
18
+ --- @include "includes/removeZSetJobs"
19
+
20
+ removeListJobs(KEYS[1], true, queueBaseKey, 0) -- wait
21
+ removeListJobs(KEYS[2], true, queueBaseKey, 0) -- paused
22
+
23
+ if KEYS[3] ~= "" then
24
+
25
+ -- We must not remove delayed jobs if they are associated to a job scheduler.
26
+ local scheduledJobs = {}
27
+ local jobSchedulers = rcall("ZRANGE", KEYS[5], 0, -1, "WITHSCORES")
28
+
29
+ -- For every job scheduler, get the current delayed job id.
30
+ for i = 1, #jobSchedulers, 2 do
31
+ local jobSchedulerId = jobSchedulers[i]
32
+ local jobSchedulerMillis = jobSchedulers[i + 1]
33
+
34
+ local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. jobSchedulerMillis
35
+ scheduledJobs[delayedJobId] = true
36
+ end
37
+
38
+ removeZSetJobs(KEYS[3], true, queueBaseKey, 0, scheduledJobs) -- delayed
39
+ end
40
+
41
+ removeZSetJobs(KEYS[4], true, queueBaseKey, 0) -- prioritized
@@ -1,45 +1,66 @@
1
1
  --[[
2
2
  Function to clean job set.
3
3
  Returns jobIds and deleted count number.
4
- ]]
4
+ ]]
5
5
 
6
6
  -- Includes
7
7
  --- @include "batches"
8
8
  --- @include "getJobsInZset"
9
9
  --- @include "getTimestamp"
10
10
  --- @include "removeJob"
11
-
12
- local function cleanSet(setKey, jobKeyPrefix, rangeEnd, timestamp, limit, attributes, isFinished)
13
- local jobs = getJobsInZset(setKey, rangeEnd, limit)
14
- local deleted = {}
15
- local deletedCount = 0
16
- local jobTS
17
- for i, job in ipairs(jobs) do
18
- if limit > 0 and deletedCount >= limit then
19
- break
11
+ local function isJobSchedulerJob(jobId, jobSchedulersKey)
12
+ if jobSchedulersKey then
13
+ local jobSchedulerId = jobId:match("repeat:(.*):%d+")
14
+ if jobSchedulerId then
15
+ return rcall("ZSCORE", jobSchedulersKey, jobSchedulerId)
16
+ end
20
17
  end
18
+ return false
19
+ end
20
+
21
+ local function cleanSet(
22
+ setKey,
23
+ jobKeyPrefix,
24
+ rangeEnd,
25
+ timestamp,
26
+ limit,
27
+ attributes,
28
+ isFinished,
29
+ jobSchedulersKey)
30
+ local jobs = getJobsInZset(setKey, rangeEnd, limit)
31
+ local deleted = {}
32
+ local deletedCount = 0
33
+ local jobTS
34
+ for i, job in ipairs(jobs) do
35
+ if limit > 0 and deletedCount >= limit then
36
+ break
37
+ end
21
38
 
22
- local jobKey = jobKeyPrefix .. job
23
- if isFinished then
24
- removeJob(job, true, jobKeyPrefix, true --[[remove debounce key]])
25
- deletedCount = deletedCount + 1
26
- table.insert(deleted, job)
27
- else
28
- -- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
29
- jobTS = getTimestamp(jobKey, attributes)
30
- if (not jobTS or jobTS <= timestamp) then
31
- removeJob(job, true, jobKeyPrefix, true --[[remove debounce key]])
32
- deletedCount = deletedCount + 1
33
- table.insert(deleted, job)
34
- end
39
+ -- Extract a Job Scheduler Id from jobId ("repeat:job-scheduler-id:millis")
40
+ -- and check if it is in the scheduled jobs
41
+ if not isJobSchedulerJob(job, jobSchedulersKey) then
42
+ local jobKey = jobKeyPrefix .. job
43
+ if isFinished then
44
+ removeJob(job, true, jobKeyPrefix, true --[[remove debounce key]] )
45
+ deletedCount = deletedCount + 1
46
+ table.insert(deleted, job)
47
+ else
48
+ -- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
49
+ jobTS = getTimestamp(jobKey, attributes)
50
+ if (not jobTS or jobTS <= timestamp) then
51
+ removeJob(job, true, jobKeyPrefix, true --[[remove debounce key]] )
52
+ deletedCount = deletedCount + 1
53
+ table.insert(deleted, job)
54
+ end
55
+ end
56
+ end
35
57
  end
36
- end
37
58
 
38
- if(#deleted > 0) then
39
- for from, to in batches(#deleted, 7000) do
40
- rcall("ZREM", setKey, unpack(deleted, from, to))
59
+ if (#deleted > 0) then
60
+ for from, to in batches(#deleted, 7000) do
61
+ rcall("ZREM", setKey, unpack(deleted, from, to))
62
+ end
41
63
  end
42
- end
43
64
 
44
- return {deleted, deletedCount}
65
+ return {deleted, deletedCount}
45
66
  end
@@ -0,0 +1,25 @@
1
+ --[[
2
+ Function to debounce a job.
3
+ ]]
4
+
5
+ local function deduplicateJob(prefixKey, deduplicationOpts, jobId, deduplicationKey, eventsKey, maxEvents)
6
+ local deduplicationId = deduplicationOpts and deduplicationOpts['id']
7
+ if deduplicationId then
8
+ local ttl = deduplicationOpts['ttl']
9
+ local deduplicationKeyExists
10
+ if ttl then
11
+ deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
12
+ else
13
+ deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
14
+ end
15
+ if deduplicationKeyExists then
16
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
17
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
18
+ "debounced", "jobId", currentDebounceJobId, "debounceId", deduplicationId)
19
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
20
+ "deduplicated", "jobId", currentDebounceJobId, "deduplicationId", deduplicationId)
21
+ return currentDebounceJobId
22
+ end
23
+ end
24
+ end
25
+
@@ -3,8 +3,20 @@
3
3
  --- @include "getZSetItems"
4
4
  --- @include "removeJobs"
5
5
 
6
- local function removeZSetJobs(keyName, hard, baseKey, max)
6
+ local function removeZSetJobs(keyName, hard, baseKey, max, jobsToIgnore)
7
7
  local jobs = getZSetItems(keyName, max)
8
+
9
+ -- filter out jobs to ignore
10
+ if jobsToIgnore then
11
+ local filteredJobs = {}
12
+ for i = 1, #jobs do
13
+ if not jobsToIgnore[jobs[i]] then
14
+ table.insert(filteredJobs, jobs[i])
15
+ end
16
+ end
17
+ jobs = filteredJobs
18
+ end
19
+
8
20
  local count = removeJobs(jobs, hard, baseKey, max)
9
21
  if(#jobs > 0) then
10
22
  for from, to in batches(#jobs, 7000) do
@@ -44,7 +44,7 @@ local metaKey = KEYS[6]
44
44
  local pausedKey = KEYS[7]
45
45
  local markerKey = KEYS[8]
46
46
  local eventStreamKey = KEYS[9]
47
- local maxStalledJobCount = ARGV[1]
47
+ local maxStalledJobCount = tonumber(ARGV[1])
48
48
  local queueKeyPrefix = ARGV[2]
49
49
  local timestamp = ARGV[3]
50
50
  local maxCheckTime = ARGV[4]
@@ -63,8 +63,6 @@ local failed = {}
63
63
  if (#stalling > 0) then
64
64
  rcall('DEL', stalledKey)
65
65
 
66
- local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
67
-
68
66
  -- Remove from active list
69
67
  for i, jobId in ipairs(stalling) do
70
68
  -- Markers in waitlist DEPRECATED in v5: Remove in v6.
@@ -83,7 +81,7 @@ if (#stalling > 0) then
83
81
  -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
84
82
  local stalledCount =
85
83
  rcall("HINCRBY", jobKey, "stalledCounter", 1)
86
- if (stalledCount > MAX_STALLED_JOB_COUNT) then
84
+ if (stalledCount > maxStalledJobCount) then
87
85
  local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
88
86
  local rawOpts = jobAttributes[1]
89
87
  local rawParentData = jobAttributes[2]
@@ -24,7 +24,7 @@ local rcall = redis.call
24
24
  --- @include "includes/removeJobKeys"
25
25
  --- @include "includes/removeParentDependencyKey"
26
26
 
27
- local function removeJob( prefix, jobId, parentKey, removeChildren)
27
+ local function removeJob(prefix, jobId, parentKey, removeChildren)
28
28
  local jobKey = prefix .. jobId;
29
29
 
30
30
  removeParentDependencyKey(jobKey, false, parentKey, nil)
@@ -33,14 +33,14 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
33
33
  -- Check if this job has children
34
34
  -- If so, we are going to try to remove the children recursively in deep first way because
35
35
  -- if some job is locked we must exit with and error.
36
- --local countProcessed = rcall("HLEN", jobKey .. ":processed")
36
+ -- local countProcessed = rcall("HLEN", jobKey .. ":processed")
37
37
  local processed = rcall("HGETALL", jobKey .. ":processed")
38
38
 
39
39
  if (#processed > 0) then
40
40
  for i = 1, #processed, 2 do
41
41
  local childJobId = getJobIdFromKey(processed[i])
42
42
  local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
43
- removeJob( childJobPrefix, childJobId, jobKey, removeChildren )
43
+ removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
44
44
  end
45
45
  end
46
46
 
@@ -50,7 +50,7 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
50
50
  -- We need to get the jobId for this job.
51
51
  local childJobId = getJobIdFromKey(childJobKey)
52
52
  local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
53
- removeJob( childJobPrefix, childJobId, jobKey, removeChildren )
53
+ removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
54
54
  end
55
55
  end
56
56
 
@@ -60,7 +60,7 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
60
60
  for i = 1, #failed, 2 do
61
61
  local childJobId = getJobIdFromKey(failed[i])
62
62
  local childJobPrefix = getJobKeyPrefix(failed[i], childJobId)
63
- removeJob( childJobPrefix, childJobId, jobKey, removeChildren )
63
+ removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
64
64
  end
65
65
  end
66
66
  end
@@ -70,15 +70,23 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
70
70
  removeDebounceKey(prefix, jobKey)
71
71
  if removeJobKeys(jobKey) > 0 then
72
72
  local maxEvents = getOrSetMaxEvents(KEYS[2])
73
- rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
74
- "jobId", jobId, "prev", prev)
73
+ rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed", "jobId", jobId, "prev",
74
+ prev)
75
75
  end
76
76
  end
77
77
 
78
78
  local prefix = KEYS[1]
79
+ local jobId = ARGV[1]
80
+ local shouldRemoveChildren = ARGV[2]
81
+ local jobKey = prefix .. jobId
79
82
 
80
- if not isLocked(prefix, ARGV[1], ARGV[2]) then
81
- removeJob(prefix, ARGV[1], nil, ARGV[2])
83
+ -- Check if the job belongs to a job scheduler and it is in delayed state.
84
+ if rcall("ZSCORE", prefix .. "delayed", jobId) and rcall("HGET", jobKey, "rjk") then
85
+ return -8 -- Return error code as the job is part of a job scheduler and is in delayed state.
86
+ end
87
+
88
+ if not isLocked(prefix, jobId, shouldRemoveChildren) then
89
+ removeJob(prefix, jobId, nil, shouldRemoveChildren)
82
90
  return 1
83
91
  end
84
92
  return 0
@@ -0,0 +1,43 @@
1
+
2
+ --[[
3
+ Removes a repeatable job
4
+ Input:
5
+ KEYS[1] job schedulers key
6
+ KEYS[2] delayed jobs key
7
+ KEYS[3] events key
8
+
9
+ ARGV[1] job scheduler id
10
+ ARGV[2] prefix key
11
+
12
+ Output:
13
+ 0 - OK
14
+ 1 - Missing repeat job
15
+
16
+ Events:
17
+ 'removed'
18
+ ]]
19
+ local rcall = redis.call
20
+
21
+ -- Includes
22
+ --- @include "includes/removeJobKeys"
23
+
24
+ local jobSchedulerId = ARGV[1]
25
+ local prefix = ARGV[2]
26
+
27
+ local millis = rcall("ZSCORE", KEYS[1], jobSchedulerId)
28
+
29
+ if millis then
30
+ -- Delete next programmed job.
31
+ local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. millis
32
+ if(rcall("ZREM", KEYS[2], delayedJobId) == 1) then
33
+ removeJobKeys(prefix .. delayedJobId)
34
+ rcall("XADD", KEYS[3], "*", "event", "removed", "jobId", delayedJobId, "prev", "delayed")
35
+ end
36
+ end
37
+
38
+ if(rcall("ZREM", KEYS[1], jobSchedulerId) == 1) then
39
+ rcall("DEL", KEYS[1] .. ":" .. jobSchedulerId)
40
+ return 0
41
+ end
42
+
43
+ return 1
@@ -5,5 +5,6 @@ export declare enum ErrorCode {
5
5
  JobPendingDependencies = -4,
6
6
  ParentJobNotExist = -5,
7
7
  JobLockMismatch = -6,
8
- ParentJobCannotBeReplaced = -7
8
+ ParentJobCannotBeReplaced = -7,
9
+ JobBelongsToJobScheduler = -8
9
10
  }
@@ -7,5 +7,6 @@ export var ErrorCode;
7
7
  ErrorCode[ErrorCode["ParentJobNotExist"] = -5] = "ParentJobNotExist";
8
8
  ErrorCode[ErrorCode["JobLockMismatch"] = -6] = "JobLockMismatch";
9
9
  ErrorCode[ErrorCode["ParentJobCannotBeReplaced"] = -7] = "ParentJobCannotBeReplaced";
10
+ ErrorCode[ErrorCode["JobBelongsToJobScheduler"] = -8] = "JobBelongsToJobScheduler";
10
11
  })(ErrorCode || (ErrorCode = {}));
11
12
  //# sourceMappingURL=error-code.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,SAQX;AARD,WAAY,SAAS;IACnB,wDAAgB,CAAA;IAChB,gEAAoB,CAAA;IACpB,4DAAkB,CAAA;IAClB,8EAA2B,CAAA;IAC3B,oEAAsB,CAAA;IACtB,gEAAoB,CAAA;IACpB,oFAA8B,CAAA;AAChC,CAAC,EARW,SAAS,KAAT,SAAS,QAQpB"}
1
+ {"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,SASX;AATD,WAAY,SAAS;IACnB,wDAAgB,CAAA;IAChB,gEAAoB,CAAA;IACpB,4DAAkB,CAAA;IAClB,8EAA2B,CAAA;IAC3B,oEAAsB,CAAA;IACtB,gEAAoB,CAAA;IACpB,oFAA8B,CAAA;IAC9B,kFAA6B,CAAA;AAC/B,CAAC,EATW,SAAS,KAAT,SAAS,QASpB"}
@@ -18,6 +18,7 @@ export interface JobJson {
18
18
  parentKey?: string;
19
19
  repeatJobKey?: string;
20
20
  debounceId?: string;
21
+ deduplicationId?: string;
21
22
  processedBy?: string;
22
23
  }
23
24
  export interface JobJsonRaw {
@@ -22,7 +22,7 @@ const content = `--[[
22
22
  [7] parent dependencies key.
23
23
  [8] parent? {id, queueKey}
24
24
  [9] repeat job key
25
- [10] debounce key
25
+ [10] deduplication key
26
26
  ARGV[2] Json stringified job data
27
27
  ARGV[3] msgpacked options
28
28
  Output:
@@ -42,7 +42,7 @@ local data = ARGV[2]
42
42
  local parentKey = args[5]
43
43
  local parent = args[8]
44
44
  local repeatJobKey = args[9]
45
- local debounceKey = args[10]
45
+ local deduplicationKey = args[10]
46
46
  local parentData
47
47
  -- Includes
48
48
  --[[
@@ -72,20 +72,22 @@ end
72
72
  --[[
73
73
  Function to debounce a job.
74
74
  ]]
75
- local function debounceJob(prefixKey, debounceOpts, jobId, debounceKey, eventsKey, maxEvents)
76
- local debounceId = debounceOpts and debounceOpts['id']
77
- if debounceId then
78
- local ttl = debounceOpts['ttl']
79
- local debounceKeyExists
75
+ local function deduplicateJob(prefixKey, deduplicationOpts, jobId, deduplicationKey, eventsKey, maxEvents)
76
+ local deduplicationId = deduplicationOpts and deduplicationOpts['id']
77
+ if deduplicationId then
78
+ local ttl = deduplicationOpts['ttl']
79
+ local deduplicationKeyExists
80
80
  if ttl then
81
- debounceKeyExists = not rcall('SET', debounceKey, jobId, 'PX', ttl, 'NX')
81
+ deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
82
82
  else
83
- debounceKeyExists = not rcall('SET', debounceKey, jobId, 'NX')
83
+ deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
84
84
  end
85
- if debounceKeyExists then
86
- local currentDebounceJobId = rcall('GET', debounceKey)
85
+ if deduplicationKeyExists then
86
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
87
87
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
88
- "debounced", "jobId", currentDebounceJobId, "debounceId", debounceId)
88
+ "debounced", "jobId", currentDebounceJobId, "debounceId", deduplicationId)
89
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
90
+ "deduplicated", "jobId", currentDebounceJobId, "deduplicationId", deduplicationId)
89
91
  return currentDebounceJobId
90
92
  end
91
93
  end
@@ -338,10 +340,10 @@ else
338
340
  maxEvents, timestamp)
339
341
  end
340
342
  end
341
- local debouncedJobId = debounceJob(args[1], opts['de'],
342
- jobId, debounceKey, eventsKey, maxEvents)
343
- if debouncedJobId then
344
- return debouncedJobId
343
+ local deduplicationJobId = deduplicateJob(args[1], opts['de'],
344
+ jobId, deduplicationKey, eventsKey, maxEvents)
345
+ if deduplicationJobId then
346
+ return deduplicationJobId
345
347
  end
346
348
  -- Store the job.
347
349
  local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
@@ -1 +1 @@
1
- {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyWf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2Wf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}