bullmq 4.17.0 → 5.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. package/dist/cjs/classes/flow-producer.js +3 -10
  2. package/dist/cjs/classes/flow-producer.js.map +1 -1
  3. package/dist/cjs/classes/job.js +21 -11
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/queue-base.js +2 -9
  6. package/dist/cjs/classes/queue-base.js.map +1 -1
  7. package/dist/cjs/classes/queue-events.js +3 -1
  8. package/dist/cjs/classes/queue-events.js.map +1 -1
  9. package/dist/cjs/classes/queue-keys.js +1 -0
  10. package/dist/cjs/classes/queue-keys.js.map +1 -1
  11. package/dist/cjs/classes/queue.js +1 -1
  12. package/dist/cjs/classes/redis-connection.js +9 -7
  13. package/dist/cjs/classes/redis-connection.js.map +1 -1
  14. package/dist/cjs/classes/repeat.js +22 -5
  15. package/dist/cjs/classes/repeat.js.map +1 -1
  16. package/dist/cjs/classes/scripts.js +34 -34
  17. package/dist/cjs/classes/scripts.js.map +1 -1
  18. package/dist/cjs/classes/worker.js +27 -32
  19. package/dist/cjs/classes/worker.js.map +1 -1
  20. package/dist/cjs/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
  21. package/dist/cjs/commands/addParentJob-4.lua +0 -2
  22. package/dist/{esm/commands/addPrioritizedJob-8.lua → cjs/commands/addPrioritizedJob-7.lua} +18 -24
  23. package/dist/cjs/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
  24. package/dist/cjs/commands/changePriority-6.lua +57 -0
  25. package/dist/cjs/commands/getCounts-1.lua +1 -0
  26. package/dist/cjs/commands/getRanges-1.lua +1 -0
  27. package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
  28. package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -7
  29. package/dist/cjs/commands/includes/checkStalledJobs.lua +1 -1
  30. package/dist/cjs/commands/includes/isQueuePaused.lua +7 -0
  31. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
  32. package/dist/cjs/commands/includes/prepareJobForProcessing.lua +1 -1
  33. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +7 -4
  34. package/dist/{esm/commands/moveToActive-10.lua → cjs/commands/moveToActive-11.lua} +18 -29
  35. package/dist/cjs/commands/moveToDelayed-7.lua +77 -0
  36. package/dist/cjs/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
  37. package/dist/cjs/commands/moveToWaitingChildren-4.lua +7 -4
  38. package/dist/cjs/commands/pause-7.lua +41 -0
  39. package/dist/cjs/commands/promote-8.lua +60 -0
  40. package/dist/cjs/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
  41. package/dist/cjs/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
  42. package/dist/cjs/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
  43. package/dist/cjs/scripts/addParentJob-4.js +51 -61
  44. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  45. package/dist/cjs/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
  46. package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -0
  47. package/dist/cjs/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
  48. package/dist/cjs/scripts/addStandardJob-7.js.map +1 -0
  49. package/dist/cjs/scripts/changePriority-6.js +85 -0
  50. package/dist/cjs/scripts/changePriority-6.js.map +1 -0
  51. package/dist/cjs/scripts/getCounts-1.js +1 -0
  52. package/dist/cjs/scripts/getCounts-1.js.map +1 -1
  53. package/dist/cjs/scripts/getRanges-1.js +1 -0
  54. package/dist/cjs/scripts/getRanges-1.js.map +1 -1
  55. package/dist/cjs/scripts/index.js +10 -10
  56. package/dist/cjs/scripts/index.js.map +1 -1
  57. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +1 -0
  58. package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
  59. package/dist/cjs/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
  60. package/dist/cjs/scripts/moveToActive-11.js.map +1 -0
  61. package/dist/cjs/scripts/moveToDelayed-7.js +103 -0
  62. package/dist/cjs/scripts/moveToDelayed-7.js.map +1 -0
  63. package/dist/cjs/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
  64. package/dist/cjs/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
  65. package/dist/cjs/scripts/moveToWaitingChildren-4.js +6 -3
  66. package/dist/cjs/scripts/moveToWaitingChildren-4.js.map +1 -1
  67. package/dist/cjs/scripts/pause-7.js +67 -0
  68. package/dist/cjs/scripts/pause-7.js.map +1 -0
  69. package/dist/cjs/scripts/promote-8.js +79 -0
  70. package/dist/cjs/scripts/promote-8.js.map +1 -0
  71. package/dist/cjs/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
  72. package/dist/cjs/scripts/retryJob-10.js.map +1 -0
  73. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  74. package/dist/esm/classes/flow-producer.js +3 -10
  75. package/dist/esm/classes/flow-producer.js.map +1 -1
  76. package/dist/esm/classes/job.d.ts +5 -0
  77. package/dist/esm/classes/job.js +21 -11
  78. package/dist/esm/classes/job.js.map +1 -1
  79. package/dist/esm/classes/queue-base.js +2 -9
  80. package/dist/esm/classes/queue-base.js.map +1 -1
  81. package/dist/esm/classes/queue-events.js +3 -1
  82. package/dist/esm/classes/queue-events.js.map +1 -1
  83. package/dist/esm/classes/queue-keys.js +1 -0
  84. package/dist/esm/classes/queue-keys.js.map +1 -1
  85. package/dist/esm/classes/queue.d.ts +1 -1
  86. package/dist/esm/classes/queue.js +1 -1
  87. package/dist/esm/classes/redis-connection.js +9 -7
  88. package/dist/esm/classes/redis-connection.js.map +1 -1
  89. package/dist/esm/classes/repeat.js +22 -5
  90. package/dist/esm/classes/repeat.js.map +1 -1
  91. package/dist/esm/classes/scripts.d.ts +4 -4
  92. package/dist/esm/classes/scripts.js +34 -34
  93. package/dist/esm/classes/scripts.js.map +1 -1
  94. package/dist/esm/classes/worker.d.ts +1 -1
  95. package/dist/esm/classes/worker.js +27 -32
  96. package/dist/esm/classes/worker.js.map +1 -1
  97. package/dist/esm/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
  98. package/dist/esm/commands/addParentJob-4.lua +0 -2
  99. package/dist/{cjs/commands/addPrioritizedJob-8.lua → esm/commands/addPrioritizedJob-7.lua} +18 -24
  100. package/dist/esm/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
  101. package/dist/esm/commands/changePriority-6.lua +57 -0
  102. package/dist/esm/commands/getCounts-1.lua +1 -0
  103. package/dist/esm/commands/getRanges-1.lua +1 -0
  104. package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
  105. package/dist/esm/commands/includes/addJobWithPriority.lua +3 -7
  106. package/dist/esm/commands/includes/checkStalledJobs.lua +1 -1
  107. package/dist/esm/commands/includes/isQueuePaused.lua +7 -0
  108. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
  109. package/dist/esm/commands/includes/prepareJobForProcessing.lua +1 -1
  110. package/dist/esm/commands/includes/promoteDelayedJobs.lua +7 -4
  111. package/dist/{cjs/commands/moveToActive-10.lua → esm/commands/moveToActive-11.lua} +18 -29
  112. package/dist/esm/commands/moveToDelayed-7.lua +77 -0
  113. package/dist/esm/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
  114. package/dist/esm/commands/moveToWaitingChildren-4.lua +7 -4
  115. package/dist/esm/commands/pause-7.lua +41 -0
  116. package/dist/esm/commands/promote-8.lua +60 -0
  117. package/dist/esm/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
  118. package/dist/esm/interfaces/job-json.d.ts +4 -1
  119. package/dist/esm/interfaces/minimal-job.d.ts +3 -0
  120. package/dist/esm/interfaces/queue-options.d.ts +1 -1
  121. package/dist/esm/interfaces/repeat-options.d.ts +7 -0
  122. package/dist/esm/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
  123. package/dist/esm/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
  124. package/dist/esm/scripts/addParentJob-4.js +51 -61
  125. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  126. package/dist/esm/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
  127. package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -0
  128. package/dist/esm/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
  129. package/dist/esm/scripts/addStandardJob-7.js.map +1 -0
  130. package/dist/esm/scripts/changePriority-6.js +82 -0
  131. package/dist/esm/scripts/changePriority-6.js.map +1 -0
  132. package/dist/esm/scripts/getCounts-1.js +1 -0
  133. package/dist/esm/scripts/getCounts-1.js.map +1 -1
  134. package/dist/esm/scripts/getRanges-1.js +1 -0
  135. package/dist/esm/scripts/getRanges-1.js.map +1 -1
  136. package/dist/esm/scripts/index.d.ts +10 -10
  137. package/dist/esm/scripts/index.js +10 -10
  138. package/dist/esm/scripts/index.js.map +1 -1
  139. package/dist/esm/scripts/moveStalledJobsToWait-8.js +1 -0
  140. package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
  141. package/dist/esm/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
  142. package/dist/esm/scripts/moveToActive-11.js.map +1 -0
  143. package/dist/esm/scripts/moveToDelayed-7.js +100 -0
  144. package/dist/esm/scripts/moveToDelayed-7.js.map +1 -0
  145. package/dist/esm/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
  146. package/dist/esm/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
  147. package/dist/esm/scripts/moveToWaitingChildren-4.js +6 -3
  148. package/dist/esm/scripts/moveToWaitingChildren-4.js.map +1 -1
  149. package/dist/esm/scripts/pause-7.js +64 -0
  150. package/dist/esm/scripts/pause-7.js.map +1 -0
  151. package/dist/esm/scripts/promote-8.js +76 -0
  152. package/dist/esm/scripts/promote-8.js.map +1 -0
  153. package/dist/esm/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
  154. package/dist/esm/scripts/retryJob-10.js.map +1 -0
  155. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  156. package/package.json +2 -1
  157. package/dist/cjs/commands/changePriority-5.lua +0 -52
  158. package/dist/cjs/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
  159. package/dist/cjs/commands/moveToDelayed-8.lua +0 -72
  160. package/dist/cjs/commands/pause-5.lua +0 -36
  161. package/dist/cjs/commands/promote-7.lua +0 -57
  162. package/dist/cjs/scripts/addDelayedJob-7.js.map +0 -1
  163. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +0 -1
  164. package/dist/cjs/scripts/addStandardJob-6.js.map +0 -1
  165. package/dist/cjs/scripts/changePriority-5.js +0 -87
  166. package/dist/cjs/scripts/changePriority-5.js.map +0 -1
  167. package/dist/cjs/scripts/moveToDelayed-8.js +0 -168
  168. package/dist/cjs/scripts/moveToDelayed-8.js.map +0 -1
  169. package/dist/cjs/scripts/pause-5.js +0 -48
  170. package/dist/cjs/scripts/pause-5.js.map +0 -1
  171. package/dist/cjs/scripts/promote-7.js +0 -87
  172. package/dist/cjs/scripts/promote-7.js.map +0 -1
  173. package/dist/cjs/scripts/retryJob-9.js.map +0 -1
  174. package/dist/esm/commands/changePriority-5.lua +0 -52
  175. package/dist/esm/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
  176. package/dist/esm/commands/moveToDelayed-8.lua +0 -72
  177. package/dist/esm/commands/pause-5.lua +0 -36
  178. package/dist/esm/commands/promote-7.lua +0 -57
  179. package/dist/esm/scripts/addDelayedJob-7.js.map +0 -1
  180. package/dist/esm/scripts/addPrioritizedJob-8.js.map +0 -1
  181. package/dist/esm/scripts/addStandardJob-6.js.map +0 -1
  182. package/dist/esm/scripts/changePriority-5.js +0 -84
  183. package/dist/esm/scripts/changePriority-5.js.map +0 -1
  184. package/dist/esm/scripts/moveToDelayed-8.js +0 -165
  185. package/dist/esm/scripts/moveToDelayed-8.js.map +0 -1
  186. package/dist/esm/scripts/pause-5.js +0 -45
  187. package/dist/esm/scripts/pause-5.js.map +0 -1
  188. package/dist/esm/scripts/promote-7.js +0 -84
  189. package/dist/esm/scripts/promote-7.js.map +0 -1
  190. package/dist/esm/scripts/retryJob-9.js.map +0 -1
  191. /package/dist/esm/scripts/{addDelayedJob-7.d.ts → addDelayedJob-6.d.ts} +0 -0
  192. /package/dist/esm/scripts/{addPrioritizedJob-8.d.ts → addPrioritizedJob-7.d.ts} +0 -0
  193. /package/dist/esm/scripts/{addStandardJob-6.d.ts → addStandardJob-7.d.ts} +0 -0
  194. /package/dist/esm/scripts/{changePriority-5.d.ts → changePriority-6.d.ts} +0 -0
  195. /package/dist/esm/scripts/{moveToActive-10.d.ts → moveToActive-11.d.ts} +0 -0
  196. /package/dist/esm/scripts/{moveToDelayed-8.d.ts → moveToDelayed-7.d.ts} +0 -0
  197. /package/dist/esm/scripts/{moveToFinished-13.d.ts → moveToFinished-14.d.ts} +0 -0
  198. /package/dist/esm/scripts/{pause-5.d.ts → pause-7.d.ts} +0 -0
  199. /package/dist/esm/scripts/{promote-7.d.ts → promote-8.d.ts} +0 -0
  200. /package/dist/esm/scripts/{retryJob-9.d.ts → retryJob-10.d.ts} +0 -0
@@ -22,22 +22,21 @@
22
22
  KEYS[11] completed/failed key
23
23
  KEYS[12] jobId key
24
24
  KEYS[13] metrics key
25
+ KEYS[14] marker key
25
26
 
26
27
  ARGV[1] jobId
27
28
  ARGV[2] timestamp
28
29
  ARGV[3] msg property returnvalue / failedReason
29
30
  ARGV[4] return value / failed reason
30
31
  ARGV[5] target (completed/failed)
31
- ARGV[6] event data (? maybe just send jobid).
32
- ARGV[7] fetch next?
33
- ARGV[8] keys prefix
34
- ARGV[9] opts
32
+ ARGV[6] fetch next?
33
+ ARGV[7] keys prefix
34
+ ARGV[8] opts
35
35
 
36
36
  opts - token - lock token
37
37
  opts - keepJobs
38
38
  opts - lockDuration - lock duration in milliseconds
39
39
  opts - attempts max attempts
40
- opts - attemptsMade
41
40
  opts - maxMetricsSize
42
41
  opts - fpof - fail parent on fail
43
42
  opts - rdof - remove dependency on fail
@@ -72,11 +71,10 @@ local rcall = redis.call
72
71
 
73
72
  local jobIdKey = KEYS[12]
74
73
  if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
75
- local opts = cmsgpack.unpack(ARGV[9])
74
+ local opts = cmsgpack.unpack(ARGV[8])
76
75
 
77
76
  local token = opts['token']
78
77
  local attempts = opts['attempts']
79
- local attemptsMade = opts['attemptsMade']
80
78
  local maxMetricsSize = opts['maxMetricsSize']
81
79
  local maxCount = opts['keepJobs']['count']
82
80
  local maxAge = opts['keepJobs']['age']
@@ -120,8 +118,9 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
120
118
 
121
119
  if (numRemovedElements < 1) then return -3 end
122
120
 
121
+ local metaKey = KEYS[9]
123
122
  -- Trim events before emiting them to avoid trimming events emitted in this script
124
- trimEvents(KEYS[9], KEYS[4])
123
+ trimEvents(metaKey, KEYS[4])
125
124
 
126
125
  -- If job has a parent we need to
127
126
  -- 1) remove this job id from parents dependencies
@@ -144,17 +143,20 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
144
143
  else
145
144
  if opts['fpof'] then
146
145
  moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
147
- parentId, jobIdKey, timestamp)
146
+ parentId, jobIdKey,
147
+ timestamp)
148
148
  elseif opts['rdof'] then
149
149
  local dependenciesSet = parentKey .. ":dependencies"
150
150
  if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
151
151
  moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
152
- parentKey, parentId, timestamp)
152
+ parentKey, parentId, timestamp)
153
153
  end
154
154
  end
155
155
  end
156
156
  end
157
157
 
158
+ local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
159
+
158
160
  -- Remove job?
159
161
  if maxCount ~= 0 then
160
162
  local targetSet = KEYS[11]
@@ -164,7 +166,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
164
166
  -- "returnvalue" / "failedReason" and "finishedOn"
165
167
 
166
168
  -- Remove old jobs?
167
- local prefix = ARGV[8]
169
+ local prefix = ARGV[7]
168
170
 
169
171
  if maxAge ~= nil then
170
172
  removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
@@ -197,13 +199,13 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
197
199
 
198
200
  -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
199
201
  -- and not rate limited.
200
- if (ARGV[7] == "1") then
202
+ if (ARGV[6] == "1") then
201
203
 
202
- local target, paused = getTargetQueueList(KEYS[9], KEYS[1], KEYS[8])
204
+ local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[8])
203
205
 
204
206
  -- Check if there are delayed jobs that can be promoted
205
- promoteDelayedJobs(KEYS[7], KEYS[1], target, KEYS[3],
206
- KEYS[4], ARGV[8], timestamp, paused, KEYS[10])
207
+ promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], KEYS[4], ARGV[7],
208
+ timestamp, KEYS[10], paused)
207
209
 
208
210
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
209
211
  -- Check if we are rate limited first.
@@ -217,25 +219,30 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
217
219
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
218
220
 
219
221
  if jobId then
222
+ -- Markers in waitlist DEPRECATED in v5: Remove in v6.
220
223
  if string.sub(jobId, 1, 2) == "0:" then
221
224
  rcall("LREM", KEYS[2], 1, jobId)
222
225
 
223
226
  -- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
224
227
  -- but if ID is 0:0, then there is at least 1 prioritized job to process
225
228
  if jobId == "0:0" then
226
- jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
227
- return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp,
228
- maxJobs, expireTime, opts)
229
+ jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
230
+ KEYS[10])
231
+ return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
232
+ timestamp, maxJobs,
233
+ expireTime, opts)
229
234
  end
230
235
  else
231
- return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
232
- expireTime, opts)
236
+ return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
237
+ timestamp, maxJobs, expireTime,
238
+ opts)
233
239
  end
234
240
  else
235
241
  jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
236
242
  if jobId then
237
- return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
238
- expireTime, opts)
243
+ return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
244
+ timestamp, maxJobs, expireTime,
245
+ opts)
239
246
  end
240
247
  end
241
248
 
@@ -11,7 +11,7 @@
11
11
  ARGV[2] child key
12
12
  ARGV[3] timestamp
13
13
  ARGV[4] the id of the job
14
-
14
+
15
15
  Output:
16
16
  0 - OK
17
17
  1 - There are not pending dependencies.
@@ -21,7 +21,8 @@
21
21
  ]]
22
22
  local rcall = redis.call
23
23
 
24
- local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp, lockKey, token)
24
+ local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp,
25
+ lockKey, jobKey, token)
25
26
  if token ~= "0" then
26
27
  if rcall("GET", lockKey) == token then
27
28
  rcall("DEL", lockKey)
@@ -46,13 +47,15 @@ end
46
47
  if rcall("EXISTS", KEYS[4]) == 1 then
47
48
  if ARGV[2] ~= "" then
48
49
  if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
49
- return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
50
+ return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], KEYS[4],
51
+ ARGV[1])
50
52
  end
51
53
 
52
54
  return 1
53
55
  else
54
56
  if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
55
- return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
57
+ return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], KEYS[4],
58
+ ARGV[1])
56
59
  end
57
60
 
58
61
  return 1
@@ -0,0 +1,41 @@
1
+ --[[
2
+ Pauses or resumes a queue globably.
3
+
4
+ Input:
5
+ KEYS[1] 'wait' or 'paused''
6
+ KEYS[2] 'paused' or 'wait'
7
+ KEYS[3] 'meta'
8
+ KEYS[4] 'prioritized'
9
+ KEYS[5] events stream key
10
+ KEYS[6] 'delayed'
11
+ KEYS|7] 'marker'
12
+
13
+ ARGV[1] 'paused' or 'resumed'
14
+
15
+ Event:
16
+ publish paused or resumed event.
17
+ ]]
18
+ local rcall = redis.call
19
+
20
+ -- Includes
21
+ --- @include "includes/addDelayMarkerIfNeeded"
22
+
23
+ local markerKey = KEYS[7]
24
+ local hasJobs = rcall("EXISTS", KEYS[1]) == 1
25
+ if hasJobs then rcall("RENAME", KEYS[1], KEYS[2]) end
26
+
27
+ if ARGV[1] == "paused" then
28
+ rcall("HSET", KEYS[3], "paused", 1)
29
+ rcall("DEL", markerKey)
30
+ else
31
+ rcall("HDEL", KEYS[3], "paused")
32
+
33
+ if hasJobs or rcall("ZCARD", KEYS[4]) > 0 then
34
+ -- Add marker if there are waiting or priority jobs
35
+ rcall("ZADD", markerKey, 0, "0")
36
+ else
37
+ addDelayMarkerIfNeeded(markerKey, KEYS[6])
38
+ end
39
+ end
40
+
41
+ rcall("XADD", KEYS[5], "*", "event", ARGV[1]);
@@ -0,0 +1,60 @@
1
+ --[[
2
+ Promotes a job that is currently "delayed" to the "waiting" state
3
+
4
+ Input:
5
+ KEYS[1] 'delayed'
6
+ KEYS[2] 'wait'
7
+ KEYS[3] 'paused'
8
+ KEYS[4] 'meta'
9
+ KEYS[5] 'prioritized'
10
+ KEYS[6] 'pc' priority counter
11
+ KEYS[7] 'event stream'
12
+ KEYS[8] 'marker'
13
+
14
+ ARGV[1] queue.toKey('')
15
+ ARGV[2] jobId
16
+
17
+ Output:
18
+ 0 - OK
19
+ -3 - Job not in delayed zset.
20
+
21
+ Events:
22
+ 'waiting'
23
+ ]]
24
+ local rcall = redis.call
25
+ local jobId = ARGV[2]
26
+
27
+ -- Includes
28
+ --- @include "includes/addJobWithPriority"
29
+ --- @include "includes/getTargetQueueList"
30
+
31
+ if rcall("ZREM", KEYS[1], jobId) == 1 then
32
+ local jobKey = ARGV[1] .. jobId
33
+ local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
34
+ local metaKey = KEYS[4]
35
+
36
+ -- Remove delayed "marker" from the wait list if there is any.
37
+ -- Since we are adding a job we do not need the marker anymore.
38
+ -- Markers in waitlist DEPRECATED in v5: Remove in v6.
39
+ local target, paused = getTargetQueueList(metaKey, KEYS[2], KEYS[3])
40
+ local marker = rcall("LINDEX", target, 0)
41
+ if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
42
+
43
+ if priority == 0 then
44
+ -- LIFO or FIFO
45
+ rcall("LPUSH", target, jobId)
46
+ if not paused then rcall("ZADD", KEYS[8], 0, "0") end
47
+ else
48
+ addJobWithPriority(KEYS[8], KEYS[5], priority, jobId, KEYS[6], paused)
49
+ end
50
+
51
+ -- Emit waiting event (wait..ing@token)
52
+ rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev",
53
+ "delayed");
54
+
55
+ rcall("HSET", jobKey, "delay", 0)
56
+
57
+ return 0
58
+ else
59
+ return -3
60
+ end
@@ -2,15 +2,16 @@
2
2
  Retries a failed job by moving it back to the wait queue.
3
3
 
4
4
  Input:
5
- KEYS[1] 'active',
6
- KEYS[2] 'wait'
7
- KEYS[3] 'paused'
8
- KEYS[4] job key
9
- KEYS[5] 'meta'
10
- KEYS[6] events stream
11
- KEYS[7] delayed key
12
- KEYS[8] prioritized key
13
- KEYS[9] 'pc' priority counter
5
+ KEYS[1] 'active',
6
+ KEYS[2] 'wait'
7
+ KEYS[3] 'paused'
8
+ KEYS[4] job key
9
+ KEYS[5] 'meta'
10
+ KEYS[6] events stream
11
+ KEYS[7] delayed key
12
+ KEYS[8] prioritized key
13
+ KEYS[9] 'pc' priority counter
14
+ KEYS[10] 'marker'
14
15
 
15
16
  ARGV[1] key prefix
16
17
  ARGV[2] timestamp
@@ -34,9 +35,11 @@ local rcall = redis.call
34
35
  --- @include "includes/promoteDelayedJobs"
35
36
 
36
37
  local target, paused = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
38
+ local markerKey = KEYS[10]
39
+
37
40
  -- Check if there are delayed jobs that we can move to wait.
38
41
  -- test example: when there are delayed jobs between retries
39
- promoteDelayedJobs(KEYS[7], KEYS[2], target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], paused, KEYS[9])
42
+ promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], paused)
40
43
 
41
44
  if rcall("EXISTS", KEYS[4]) == 1 then
42
45
 
@@ -57,9 +60,11 @@ if rcall("EXISTS", KEYS[4]) == 1 then
57
60
  if priority == 0 then
58
61
  rcall(ARGV[3], target, ARGV[4])
59
62
  else
60
- addJobWithPriority(KEYS[2], KEYS[8], priority, paused, ARGV[4], KEYS[9])
63
+ addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], paused)
61
64
  end
62
65
 
66
+ rcall("HINCRBY", KEYS[4], "atm", 1)
67
+
63
68
  local maxEvents = rcall("HGET", KEYS[5], "opts.maxLenEvents") or 10000
64
69
 
65
70
  -- Emit waiting event
@@ -9,13 +9,12 @@ const content = `--[[
9
9
  - adds to delayed zset.
10
10
  - Emits a global event 'delayed' if the job is delayed.
11
11
  Input:
12
- KEYS[1] 'wait',
13
- KEYS[2] 'paused'
14
- KEYS[3] 'meta'
15
- KEYS[4] 'id'
16
- KEYS[5] 'delayed'
17
- KEYS[6] 'completed'
18
- KEYS[7] events stream key
12
+ KEYS[1] 'marker',
13
+ KEYS[2] 'meta'
14
+ KEYS[3] 'id'
15
+ KEYS[4] 'delayed'
16
+ KEYS[5] 'completed'
17
+ KEYS[6] events stream key
19
18
  ARGV[1] msgpacked arguments array
20
19
  [1] key prefix,
21
20
  [2] custom id (use custom instead of one generated automatically)
@@ -32,13 +31,11 @@ const content = `--[[
32
31
  jobId - OK
33
32
  -5 - Missing parent key
34
33
  ]]
35
- local waitKey = KEYS[1]
36
- local pausedKey = KEYS[2]
37
- local metaKey = KEYS[3]
38
- local idKey = KEYS[4]
39
- local delayedKey = KEYS[5]
40
- local completedKey = KEYS[6]
41
- local eventsKey = KEYS[7]
34
+ local metaKey = KEYS[2]
35
+ local idKey = KEYS[3]
36
+ local delayedKey = KEYS[4]
37
+ local completedKey = KEYS[5]
38
+ local eventsKey = KEYS[6]
42
39
  local jobId
43
40
  local jobIdKey
44
41
  local rcall = redis.call
@@ -77,7 +74,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
77
74
  end
78
75
  --[[
79
76
  Add delay marker if needed.
80
- ]]
77
+ ]]
81
78
  -- Includes
82
79
  --[[
83
80
  Function to return the next delayed job timestamp.
@@ -92,36 +89,20 @@ local function getNextDelayedTimestamp(delayedKey)
92
89
  return nextTimestamp
93
90
  end
94
91
  end
95
- local function addDelayMarkerIfNeeded(targetKey, delayedKey)
96
- local waitLen = rcall("LLEN", targetKey)
97
- if waitLen <= 1 then
92
+ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
98
93
  local nextTimestamp = getNextDelayedTimestamp(delayedKey)
99
94
  if nextTimestamp ~= nil then
100
- -- Check if there is already a marker with older timestamp
101
- -- if there is, we need to replace it.
102
- if waitLen == 1 then
103
- local marker = rcall("LINDEX", targetKey, 0)
104
- local oldTimestamp = tonumber(marker:sub(3))
105
- if oldTimestamp and oldTimestamp > nextTimestamp then
106
- rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
107
- end
108
- else
109
- -- if there is no marker, then we need to add one
110
- rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
111
- end
95
+ -- Replace the score of the marker with the newest known
96
+ -- next timestamp.
97
+ rcall("ZADD", markerKey, nextTimestamp, "0")
112
98
  end
113
- end
114
99
  end
115
100
  --[[
116
101
  Function to check for the meta.paused key to decide if we are paused or not
117
102
  (since an empty list and !EXISTS are not really the same).
118
103
  ]]
119
- local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
120
- if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
121
- return waitKey, false
122
- else
123
- return pausedKey, true
124
- end
104
+ local function isQueuePaused(queueMetaKey)
105
+ return rcall("HEXISTS", queueMetaKey, "paused") == 1
125
106
  end
126
107
  --[[
127
108
  Validate and move or add dependencies to parent.
@@ -134,56 +115,64 @@ end
134
115
  --[[
135
116
  Function to add job considering priority.
136
117
  ]]
137
- -- Includes
138
- --[[
139
- Function priority marker to wait if needed
140
- in order to wake up our workers and to respect priority
141
- order as much as possible
142
- ]]
143
- local function addPriorityMarkerIfNeeded(waitKey)
144
- local waitLen = rcall("LLEN", waitKey)
145
- if waitLen == 0 then
146
- rcall("LPUSH", waitKey, "0:0")
147
- end
148
- end
149
- local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
118
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
150
119
  local prioCounter = rcall("INCR", priorityCounterKey)
151
120
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
152
121
  rcall("ZADD", prioritizedKey, score, jobId)
153
- if not paused then
154
- addPriorityMarkerIfNeeded(waitKey)
122
+ if not isPaused then
123
+ rcall("ZADD", markerKey, 0, "0")
155
124
  end
156
125
  end
157
- local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
158
- local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
159
- if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
160
- rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
161
- local parentWaitKey = parentQueueKey .. ":wait"
162
- local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
163
- parentQueueKey .. ":paused")
164
- local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
165
- local priority = tonumber(jobAttributes[1]) or 0
166
- local delay = tonumber(jobAttributes[2]) or 0
167
- if delay > 0 then
168
- local delayedTimestamp = tonumber(timestamp) + delay
169
- local score = delayedTimestamp * 0x1000
170
- local parentDelayedKey = parentQueueKey .. ":delayed"
171
- rcall("ZADD", parentDelayedKey, score, parentId)
172
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
173
- "delay", delayedTimestamp)
174
- addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
175
- else
176
- if priority == 0 then
177
- rcall("RPUSH", parentTarget, parentId)
178
- else
179
- addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
180
- parentId, parentQueueKey .. ":pc")
181
- end
182
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
183
- "prev", "waiting-children")
184
- end
126
+ --[[
127
+ Function to check for the meta.paused key to decide if we are paused or not
128
+ (since an empty list and !EXISTS are not really the same).
129
+ ]]
130
+ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
131
+ if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
132
+ return waitKey, false
133
+ else
134
+ return pausedKey, true
185
135
  end
186
136
  end
137
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
138
+ parentKey, parentId, timestamp)
139
+ local isParentActive = rcall("ZSCORE",
140
+ parentQueueKey .. ":waiting-children", parentId)
141
+ if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
142
+ rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
143
+ local parentWaitKey = parentQueueKey .. ":wait"
144
+ local parentPausedKey = parentQueueKey .. ":paused"
145
+ local parentMetaKey = parentQueueKey .. ":meta"
146
+ local parentMarkerKey = parentQueueKey .. ":marker"
147
+ local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
148
+ local priority = tonumber(jobAttributes[1]) or 0
149
+ local delay = tonumber(jobAttributes[2]) or 0
150
+ if delay > 0 then
151
+ local delayedTimestamp = tonumber(timestamp) + delay
152
+ local score = delayedTimestamp * 0x1000
153
+ local parentDelayedKey = parentQueueKey .. ":delayed"
154
+ rcall("ZADD", parentDelayedKey, score, parentId)
155
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
156
+ "jobId", parentId, "delay", delayedTimestamp)
157
+ addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
158
+ else
159
+ if priority == 0 then
160
+ local parentTarget, _paused =
161
+ getTargetQueueList(parentMetaKey, parentWaitKey,
162
+ parentPausedKey)
163
+ rcall("RPUSH", parentTarget, parentId)
164
+ rcall("ZADD", parentMarkerKey, 0, "0")
165
+ else
166
+ local isPaused = isQueuePaused(parentMetaKey)
167
+ addJobWithPriority(parentMarkerKey,
168
+ parentQueueKey .. ":prioritized", priority,
169
+ parentId, parentQueueKey .. ":pc", isPaused)
170
+ end
171
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
172
+ "jobId", parentId, "prev", "waiting-children")
173
+ end
174
+ end
175
+ end
187
176
  local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
188
177
  parentId, jobIdKey, returnvalue, timestamp )
189
178
  local processedSet = parentKey .. ":processed"
@@ -255,13 +244,13 @@ local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
255
244
  rcall("ZADD", delayedKey, score, jobId)
256
245
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
257
246
  "jobId", jobId, "delay", delayedTimestamp)
258
- -- If wait list is empty, and this delayed job is the next one to be processed,
259
- -- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
260
- local target = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
261
- addDelayMarkerIfNeeded(target, delayedKey)
247
+ -- mark that a delayed job is available
248
+ local isPaused = isQueuePaused(metaKey)
249
+ if not isPaused then
250
+ local markerKey = KEYS[1]
251
+ addDelayMarkerIfNeeded(markerKey, delayedKey)
252
+ end
262
253
  -- Check if this job is a child of another job, if so add it to the parents dependencies
263
- -- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
264
- -- fail in this case.
265
254
  if parentDependenciesKey ~= nil then
266
255
  rcall("SADD", parentDependenciesKey, jobIdKey)
267
256
  end
@@ -270,6 +259,6 @@ return jobId .. "" -- convert to string
270
259
  exports.addDelayedJob = {
271
260
  name: 'addDelayedJob',
272
261
  content,
273
- keys: 7,
262
+ keys: 6,
274
263
  };
275
- //# sourceMappingURL=addDelayedJob-7.js.map
264
+ //# sourceMappingURL=addDelayedJob-6.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"moveToActive-10.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-10.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmPf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Pf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}