bullmq 4.16.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/dist/cjs/classes/flow-producer.js +3 -10
  2. package/dist/cjs/classes/flow-producer.js.map +1 -1
  3. package/dist/cjs/classes/job.js +21 -11
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/queue-base.js +2 -9
  6. package/dist/cjs/classes/queue-base.js.map +1 -1
  7. package/dist/cjs/classes/queue-events.js +3 -1
  8. package/dist/cjs/classes/queue-events.js.map +1 -1
  9. package/dist/cjs/classes/queue-keys.js +1 -0
  10. package/dist/cjs/classes/queue-keys.js.map +1 -1
  11. package/dist/cjs/classes/redis-connection.js +9 -7
  12. package/dist/cjs/classes/redis-connection.js.map +1 -1
  13. package/dist/cjs/classes/scripts.js +34 -34
  14. package/dist/cjs/classes/scripts.js.map +1 -1
  15. package/dist/cjs/classes/worker.js +41 -37
  16. package/dist/cjs/classes/worker.js.map +1 -1
  17. package/dist/cjs/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
  18. package/dist/cjs/commands/addParentJob-4.lua +0 -2
  19. package/dist/{esm/commands/addPrioritizedJob-8.lua → cjs/commands/addPrioritizedJob-7.lua} +18 -24
  20. package/dist/cjs/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
  21. package/dist/cjs/commands/changePriority-6.lua +57 -0
  22. package/dist/cjs/commands/getCounts-1.lua +1 -0
  23. package/dist/cjs/commands/getRanges-1.lua +1 -0
  24. package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
  25. package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -7
  26. package/dist/cjs/commands/includes/checkStalledJobs.lua +1 -1
  27. package/dist/cjs/commands/includes/isQueuePaused.lua +7 -0
  28. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
  29. package/dist/cjs/commands/includes/prepareJobForProcessing.lua +1 -1
  30. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +7 -4
  31. package/dist/{esm/commands/moveToActive-10.lua → cjs/commands/moveToActive-11.lua} +18 -29
  32. package/dist/cjs/commands/moveToDelayed-7.lua +77 -0
  33. package/dist/cjs/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
  34. package/dist/cjs/commands/moveToWaitingChildren-4.lua +7 -4
  35. package/dist/cjs/commands/pause-7.lua +41 -0
  36. package/dist/cjs/commands/promote-8.lua +60 -0
  37. package/dist/cjs/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
  38. package/dist/cjs/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
  39. package/dist/cjs/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
  40. package/dist/cjs/scripts/addParentJob-4.js +51 -61
  41. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  42. package/dist/cjs/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
  43. package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -0
  44. package/dist/cjs/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
  45. package/dist/cjs/scripts/addStandardJob-7.js.map +1 -0
  46. package/dist/cjs/scripts/changePriority-6.js +85 -0
  47. package/dist/cjs/scripts/changePriority-6.js.map +1 -0
  48. package/dist/cjs/scripts/getCounts-1.js +1 -0
  49. package/dist/cjs/scripts/getCounts-1.js.map +1 -1
  50. package/dist/cjs/scripts/getRanges-1.js +1 -0
  51. package/dist/cjs/scripts/getRanges-1.js.map +1 -1
  52. package/dist/cjs/scripts/index.js +10 -10
  53. package/dist/cjs/scripts/index.js.map +1 -1
  54. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +1 -0
  55. package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
  56. package/dist/cjs/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
  57. package/dist/cjs/scripts/moveToActive-11.js.map +1 -0
  58. package/dist/cjs/scripts/moveToDelayed-7.js +103 -0
  59. package/dist/cjs/scripts/moveToDelayed-7.js.map +1 -0
  60. package/dist/cjs/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
  61. package/dist/cjs/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
  62. package/dist/cjs/scripts/moveToWaitingChildren-4.js +6 -3
  63. package/dist/cjs/scripts/moveToWaitingChildren-4.js.map +1 -1
  64. package/dist/cjs/scripts/pause-7.js +67 -0
  65. package/dist/cjs/scripts/pause-7.js.map +1 -0
  66. package/dist/cjs/scripts/promote-8.js +79 -0
  67. package/dist/cjs/scripts/promote-8.js.map +1 -0
  68. package/dist/cjs/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
  69. package/dist/cjs/scripts/retryJob-10.js.map +1 -0
  70. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  71. package/dist/esm/classes/flow-producer.js +3 -10
  72. package/dist/esm/classes/flow-producer.js.map +1 -1
  73. package/dist/esm/classes/job.d.ts +5 -0
  74. package/dist/esm/classes/job.js +21 -11
  75. package/dist/esm/classes/job.js.map +1 -1
  76. package/dist/esm/classes/queue-base.js +2 -9
  77. package/dist/esm/classes/queue-base.js.map +1 -1
  78. package/dist/esm/classes/queue-events.js +3 -1
  79. package/dist/esm/classes/queue-events.js.map +1 -1
  80. package/dist/esm/classes/queue-keys.js +1 -0
  81. package/dist/esm/classes/queue-keys.js.map +1 -1
  82. package/dist/esm/classes/redis-connection.js +9 -7
  83. package/dist/esm/classes/redis-connection.js.map +1 -1
  84. package/dist/esm/classes/scripts.d.ts +4 -4
  85. package/dist/esm/classes/scripts.js +34 -34
  86. package/dist/esm/classes/scripts.js.map +1 -1
  87. package/dist/esm/classes/worker.d.ts +4 -2
  88. package/dist/esm/classes/worker.js +41 -37
  89. package/dist/esm/classes/worker.js.map +1 -1
  90. package/dist/esm/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
  91. package/dist/esm/commands/addParentJob-4.lua +0 -2
  92. package/dist/{cjs/commands/addPrioritizedJob-8.lua → esm/commands/addPrioritizedJob-7.lua} +18 -24
  93. package/dist/esm/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
  94. package/dist/esm/commands/changePriority-6.lua +57 -0
  95. package/dist/esm/commands/getCounts-1.lua +1 -0
  96. package/dist/esm/commands/getRanges-1.lua +1 -0
  97. package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
  98. package/dist/esm/commands/includes/addJobWithPriority.lua +3 -7
  99. package/dist/esm/commands/includes/checkStalledJobs.lua +1 -1
  100. package/dist/esm/commands/includes/isQueuePaused.lua +7 -0
  101. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
  102. package/dist/esm/commands/includes/prepareJobForProcessing.lua +1 -1
  103. package/dist/esm/commands/includes/promoteDelayedJobs.lua +7 -4
  104. package/dist/{cjs/commands/moveToActive-10.lua → esm/commands/moveToActive-11.lua} +18 -29
  105. package/dist/esm/commands/moveToDelayed-7.lua +77 -0
  106. package/dist/esm/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
  107. package/dist/esm/commands/moveToWaitingChildren-4.lua +7 -4
  108. package/dist/esm/commands/pause-7.lua +41 -0
  109. package/dist/esm/commands/promote-8.lua +60 -0
  110. package/dist/esm/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
  111. package/dist/esm/interfaces/job-json.d.ts +4 -1
  112. package/dist/esm/interfaces/minimal-job.d.ts +3 -0
  113. package/dist/esm/interfaces/queue-options.d.ts +1 -1
  114. package/dist/esm/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
  115. package/dist/esm/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
  116. package/dist/esm/scripts/addParentJob-4.js +51 -61
  117. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  118. package/dist/esm/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
  119. package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -0
  120. package/dist/esm/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
  121. package/dist/esm/scripts/addStandardJob-7.js.map +1 -0
  122. package/dist/esm/scripts/changePriority-6.js +82 -0
  123. package/dist/esm/scripts/changePriority-6.js.map +1 -0
  124. package/dist/esm/scripts/getCounts-1.js +1 -0
  125. package/dist/esm/scripts/getCounts-1.js.map +1 -1
  126. package/dist/esm/scripts/getRanges-1.js +1 -0
  127. package/dist/esm/scripts/getRanges-1.js.map +1 -1
  128. package/dist/esm/scripts/index.d.ts +10 -10
  129. package/dist/esm/scripts/index.js +10 -10
  130. package/dist/esm/scripts/index.js.map +1 -1
  131. package/dist/esm/scripts/moveStalledJobsToWait-8.js +1 -0
  132. package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
  133. package/dist/esm/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
  134. package/dist/esm/scripts/moveToActive-11.js.map +1 -0
  135. package/dist/esm/scripts/moveToDelayed-7.js +100 -0
  136. package/dist/esm/scripts/moveToDelayed-7.js.map +1 -0
  137. package/dist/esm/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
  138. package/dist/esm/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
  139. package/dist/esm/scripts/moveToWaitingChildren-4.js +6 -3
  140. package/dist/esm/scripts/moveToWaitingChildren-4.js.map +1 -1
  141. package/dist/esm/scripts/pause-7.js +64 -0
  142. package/dist/esm/scripts/pause-7.js.map +1 -0
  143. package/dist/esm/scripts/promote-8.js +76 -0
  144. package/dist/esm/scripts/promote-8.js.map +1 -0
  145. package/dist/esm/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
  146. package/dist/esm/scripts/retryJob-10.js.map +1 -0
  147. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  148. package/package.json +1 -1
  149. package/dist/cjs/commands/changePriority-5.lua +0 -52
  150. package/dist/cjs/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
  151. package/dist/cjs/commands/moveToDelayed-8.lua +0 -72
  152. package/dist/cjs/commands/pause-5.lua +0 -36
  153. package/dist/cjs/commands/promote-7.lua +0 -57
  154. package/dist/cjs/scripts/addDelayedJob-7.js.map +0 -1
  155. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +0 -1
  156. package/dist/cjs/scripts/addStandardJob-6.js.map +0 -1
  157. package/dist/cjs/scripts/changePriority-5.js +0 -87
  158. package/dist/cjs/scripts/changePriority-5.js.map +0 -1
  159. package/dist/cjs/scripts/moveToDelayed-8.js +0 -168
  160. package/dist/cjs/scripts/moveToDelayed-8.js.map +0 -1
  161. package/dist/cjs/scripts/pause-5.js +0 -48
  162. package/dist/cjs/scripts/pause-5.js.map +0 -1
  163. package/dist/cjs/scripts/promote-7.js +0 -87
  164. package/dist/cjs/scripts/promote-7.js.map +0 -1
  165. package/dist/cjs/scripts/retryJob-9.js.map +0 -1
  166. package/dist/esm/commands/changePriority-5.lua +0 -52
  167. package/dist/esm/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
  168. package/dist/esm/commands/moveToDelayed-8.lua +0 -72
  169. package/dist/esm/commands/pause-5.lua +0 -36
  170. package/dist/esm/commands/promote-7.lua +0 -57
  171. package/dist/esm/scripts/addDelayedJob-7.js.map +0 -1
  172. package/dist/esm/scripts/addPrioritizedJob-8.js.map +0 -1
  173. package/dist/esm/scripts/addStandardJob-6.js.map +0 -1
  174. package/dist/esm/scripts/changePriority-5.js +0 -84
  175. package/dist/esm/scripts/changePriority-5.js.map +0 -1
  176. package/dist/esm/scripts/moveToDelayed-8.js +0 -165
  177. package/dist/esm/scripts/moveToDelayed-8.js.map +0 -1
  178. package/dist/esm/scripts/pause-5.js +0 -45
  179. package/dist/esm/scripts/pause-5.js.map +0 -1
  180. package/dist/esm/scripts/promote-7.js +0 -84
  181. package/dist/esm/scripts/promote-7.js.map +0 -1
  182. package/dist/esm/scripts/retryJob-9.js.map +0 -1
  183. /package/dist/esm/scripts/{addDelayedJob-7.d.ts → addDelayedJob-6.d.ts} +0 -0
  184. /package/dist/esm/scripts/{addPrioritizedJob-8.d.ts → addPrioritizedJob-7.d.ts} +0 -0
  185. /package/dist/esm/scripts/{addStandardJob-6.d.ts → addStandardJob-7.d.ts} +0 -0
  186. /package/dist/esm/scripts/{changePriority-5.d.ts → changePriority-6.d.ts} +0 -0
  187. /package/dist/esm/scripts/{moveToActive-10.d.ts → moveToActive-11.d.ts} +0 -0
  188. /package/dist/esm/scripts/{moveToDelayed-8.d.ts → moveToDelayed-7.d.ts} +0 -0
  189. /package/dist/esm/scripts/{moveToFinished-13.d.ts → moveToFinished-14.d.ts} +0 -0
  190. /package/dist/esm/scripts/{pause-5.d.ts → pause-7.d.ts} +0 -0
  191. /package/dist/esm/scripts/{promote-7.d.ts → promote-8.d.ts} +0 -0
  192. /package/dist/esm/scripts/{retryJob-9.d.ts → retryJob-10.d.ts} +0 -0
@@ -22,22 +22,21 @@
22
22
  KEYS[11] completed/failed key
23
23
  KEYS[12] jobId key
24
24
  KEYS[13] metrics key
25
+ KEYS[14] marker key
25
26
 
26
27
  ARGV[1] jobId
27
28
  ARGV[2] timestamp
28
29
  ARGV[3] msg property returnvalue / failedReason
29
30
  ARGV[4] return value / failed reason
30
31
  ARGV[5] target (completed/failed)
31
- ARGV[6] event data (? maybe just send jobid).
32
- ARGV[7] fetch next?
33
- ARGV[8] keys prefix
34
- ARGV[9] opts
32
+ ARGV[6] fetch next?
33
+ ARGV[7] keys prefix
34
+ ARGV[8] opts
35
35
 
36
36
  opts - token - lock token
37
37
  opts - keepJobs
38
38
  opts - lockDuration - lock duration in milliseconds
39
39
  opts - attempts max attempts
40
- opts - attemptsMade
41
40
  opts - maxMetricsSize
42
41
  opts - fpof - fail parent on fail
43
42
  opts - rdof - remove dependency on fail
@@ -72,11 +71,10 @@ local rcall = redis.call
72
71
 
73
72
  local jobIdKey = KEYS[12]
74
73
  if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
75
- local opts = cmsgpack.unpack(ARGV[9])
74
+ local opts = cmsgpack.unpack(ARGV[8])
76
75
 
77
76
  local token = opts['token']
78
77
  local attempts = opts['attempts']
79
- local attemptsMade = opts['attemptsMade']
80
78
  local maxMetricsSize = opts['maxMetricsSize']
81
79
  local maxCount = opts['keepJobs']['count']
82
80
  local maxAge = opts['keepJobs']['age']
@@ -120,8 +118,9 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
120
118
 
121
119
  if (numRemovedElements < 1) then return -3 end
122
120
 
121
+ local metaKey = KEYS[9]
123
122
  -- Trim events before emiting them to avoid trimming events emitted in this script
124
- trimEvents(KEYS[9], KEYS[4])
123
+ trimEvents(metaKey, KEYS[4])
125
124
 
126
125
  -- If job has a parent we need to
127
126
  -- 1) remove this job id from parents dependencies
@@ -144,17 +143,20 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
144
143
  else
145
144
  if opts['fpof'] then
146
145
  moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
147
- parentId, jobIdKey, timestamp)
146
+ parentId, jobIdKey,
147
+ timestamp)
148
148
  elseif opts['rdof'] then
149
149
  local dependenciesSet = parentKey .. ":dependencies"
150
150
  if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
151
151
  moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
152
- parentKey, parentId, timestamp)
152
+ parentKey, parentId, timestamp)
153
153
  end
154
154
  end
155
155
  end
156
156
  end
157
157
 
158
+ local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
159
+
158
160
  -- Remove job?
159
161
  if maxCount ~= 0 then
160
162
  local targetSet = KEYS[11]
@@ -164,7 +166,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
164
166
  -- "returnvalue" / "failedReason" and "finishedOn"
165
167
 
166
168
  -- Remove old jobs?
167
- local prefix = ARGV[8]
169
+ local prefix = ARGV[7]
168
170
 
169
171
  if maxAge ~= nil then
170
172
  removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
@@ -197,13 +199,13 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
197
199
 
198
200
  -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
199
201
  -- and not rate limited.
200
- if (ARGV[7] == "1") then
202
+ if (ARGV[6] == "1") then
201
203
 
202
- local target, paused = getTargetQueueList(KEYS[9], KEYS[1], KEYS[8])
204
+ local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[8])
203
205
 
204
206
  -- Check if there are delayed jobs that can be promoted
205
- promoteDelayedJobs(KEYS[7], KEYS[1], target, KEYS[3],
206
- KEYS[4], ARGV[8], timestamp, paused, KEYS[10])
207
+ promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], KEYS[4], ARGV[7],
208
+ timestamp, KEYS[10], paused)
207
209
 
208
210
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
209
211
  -- Check if we are rate limited first.
@@ -217,25 +219,30 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
217
219
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
218
220
 
219
221
  if jobId then
222
+ -- Markers in waitlist DEPRECATED in v5: Remove in v6.
220
223
  if string.sub(jobId, 1, 2) == "0:" then
221
224
  rcall("LREM", KEYS[2], 1, jobId)
222
225
 
223
226
  -- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
224
227
  -- but if ID is 0:0, then there is at least 1 prioritized job to process
225
228
  if jobId == "0:0" then
226
- jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
227
- return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp,
228
- maxJobs, expireTime, opts)
229
+ jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
230
+ KEYS[10])
231
+ return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
232
+ timestamp, maxJobs,
233
+ expireTime, opts)
229
234
  end
230
235
  else
231
- return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
232
- expireTime, opts)
236
+ return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
237
+ timestamp, maxJobs, expireTime,
238
+ opts)
233
239
  end
234
240
  else
235
241
  jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
236
242
  if jobId then
237
- return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
238
- expireTime, opts)
243
+ return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
244
+ timestamp, maxJobs, expireTime,
245
+ opts)
239
246
  end
240
247
  end
241
248
 
@@ -11,7 +11,7 @@
11
11
  ARGV[2] child key
12
12
  ARGV[3] timestamp
13
13
  ARGV[4] the id of the job
14
-
14
+
15
15
  Output:
16
16
  0 - OK
17
17
  1 - There are not pending dependencies.
@@ -21,7 +21,8 @@
21
21
  ]]
22
22
  local rcall = redis.call
23
23
 
24
- local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp, lockKey, token)
24
+ local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp,
25
+ lockKey, jobKey, token)
25
26
  if token ~= "0" then
26
27
  if rcall("GET", lockKey) == token then
27
28
  rcall("DEL", lockKey)
@@ -46,13 +47,15 @@ end
46
47
  if rcall("EXISTS", KEYS[4]) == 1 then
47
48
  if ARGV[2] ~= "" then
48
49
  if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
49
- return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
50
+ return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], KEYS[4],
51
+ ARGV[1])
50
52
  end
51
53
 
52
54
  return 1
53
55
  else
54
56
  if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
55
- return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
57
+ return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], KEYS[4],
58
+ ARGV[1])
56
59
  end
57
60
 
58
61
  return 1
@@ -0,0 +1,41 @@
1
+ --[[
2
+ Pauses or resumes a queue globably.
3
+
4
+ Input:
5
+ KEYS[1] 'wait' or 'paused''
6
+ KEYS[2] 'paused' or 'wait'
7
+ KEYS[3] 'meta'
8
+ KEYS[4] 'prioritized'
9
+ KEYS[5] events stream key
10
+ KEYS[6] 'delayed'
11
+ KEYS|7] 'marker'
12
+
13
+ ARGV[1] 'paused' or 'resumed'
14
+
15
+ Event:
16
+ publish paused or resumed event.
17
+ ]]
18
+ local rcall = redis.call
19
+
20
+ -- Includes
21
+ --- @include "includes/addDelayMarkerIfNeeded"
22
+
23
+ local markerKey = KEYS[7]
24
+ local hasJobs = rcall("EXISTS", KEYS[1]) == 1
25
+ if hasJobs then rcall("RENAME", KEYS[1], KEYS[2]) end
26
+
27
+ if ARGV[1] == "paused" then
28
+ rcall("HSET", KEYS[3], "paused", 1)
29
+ rcall("DEL", markerKey)
30
+ else
31
+ rcall("HDEL", KEYS[3], "paused")
32
+
33
+ if hasJobs or rcall("ZCARD", KEYS[4]) > 0 then
34
+ -- Add marker if there are waiting or priority jobs
35
+ rcall("ZADD", markerKey, 0, "0")
36
+ else
37
+ addDelayMarkerIfNeeded(markerKey, KEYS[6])
38
+ end
39
+ end
40
+
41
+ rcall("XADD", KEYS[5], "*", "event", ARGV[1]);
@@ -0,0 +1,60 @@
1
+ --[[
2
+ Promotes a job that is currently "delayed" to the "waiting" state
3
+
4
+ Input:
5
+ KEYS[1] 'delayed'
6
+ KEYS[2] 'wait'
7
+ KEYS[3] 'paused'
8
+ KEYS[4] 'meta'
9
+ KEYS[5] 'prioritized'
10
+ KEYS[6] 'pc' priority counter
11
+ KEYS[7] 'event stream'
12
+ KEYS[8] 'marker'
13
+
14
+ ARGV[1] queue.toKey('')
15
+ ARGV[2] jobId
16
+
17
+ Output:
18
+ 0 - OK
19
+ -3 - Job not in delayed zset.
20
+
21
+ Events:
22
+ 'waiting'
23
+ ]]
24
+ local rcall = redis.call
25
+ local jobId = ARGV[2]
26
+
27
+ -- Includes
28
+ --- @include "includes/addJobWithPriority"
29
+ --- @include "includes/getTargetQueueList"
30
+
31
+ if rcall("ZREM", KEYS[1], jobId) == 1 then
32
+ local jobKey = ARGV[1] .. jobId
33
+ local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
34
+ local metaKey = KEYS[4]
35
+
36
+ -- Remove delayed "marker" from the wait list if there is any.
37
+ -- Since we are adding a job we do not need the marker anymore.
38
+ -- Markers in waitlist DEPRECATED in v5: Remove in v6.
39
+ local target, paused = getTargetQueueList(metaKey, KEYS[2], KEYS[3])
40
+ local marker = rcall("LINDEX", target, 0)
41
+ if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
42
+
43
+ if priority == 0 then
44
+ -- LIFO or FIFO
45
+ rcall("LPUSH", target, jobId)
46
+ if not paused then rcall("ZADD", KEYS[8], 0, "0") end
47
+ else
48
+ addJobWithPriority(KEYS[8], KEYS[5], priority, jobId, KEYS[6], paused)
49
+ end
50
+
51
+ -- Emit waiting event (wait..ing@token)
52
+ rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev",
53
+ "delayed");
54
+
55
+ rcall("HSET", jobKey, "delay", 0)
56
+
57
+ return 0
58
+ else
59
+ return -3
60
+ end
@@ -2,15 +2,16 @@
2
2
  Retries a failed job by moving it back to the wait queue.
3
3
 
4
4
  Input:
5
- KEYS[1] 'active',
6
- KEYS[2] 'wait'
7
- KEYS[3] 'paused'
8
- KEYS[4] job key
9
- KEYS[5] 'meta'
10
- KEYS[6] events stream
11
- KEYS[7] delayed key
12
- KEYS[8] prioritized key
13
- KEYS[9] 'pc' priority counter
5
+ KEYS[1] 'active',
6
+ KEYS[2] 'wait'
7
+ KEYS[3] 'paused'
8
+ KEYS[4] job key
9
+ KEYS[5] 'meta'
10
+ KEYS[6] events stream
11
+ KEYS[7] delayed key
12
+ KEYS[8] prioritized key
13
+ KEYS[9] 'pc' priority counter
14
+ KEYS[10] 'marker'
14
15
 
15
16
  ARGV[1] key prefix
16
17
  ARGV[2] timestamp
@@ -34,9 +35,11 @@ local rcall = redis.call
34
35
  --- @include "includes/promoteDelayedJobs"
35
36
 
36
37
  local target, paused = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
38
+ local markerKey = KEYS[10]
39
+
37
40
  -- Check if there are delayed jobs that we can move to wait.
38
41
  -- test example: when there are delayed jobs between retries
39
- promoteDelayedJobs(KEYS[7], KEYS[2], target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], paused, KEYS[9])
42
+ promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], paused)
40
43
 
41
44
  if rcall("EXISTS", KEYS[4]) == 1 then
42
45
 
@@ -57,9 +60,11 @@ if rcall("EXISTS", KEYS[4]) == 1 then
57
60
  if priority == 0 then
58
61
  rcall(ARGV[3], target, ARGV[4])
59
62
  else
60
- addJobWithPriority(KEYS[2], KEYS[8], priority, paused, ARGV[4], KEYS[9])
63
+ addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], paused)
61
64
  end
62
65
 
66
+ rcall("HINCRBY", KEYS[4], "atm", 1)
67
+
63
68
  local maxEvents = rcall("HGET", KEYS[5], "opts.maxLenEvents") or 10000
64
69
 
65
70
  -- Emit waiting event
@@ -7,6 +7,7 @@ export interface JobJson {
7
7
  opts: RedisJobOptions;
8
8
  progress: number | object;
9
9
  attemptsMade: number;
10
+ attemptsStarted: number;
10
11
  finishedOn?: number;
11
12
  processedOn?: number;
12
13
  timestamp: number;
@@ -24,7 +25,7 @@ export interface JobJsonRaw {
24
25
  delay: string;
25
26
  opts: string;
26
27
  progress: string;
27
- attemptsMade: string;
28
+ attemptsMade?: string;
28
29
  finishedOn?: string;
29
30
  processedOn?: string;
30
31
  timestamp: string;
@@ -34,4 +35,6 @@ export interface JobJsonRaw {
34
35
  parentKey?: string;
35
36
  parent?: string;
36
37
  rjk?: string;
38
+ atm?: string;
39
+ ats?: string;
37
40
  }
@@ -2,6 +2,9 @@ import { JobsOptions, JobJsonSandbox } from '../types';
2
2
  import { JobJson } from './job-json';
3
3
  import { ParentKeys } from './parent';
4
4
  export type BulkJobOptions = Omit<JobsOptions, 'repeat'>;
5
+ export interface MoveToDelayedOpts {
6
+ skipAttempt?: boolean;
7
+ }
5
8
  export interface MoveToWaitingChildrenOpts {
6
9
  child?: {
7
10
  id: string;
@@ -12,7 +12,7 @@ export interface QueueBaseOptions {
12
12
  /**
13
13
  * Options for connecting to a Redis instance.
14
14
  */
15
- connection?: ConnectionOptions;
15
+ connection: ConnectionOptions;
16
16
  /**
17
17
  * Denotes commands should retry indefinitely.
18
18
  */
@@ -6,13 +6,12 @@ const content = `--[[
6
6
  - adds to delayed zset.
7
7
  - Emits a global event 'delayed' if the job is delayed.
8
8
  Input:
9
- KEYS[1] 'wait',
10
- KEYS[2] 'paused'
11
- KEYS[3] 'meta'
12
- KEYS[4] 'id'
13
- KEYS[5] 'delayed'
14
- KEYS[6] 'completed'
15
- KEYS[7] events stream key
9
+ KEYS[1] 'marker',
10
+ KEYS[2] 'meta'
11
+ KEYS[3] 'id'
12
+ KEYS[4] 'delayed'
13
+ KEYS[5] 'completed'
14
+ KEYS[6] events stream key
16
15
  ARGV[1] msgpacked arguments array
17
16
  [1] key prefix,
18
17
  [2] custom id (use custom instead of one generated automatically)
@@ -29,13 +28,11 @@ const content = `--[[
29
28
  jobId - OK
30
29
  -5 - Missing parent key
31
30
  ]]
32
- local waitKey = KEYS[1]
33
- local pausedKey = KEYS[2]
34
- local metaKey = KEYS[3]
35
- local idKey = KEYS[4]
36
- local delayedKey = KEYS[5]
37
- local completedKey = KEYS[6]
38
- local eventsKey = KEYS[7]
31
+ local metaKey = KEYS[2]
32
+ local idKey = KEYS[3]
33
+ local delayedKey = KEYS[4]
34
+ local completedKey = KEYS[5]
35
+ local eventsKey = KEYS[6]
39
36
  local jobId
40
37
  local jobIdKey
41
38
  local rcall = redis.call
@@ -74,7 +71,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
74
71
  end
75
72
  --[[
76
73
  Add delay marker if needed.
77
- ]]
74
+ ]]
78
75
  -- Includes
79
76
  --[[
80
77
  Function to return the next delayed job timestamp.
@@ -89,36 +86,20 @@ local function getNextDelayedTimestamp(delayedKey)
89
86
  return nextTimestamp
90
87
  end
91
88
  end
92
- local function addDelayMarkerIfNeeded(targetKey, delayedKey)
93
- local waitLen = rcall("LLEN", targetKey)
94
- if waitLen <= 1 then
89
+ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
95
90
  local nextTimestamp = getNextDelayedTimestamp(delayedKey)
96
91
  if nextTimestamp ~= nil then
97
- -- Check if there is already a marker with older timestamp
98
- -- if there is, we need to replace it.
99
- if waitLen == 1 then
100
- local marker = rcall("LINDEX", targetKey, 0)
101
- local oldTimestamp = tonumber(marker:sub(3))
102
- if oldTimestamp and oldTimestamp > nextTimestamp then
103
- rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
104
- end
105
- else
106
- -- if there is no marker, then we need to add one
107
- rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
108
- end
92
+ -- Replace the score of the marker with the newest known
93
+ -- next timestamp.
94
+ rcall("ZADD", markerKey, nextTimestamp, "0")
109
95
  end
110
- end
111
96
  end
112
97
  --[[
113
98
  Function to check for the meta.paused key to decide if we are paused or not
114
99
  (since an empty list and !EXISTS are not really the same).
115
100
  ]]
116
- local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
117
- if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
118
- return waitKey, false
119
- else
120
- return pausedKey, true
121
- end
101
+ local function isQueuePaused(queueMetaKey)
102
+ return rcall("HEXISTS", queueMetaKey, "paused") == 1
122
103
  end
123
104
  --[[
124
105
  Validate and move or add dependencies to parent.
@@ -131,56 +112,64 @@ end
131
112
  --[[
132
113
  Function to add job considering priority.
133
114
  ]]
134
- -- Includes
135
- --[[
136
- Function priority marker to wait if needed
137
- in order to wake up our workers and to respect priority
138
- order as much as possible
139
- ]]
140
- local function addPriorityMarkerIfNeeded(waitKey)
141
- local waitLen = rcall("LLEN", waitKey)
142
- if waitLen == 0 then
143
- rcall("LPUSH", waitKey, "0:0")
144
- end
145
- end
146
- local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
115
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
147
116
  local prioCounter = rcall("INCR", priorityCounterKey)
148
117
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
149
118
  rcall("ZADD", prioritizedKey, score, jobId)
150
- if not paused then
151
- addPriorityMarkerIfNeeded(waitKey)
119
+ if not isPaused then
120
+ rcall("ZADD", markerKey, 0, "0")
152
121
  end
153
122
  end
154
- local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
155
- local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
156
- if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
157
- rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
158
- local parentWaitKey = parentQueueKey .. ":wait"
159
- local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
160
- parentQueueKey .. ":paused")
161
- local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
162
- local priority = tonumber(jobAttributes[1]) or 0
163
- local delay = tonumber(jobAttributes[2]) or 0
164
- if delay > 0 then
165
- local delayedTimestamp = tonumber(timestamp) + delay
166
- local score = delayedTimestamp * 0x1000
167
- local parentDelayedKey = parentQueueKey .. ":delayed"
168
- rcall("ZADD", parentDelayedKey, score, parentId)
169
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
170
- "delay", delayedTimestamp)
171
- addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
172
- else
173
- if priority == 0 then
174
- rcall("RPUSH", parentTarget, parentId)
175
- else
176
- addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
177
- parentId, parentQueueKey .. ":pc")
178
- end
179
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
180
- "prev", "waiting-children")
181
- end
123
+ --[[
124
+ Function to check for the meta.paused key to decide if we are paused or not
125
+ (since an empty list and !EXISTS are not really the same).
126
+ ]]
127
+ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
128
+ if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
129
+ return waitKey, false
130
+ else
131
+ return pausedKey, true
182
132
  end
183
133
  end
134
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
135
+ parentKey, parentId, timestamp)
136
+ local isParentActive = rcall("ZSCORE",
137
+ parentQueueKey .. ":waiting-children", parentId)
138
+ if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
139
+ rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
140
+ local parentWaitKey = parentQueueKey .. ":wait"
141
+ local parentPausedKey = parentQueueKey .. ":paused"
142
+ local parentMetaKey = parentQueueKey .. ":meta"
143
+ local parentMarkerKey = parentQueueKey .. ":marker"
144
+ local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
145
+ local priority = tonumber(jobAttributes[1]) or 0
146
+ local delay = tonumber(jobAttributes[2]) or 0
147
+ if delay > 0 then
148
+ local delayedTimestamp = tonumber(timestamp) + delay
149
+ local score = delayedTimestamp * 0x1000
150
+ local parentDelayedKey = parentQueueKey .. ":delayed"
151
+ rcall("ZADD", parentDelayedKey, score, parentId)
152
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
153
+ "jobId", parentId, "delay", delayedTimestamp)
154
+ addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
155
+ else
156
+ if priority == 0 then
157
+ local parentTarget, _paused =
158
+ getTargetQueueList(parentMetaKey, parentWaitKey,
159
+ parentPausedKey)
160
+ rcall("RPUSH", parentTarget, parentId)
161
+ rcall("ZADD", parentMarkerKey, 0, "0")
162
+ else
163
+ local isPaused = isQueuePaused(parentMetaKey)
164
+ addJobWithPriority(parentMarkerKey,
165
+ parentQueueKey .. ":prioritized", priority,
166
+ parentId, parentQueueKey .. ":pc", isPaused)
167
+ end
168
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
169
+ "jobId", parentId, "prev", "waiting-children")
170
+ end
171
+ end
172
+ end
184
173
  local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
185
174
  parentId, jobIdKey, returnvalue, timestamp )
186
175
  local processedSet = parentKey .. ":processed"
@@ -252,13 +241,13 @@ local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
252
241
  rcall("ZADD", delayedKey, score, jobId)
253
242
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
254
243
  "jobId", jobId, "delay", delayedTimestamp)
255
- -- If wait list is empty, and this delayed job is the next one to be processed,
256
- -- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
257
- local target = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
258
- addDelayMarkerIfNeeded(target, delayedKey)
244
+ -- mark that a delayed job is available
245
+ local isPaused = isQueuePaused(metaKey)
246
+ if not isPaused then
247
+ local markerKey = KEYS[1]
248
+ addDelayMarkerIfNeeded(markerKey, delayedKey)
249
+ end
259
250
  -- Check if this job is a child of another job, if so add it to the parents dependencies
260
- -- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
261
- -- fail in this case.
262
251
  if parentDependenciesKey ~= nil then
263
252
  rcall("SADD", parentDependenciesKey, jobIdKey)
264
253
  end
@@ -267,6 +256,6 @@ return jobId .. "" -- convert to string
267
256
  export const addDelayedJob = {
268
257
  name: 'addDelayedJob',
269
258
  content,
270
- keys: 7,
259
+ keys: 6,
271
260
  };
272
- //# sourceMappingURL=addDelayedJob-7.js.map
261
+ //# sourceMappingURL=addDelayedJob-6.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"moveToActive-10.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-10.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmPf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Pf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}