bullmq 3.1.2 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/dist/cjs/classes/job.js +4 -0
  2. package/dist/cjs/classes/job.js.map +1 -1
  3. package/dist/cjs/classes/queue-getters.js +1 -1
  4. package/dist/cjs/classes/queue-getters.js.map +1 -1
  5. package/dist/cjs/classes/queue.js +3 -0
  6. package/dist/cjs/classes/queue.js.map +1 -1
  7. package/dist/cjs/classes/worker.js +14 -6
  8. package/dist/cjs/classes/worker.js.map +1 -1
  9. package/dist/cjs/commands/addJob-8.lua +8 -7
  10. package/dist/cjs/commands/includes/checkStalledJobs.lua +22 -21
  11. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +1 -2
  12. package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +18 -3
  13. package/dist/cjs/commands/moveToActive-9.lua +13 -7
  14. package/dist/cjs/commands/moveToDelayed-8.lua +31 -23
  15. package/dist/cjs/commands/moveToFinished-12.lua +10 -7
  16. package/dist/cjs/commands/promote-6.lua +2 -1
  17. package/dist/cjs/interfaces/base-job-options.d.ts +2 -2
  18. package/dist/cjs/scripts/addJob-8.js +24 -10
  19. package/dist/cjs/scripts/addJob-8.js.map +1 -1
  20. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +22 -21
  21. package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
  22. package/dist/cjs/scripts/moveToActive-9.js +13 -9
  23. package/dist/cjs/scripts/moveToActive-9.js.map +1 -1
  24. package/dist/cjs/scripts/moveToDelayed-8.js +43 -22
  25. package/dist/cjs/scripts/moveToDelayed-8.js.map +1 -1
  26. package/dist/cjs/scripts/moveToFinished-12.js +27 -12
  27. package/dist/cjs/scripts/moveToFinished-12.js.map +1 -1
  28. package/dist/cjs/scripts/promote-6.js +2 -1
  29. package/dist/cjs/scripts/promote-6.js.map +1 -1
  30. package/dist/cjs/scripts/retryJob-8.js +1 -2
  31. package/dist/cjs/scripts/retryJob-8.js.map +1 -1
  32. package/dist/esm/classes/job.js +4 -0
  33. package/dist/esm/classes/job.js.map +1 -1
  34. package/dist/esm/classes/queue-getters.js +1 -1
  35. package/dist/esm/classes/queue-getters.js.map +1 -1
  36. package/dist/esm/classes/queue.js +3 -0
  37. package/dist/esm/classes/queue.js.map +1 -1
  38. package/dist/esm/classes/worker.js +14 -6
  39. package/dist/esm/classes/worker.js.map +1 -1
  40. package/dist/esm/commands/addJob-8.lua +8 -7
  41. package/dist/esm/commands/includes/checkStalledJobs.lua +22 -21
  42. package/dist/esm/commands/includes/promoteDelayedJobs.lua +1 -2
  43. package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +18 -3
  44. package/dist/esm/commands/moveToActive-9.lua +13 -7
  45. package/dist/esm/commands/moveToDelayed-8.lua +31 -23
  46. package/dist/esm/commands/moveToFinished-12.lua +10 -7
  47. package/dist/esm/commands/promote-6.lua +2 -1
  48. package/dist/esm/interfaces/base-job-options.d.ts +2 -2
  49. package/dist/esm/scripts/addJob-8.js +24 -10
  50. package/dist/esm/scripts/addJob-8.js.map +1 -1
  51. package/dist/esm/scripts/moveStalledJobsToWait-8.js +22 -21
  52. package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
  53. package/dist/esm/scripts/moveToActive-9.js +13 -9
  54. package/dist/esm/scripts/moveToActive-9.js.map +1 -1
  55. package/dist/esm/scripts/moveToDelayed-8.js +43 -22
  56. package/dist/esm/scripts/moveToDelayed-8.js.map +1 -1
  57. package/dist/esm/scripts/moveToFinished-12.js +27 -12
  58. package/dist/esm/scripts/moveToFinished-12.js.map +1 -1
  59. package/dist/esm/scripts/promote-6.js +2 -1
  60. package/dist/esm/scripts/promote-6.js.map +1 -1
  61. package/dist/esm/scripts/retryJob-8.js +1 -2
  62. package/dist/esm/scripts/retryJob-8.js.map +1 -1
  63. package/package.json +1 -1
@@ -110,7 +110,7 @@ local jobCounter = rcall("INCR", KEYS[4])
110
110
  ]]
111
111
  -- Includes
112
112
  local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
113
- parentId, jobIdKey, returnvalue )
113
+ parentId, jobIdKey, returnvalue, timestamp )
114
114
  local processedSet = parentKey .. ":processed"
115
115
  rcall("HSET", processedSet, jobIdKey, returnvalue)
116
116
  local activeParent = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
@@ -118,9 +118,22 @@ local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDepende
118
118
  rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
119
119
  local parentTarget = getTargetQueueList(parentQueueKey .. ":meta", parentQueueKey .. ":wait",
120
120
  parentQueueKey .. ":paused")
121
- local priority = tonumber(rcall("HGET", parentKey, "priority"))
121
+ local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
122
+ local priority = tonumber(jobAttributes[1]) or 0
123
+ local delay = tonumber(jobAttributes[2]) or 0
124
+ if delay > 0 then
125
+ local delayedTimestamp = tonumber(timestamp) + delay
126
+ local score = delayedTimestamp * 0x1000
127
+ local parentDelayedKey = parentQueueKey .. ":delayed"
128
+ rcall("ZADD", parentDelayedKey, score, parentId)
129
+ if rcall("LLEN", parentTarget) == 0 then
130
+ local nextTimestamp = getNextDelayedTimestamp(parentDelayedKey)
131
+ if not nextTimestamp or (delayedTimestamp <= nextTimestamp) then
132
+ rcall("LPUSH", parentTarget, "0:" .. delayedTimestamp - tonumber(timestamp))
133
+ end
134
+ end
122
135
  -- Standard or priority add
123
- if priority == 0 then
136
+ elseif priority == 0 then
124
137
  rcall("RPUSH", parentTarget, parentId)
125
138
  else
126
139
  addJobWithPriority(parentQueueKey .. ":priority", priority, parentTarget, parentId)
@@ -131,6 +144,7 @@ end
131
144
  -- Trim events before emiting them to avoid trimming events emitted in this script
132
145
  trimEvents(KEYS[3], KEYS[8])
133
146
  local parentDependenciesKey = args[7]
147
+ local timestamp = args[4]
134
148
  if args[2] == "" then
135
149
  jobId = jobCounter
136
150
  jobIdKey = args[1] .. jobId
@@ -142,7 +156,7 @@ else
142
156
  if rcall("ZSCORE", KEYS[7], jobId) ~= false then
143
157
  local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
144
158
  updateParentDepsIfNeeded(parentKey, parent['queueKey'], parentDependenciesKey,
145
- parent['id'], jobIdKey, returnvalue)
159
+ parent['id'], jobIdKey, returnvalue, timestamp)
146
160
  else
147
161
  if parentDependenciesKey ~= nil then
148
162
  rcall("SADD", parentDependenciesKey, jobIdKey)
@@ -157,7 +171,6 @@ end
157
171
  local jsonOpts = cjson.encode(opts)
158
172
  local delay = opts['delay'] or 0
159
173
  local priority = opts['priority'] or 0
160
- local timestamp = args[4]
161
174
  local optionalValues = {}
162
175
  if parentKey ~= nil then
163
176
  table.insert(optionalValues, "parentKey")
@@ -180,17 +193,18 @@ if waitChildrenKey ~= nil then
180
193
  rcall("ZADD", waitChildrenKey, timestamp, jobId)
181
194
  rcall("XADD", KEYS[8], "*", "event", "waiting-children", "jobId", jobId)
182
195
  elseif (delayedTimestamp ~= 0) then
183
- local timestamp = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
184
- rcall("ZADD", KEYS[5], timestamp, jobId)
196
+ local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
197
+ rcall("ZADD", KEYS[5], score, jobId)
185
198
  rcall("XADD", KEYS[8], "*", "event", "delayed", "jobId", jobId, "delay",
186
199
  delayedTimestamp)
187
200
  -- If wait list is empty, and this delayed job is the next one to be processed,
188
- -- then we need to signal the workers by adding a dummy job (jobId 0) to the wait list.
201
+ -- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
189
202
  local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
190
203
  if rcall("LLEN", target) == 0 then
191
204
  local nextTimestamp = getNextDelayedTimestamp(KEYS[5])
192
- if delayedTimestamp < nextTimestamp then
193
- rcall("LPUSH", target, 0)
205
+ if not nextTimestamp or (delayedTimestamp < nextTimestamp) then
206
+ local delay = delayedTimestamp - tonumber(timestamp)
207
+ rcall("LPUSH", target, "0:" .. delay)
194
208
  end
195
209
  end
196
210
  else
@@ -1 +1 @@
1
- {"version":3,"file":"addJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addJob-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwNf,CAAC;AACF,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,IAAI,EAAE,QAAQ;IACd,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addJob-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsOf,CAAC;AACF,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,IAAI,EAAE,QAAQ;IACd,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -189,11 +189,11 @@ local function trimEvents(metaKey, eventStreamKey)
189
189
  end
190
190
  end
191
191
  -- Check if we need to check for stalled jobs now.
192
- local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey, stalledCheckKey,
193
- metaKey, pausedKey, eventStreamKey, maxStalledJobCount, queueKeyPrefix, timestamp, maxCheckTime)
194
- if rcall("EXISTS", stalledCheckKey) == 1 then
195
- return {{}, {}}
196
- end
192
+ local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey,
193
+ stalledCheckKey, metaKey, pausedKey,
194
+ eventStreamKey, maxStalledJobCount,
195
+ queueKeyPrefix, timestamp, maxCheckTime)
196
+ if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
197
197
  rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
198
198
  -- Trim events before emiting them to avoid trimming events emitted in this script
199
199
  trimEvents(metaKey, eventStreamKey)
@@ -206,7 +206,7 @@ local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey, stall
206
206
  local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
207
207
  -- Remove from active list
208
208
  for i, jobId in ipairs(stalling) do
209
- if jobId == '0' then
209
+ if string.sub(jobId, 1, 2) == "0:" then
210
210
  -- If the jobId is a delay marker ID we just remove it.
211
211
  local removed = rcall("LREM", activeKey, 1, jobId)
212
212
  else
@@ -227,13 +227,13 @@ local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey, stall
227
227
  local failedReason =
228
228
  "job stalled more than allowable limit"
229
229
  rcall("HMSET", jobKey, "failedReason", failedReason,
230
- "finishedOn", timestamp)
231
- rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId",
232
- jobId, 'prev', 'active', 'failedReason',
233
- failedReason)
230
+ "finishedOn", timestamp)
231
+ rcall("XADD", eventStreamKey, "*", "event",
232
+ "failed", "jobId", jobId, 'prev', 'active',
233
+ 'failedReason', failedReason)
234
234
  if removeOnFailType == "number" then
235
- removeJobsByMaxCount(opts["removeOnFail"], failedKey,
236
- queueKeyPrefix)
235
+ removeJobsByMaxCount(opts["removeOnFail"],
236
+ failedKey, queueKeyPrefix)
237
237
  elseif removeOnFailType == "boolean" then
238
238
  if opts["removeOnFail"] then
239
239
  removeJob(jobId, false, queueKeyPrefix)
@@ -243,24 +243,25 @@ local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey, stall
243
243
  local maxAge = opts["removeOnFail"]["age"]
244
244
  local maxCount = opts["removeOnFail"]["count"]
245
245
  if maxAge ~= nil then
246
- removeJobsByMaxAge(timestamp, maxAge, failedKey,
247
- queueKeyPrefix)
246
+ removeJobsByMaxAge(timestamp, maxAge,
247
+ failedKey, queueKeyPrefix)
248
248
  end
249
249
  if maxCount ~= nil and maxCount > 0 then
250
- removeJobsByMaxCount(maxCount, failedKey, queueKeyPrefix)
250
+ removeJobsByMaxCount(maxCount, failedKey,
251
+ queueKeyPrefix)
251
252
  end
252
253
  end
253
254
  table.insert(failed, jobId)
254
255
  else
255
- local target = getTargetQueueList(metaKey, waitKey,
256
- pausedKey)
256
+ local target =
257
+ getTargetQueueList(metaKey, waitKey, pausedKey)
257
258
  -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
258
259
  rcall("RPUSH", target, jobId)
259
- rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
260
- jobId, 'prev', 'active')
260
+ rcall("XADD", eventStreamKey, "*", "event",
261
+ "waiting", "jobId", jobId, 'prev', 'active')
261
262
  -- Emit the stalled event
262
- rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId",
263
- jobId)
263
+ rcall("XADD", eventStreamKey, "*", "event",
264
+ "stalled", "jobId", jobId)
264
265
  table.insert(stalled, jobId)
265
266
  end
266
267
  end
@@ -1 +1 @@
1
- {"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyRf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Rf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -126,8 +126,7 @@ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
126
126
  return pausedKey
127
127
  end
128
128
  end
129
- -- Try to get as much as 1000 jobs at once, and returns the nextTimestamp if
130
- -- there are more delayed jobs to process.
129
+ -- Try to get as much as 1000 jobs at once
131
130
  local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
132
131
  metaKey, eventStreamKey, prefix, timestamp)
133
132
  local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
@@ -181,13 +180,18 @@ else
181
180
  -- no job ID, try non-blocking move from wait to active
182
181
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
183
182
  end
184
- -- If jobId is special ID 0, then there is no job to process
185
- if jobId == "0" then
186
- rcall("LREM", KEYS[2], 1, 0)
187
- elseif jobId then
188
- opts = opts or cmsgpack.unpack(ARGV[4])
189
- -- this script is not really moving, it is preparing the job for processing
190
- return moveJobFromWaitToActive(KEYS, ARGV[1], jobId, ARGV[2], opts)
183
+ -- If jobId is special ID 0:delay, then there is no job to process
184
+ if jobId then
185
+ if string.sub(jobId, 1, 2) == "0:" then
186
+ rcall("LREM", KEYS[2], 1, jobId)
187
+ -- Move again since we just got the marker job.
188
+ jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
189
+ end
190
+ if jobId then
191
+ opts = opts or cmsgpack.unpack(ARGV[4])
192
+ -- this script is not really moving, it is preparing the job for processing
193
+ return moveJobFromWaitToActive(KEYS, ARGV[1], jobId, ARGV[2], opts)
194
+ end
191
195
  end
192
196
  -- Return the timestamp for the next delayed job if any.
193
197
  local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
@@ -1 +1 @@
1
- {"version":3,"file":"moveToActive-9.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoMf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveToActive-9.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwMf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -56,8 +56,7 @@ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
56
56
  return pausedKey
57
57
  end
58
58
  end
59
- -- Try to get as much as 1000 jobs at once, and returns the nextTimestamp if
60
- -- there are more delayed jobs to process.
59
+ -- Try to get as much as 1000 jobs at once
61
60
  local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
62
61
  metaKey, eventStreamKey, prefix, timestamp)
63
62
  local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
@@ -81,28 +80,50 @@ local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
81
80
  end
82
81
  end
83
82
  end
84
- promoteDelayedJobs(KEYS[4], KEYS[1], KEYS[3], KEYS[7], KEYS[8], KEYS[6], ARGV[1], ARGV[2])
85
- if rcall("EXISTS", KEYS[5]) == 1 then
86
- if ARGV[5] ~= "0" then
87
- local lockKey = KEYS[5] .. ':lock'
88
- if rcall("GET", lockKey) == ARGV[5] then
89
- rcall("DEL", lockKey)
90
- else
91
- return -2
83
+ --[[
84
+ Function to return the next delayed job timestamp.
85
+ ]]
86
+ local function getNextDelayedTimestamp(delayedKey)
87
+ local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
88
+ if #result then
89
+ local nextTimestamp = tonumber(result[2])
90
+ if (nextTimestamp ~= nil) then
91
+ nextTimestamp = nextTimestamp / 0x1000
92
+ end
93
+ return nextTimestamp
92
94
  end
93
- end
94
- local jobId = ARGV[4]
95
- local score = tonumber(ARGV[3])
96
- local delayedTimestamp = (score / 0x1000)
97
- local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
98
- if(numRemovedElements < 1) then
99
- return -3
100
- end
101
- rcall("ZADD", KEYS[4], score, jobId)
102
- rcall("XADD", KEYS[6], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
103
- return 0
95
+ end
96
+ local jobKey = KEYS[5]
97
+ if rcall("EXISTS", jobKey) == 1 then
98
+ local delayedKey = KEYS[4]
99
+ if ARGV[5] ~= "0" then
100
+ local lockKey = jobKey .. ':lock'
101
+ if rcall("GET", lockKey) == ARGV[5] then
102
+ rcall("DEL", lockKey)
103
+ else
104
+ return -2
105
+ end
106
+ end
107
+ local jobId = ARGV[4]
108
+ local score = tonumber(ARGV[3])
109
+ local delayedTimestamp = (score / 0x1000)
110
+ local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
111
+ if (numRemovedElements < 1) then
112
+ return -3
113
+ end
114
+ -- Check if we need to push a marker job to wake up sleeping workers.
115
+ local target = getTargetQueueList(KEYS[8], KEYS[1], KEYS[7])
116
+ if rcall("LLEN", target) == 0 then
117
+ local nextTimestamp = getNextDelayedTimestamp(delayedKey)
118
+ if not nextTimestamp or (delayedTimestamp < nextTimestamp) then
119
+ rcall("LPUSH", target, "0:" .. delayedTimestamp - tonumber(ARGV[2]))
120
+ end
121
+ end
122
+ rcall("ZADD", delayedKey, score, jobId)
123
+ rcall("XADD", KEYS[6], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp)
124
+ return 0
104
125
  else
105
- return -1
126
+ return -1
106
127
  end
107
128
  `;
108
129
  export const moveToDelayed = {
@@ -1 +1 @@
1
- {"version":3,"file":"moveToDelayed-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToDelayed-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Gf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveToDelayed-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToDelayed-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Hf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -224,8 +224,7 @@ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
224
224
  return pausedKey
225
225
  end
226
226
  end
227
- -- Try to get as much as 1000 jobs at once, and returns the nextTimestamp if
228
- -- there are more delayed jobs to process.
227
+ -- Try to get as much as 1000 jobs at once
229
228
  local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
230
229
  metaKey, eventStreamKey, prefix, timestamp)
231
230
  local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
@@ -380,7 +379,7 @@ end
380
379
  ]]
381
380
  -- Includes
382
381
  local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
383
- parentId, jobIdKey, returnvalue )
382
+ parentId, jobIdKey, returnvalue, timestamp )
384
383
  local processedSet = parentKey .. ":processed"
385
384
  rcall("HSET", processedSet, jobIdKey, returnvalue)
386
385
  local activeParent = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
@@ -388,9 +387,22 @@ local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDepende
388
387
  rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
389
388
  local parentTarget = getTargetQueueList(parentQueueKey .. ":meta", parentQueueKey .. ":wait",
390
389
  parentQueueKey .. ":paused")
391
- local priority = tonumber(rcall("HGET", parentKey, "priority"))
390
+ local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
391
+ local priority = tonumber(jobAttributes[1]) or 0
392
+ local delay = tonumber(jobAttributes[2]) or 0
393
+ if delay > 0 then
394
+ local delayedTimestamp = tonumber(timestamp) + delay
395
+ local score = delayedTimestamp * 0x1000
396
+ local parentDelayedKey = parentQueueKey .. ":delayed"
397
+ rcall("ZADD", parentDelayedKey, score, parentId)
398
+ if rcall("LLEN", parentTarget) == 0 then
399
+ local nextTimestamp = getNextDelayedTimestamp(parentDelayedKey)
400
+ if not nextTimestamp or (delayedTimestamp <= nextTimestamp) then
401
+ rcall("LPUSH", parentTarget, "0:" .. delayedTimestamp - tonumber(timestamp))
402
+ end
403
+ end
392
404
  -- Standard or priority add
393
- if priority == 0 then
405
+ elseif priority == 0 then
394
406
  rcall("RPUSH", parentTarget, parentId)
395
407
  else
396
408
  addJobWithPriority(parentQueueKey .. ":priority", priority, parentTarget, parentId)
@@ -431,7 +443,6 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
431
443
  rcall("SREM", KEYS[5], ARGV[1])
432
444
  else
433
445
  if lockToken then
434
- rcall("SET", "DEBUG", lockToken)
435
446
  -- Lock exists but token does not match
436
447
  return -6
437
448
  else
@@ -465,7 +476,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
465
476
  if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
466
477
  updateParentDepsIfNeeded(parentKey, parentQueueKey,
467
478
  dependenciesSet, parentId, jobIdKey,
468
- ARGV[4])
479
+ ARGV[4], timestamp)
469
480
  end
470
481
  elseif opts['fpof'] then
471
482
  moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
@@ -512,11 +523,15 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
512
523
  local pttl = getRateLimitTTL(opts, KEYS[6])
513
524
  if pttl > 0 then return {0, 0, pttl} end
514
525
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
515
- if jobId == "0" then
516
- rcall("LREM", KEYS[2], 1, 0)
517
- elseif jobId then
518
- return
519
- moveJobFromWaitToActive(KEYS, ARGV[8], jobId, timestamp, opts)
526
+ -- If jobId is special ID 0:delay, then there is no job to process
527
+ if jobId then
528
+ if string.sub(jobId, 1, 2) == "0:" then
529
+ rcall("LREM", KEYS[2], 1, jobId)
530
+ else
531
+ opts = opts or cmsgpack.unpack(ARGV[4])
532
+ -- this script is not really moving, it is preparing the job for processing
533
+ return moveJobFromWaitToActive(KEYS, ARGV[8], jobId, timestamp, opts)
534
+ end
520
535
  end
521
536
  -- Return the timestamp for the next delayed job if any.
522
537
  local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
@@ -1 +1 @@
1
- {"version":3,"file":"moveToFinished-12.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-12.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2hBf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToFinished-12.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-12.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0iBf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -45,7 +45,8 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then
45
45
  local target = getTargetQueueList(KEYS[4], KEYS[2], KEYS[3])
46
46
  -- Remove delayed "marker" from the wait list if there is any.
47
47
  -- Since we are adding a job we do not need the marker anymore.
48
- if rcall("LINDEX", target, 0) == "0" then
48
+ local marker = rcall("LINDEX", target, 0)
49
+ if marker and string.sub(marker, 1, 2) == "0:" then
49
50
  rcall("LPOP", target)
50
51
  end
51
52
  if priority == 0 then
@@ -1 +1 @@
1
- {"version":3,"file":"promote-6.js","sourceRoot":"","sources":["../../../src/scripts/promote-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA+DZ,CAAC;AACL,MAAM,CAAC,MAAM,OAAO,GAAG;IACrB,IAAI,EAAE,SAAS;IACf,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"promote-6.js","sourceRoot":"","sources":["../../../src/scripts/promote-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAgEZ,CAAC;AACL,MAAM,CAAC,MAAM,OAAO,GAAG;IACrB,IAAI,EAAE,SAAS;IACf,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -56,8 +56,7 @@ local function addJobWithPriority(priorityKey, priority, targetKey, jobId)
56
56
  rcall("RPUSH", targetKey, jobId)
57
57
  end
58
58
  end
59
- -- Try to get as much as 1000 jobs at once, and returns the nextTimestamp if
60
- -- there are more delayed jobs to process.
59
+ -- Try to get as much as 1000 jobs at once
61
60
  local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
62
61
  metaKey, eventStreamKey, prefix, timestamp)
63
62
  local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
@@ -1 +1 @@
1
- {"version":3,"file":"retryJob-8.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsGf,CAAC;AACF,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"retryJob-8.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqGf,CAAC;AACF,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "bullmq",
3
- "version": "3.1.2",
3
+ "version": "3.2.0",
4
4
  "description": "Queue for messages and jobs based on Redis",
5
5
  "main": "./dist/cjs/index.js",
6
6
  "module": "./dist/esm/index.js",