bullmq 5.1.2 → 5.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/dist/cjs/classes/scripts.js +3 -1
  2. package/dist/cjs/classes/scripts.js.map +1 -1
  3. package/dist/cjs/commands/addStandardJob-7.lua +4 -8
  4. package/dist/cjs/commands/changeDelay-3.lua +1 -0
  5. package/dist/cjs/commands/changePriority-6.lua +0 -1
  6. package/dist/cjs/commands/includes/addBaseMarkerIfNeeded.lua +9 -0
  7. package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +3 -1
  8. package/dist/cjs/commands/includes/addJobInTargetList.lua +11 -0
  9. package/dist/cjs/commands/includes/addJobWithPriority.lua +5 -3
  10. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +6 -4
  11. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +2 -4
  12. package/dist/{esm/commands/moveJobsToWait-6.lua → cjs/commands/moveJobsToWait-7.lua} +5 -1
  13. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +150 -0
  14. package/dist/cjs/commands/moveToFinished-14.lua +1 -0
  15. package/dist/cjs/commands/pause-7.lua +1 -0
  16. package/dist/cjs/commands/promote-8.lua +2 -2
  17. package/dist/cjs/commands/retryJob-10.lua +1 -0
  18. package/dist/cjs/scripts/addDelayedJob-6.js +21 -7
  19. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  20. package/dist/cjs/scripts/addParentJob-4.js +25 -11
  21. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  22. package/dist/cjs/scripts/addPrioritizedJob-7.js +21 -7
  23. package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -1
  24. package/dist/cjs/scripts/addStandardJob-7.js +46 -36
  25. package/dist/cjs/scripts/addStandardJob-7.js.map +1 -1
  26. package/dist/cjs/scripts/changeDelay-3.js +1 -0
  27. package/dist/cjs/scripts/changeDelay-3.js.map +1 -1
  28. package/dist/cjs/scripts/changePriority-6.js +10 -14
  29. package/dist/cjs/scripts/changePriority-6.js.map +1 -1
  30. package/dist/cjs/scripts/index.js +2 -2
  31. package/dist/cjs/scripts/{moveJobsToWait-6.js → moveJobsToWait-7.js} +13 -3
  32. package/dist/cjs/scripts/moveJobsToWait-7.js.map +1 -0
  33. package/dist/cjs/scripts/{moveStalledJobsToWait-8.js → moveStalledJobsToWait-9.js} +109 -106
  34. package/dist/cjs/scripts/{moveStalledJobsToWait-8.js.map → moveStalledJobsToWait-9.js.map} +1 -1
  35. package/dist/cjs/scripts/moveToActive-11.js +19 -7
  36. package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
  37. package/dist/cjs/scripts/moveToDelayed-7.js +1 -1
  38. package/dist/cjs/scripts/moveToFinished-14.js +32 -21
  39. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  40. package/dist/cjs/scripts/pause-7.js +2 -1
  41. package/dist/cjs/scripts/pause-7.js.map +1 -1
  42. package/dist/cjs/scripts/promote-8.js +19 -5
  43. package/dist/cjs/scripts/promote-8.js.map +1 -1
  44. package/dist/cjs/scripts/retryJob-10.js +20 -7
  45. package/dist/cjs/scripts/retryJob-10.js.map +1 -1
  46. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  47. package/dist/esm/classes/scripts.js +3 -1
  48. package/dist/esm/classes/scripts.js.map +1 -1
  49. package/dist/esm/commands/addStandardJob-7.lua +4 -8
  50. package/dist/esm/commands/changeDelay-3.lua +1 -0
  51. package/dist/esm/commands/changePriority-6.lua +0 -1
  52. package/dist/esm/commands/includes/addBaseMarkerIfNeeded.lua +9 -0
  53. package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +3 -1
  54. package/dist/esm/commands/includes/addJobInTargetList.lua +11 -0
  55. package/dist/esm/commands/includes/addJobWithPriority.lua +5 -3
  56. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +6 -4
  57. package/dist/esm/commands/includes/promoteDelayedJobs.lua +2 -4
  58. package/dist/{cjs/commands/moveJobsToWait-6.lua → esm/commands/moveJobsToWait-7.lua} +5 -1
  59. package/dist/esm/commands/moveStalledJobsToWait-9.lua +150 -0
  60. package/dist/esm/commands/moveToFinished-14.lua +1 -0
  61. package/dist/esm/commands/pause-7.lua +1 -0
  62. package/dist/esm/commands/promote-8.lua +2 -2
  63. package/dist/esm/commands/retryJob-10.lua +1 -0
  64. package/dist/esm/scripts/addDelayedJob-6.js +21 -7
  65. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  66. package/dist/esm/scripts/addParentJob-4.js +25 -11
  67. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  68. package/dist/esm/scripts/addPrioritizedJob-7.js +21 -7
  69. package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -1
  70. package/dist/esm/scripts/addStandardJob-7.js +46 -36
  71. package/dist/esm/scripts/addStandardJob-7.js.map +1 -1
  72. package/dist/esm/scripts/changeDelay-3.js +1 -0
  73. package/dist/esm/scripts/changeDelay-3.js.map +1 -1
  74. package/dist/esm/scripts/changePriority-6.js +10 -14
  75. package/dist/esm/scripts/changePriority-6.js.map +1 -1
  76. package/dist/esm/scripts/index.d.ts +2 -2
  77. package/dist/esm/scripts/index.js +2 -2
  78. package/dist/esm/scripts/{moveJobsToWait-6.js → moveJobsToWait-7.js} +13 -3
  79. package/dist/esm/scripts/moveJobsToWait-7.js.map +1 -0
  80. package/dist/esm/scripts/{moveStalledJobsToWait-8.js → moveStalledJobsToWait-9.js} +109 -106
  81. package/dist/esm/scripts/{moveStalledJobsToWait-8.js.map → moveStalledJobsToWait-9.js.map} +1 -1
  82. package/dist/esm/scripts/moveToActive-11.js +19 -7
  83. package/dist/esm/scripts/moveToActive-11.js.map +1 -1
  84. package/dist/esm/scripts/moveToDelayed-7.js +1 -1
  85. package/dist/esm/scripts/moveToFinished-14.js +32 -21
  86. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  87. package/dist/esm/scripts/pause-7.js +2 -1
  88. package/dist/esm/scripts/pause-7.js.map +1 -1
  89. package/dist/esm/scripts/promote-8.js +19 -5
  90. package/dist/esm/scripts/promote-8.js.map +1 -1
  91. package/dist/esm/scripts/retryJob-10.js +20 -7
  92. package/dist/esm/scripts/retryJob-10.js.map +1 -1
  93. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  94. package/package.json +1 -1
  95. package/dist/cjs/commands/includes/checkStalledJobs.lua +0 -139
  96. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -24
  97. package/dist/cjs/scripts/moveJobsToWait-6.js.map +0 -1
  98. package/dist/esm/commands/includes/checkStalledJobs.lua +0 -139
  99. package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -24
  100. package/dist/esm/scripts/moveJobsToWait-6.js.map +0 -1
  101. /package/dist/esm/scripts/{moveJobsToWait-6.d.ts → moveJobsToWait-7.d.ts} +0 -0
  102. /package/dist/esm/scripts/{moveStalledJobsToWait-8.d.ts → moveStalledJobsToWait-9.d.ts} +0 -0
@@ -11,34 +11,33 @@ const content = `--[[
11
11
  KEYS[5] 'stalled-check', (KEY)
12
12
  KEYS[6] 'meta', (KEY)
13
13
  KEYS[7] 'paused', (LIST)
14
- KEYS[8] 'event stream' (STREAM)
14
+ KEYS[8] 'marker'
15
+ KEYS[9] 'event stream' (STREAM)
15
16
  ARGV[1] Max stalled job count
16
17
  ARGV[2] queue.toKey('')
17
18
  ARGV[3] timestamp
18
19
  ARGV[4] max check time
19
20
  Events:
20
21
  'stalled' with stalled job id.
21
- ]] -- Includes
22
- --[[
23
- Move stalled jobs to wait.
24
- Input:
25
- stalledKey 'stalled' (SET)
26
- waitKey 'wait', (LIST)
27
- activeKey 'active', (LIST)
28
- failedKey 'failed', (ZSET)
29
- stalledCheckKey 'stalled-check', (KEY)
30
- metaKey 'meta', (KEY)
31
- pausedKey 'paused', (LIST)
32
- eventStreamKey 'event stream' (STREAM)
33
- maxStalledJobCount Max stalled job count
34
- queueKeyPrefix queue.toKey('')
35
- timestamp timestamp
36
- maxCheckTime max check time
37
- Events:
38
- 'stalled' with stalled job id.
39
22
  ]]
40
23
  local rcall = redis.call
41
24
  -- Includes
25
+ --[[
26
+ Function to add job in target list and add marker if needed.
27
+ ]]
28
+ -- Includes
29
+ --[[
30
+ Add marker if needed when a job is available.
31
+ ]]
32
+ local function addBaseMarkerIfNeeded(markerKey, isPaused)
33
+ if not isPaused then
34
+ rcall("ZADD", markerKey, 0, "0")
35
+ end
36
+ end
37
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
38
+ rcall(pushCmd, targetKey, jobId)
39
+ addBaseMarkerIfNeeded(markerKey, isPaused)
40
+ end
42
41
  --[[
43
42
  Function to loop in batches.
44
43
  Just a bit of warning, some commands as ZREM
@@ -200,103 +199,107 @@ local function trimEvents(metaKey, eventStreamKey)
200
199
  rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
201
200
  end
202
201
  end
203
- -- Check if we need to check for stalled jobs now.
204
- local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey,
205
- stalledCheckKey, metaKey, pausedKey,
206
- eventStreamKey, maxStalledJobCount,
207
- queueKeyPrefix, timestamp, maxCheckTime)
208
- if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
209
- rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
210
- -- Trim events before emiting them to avoid trimming events emitted in this script
211
- trimEvents(metaKey, eventStreamKey)
212
- -- Move all stalled jobs to wait
213
- local stalling = rcall('SMEMBERS', stalledKey)
214
- local stalled = {}
215
- local failed = {}
216
- if (#stalling > 0) then
217
- rcall('DEL', stalledKey)
218
- local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
219
- -- Remove from active list
220
- for i, jobId in ipairs(stalling) do
221
- -- Markers in waitlist DEPRECATED in v5: Remove in v6.
222
- if string.sub(jobId, 1, 2) == "0:" then
223
- -- If the jobId is a delay marker ID we just remove it.
224
- rcall("LREM", activeKey, 1, jobId)
225
- else
226
- local jobKey = queueKeyPrefix .. jobId
227
- -- Check that the lock is also missing, then we can handle this job as really stalled.
228
- if (rcall("EXISTS", jobKey .. ":lock") == 0) then
229
- -- Remove from the active queue.
230
- local removed = rcall("LREM", activeKey, 1, jobId)
231
- if (removed > 0) then
232
- -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
233
- local stalledCount =
234
- rcall("HINCRBY", jobKey, "stalledCounter", 1)
235
- if (stalledCount > MAX_STALLED_JOB_COUNT) then
236
- local rawOpts = rcall("HGET", jobKey, "opts")
237
- local opts = cjson.decode(rawOpts)
238
- local removeOnFailType = type(opts["removeOnFail"])
239
- rcall("ZADD", failedKey, timestamp, jobId)
240
- local failedReason =
241
- "job stalled more than allowable limit"
242
- rcall("HMSET", jobKey, "failedReason", failedReason,
243
- "finishedOn", timestamp)
244
- rcall("XADD", eventStreamKey, "*", "event",
245
- "failed", "jobId", jobId, 'prev', 'active',
246
- 'failedReason', failedReason)
247
- if removeOnFailType == "number" then
248
- removeJobsByMaxCount(opts["removeOnFail"],
249
- failedKey, queueKeyPrefix)
250
- elseif removeOnFailType == "boolean" then
251
- if opts["removeOnFail"] then
252
- removeJob(jobId, false, queueKeyPrefix)
253
- rcall("ZREM", failedKey, jobId)
254
- end
255
- elseif removeOnFailType ~= "nil" then
256
- local maxAge = opts["removeOnFail"]["age"]
257
- local maxCount = opts["removeOnFail"]["count"]
258
- if maxAge ~= nil then
259
- removeJobsByMaxAge(timestamp, maxAge,
260
- failedKey, queueKeyPrefix)
261
- end
262
- if maxCount ~= nil and maxCount > 0 then
263
- removeJobsByMaxCount(maxCount, failedKey,
264
- queueKeyPrefix)
265
- end
202
+ local stalledKey = KEYS[1]
203
+ local waitKey = KEYS[2]
204
+ local activeKey = KEYS[3]
205
+ local failedKey = KEYS[4]
206
+ local stalledCheckKey = KEYS[5]
207
+ local metaKey = KEYS[6]
208
+ local pausedKey = KEYS[7]
209
+ local markerKey = KEYS[8]
210
+ local eventStreamKey = KEYS[9]
211
+ local maxStalledJobCount = ARGV[1]
212
+ local queueKeyPrefix = ARGV[2]
213
+ local timestamp = ARGV[3]
214
+ local maxCheckTime = ARGV[4]
215
+ if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
216
+ rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
217
+ -- Trim events before emiting them to avoid trimming events emitted in this script
218
+ trimEvents(metaKey, eventStreamKey)
219
+ -- Move all stalled jobs to wait
220
+ local stalling = rcall('SMEMBERS', stalledKey)
221
+ local stalled = {}
222
+ local failed = {}
223
+ if (#stalling > 0) then
224
+ rcall('DEL', stalledKey)
225
+ local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
226
+ -- Remove from active list
227
+ for i, jobId in ipairs(stalling) do
228
+ -- Markers in waitlist DEPRECATED in v5: Remove in v6.
229
+ if string.sub(jobId, 1, 2) == "0:" then
230
+ -- If the jobId is a delay marker ID we just remove it.
231
+ rcall("LREM", activeKey, 1, jobId)
232
+ else
233
+ local jobKey = queueKeyPrefix .. jobId
234
+ -- Check that the lock is also missing, then we can handle this job as really stalled.
235
+ if (rcall("EXISTS", jobKey .. ":lock") == 0) then
236
+ -- Remove from the active queue.
237
+ local removed = rcall("LREM", activeKey, 1, jobId)
238
+ if (removed > 0) then
239
+ -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
240
+ local stalledCount =
241
+ rcall("HINCRBY", jobKey, "stalledCounter", 1)
242
+ if (stalledCount > MAX_STALLED_JOB_COUNT) then
243
+ local rawOpts = rcall("HGET", jobKey, "opts")
244
+ local opts = cjson.decode(rawOpts)
245
+ local removeOnFailType = type(opts["removeOnFail"])
246
+ rcall("ZADD", failedKey, timestamp, jobId)
247
+ local failedReason =
248
+ "job stalled more than allowable limit"
249
+ rcall("HMSET", jobKey, "failedReason", failedReason,
250
+ "finishedOn", timestamp)
251
+ rcall("XADD", eventStreamKey, "*", "event",
252
+ "failed", "jobId", jobId, 'prev', 'active',
253
+ 'failedReason', failedReason)
254
+ if removeOnFailType == "number" then
255
+ removeJobsByMaxCount(opts["removeOnFail"],
256
+ failedKey, queueKeyPrefix)
257
+ elseif removeOnFailType == "boolean" then
258
+ if opts["removeOnFail"] then
259
+ removeJob(jobId, false, queueKeyPrefix)
260
+ rcall("ZREM", failedKey, jobId)
261
+ end
262
+ elseif removeOnFailType ~= "nil" then
263
+ local maxAge = opts["removeOnFail"]["age"]
264
+ local maxCount = opts["removeOnFail"]["count"]
265
+ if maxAge ~= nil then
266
+ removeJobsByMaxAge(timestamp, maxAge,
267
+ failedKey, queueKeyPrefix)
268
+ end
269
+ if maxCount ~= nil and maxCount > 0 then
270
+ removeJobsByMaxCount(maxCount, failedKey,
271
+ queueKeyPrefix)
266
272
  end
267
- table.insert(failed, jobId)
268
- else
269
- local target =
270
- getTargetQueueList(metaKey, waitKey, pausedKey)
271
- -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
272
- rcall("RPUSH", target, jobId)
273
- rcall("XADD", eventStreamKey, "*", "event",
274
- "waiting", "jobId", jobId, 'prev', 'active')
275
- -- Emit the stalled event
276
- rcall("XADD", eventStreamKey, "*", "event",
277
- "stalled", "jobId", jobId)
278
- table.insert(stalled, jobId)
279
273
  end
274
+ table.insert(failed, jobId)
275
+ else
276
+ local target, isPaused=
277
+ getTargetQueueList(metaKey, waitKey, pausedKey)
278
+ -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
279
+ addJobInTargetList(target, markerKey, "RPUSH", isPaused, jobId)
280
+ rcall("XADD", eventStreamKey, "*", "event",
281
+ "waiting", "jobId", jobId, 'prev', 'active')
282
+ -- Emit the stalled event
283
+ rcall("XADD", eventStreamKey, "*", "event",
284
+ "stalled", "jobId", jobId)
285
+ table.insert(stalled, jobId)
280
286
  end
281
287
  end
282
288
  end
283
289
  end
284
290
  end
285
- -- Mark potentially stalled jobs
286
- local active = rcall('LRANGE', activeKey, 0, -1)
287
- if (#active > 0) then
288
- for from, to in batches(#active, 7000) do
289
- rcall('SADD', stalledKey, unpack(active, from, to))
290
- end
291
+ end
292
+ -- Mark potentially stalled jobs
293
+ local active = rcall('LRANGE', activeKey, 0, -1)
294
+ if (#active > 0) then
295
+ for from, to in batches(#active, 7000) do
296
+ rcall('SADD', stalledKey, unpack(active, from, to))
291
297
  end
292
- return {failed, stalled}
293
298
  end
294
- return checkStalledJobs(KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6],
295
- KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])
296
- `;
299
+ return {failed, stalled}`;
297
300
  exports.moveStalledJobsToWait = {
298
301
  name: 'moveStalledJobsToWait',
299
302
  content,
300
- keys: 8,
303
+ keys: 9,
301
304
  };
302
- //# sourceMappingURL=moveStalledJobsToWait-8.js.map
305
+ //# sourceMappingURL=moveStalledJobsToWait-9.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoSf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAuSS,CAAC;AACb,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -159,16 +159,31 @@ end
159
159
  'waiting'
160
160
  ]]
161
161
  -- Includes
162
+ --[[
163
+ Function to add job in target list and add marker if needed.
164
+ ]]
165
+ -- Includes
166
+ --[[
167
+ Add marker if needed when a job is available.
168
+ ]]
169
+ local function addBaseMarkerIfNeeded(markerKey, isPaused)
170
+ if not isPaused then
171
+ rcall("ZADD", markerKey, 0, "0")
172
+ end
173
+ end
174
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
175
+ rcall(pushCmd, targetKey, jobId)
176
+ addBaseMarkerIfNeeded(markerKey, isPaused)
177
+ end
162
178
  --[[
163
179
  Function to add job considering priority.
164
180
  ]]
181
+ -- Includes
165
182
  local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
166
183
  local prioCounter = rcall("INCR", priorityCounterKey)
167
184
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
168
185
  rcall("ZADD", prioritizedKey, score, jobId)
169
- if not isPaused then
170
- rcall("ZADD", markerKey, 0, "0")
171
- end
186
+ addBaseMarkerIfNeeded(markerKey, isPaused)
172
187
  end
173
188
  -- Try to get as much as 1000 jobs at once
174
189
  local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
@@ -182,10 +197,7 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
182
197
  tonumber(rcall("HGET", jobKey, "priority")) or 0
183
198
  if priority == 0 then
184
199
  -- LIFO or FIFO
185
- rcall("LPUSH", targetKey, jobId)
186
- if not isPaused then
187
- rcall("ZADD", markerKey, 0, "0")
188
- end
200
+ addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
189
201
  else
190
202
  addJobWithPriority(markerKey, prioritizedKey, priority,
191
203
  jobId, priorityCounterKey, isPaused)
@@ -1 +1 @@
1
- {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoOf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgPf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -29,7 +29,7 @@ local rcall = redis.call
29
29
  -- Includes
30
30
  --[[
31
31
  Add delay marker if needed.
32
- ]]
32
+ ]]
33
33
  -- Includes
34
34
  --[[
35
35
  Function to return the next delayed job timestamp.
@@ -116,6 +116,17 @@ local function getNextDelayedTimestamp(delayedKey)
116
116
  return nextTimestamp
117
117
  end
118
118
  end
119
+ --[[
120
+ Function to check for the meta.paused key to decide if we are paused or not
121
+ (since an empty list and !EXISTS are not really the same).
122
+ ]]
123
+ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
124
+ if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
125
+ return waitKey, false
126
+ else
127
+ return pausedKey, true
128
+ end
129
+ end
119
130
  --[[
120
131
  Function to move job from prioritized state to active.
121
132
  ]]
@@ -202,7 +213,7 @@ end
202
213
  -- Includes
203
214
  --[[
204
215
  Add delay marker if needed.
205
- ]]
216
+ ]]
206
217
  -- Includes
207
218
  local function addDelayMarkerIfNeeded(markerKey, delayedKey)
208
219
  local nextTimestamp = getNextDelayedTimestamp(delayedKey)
@@ -213,33 +224,37 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
213
224
  end
214
225
  end
215
226
  --[[
216
- Function to check for the meta.paused key to decide if we are paused or not
217
- (since an empty list and !EXISTS are not really the same).
227
+ Function to add job in target list and add marker if needed.
218
228
  ]]
219
- local function isQueuePaused(queueMetaKey)
220
- return rcall("HEXISTS", queueMetaKey, "paused") == 1
229
+ -- Includes
230
+ --[[
231
+ Add marker if needed when a job is available.
232
+ ]]
233
+ local function addBaseMarkerIfNeeded(markerKey, isPaused)
234
+ if not isPaused then
235
+ rcall("ZADD", markerKey, 0, "0")
236
+ end
237
+ end
238
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
239
+ rcall(pushCmd, targetKey, jobId)
240
+ addBaseMarkerIfNeeded(markerKey, isPaused)
221
241
  end
222
242
  --[[
223
243
  Function to add job considering priority.
224
244
  ]]
245
+ -- Includes
225
246
  local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
226
247
  local prioCounter = rcall("INCR", priorityCounterKey)
227
248
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
228
249
  rcall("ZADD", prioritizedKey, score, jobId)
229
- if not isPaused then
230
- rcall("ZADD", markerKey, 0, "0")
231
- end
250
+ addBaseMarkerIfNeeded(markerKey, isPaused)
232
251
  end
233
252
  --[[
234
253
  Function to check for the meta.paused key to decide if we are paused or not
235
254
  (since an empty list and !EXISTS are not really the same).
236
255
  ]]
237
- local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
238
- if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
239
- return waitKey, false
240
- else
241
- return pausedKey, true
242
- end
256
+ local function isQueuePaused(queueMetaKey)
257
+ return rcall("HEXISTS", queueMetaKey, "paused") == 1
243
258
  end
244
259
  local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
245
260
  parentKey, parentId, timestamp)
@@ -264,11 +279,10 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
264
279
  addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
265
280
  else
266
281
  if priority == 0 then
267
- local parentTarget, _paused =
282
+ local parentTarget, isParentPaused =
268
283
  getTargetQueueList(parentMetaKey, parentWaitKey,
269
284
  parentPausedKey)
270
- rcall("RPUSH", parentTarget, parentId)
271
- rcall("ZADD", parentMarkerKey, 0, "0")
285
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPaused, parentId)
272
286
  else
273
287
  local isPaused = isQueuePaused(parentMetaKey)
274
288
  addJobWithPriority(parentMarkerKey,
@@ -328,10 +342,7 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
328
342
  tonumber(rcall("HGET", jobKey, "priority")) or 0
329
343
  if priority == 0 then
330
344
  -- LIFO or FIFO
331
- rcall("LPUSH", targetKey, jobId)
332
- if not isPaused then
333
- rcall("ZADD", markerKey, 0, "0")
334
- end
345
+ addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
335
346
  else
336
347
  addJobWithPriority(markerKey, prioritizedKey, priority,
337
348
  jobId, priorityCounterKey, isPaused)
@@ -1 +1 @@
1
- {"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgqBf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2qBf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -19,7 +19,7 @@ local rcall = redis.call
19
19
  -- Includes
20
20
  --[[
21
21
  Add delay marker if needed.
22
- ]]
22
+ ]]
23
23
  -- Includes
24
24
  --[[
25
25
  Function to return the next delayed job timestamp.
@@ -44,6 +44,7 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
44
44
  end
45
45
  local markerKey = KEYS[7]
46
46
  local hasJobs = rcall("EXISTS", KEYS[1]) == 1
47
+ --TODO: check this logic to be reused when changing a delay
47
48
  if hasJobs then rcall("RENAME", KEYS[1], KEYS[2]) end
48
49
  if ARGV[1] == "paused" then
49
50
  rcall("HSET", KEYS[3], "paused", 1)
@@ -1 +1 @@
1
- {"version":3,"file":"pause-7.js","sourceRoot":"","sources":["../../../src/scripts/pause-7.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyDf,CAAC;AACW,QAAA,KAAK,GAAG;IACnB,IAAI,EAAE,OAAO;IACb,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"pause-7.js","sourceRoot":"","sources":["../../../src/scripts/pause-7.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Df,CAAC;AACW,QAAA,KAAK,GAAG;IACnB,IAAI,EAAE,OAAO;IACb,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -23,16 +23,31 @@ const content = `--[[
23
23
  local rcall = redis.call
24
24
  local jobId = ARGV[2]
25
25
  -- Includes
26
+ --[[
27
+ Function to add job in target list and add marker if needed.
28
+ ]]
29
+ -- Includes
30
+ --[[
31
+ Add marker if needed when a job is available.
32
+ ]]
33
+ local function addBaseMarkerIfNeeded(markerKey, isPaused)
34
+ if not isPaused then
35
+ rcall("ZADD", markerKey, 0, "0")
36
+ end
37
+ end
38
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
39
+ rcall(pushCmd, targetKey, jobId)
40
+ addBaseMarkerIfNeeded(markerKey, isPaused)
41
+ end
26
42
  --[[
27
43
  Function to add job considering priority.
28
44
  ]]
45
+ -- Includes
29
46
  local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
30
47
  local prioCounter = rcall("INCR", priorityCounterKey)
31
48
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
32
49
  rcall("ZADD", prioritizedKey, score, jobId)
33
- if not isPaused then
34
- rcall("ZADD", markerKey, 0, "0")
35
- end
50
+ addBaseMarkerIfNeeded(markerKey, isPaused)
36
51
  end
37
52
  --[[
38
53
  Function to check for the meta.paused key to decide if we are paused or not
@@ -57,8 +72,7 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then
57
72
  if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
58
73
  if priority == 0 then
59
74
  -- LIFO or FIFO
60
- rcall("LPUSH", target, jobId)
61
- if not paused then rcall("ZADD", KEYS[8], 0, "0") end
75
+ addJobInTargetList(target, KEYS[8], "LPUSH", paused, jobId)
62
76
  else
63
77
  addJobWithPriority(KEYS[8], KEYS[5], priority, jobId, KEYS[6], paused)
64
78
  end
@@ -1 +1 @@
1
- {"version":3,"file":"promote-8.js","sourceRoot":"","sources":["../../../src/scripts/promote-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqEf,CAAC;AACW,QAAA,OAAO,GAAG;IACrB,IAAI,EAAE,SAAS;IACf,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"promote-8.js","sourceRoot":"","sources":["../../../src/scripts/promote-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmFf,CAAC;AACW,QAAA,OAAO,GAAG;IACrB,IAAI,EAAE,SAAS;IACf,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -31,13 +31,20 @@ local rcall = redis.call
31
31
  --[[
32
32
  Function to add job considering priority.
33
33
  ]]
34
+ -- Includes
35
+ --[[
36
+ Add marker if needed when a job is available.
37
+ ]]
38
+ local function addBaseMarkerIfNeeded(markerKey, isPaused)
39
+ if not isPaused then
40
+ rcall("ZADD", markerKey, 0, "0")
41
+ end
42
+ end
34
43
  local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
35
44
  local prioCounter = rcall("INCR", priorityCounterKey)
36
45
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
37
46
  rcall("ZADD", prioritizedKey, score, jobId)
38
- if not isPaused then
39
- rcall("ZADD", markerKey, 0, "0")
40
- end
47
+ addBaseMarkerIfNeeded(markerKey, isPaused)
41
48
  end
42
49
  --[[
43
50
  Function to get max events value or set by default 10000.
@@ -68,6 +75,14 @@ end
68
75
  'waiting'
69
76
  ]]
70
77
  -- Includes
78
+ --[[
79
+ Function to add job in target list and add marker if needed.
80
+ ]]
81
+ -- Includes
82
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
83
+ rcall(pushCmd, targetKey, jobId)
84
+ addBaseMarkerIfNeeded(markerKey, isPaused)
85
+ end
71
86
  -- Try to get as much as 1000 jobs at once
72
87
  local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
73
88
  eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
@@ -80,10 +95,7 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
80
95
  tonumber(rcall("HGET", jobKey, "priority")) or 0
81
96
  if priority == 0 then
82
97
  -- LIFO or FIFO
83
- rcall("LPUSH", targetKey, jobId)
84
- if not isPaused then
85
- rcall("ZADD", markerKey, 0, "0")
86
- end
98
+ addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
87
99
  else
88
100
  addJobWithPriority(markerKey, prioritizedKey, priority,
89
101
  jobId, priorityCounterKey, isPaused)
@@ -114,6 +126,7 @@ if rcall("EXISTS", KEYS[4]) == 1 then
114
126
  -- Standard or priority add
115
127
  if priority == 0 then
116
128
  rcall(ARGV[3], target, ARGV[4])
129
+ -- TODO: check if we need to add marker in this case too
117
130
  else
118
131
  addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], paused)
119
132
  end
@@ -1 +1 @@
1
- {"version":3,"file":"retryJob-10.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-10.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6Hf,CAAC;AACW,QAAA,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"retryJob-10.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-10.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0If,CAAC;AACW,QAAA,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}