bullmq 5.46.0 → 5.47.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/dist/cjs/classes/job.js +11 -9
  2. package/dist/cjs/classes/job.js.map +1 -1
  3. package/dist/cjs/classes/scripts.js +4 -3
  4. package/dist/cjs/classes/scripts.js.map +1 -1
  5. package/dist/cjs/classes/worker.js.map +1 -1
  6. package/dist/cjs/commands/includes/moveParentToFailedIfNeeded.lua +3 -0
  7. package/dist/cjs/commands/includes/moveParentToWait.lua +49 -0
  8. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +6 -51
  9. package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
  10. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +26 -30
  11. package/dist/cjs/commands/moveToFinished-14.lua +38 -30
  12. package/dist/cjs/commands/moveToWaitingChildren-8.lua +13 -8
  13. package/dist/cjs/scripts/addDelayedJob-6.js +24 -20
  14. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  15. package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
  16. package/dist/cjs/scripts/addParentJob-4.js +24 -20
  17. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  18. package/dist/cjs/scripts/addPrioritizedJob-8.js +24 -20
  19. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
  20. package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
  21. package/dist/cjs/scripts/addStandardJob-8.js +24 -20
  22. package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
  23. package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
  24. package/dist/cjs/scripts/drain-5.js +5 -5
  25. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +56 -54
  26. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  27. package/dist/cjs/scripts/moveToFinished-14.js +67 -55
  28. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  29. package/dist/cjs/scripts/moveToWaitingChildren-8.js +43 -33
  30. package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
  31. package/dist/cjs/scripts/obliterate-2.js +5 -5
  32. package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
  33. package/dist/cjs/scripts/removeJob-3.js +5 -5
  34. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  35. package/dist/cjs/version.js +1 -1
  36. package/dist/esm/classes/job.js +11 -9
  37. package/dist/esm/classes/job.js.map +1 -1
  38. package/dist/esm/classes/queue-events.d.ts +1 -1
  39. package/dist/esm/classes/scripts.js +4 -3
  40. package/dist/esm/classes/scripts.js.map +1 -1
  41. package/dist/esm/classes/worker.js.map +1 -1
  42. package/dist/esm/commands/includes/moveParentToFailedIfNeeded.lua +3 -0
  43. package/dist/esm/commands/includes/moveParentToWait.lua +49 -0
  44. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +6 -51
  45. package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
  46. package/dist/esm/commands/moveStalledJobsToWait-9.lua +26 -30
  47. package/dist/esm/commands/moveToFinished-14.lua +38 -30
  48. package/dist/esm/commands/moveToWaitingChildren-8.lua +13 -8
  49. package/dist/esm/scripts/addDelayedJob-6.js +24 -20
  50. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  51. package/dist/esm/scripts/addJobScheduler-11.js +5 -5
  52. package/dist/esm/scripts/addParentJob-4.js +24 -20
  53. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  54. package/dist/esm/scripts/addPrioritizedJob-8.js +24 -20
  55. package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
  56. package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
  57. package/dist/esm/scripts/addStandardJob-8.js +24 -20
  58. package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
  59. package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
  60. package/dist/esm/scripts/drain-5.js +5 -5
  61. package/dist/esm/scripts/moveStalledJobsToWait-9.js +56 -54
  62. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  63. package/dist/esm/scripts/moveToFinished-14.js +67 -55
  64. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  65. package/dist/esm/scripts/moveToWaitingChildren-8.js +43 -33
  66. package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
  67. package/dist/esm/scripts/obliterate-2.js +5 -5
  68. package/dist/esm/scripts/removeChildDependency-1.js +5 -5
  69. package/dist/esm/scripts/removeJob-3.js +5 -5
  70. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  71. package/dist/esm/types/job-options.d.ts +12 -1
  72. package/dist/esm/version.d.ts +1 -1
  73. package/dist/esm/version.js +1 -1
  74. package/package.json +1 -1
@@ -105,7 +105,7 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
105
105
  end
106
106
  return waitKey, false
107
107
  end
108
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
108
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
109
109
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
110
110
  parentPrefix .. "wait", parentPrefix .. "paused")
111
111
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -133,10 +133,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
133
133
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
134
134
  end
135
135
  else
136
- moveParentToWait(parentPrefix, parentId)
136
+ _moveParentToWait(parentPrefix, parentId)
137
137
  end
138
138
  else
139
- moveParentToWait(parentPrefix, parentId, true)
139
+ _moveParentToWait(parentPrefix, parentId, true)
140
140
  end
141
141
  end
142
142
  end
@@ -164,10 +164,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
164
164
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
165
165
  end
166
166
  else
167
- moveParentToWait(parentPrefix, parentId)
167
+ _moveParentToWait(parentPrefix, parentId)
168
168
  end
169
169
  else
170
- moveParentToWait(parentPrefix, parentId, true)
170
+ _moveParentToWait(parentPrefix, parentId, true)
171
171
  end
172
172
  end
173
173
  end
@@ -76,9 +76,11 @@ end
76
76
  ]]
77
77
  -- Includes
78
78
  --[[
79
- Validate and move parent to active if needed.
79
+ Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
80
+ ]]
81
+ --[[
82
+ Validate and move parent to a wait status (wait, prioritized or delayed)
80
83
  ]]
81
- -- Includes
82
84
  --[[
83
85
  Add delay marker if needed.
84
86
  ]]
@@ -136,12 +138,9 @@ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
136
138
  end
137
139
  return false
138
140
  end
139
- local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
140
- parentKey, parentId, timestamp)
141
- local isParentActive = rcall("ZSCORE",
142
- parentQueueKey .. ":waiting-children", parentId)
143
- if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
144
- rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
141
+ local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
142
+ local isParentWaitingChildren = rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
143
+ if isParentWaitingChildren > 0 then
145
144
  local parentWaitKey = parentQueueKey .. ":wait"
146
145
  local parentPausedKey = parentQueueKey .. ":paused"
147
146
  local parentActiveKey = parentQueueKey .. ":active"
@@ -150,32 +149,37 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
150
149
  local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
151
150
  local priority = tonumber(jobAttributes[1]) or 0
152
151
  local delay = tonumber(jobAttributes[2]) or 0
152
+ -- ignore dependencies if any left
153
+ rcall("HSET", parentKey, "igdp", 1)
153
154
  if delay > 0 then
154
155
  local delayedTimestamp = tonumber(timestamp) + delay
155
156
  local score = delayedTimestamp * 0x1000
156
157
  local parentDelayedKey = parentQueueKey .. ":delayed"
157
158
  rcall("ZADD", parentDelayedKey, score, parentId)
158
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
159
- "jobId", parentId, "delay", delayedTimestamp)
159
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
160
+ delayedTimestamp)
160
161
  addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
161
162
  else
162
163
  if priority == 0 then
163
- local parentTarget, isParentPausedOrMaxed =
164
- getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
165
- parentPausedKey)
166
- addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
167
- parentId)
164
+ local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
165
+ parentWaitKey, parentPausedKey)
166
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
168
167
  else
169
168
  local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
170
- addJobWithPriority(parentMarkerKey,
171
- parentQueueKey .. ":prioritized", priority,
172
- parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
169
+ addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
170
+ parentQueueKey .. ":pc", isPausedOrMaxed)
173
171
  end
174
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
175
- "jobId", parentId, "prev", "waiting-children")
172
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
173
+ "waiting-children")
176
174
  end
177
175
  end
178
176
  end
177
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
178
+ local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
179
+ if hasPendingDependencies then
180
+ moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
181
+ end
182
+ end
179
183
  --[[
180
184
  Function to remove deduplication key if needed.
181
185
  ]]
@@ -229,7 +233,7 @@ end
229
233
  local getJobKeyPrefix = function (jobKey, jobId)
230
234
  return string.sub(jobKey, 0, #jobKey - #jobId)
231
235
  end
232
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
236
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
233
237
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
234
238
  parentPrefix .. "wait", parentPrefix .. "paused")
235
239
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -257,10 +261,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
257
261
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
258
262
  end
259
263
  else
260
- moveParentToWait(parentPrefix, parentId)
264
+ _moveParentToWait(parentPrefix, parentId)
261
265
  end
262
266
  else
263
- moveParentToWait(parentPrefix, parentId, true)
267
+ _moveParentToWait(parentPrefix, parentId, true)
264
268
  end
265
269
  end
266
270
  end
@@ -288,10 +292,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
288
292
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
289
293
  end
290
294
  else
291
- moveParentToWait(parentPrefix, parentId)
295
+ _moveParentToWait(parentPrefix, parentId)
292
296
  end
293
297
  else
294
- moveParentToWait(parentPrefix, parentId, true)
298
+ _moveParentToWait(parentPrefix, parentId, true)
295
299
  end
296
300
  end
297
301
  end
@@ -399,6 +403,8 @@ local function moveParentToFailedIfNeeded(parentQueueKey, parentKey, parentId, j
399
403
  parentKey,
400
404
  timestamp
401
405
  )
406
+ elseif parentData['cpof'] then
407
+ moveParentToWait(parentData['queueKey'], parentKey, parentData['id'], timestamp)
402
408
  elseif parentData['idof'] or parentData['rdof'] then
403
409
  local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
404
410
  local grandParentDependenciesSet = grandParentKey .. ":dependencies"
@@ -460,7 +466,9 @@ local maxStalledJobCount = tonumber(ARGV[1])
460
466
  local queueKeyPrefix = ARGV[2]
461
467
  local timestamp = ARGV[3]
462
468
  local maxCheckTime = ARGV[4]
463
- if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
469
+ if rcall("EXISTS", stalledCheckKey) == 1 then
470
+ return {{}, {}}
471
+ end
464
472
  rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
465
473
  -- Trim events before emiting them to avoid trimming events emitted in this script
466
474
  trimEvents(metaKey, eventStreamKey)
@@ -484,8 +492,7 @@ if (#stalling > 0) then
484
492
  local removed = rcall("LREM", activeKey, 1, jobId)
485
493
  if (removed > 0) then
486
494
  -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
487
- local stalledCount =
488
- rcall("HINCRBY", jobKey, "stalledCounter", 1)
495
+ local stalledCount = rcall("HINCRBY", jobKey, "stalledCounter", 1)
489
496
  if (stalledCount > maxStalledJobCount) then
490
497
  local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
491
498
  local rawOpts = jobAttributes[1]
@@ -493,38 +500,35 @@ if (#stalling > 0) then
493
500
  local opts = cjson.decode(rawOpts)
494
501
  rcall("ZADD", failedKey, timestamp, jobId)
495
502
  removeDeduplicationKeyIfNeeded(queueKeyPrefix, jobAttributes[3])
496
- local failedReason =
497
- "job stalled more than allowable limit"
498
- rcall("HMSET", jobKey, "failedReason", failedReason,
499
- "finishedOn", timestamp)
500
- rcall("XADD", eventStreamKey, "*", "event",
501
- "failed", "jobId", jobId, 'prev', 'active',
502
- 'failedReason', failedReason)
503
+ local failedReason = "job stalled more than allowable limit"
504
+ rcall("HMSET", jobKey, "failedReason", failedReason, "finishedOn", timestamp)
505
+ rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId", jobId, 'prev', 'active',
506
+ 'failedReason', failedReason)
503
507
  if rawParentData then
504
- if opts['fpof'] then
508
+ if opts['fpof'] or opts['cpof'] then
505
509
  local parentData = cjson.decode(rawParentData)
506
510
  -- TODO: need to remove this job from dependencies set in next breaking change
507
511
  -- no for now as it would imply a breaking change
508
512
  local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
509
513
  local unsuccesssfulSet = parentKey .. ":unsuccessful"
510
514
  rcall("ZADD", unsuccesssfulSet, timestamp, jobKey)
511
- moveParentToFailedIfNeeded(
512
- parentData['queueKey'],
513
- parentData['queueKey'] .. ':' .. parentData['id'],
514
- parentData['id'],
515
- jobKey,
516
- timestamp
517
- )
515
+ if opts['fpof'] then
516
+ moveParentToFailedIfNeeded(parentData['queueKey'],
517
+ parentData['queueKey'] .. ':' .. parentData['id'], parentData['id'], jobKey,
518
+ timestamp)
519
+ elseif opts['cpof'] then
520
+ moveParentToWait(parentData['queueKey'], parentKey, parentData['id'], timestamp)
521
+ end
518
522
  elseif opts['idof'] or opts['rdof'] then
519
523
  local parentData = cjson.decode(rawParentData)
520
524
  local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
521
525
  local dependenciesSet = parentKey .. ":dependencies"
522
526
  if rcall("SREM", dependenciesSet, jobKey) == 1 then
523
- moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
524
- parentKey, parentData['id'], timestamp)
527
+ moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet, parentKey,
528
+ parentData['id'], timestamp)
525
529
  if opts['idof'] then
526
- local failedSet = parentKey .. ":failed"
527
- rcall("HSET", failedSet, jobKey, failedReason)
530
+ local failedSet = parentKey .. ":failed"
531
+ rcall("HSET", failedSet, jobKey, failedReason)
528
532
  end
529
533
  end
530
534
  end
@@ -532,15 +536,12 @@ if (#stalling > 0) then
532
536
  removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
533
537
  table.insert(failed, jobId)
534
538
  else
535
- local target, isPausedOrMaxed =
536
- getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
539
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
537
540
  -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
538
541
  addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
539
- rcall("XADD", eventStreamKey, "*", "event",
540
- "waiting", "jobId", jobId, 'prev', 'active')
542
+ rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, 'prev', 'active')
541
543
  -- Emit the stalled event
542
- rcall("XADD", eventStreamKey, "*", "event",
543
- "stalled", "jobId", jobId)
544
+ rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId", jobId)
544
545
  table.insert(stalled, jobId)
545
546
  end
546
547
  end
@@ -555,7 +556,8 @@ if (#active > 0) then
555
556
  rcall('SADD', stalledKey, unpack(active, from, to))
556
557
  end
557
558
  end
558
- return {failed, stalled}`;
559
+ return {failed, stalled}
560
+ `;
559
561
  export const moveStalledJobsToWait = {
560
562
  name: 'moveStalledJobsToWait',
561
563
  content,
@@ -1 +1 @@
1
- {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBA6iBS,CAAC;AAC1B,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+iBf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -34,6 +34,7 @@ const content = `--[[
34
34
  opts - attempts max attempts
35
35
  opts - maxMetricsSize
36
36
  opts - fpof - fail parent on fail
37
+ opts - cpof - continue parent on fail
37
38
  opts - idof - ignore dependency on fail
38
39
  opts - rdof - remove dependency on fail
39
40
  opts - name - worker name
@@ -167,9 +168,11 @@ end
167
168
  ]]
168
169
  -- Includes
169
170
  --[[
170
- Validate and move parent to active if needed.
171
+ Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
172
+ ]]
173
+ --[[
174
+ Validate and move parent to a wait status (wait, prioritized or delayed)
171
175
  ]]
172
- -- Includes
173
176
  --[[
174
177
  Add delay marker if needed.
175
178
  ]]
@@ -231,12 +234,9 @@ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
231
234
  end
232
235
  return false
233
236
  end
234
- local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
235
- parentKey, parentId, timestamp)
236
- local isParentActive = rcall("ZSCORE",
237
- parentQueueKey .. ":waiting-children", parentId)
238
- if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
239
- rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
237
+ local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
238
+ local isParentWaitingChildren = rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
239
+ if isParentWaitingChildren > 0 then
240
240
  local parentWaitKey = parentQueueKey .. ":wait"
241
241
  local parentPausedKey = parentQueueKey .. ":paused"
242
242
  local parentActiveKey = parentQueueKey .. ":active"
@@ -245,32 +245,37 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
245
245
  local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
246
246
  local priority = tonumber(jobAttributes[1]) or 0
247
247
  local delay = tonumber(jobAttributes[2]) or 0
248
+ -- ignore dependencies if any left
249
+ rcall("HSET", parentKey, "igdp", 1)
248
250
  if delay > 0 then
249
251
  local delayedTimestamp = tonumber(timestamp) + delay
250
252
  local score = delayedTimestamp * 0x1000
251
253
  local parentDelayedKey = parentQueueKey .. ":delayed"
252
254
  rcall("ZADD", parentDelayedKey, score, parentId)
253
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
254
- "jobId", parentId, "delay", delayedTimestamp)
255
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
256
+ delayedTimestamp)
255
257
  addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
256
258
  else
257
259
  if priority == 0 then
258
- local parentTarget, isParentPausedOrMaxed =
259
- getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
260
- parentPausedKey)
261
- addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
262
- parentId)
260
+ local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
261
+ parentWaitKey, parentPausedKey)
262
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
263
263
  else
264
264
  local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
265
- addJobWithPriority(parentMarkerKey,
266
- parentQueueKey .. ":prioritized", priority,
267
- parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
265
+ addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
266
+ parentQueueKey .. ":pc", isPausedOrMaxed)
268
267
  end
269
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
270
- "jobId", parentId, "prev", "waiting-children")
268
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
269
+ "waiting-children")
271
270
  end
272
271
  end
273
272
  end
273
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
274
+ local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
275
+ if hasPendingDependencies then
276
+ moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
277
+ end
278
+ end
274
279
  --[[
275
280
  Function to remove deduplication key if needed.
276
281
  ]]
@@ -324,7 +329,7 @@ end
324
329
  local getJobKeyPrefix = function (jobKey, jobId)
325
330
  return string.sub(jobKey, 0, #jobKey - #jobId)
326
331
  end
327
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
332
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
328
333
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
329
334
  parentPrefix .. "wait", parentPrefix .. "paused")
330
335
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -352,10 +357,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
352
357
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
353
358
  end
354
359
  else
355
- moveParentToWait(parentPrefix, parentId)
360
+ _moveParentToWait(parentPrefix, parentId)
356
361
  end
357
362
  else
358
- moveParentToWait(parentPrefix, parentId, true)
363
+ _moveParentToWait(parentPrefix, parentId, true)
359
364
  end
360
365
  end
361
366
  end
@@ -383,10 +388,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
383
388
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
384
389
  end
385
390
  else
386
- moveParentToWait(parentPrefix, parentId)
391
+ _moveParentToWait(parentPrefix, parentId)
387
392
  end
388
393
  else
389
- moveParentToWait(parentPrefix, parentId, true)
394
+ _moveParentToWait(parentPrefix, parentId, true)
390
395
  end
391
396
  end
392
397
  end
@@ -494,6 +499,8 @@ local function moveParentToFailedIfNeeded(parentQueueKey, parentKey, parentId, j
494
499
  parentKey,
495
500
  timestamp
496
501
  )
502
+ elseif parentData['cpof'] then
503
+ moveParentToWait(parentData['queueKey'], parentKey, parentData['id'], timestamp)
497
504
  elseif parentData['idof'] or parentData['rdof'] then
498
505
  local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
499
506
  local grandParentDependenciesSet = grandParentKey .. ":dependencies"
@@ -655,7 +662,9 @@ local jobIdKey = KEYS[12]
655
662
  if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
656
663
  -- Make sure it does not have pending dependencies
657
664
  -- It must happen before removing lock
658
- if ARGV[5] == "completed" and rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then
665
+ if ARGV[5] == "completed" and
666
+ not rcall("HGET", jobIdKey, "igdp") and -- check if we should ignore this check
667
+ rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then
659
668
  return -4
660
669
  end
661
670
  local opts = cmsgpack.unpack(ARGV[8])
@@ -682,7 +691,9 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
682
691
  local timestamp = ARGV[2]
683
692
  -- Remove from active list (if not active we shall return error)
684
693
  local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
685
- if (numRemovedElements < 1) then return -3 end
694
+ if (numRemovedElements < 1) then
695
+ return -3
696
+ end
686
697
  local eventStreamKey = KEYS[4]
687
698
  local metaKey = KEYS[9]
688
699
  -- Trim events before emiting them to avoid trimming events emitted in this script
@@ -702,22 +713,24 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
702
713
  if ARGV[5] == "completed" then
703
714
  local dependenciesSet = parentKey .. ":dependencies"
704
715
  if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
705
- updateParentDepsIfNeeded(parentKey, parentQueueKey,
706
- dependenciesSet, parentId, jobIdKey,
707
- ARGV[4], timestamp)
716
+ updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet, parentId, jobIdKey, ARGV[4],
717
+ timestamp)
708
718
  end
709
719
  else
710
- if opts['fpof'] then
720
+ if opts['fpof'] or opts['cpof'] then
711
721
  local unsuccesssfulSet = parentKey .. ":unsuccessful"
712
722
  rcall("ZADD", unsuccesssfulSet, timestamp, jobIdKey)
713
- moveParentToFailedIfNeeded(parentQueueKey, parentKey,
714
- parentId, jobIdKey,
715
- timestamp)
723
+ if opts['fpof'] then
724
+ moveParentToFailedIfNeeded(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
725
+ elseif opts['cpof'] then
726
+ local failedSet = parentKey .. ":failed"
727
+ rcall("HSET", failedSet, jobIdKey, ARGV[4])
728
+ moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
729
+ end
716
730
  elseif opts['idof'] or opts['rdof'] then
717
731
  local dependenciesSet = parentKey .. ":dependencies"
718
732
  if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
719
- moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
720
- parentKey, parentId, timestamp)
733
+ moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet, parentKey, parentId, timestamp)
721
734
  if opts['idof'] then
722
735
  local failedSet = parentKey .. ":failed"
723
736
  rcall("HSET", failedSet, jobIdKey, ARGV[4])
@@ -750,12 +763,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
750
763
  removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3])
751
764
  end
752
765
  end
753
- rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
754
- ARGV[4], "prev", "active")
766
+ rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3], ARGV[4], "prev", "active")
755
767
  if ARGV[5] == "failed" then
756
768
  if tonumber(attemptsMade) >= tonumber(attempts) then
757
- rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId",
758
- jobId, "attemptsMade", attemptsMade)
769
+ rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId", jobId, "attemptsMade",
770
+ attemptsMade)
759
771
  end
760
772
  end
761
773
  -- Collect metrics
@@ -768,14 +780,18 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
768
780
  local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
769
781
  local markerKey = KEYS[14]
770
782
  -- Check if there are delayed jobs that can be promoted
771
- promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix,
772
- timestamp, KEYS[10], isPausedOrMaxed)
783
+ promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10],
784
+ isPausedOrMaxed)
773
785
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
774
786
  -- Check if we are rate limited first.
775
787
  local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
776
- if expireTime > 0 then return {0, 0, expireTime, 0} end
788
+ if expireTime > 0 then
789
+ return {0, 0, expireTime, 0}
790
+ end
777
791
  -- paused or maxed queue
778
- if isPausedOrMaxed then return {0, 0, 0, 0} end
792
+ if isPausedOrMaxed then
793
+ return {0, 0, 0, 0}
794
+ end
779
795
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
780
796
  if jobId then
781
797
  -- Markers in waitlist DEPRECATED in v5: Remove in v6.
@@ -784,23 +800,19 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
784
800
  -- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
785
801
  -- but if ID is 0:0, then there is at least 1 prioritized job to process
786
802
  if jobId == "0:0" then
787
- jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
788
- KEYS[10])
789
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
790
- timestamp, maxJobs, markerKey,
791
- opts)
803
+ jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
804
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
805
+ markerKey, opts)
792
806
  end
793
807
  else
794
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
795
- timestamp, maxJobs, markerKey,
796
- opts)
808
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
809
+ opts)
797
810
  end
798
811
  else
799
812
  jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
800
813
  if jobId then
801
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
802
- timestamp, maxJobs, markerKey,
803
- opts)
814
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
815
+ opts)
804
816
  end
805
817
  end
806
818
  -- Return the timestamp for the next delayed job if any.
@@ -1 +1 @@
1
- {"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2zBf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAu0Bf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}