bullmq 5.8.7 → 5.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/dist/cjs/classes/queue.js +29 -0
  2. package/dist/cjs/classes/queue.js.map +1 -1
  3. package/dist/cjs/classes/scripts.js +17 -1
  4. package/dist/cjs/classes/scripts.js.map +1 -1
  5. package/dist/cjs/classes/worker.js +1 -1
  6. package/dist/cjs/classes/worker.js.map +1 -1
  7. package/dist/cjs/commands/{addPrioritizedJob-7.lua → addPrioritizedJob-8.lua} +9 -7
  8. package/dist/cjs/commands/{addStandardJob-7.lua → addStandardJob-8.lua} +6 -5
  9. package/dist/cjs/commands/{changePriority-6.lua → changePriority-7.lua} +12 -10
  10. package/dist/cjs/commands/getCountsPerPriority-4.lua +6 -3
  11. package/dist/cjs/commands/includes/addBaseMarkerIfNeeded.lua +2 -2
  12. package/dist/cjs/commands/includes/addJobInTargetList.lua +2 -2
  13. package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -2
  14. package/dist/cjs/commands/includes/getTargetQueueList.lua +14 -4
  15. package/dist/cjs/commands/includes/isQueueMaxed.lua +15 -0
  16. package/dist/cjs/commands/includes/isQueuePaused.lua +1 -1
  17. package/dist/cjs/commands/includes/isQueuePausedOrMaxed.lua +18 -0
  18. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +8 -6
  19. package/dist/cjs/commands/includes/removeParentDependencyKey.lua +3 -3
  20. package/dist/cjs/commands/isMaxed-2.lua +17 -0
  21. package/dist/cjs/commands/moveJobFromActiveToWait-10.lua +2 -2
  22. package/dist/{esm/commands/moveJobsToWait-7.lua → cjs/commands/moveJobsToWait-8.lua} +4 -3
  23. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +3 -3
  24. package/dist/cjs/commands/moveToActive-11.lua +4 -4
  25. package/dist/cjs/commands/moveToFinished-14.lua +4 -4
  26. package/dist/cjs/commands/moveToWaitingChildren-5.lua +4 -3
  27. package/dist/{esm/commands/promote-8.lua → cjs/commands/promote-9.lua} +9 -7
  28. package/dist/{esm/commands/removeJob-1.lua → cjs/commands/removeJob-2.lua} +2 -1
  29. package/dist/{esm/commands/reprocessJob-7.lua → cjs/commands/reprocessJob-8.lua} +4 -3
  30. package/dist/cjs/commands/retryJob-11.lua +8 -4
  31. package/dist/cjs/scripts/addDelayedJob-6.js +39 -18
  32. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  33. package/dist/cjs/scripts/addParentJob-4.js +39 -18
  34. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  35. package/dist/cjs/scripts/{addPrioritizedJob-7.js → addPrioritizedJob-8.js} +49 -26
  36. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -0
  37. package/dist/cjs/scripts/{addStandardJob-7.js → addStandardJob-8.js} +47 -25
  38. package/dist/cjs/scripts/addStandardJob-8.js.map +1 -0
  39. package/dist/cjs/scripts/{changePriority-6.js → changePriority-7.js} +33 -21
  40. package/dist/cjs/scripts/changePriority-7.js.map +1 -0
  41. package/dist/cjs/scripts/cleanJobsInSet-2.js +20 -11
  42. package/dist/cjs/scripts/cleanJobsInSet-2.js.map +1 -1
  43. package/dist/cjs/scripts/drain-4.js +20 -11
  44. package/dist/cjs/scripts/drain-4.js.map +1 -1
  45. package/dist/cjs/scripts/getCountsPerPriority-4.js +7 -8
  46. package/dist/cjs/scripts/getCountsPerPriority-4.js.map +1 -1
  47. package/dist/cjs/scripts/index.js +8 -7
  48. package/dist/cjs/scripts/index.js.map +1 -1
  49. package/dist/cjs/scripts/isMaxed-2.js +34 -0
  50. package/dist/cjs/scripts/isMaxed-2.js.map +1 -0
  51. package/dist/cjs/scripts/moveJobFromActiveToWait-10.js +19 -10
  52. package/dist/cjs/scripts/moveJobFromActiveToWait-10.js.map +1 -1
  53. package/dist/cjs/scripts/{moveJobsToWait-7.js → moveJobsToWait-8.js} +21 -11
  54. package/dist/cjs/scripts/moveJobsToWait-8.js.map +1 -0
  55. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +45 -24
  56. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  57. package/dist/cjs/scripts/moveToActive-11.js +24 -14
  58. package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
  59. package/dist/cjs/scripts/moveToFinished-14.js +46 -25
  60. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  61. package/dist/cjs/scripts/moveToWaitingChildren-5.js +4 -3
  62. package/dist/cjs/scripts/moveToWaitingChildren-5.js.map +1 -1
  63. package/dist/cjs/scripts/obliterate-2.js +20 -11
  64. package/dist/cjs/scripts/obliterate-2.js.map +1 -1
  65. package/dist/cjs/scripts/{promote-8.js → promote-9.js} +31 -19
  66. package/dist/cjs/scripts/promote-9.js.map +1 -0
  67. package/dist/cjs/scripts/removeChildDependency-1.js +20 -11
  68. package/dist/cjs/scripts/removeChildDependency-1.js.map +1 -1
  69. package/dist/cjs/scripts/{removeJob-1.js → removeJob-2.js} +24 -14
  70. package/dist/cjs/scripts/{removeJob-1.js.map → removeJob-2.js.map} +1 -1
  71. package/dist/cjs/scripts/{reprocessJob-7.js → reprocessJob-8.js} +23 -13
  72. package/dist/cjs/scripts/reprocessJob-8.js.map +1 -0
  73. package/dist/cjs/scripts/retryJob-11.js +42 -14
  74. package/dist/cjs/scripts/retryJob-11.js.map +1 -1
  75. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  76. package/dist/esm/classes/queue.d.ts +17 -0
  77. package/dist/esm/classes/queue.js +29 -0
  78. package/dist/esm/classes/queue.js.map +1 -1
  79. package/dist/esm/classes/scripts.d.ts +2 -0
  80. package/dist/esm/classes/scripts.js +17 -1
  81. package/dist/esm/classes/scripts.js.map +1 -1
  82. package/dist/esm/classes/worker.js +1 -1
  83. package/dist/esm/classes/worker.js.map +1 -1
  84. package/dist/esm/commands/{addPrioritizedJob-7.lua → addPrioritizedJob-8.lua} +9 -7
  85. package/dist/esm/commands/{addStandardJob-7.lua → addStandardJob-8.lua} +6 -5
  86. package/dist/esm/commands/{changePriority-6.lua → changePriority-7.lua} +12 -10
  87. package/dist/esm/commands/getCountsPerPriority-4.lua +6 -3
  88. package/dist/esm/commands/includes/addBaseMarkerIfNeeded.lua +2 -2
  89. package/dist/esm/commands/includes/addJobInTargetList.lua +2 -2
  90. package/dist/esm/commands/includes/addJobWithPriority.lua +3 -2
  91. package/dist/esm/commands/includes/getTargetQueueList.lua +14 -4
  92. package/dist/esm/commands/includes/isQueueMaxed.lua +15 -0
  93. package/dist/esm/commands/includes/isQueuePaused.lua +1 -1
  94. package/dist/esm/commands/includes/isQueuePausedOrMaxed.lua +18 -0
  95. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +8 -6
  96. package/dist/esm/commands/includes/removeParentDependencyKey.lua +3 -3
  97. package/dist/esm/commands/isMaxed-2.lua +17 -0
  98. package/dist/esm/commands/moveJobFromActiveToWait-10.lua +2 -2
  99. package/dist/{cjs/commands/moveJobsToWait-7.lua → esm/commands/moveJobsToWait-8.lua} +4 -3
  100. package/dist/esm/commands/moveStalledJobsToWait-9.lua +3 -3
  101. package/dist/esm/commands/moveToActive-11.lua +4 -4
  102. package/dist/esm/commands/moveToFinished-14.lua +4 -4
  103. package/dist/esm/commands/moveToWaitingChildren-5.lua +4 -3
  104. package/dist/{cjs/commands/promote-8.lua → esm/commands/promote-9.lua} +9 -7
  105. package/dist/{cjs/commands/removeJob-1.lua → esm/commands/removeJob-2.lua} +2 -1
  106. package/dist/{cjs/commands/reprocessJob-7.lua → esm/commands/reprocessJob-8.lua} +4 -3
  107. package/dist/esm/commands/retryJob-11.lua +8 -4
  108. package/dist/esm/scripts/addDelayedJob-6.js +39 -18
  109. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  110. package/dist/esm/scripts/addParentJob-4.js +39 -18
  111. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  112. package/dist/esm/scripts/{addPrioritizedJob-7.js → addPrioritizedJob-8.js} +49 -26
  113. package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -0
  114. package/dist/esm/scripts/{addStandardJob-7.js → addStandardJob-8.js} +47 -25
  115. package/dist/esm/scripts/addStandardJob-8.js.map +1 -0
  116. package/dist/esm/scripts/{changePriority-6.js → changePriority-7.js} +33 -21
  117. package/dist/esm/scripts/changePriority-7.js.map +1 -0
  118. package/dist/esm/scripts/cleanJobsInSet-2.js +20 -11
  119. package/dist/esm/scripts/cleanJobsInSet-2.js.map +1 -1
  120. package/dist/esm/scripts/drain-4.js +20 -11
  121. package/dist/esm/scripts/drain-4.js.map +1 -1
  122. package/dist/esm/scripts/getCountsPerPriority-4.js +7 -8
  123. package/dist/esm/scripts/getCountsPerPriority-4.js.map +1 -1
  124. package/dist/esm/scripts/index.d.ts +8 -7
  125. package/dist/esm/scripts/index.js +8 -7
  126. package/dist/esm/scripts/index.js.map +1 -1
  127. package/dist/esm/scripts/isMaxed-2.d.ts +5 -0
  128. package/dist/esm/scripts/isMaxed-2.js +31 -0
  129. package/dist/esm/scripts/isMaxed-2.js.map +1 -0
  130. package/dist/esm/scripts/moveJobFromActiveToWait-10.js +19 -10
  131. package/dist/esm/scripts/moveJobFromActiveToWait-10.js.map +1 -1
  132. package/dist/esm/scripts/{moveJobsToWait-7.js → moveJobsToWait-8.js} +21 -11
  133. package/dist/esm/scripts/moveJobsToWait-8.js.map +1 -0
  134. package/dist/esm/scripts/moveStalledJobsToWait-9.js +45 -24
  135. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  136. package/dist/esm/scripts/moveToActive-11.js +24 -14
  137. package/dist/esm/scripts/moveToActive-11.js.map +1 -1
  138. package/dist/esm/scripts/moveToFinished-14.js +46 -25
  139. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  140. package/dist/esm/scripts/moveToWaitingChildren-5.js +4 -3
  141. package/dist/esm/scripts/moveToWaitingChildren-5.js.map +1 -1
  142. package/dist/esm/scripts/obliterate-2.js +20 -11
  143. package/dist/esm/scripts/obliterate-2.js.map +1 -1
  144. package/dist/esm/scripts/{promote-8.js → promote-9.js} +31 -19
  145. package/dist/esm/scripts/promote-9.js.map +1 -0
  146. package/dist/esm/scripts/removeChildDependency-1.js +20 -11
  147. package/dist/esm/scripts/removeChildDependency-1.js.map +1 -1
  148. package/dist/esm/scripts/{removeJob-1.js → removeJob-2.js} +24 -14
  149. package/dist/esm/scripts/{removeJob-1.js.map → removeJob-2.js.map} +1 -1
  150. package/dist/esm/scripts/{reprocessJob-7.js → reprocessJob-8.js} +23 -13
  151. package/dist/esm/scripts/reprocessJob-8.js.map +1 -0
  152. package/dist/esm/scripts/retryJob-11.js +42 -14
  153. package/dist/esm/scripts/retryJob-11.js.map +1 -1
  154. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  155. package/package.json +2 -1
  156. package/dist/cjs/scripts/addPrioritizedJob-7.js.map +0 -1
  157. package/dist/cjs/scripts/addStandardJob-7.js.map +0 -1
  158. package/dist/cjs/scripts/changePriority-6.js.map +0 -1
  159. package/dist/cjs/scripts/moveJobsToWait-7.js.map +0 -1
  160. package/dist/cjs/scripts/promote-8.js.map +0 -1
  161. package/dist/cjs/scripts/reprocessJob-7.js.map +0 -1
  162. package/dist/esm/scripts/addPrioritizedJob-7.js.map +0 -1
  163. package/dist/esm/scripts/addStandardJob-7.js.map +0 -1
  164. package/dist/esm/scripts/changePriority-6.js.map +0 -1
  165. package/dist/esm/scripts/moveJobsToWait-7.js.map +0 -1
  166. package/dist/esm/scripts/promote-8.js.map +0 -1
  167. package/dist/esm/scripts/reprocessJob-7.js.map +0 -1
  168. /package/dist/esm/scripts/{addPrioritizedJob-7.d.ts → addPrioritizedJob-8.d.ts} +0 -0
  169. /package/dist/esm/scripts/{addStandardJob-7.d.ts → addStandardJob-8.d.ts} +0 -0
  170. /package/dist/esm/scripts/{changePriority-6.d.ts → changePriority-7.d.ts} +0 -0
  171. /package/dist/esm/scripts/{moveJobsToWait-7.d.ts → moveJobsToWait-8.d.ts} +0 -0
  172. /package/dist/esm/scripts/{promote-8.d.ts → promote-9.d.ts} +0 -0
  173. /package/dist/esm/scripts/{removeJob-1.d.ts → removeJob-2.d.ts} +0 -0
  174. /package/dist/esm/scripts/{reprocessJob-7.d.ts → reprocessJob-8.d.ts} +0 -0
@@ -9,9 +9,10 @@ const content = `--[[
9
9
  KEYS[3] 'paused'
10
10
  KEYS[4] 'meta'
11
11
  KEYS[5] 'prioritized'
12
- KEYS[6] 'pc' priority counter
13
- KEYS[7] 'event stream'
14
- KEYS[8] 'marker'
12
+ KEYS[6] 'active'
13
+ KEYS[7] 'pc' priority counter
14
+ KEYS[8] 'event stream'
15
+ KEYS[9] 'marker'
15
16
  ARGV[1] queue.toKey('')
16
17
  ARGV[2] jobId
17
18
  Output:
@@ -30,54 +31,65 @@ local jobId = ARGV[2]
30
31
  --[[
31
32
  Add marker if needed when a job is available.
32
33
  ]]
33
- local function addBaseMarkerIfNeeded(markerKey, isPaused)
34
- if not isPaused then
34
+ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
35
+ if not isPausedOrMaxed then
35
36
  rcall("ZADD", markerKey, 0, "0")
36
37
  end
37
38
  end
38
- local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
39
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
39
40
  rcall(pushCmd, targetKey, jobId)
40
- addBaseMarkerIfNeeded(markerKey, isPaused)
41
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
41
42
  end
42
43
  --[[
43
44
  Function to add job considering priority.
44
45
  ]]
45
46
  -- Includes
46
- local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
47
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
48
+ isPausedOrMaxed)
47
49
  local prioCounter = rcall("INCR", priorityCounterKey)
48
50
  local score = priority * 0x100000000 + prioCounter % 0x100000000
49
51
  rcall("ZADD", prioritizedKey, score, jobId)
50
- addBaseMarkerIfNeeded(markerKey, isPaused)
52
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
51
53
  end
52
54
  --[[
53
55
  Function to check for the meta.paused key to decide if we are paused or not
54
56
  (since an empty list and !EXISTS are not really the same).
55
57
  ]]
56
- local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
57
- if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
58
- return waitKey, false
59
- else
58
+ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
59
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
60
+ if queueAttributes[1] then
60
61
  return pausedKey, true
62
+ else
63
+ if queueAttributes[2] then
64
+ local activeCount = rcall("LLEN", activeKey)
65
+ if activeCount >= tonumber(queueAttributes[2]) then
66
+ return waitKey, true
67
+ else
68
+ return waitKey, false
69
+ end
70
+ end
61
71
  end
72
+ return waitKey, false
62
73
  end
63
74
  if rcall("ZREM", KEYS[1], jobId) == 1 then
64
75
  local jobKey = ARGV[1] .. jobId
65
76
  local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
66
77
  local metaKey = KEYS[4]
78
+ local markerKey = KEYS[9]
67
79
  -- Remove delayed "marker" from the wait list if there is any.
68
80
  -- Since we are adding a job we do not need the marker anymore.
69
81
  -- Markers in waitlist DEPRECATED in v5: Remove in v6.
70
- local target, paused = getTargetQueueList(metaKey, KEYS[2], KEYS[3])
82
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[6], KEYS[2], KEYS[3])
71
83
  local marker = rcall("LINDEX", target, 0)
72
84
  if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
73
85
  if priority == 0 then
74
86
  -- LIFO or FIFO
75
- addJobInTargetList(target, KEYS[8], "LPUSH", paused, jobId)
87
+ addJobInTargetList(target, markerKey, "LPUSH", isPausedOrMaxed, jobId)
76
88
  else
77
- addJobWithPriority(KEYS[8], KEYS[5], priority, jobId, KEYS[6], paused)
89
+ addJobWithPriority(markerKey, KEYS[5], priority, jobId, KEYS[7], isPausedOrMaxed)
78
90
  end
79
91
  -- Emit waiting event (wait..ing@token)
80
- rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev",
92
+ rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId, "prev",
81
93
  "delayed");
82
94
  rcall("HSET", jobKey, "delay", 0)
83
95
  return 0
@@ -88,6 +100,6 @@ end
88
100
  exports.promote = {
89
101
  name: 'promote',
90
102
  content,
91
- keys: 8,
103
+ keys: 9,
92
104
  };
93
- //# sourceMappingURL=promote-8.js.map
105
+ //# sourceMappingURL=promote-9.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"promote-9.js","sourceRoot":"","sources":["../../../src/scripts/promote-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Ff,CAAC;AACW,QAAA,OAAO,GAAG;IACrB,IAAI,EAAE,SAAS;IACf,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -31,14 +31,14 @@ local parentKey = ARGV[2]
31
31
  --[[
32
32
  Add marker if needed when a job is available.
33
33
  ]]
34
- local function addBaseMarkerIfNeeded(markerKey, isPaused)
35
- if not isPaused then
34
+ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
35
+ if not isPausedOrMaxed then
36
36
  rcall("ZADD", markerKey, 0, "0")
37
37
  end
38
38
  end
39
- local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
39
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
40
40
  rcall(pushCmd, targetKey, jobId)
41
- addBaseMarkerIfNeeded(markerKey, isPaused)
41
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
42
42
  end
43
43
  --[[
44
44
  Functions to destructure job key.
@@ -54,12 +54,21 @@ end
54
54
  Function to check for the meta.paused key to decide if we are paused or not
55
55
  (since an empty list and !EXISTS are not really the same).
56
56
  ]]
57
- local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
58
- if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
59
- return waitKey, false
60
- else
57
+ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
58
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
59
+ if queueAttributes[1] then
61
60
  return pausedKey, true
61
+ else
62
+ if queueAttributes[2] then
63
+ local activeCount = rcall("LLEN", activeKey)
64
+ if activeCount >= tonumber(queueAttributes[2]) then
65
+ return waitKey, true
66
+ else
67
+ return waitKey, false
68
+ end
69
+ end
62
70
  end
71
+ return waitKey, false
63
72
  end
64
73
  --[[
65
74
  Function to remove job keys.
@@ -69,9 +78,9 @@ local function removeJobKeys(jobKey)
69
78
  jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
70
79
  end
71
80
  local function moveParentToWait(parentPrefix, parentId, emitEvent)
72
- local parentTarget, isPaused = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait",
73
- parentPrefix .. "paused")
74
- addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPaused, parentId)
81
+ local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
82
+ parentPrefix .. "wait", parentPrefix .. "paused")
83
+ addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
75
84
  if emitEvent then
76
85
  local parentEventStream = parentPrefix .. "events"
77
86
  rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
@@ -1 +1 @@
1
- {"version":3,"file":"removeChildDependency-1.js","sourceRoot":"","sources":["../../../src/scripts/removeChildDependency-1.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA2IZ,CAAC;AACQ,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"removeChildDependency-1.js","sourceRoot":"","sources":["../../../src/scripts/removeChildDependency-1.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAoJZ,CAAC;AACQ,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -6,6 +6,7 @@ const content = `--[[
6
6
  In order to be able to remove a job, it cannot be active.
7
7
  Input:
8
8
  KEYS[1] queue prefix
9
+ KEYS[2] meta key
9
10
  ARGV[1] jobId
10
11
  ARGV[2] remove children
11
12
  Events:
@@ -116,30 +117,39 @@ end
116
117
  --[[
117
118
  Add marker if needed when a job is available.
118
119
  ]]
119
- local function addBaseMarkerIfNeeded(markerKey, isPaused)
120
- if not isPaused then
120
+ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
121
+ if not isPausedOrMaxed then
121
122
  rcall("ZADD", markerKey, 0, "0")
122
123
  end
123
124
  end
124
- local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
125
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
125
126
  rcall(pushCmd, targetKey, jobId)
126
- addBaseMarkerIfNeeded(markerKey, isPaused)
127
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
127
128
  end
128
129
  --[[
129
130
  Function to check for the meta.paused key to decide if we are paused or not
130
131
  (since an empty list and !EXISTS are not really the same).
131
132
  ]]
132
- local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
133
- if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
134
- return waitKey, false
135
- else
133
+ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
134
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
135
+ if queueAttributes[1] then
136
136
  return pausedKey, true
137
+ else
138
+ if queueAttributes[2] then
139
+ local activeCount = rcall("LLEN", activeKey)
140
+ if activeCount >= tonumber(queueAttributes[2]) then
141
+ return waitKey, true
142
+ else
143
+ return waitKey, false
144
+ end
145
+ end
137
146
  end
147
+ return waitKey, false
138
148
  end
139
149
  local function moveParentToWait(parentPrefix, parentId, emitEvent)
140
- local parentTarget, isPaused = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait",
141
- parentPrefix .. "paused")
142
- addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPaused, parentId)
150
+ local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
151
+ parentPrefix .. "wait", parentPrefix .. "paused")
152
+ addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
143
153
  if emitEvent then
144
154
  local parentEventStream = parentPrefix .. "events"
145
155
  rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
@@ -237,7 +247,7 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
237
247
  end
238
248
  local prev = removeJobFromAnyState(prefix, jobId)
239
249
  if removeJobKeys(jobKey) > 0 then
240
- local maxEvents = getOrSetMaxEvents(prefix .. "meta")
250
+ local maxEvents = getOrSetMaxEvents(KEYS[2])
241
251
  rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
242
252
  "jobId", jobId, "prev", prev)
243
253
  end
@@ -252,6 +262,6 @@ return 0
252
262
  exports.removeJob = {
253
263
  name: 'removeJob',
254
264
  content,
255
- keys: 1,
265
+ keys: 2,
256
266
  };
257
- //# sourceMappingURL=removeJob-1.js.map
267
+ //# sourceMappingURL=removeJob-2.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"removeJob-1.js","sourceRoot":"","sources":["../../../src/scripts/removeJob-1.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuPf,CAAC;AACW,QAAA,SAAS,GAAG;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"removeJob-2.js","sourceRoot":"","sources":["../../../src/scripts/removeJob-2.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiQf,CAAC;AACW,QAAA,SAAS,GAAG;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -10,7 +10,8 @@ const content = `--[[
10
10
  KEYS[4] wait key
11
11
  KEYS[5] meta
12
12
  KEYS[6] paused key
13
- KEYS[7] marker key
13
+ KEYS[7] active key
14
+ KEYS[8] marker key
14
15
  ARGV[1] job.id
15
16
  ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
16
17
  ARGV[3] propVal - failedReason/returnvalue
@@ -29,14 +30,14 @@ local rcall = redis.call;
29
30
  --[[
30
31
  Add marker if needed when a job is available.
31
32
  ]]
32
- local function addBaseMarkerIfNeeded(markerKey, isPaused)
33
- if not isPaused then
33
+ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
34
+ if not isPausedOrMaxed then
34
35
  rcall("ZADD", markerKey, 0, "0")
35
36
  end
36
37
  end
37
- local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
38
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
38
39
  rcall(pushCmd, targetKey, jobId)
39
- addBaseMarkerIfNeeded(markerKey, isPaused)
40
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
40
41
  end
41
42
  --[[
42
43
  Function to get max events value or set by default 10000.
@@ -53,19 +54,28 @@ end
53
54
  Function to check for the meta.paused key to decide if we are paused or not
54
55
  (since an empty list and !EXISTS are not really the same).
55
56
  ]]
56
- local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
57
- if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
58
- return waitKey, false
59
- else
57
+ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
58
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
59
+ if queueAttributes[1] then
60
60
  return pausedKey, true
61
+ else
62
+ if queueAttributes[2] then
63
+ local activeCount = rcall("LLEN", activeKey)
64
+ if activeCount >= tonumber(queueAttributes[2]) then
65
+ return waitKey, true
66
+ else
67
+ return waitKey, false
68
+ end
69
+ end
61
70
  end
71
+ return waitKey, false
62
72
  end
63
73
  if rcall("EXISTS", KEYS[1]) == 1 then
64
74
  local jobId = ARGV[1]
65
75
  if (rcall("ZREM", KEYS[3], jobId) == 1) then
66
76
  rcall("HDEL", KEYS[1], "finishedOn", "processedOn", ARGV[3])
67
- local target, isPaused = getTargetQueueList(KEYS[5], KEYS[4], KEYS[6])
68
- addJobInTargetList(target, KEYS[7], ARGV[2], isPaused, jobId)
77
+ local target, isPausedOrMaxed = getTargetQueueList(KEYS[5], KEYS[7], KEYS[4], KEYS[6])
78
+ addJobInTargetList(target, KEYS[8], ARGV[2], isPausedOrMaxed, jobId)
69
79
  local maxEvents = getOrSetMaxEvents(KEYS[5])
70
80
  -- Emit waiting event
71
81
  rcall("XADD", KEYS[2], "MAXLEN", "~", maxEvents, "*", "event", "waiting",
@@ -81,6 +91,6 @@ end
81
91
  exports.reprocessJob = {
82
92
  name: 'reprocessJob',
83
93
  content,
84
- keys: 7,
94
+ keys: 8,
85
95
  };
86
- //# sourceMappingURL=reprocessJob-7.js.map
96
+ //# sourceMappingURL=reprocessJob-8.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"reprocessJob-8.js","sourceRoot":"","sources":["../../../src/scripts/reprocessJob-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsFf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -37,24 +37,25 @@ local rcall = redis.call
37
37
  --[[
38
38
  Add marker if needed when a job is available.
39
39
  ]]
40
- local function addBaseMarkerIfNeeded(markerKey, isPaused)
41
- if not isPaused then
40
+ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
41
+ if not isPausedOrMaxed then
42
42
  rcall("ZADD", markerKey, 0, "0")
43
43
  end
44
44
  end
45
- local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
45
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
46
46
  rcall(pushCmd, targetKey, jobId)
47
- addBaseMarkerIfNeeded(markerKey, isPaused)
47
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
48
48
  end
49
49
  --[[
50
50
  Function to add job considering priority.
51
51
  ]]
52
52
  -- Includes
53
- local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
53
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
54
+ isPausedOrMaxed)
54
55
  local prioCounter = rcall("INCR", priorityCounterKey)
55
56
  local score = priority * 0x100000000 + prioCounter % 0x100000000
56
57
  rcall("ZADD", prioritizedKey, score, jobId)
57
- addBaseMarkerIfNeeded(markerKey, isPaused)
58
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
58
59
  end
59
60
  --[[
60
61
  Function to get max events value or set by default 10000.
@@ -71,12 +72,21 @@ end
71
72
  Function to check for the meta.paused key to decide if we are paused or not
72
73
  (since an empty list and !EXISTS are not really the same).
73
74
  ]]
74
- local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
75
- if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
76
- return waitKey, false
77
- else
75
+ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
76
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
77
+ if queueAttributes[1] then
78
78
  return pausedKey, true
79
+ else
80
+ if queueAttributes[2] then
81
+ local activeCount = rcall("LLEN", activeKey)
82
+ if activeCount >= tonumber(queueAttributes[2]) then
83
+ return waitKey, true
84
+ else
85
+ return waitKey, false
86
+ end
87
+ end
79
88
  end
89
+ return waitKey, false
80
90
  end
81
91
  --[[
82
92
  Updates the delay set, by moving delayed jobs that should
@@ -128,11 +138,27 @@ local function removeLock(jobKey, stalledKey, token, jobId)
128
138
  end
129
139
  return 0
130
140
  end
131
- local target, paused = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
141
+ --[[
142
+ Function to check if queue is paused or maxed
143
+ (since an empty list and !EXISTS are not really the same).
144
+ ]]
145
+ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
146
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
147
+ if queueAttributes[1] then
148
+ return true
149
+ else
150
+ if queueAttributes[2] then
151
+ local activeCount = rcall("LLEN", activeKey)
152
+ return activeCount >= tonumber(queueAttributes[2])
153
+ end
154
+ end
155
+ return false
156
+ end
157
+ local target, isPausedOrMaxed = getTargetQueueList(KEYS[5], KEYS[1], KEYS[2], KEYS[3])
132
158
  local markerKey = KEYS[10]
133
159
  -- Check if there are delayed jobs that we can move to wait.
134
160
  -- test example: when there are delayed jobs between retries
135
- promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], paused)
161
+ promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], isPausedOrMaxed)
136
162
  if rcall("EXISTS", KEYS[4]) == 1 then
137
163
  local errorCode = removeLock(KEYS[4], KEYS[11], ARGV[5], ARGV[4])
138
164
  if errorCode < 0 then
@@ -141,11 +167,13 @@ if rcall("EXISTS", KEYS[4]) == 1 then
141
167
  local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[4])
142
168
  if (numRemovedElements < 1) then return -3 end
143
169
  local priority = tonumber(rcall("HGET", KEYS[4], "priority")) or 0
170
+ --need to re-evaluate after removing job from active
171
+ isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[5], KEYS[1])
144
172
  -- Standard or priority add
145
173
  if priority == 0 then
146
- addJobInTargetList(target, KEYS[10], ARGV[3], paused, ARGV[4])
174
+ addJobInTargetList(target, markerKey, ARGV[3], isPausedOrMaxed, ARGV[4])
147
175
  else
148
- addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], paused)
176
+ addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], isPausedOrMaxed)
149
177
  end
150
178
  rcall("HINCRBY", KEYS[4], "atm", 1)
151
179
  local maxEvents = getOrSetMaxEvents(KEYS[5])
@@ -1 +1 @@
1
- {"version":3,"file":"retryJob-11.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2Jf,CAAC;AACW,QAAA,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"retryJob-11.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuLf,CAAC;AACW,QAAA,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}