bullmq 5.8.6 → 5.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/dist/cjs/classes/queue.js +29 -0
  2. package/dist/cjs/classes/queue.js.map +1 -1
  3. package/dist/cjs/classes/scripts.js +17 -1
  4. package/dist/cjs/classes/scripts.js.map +1 -1
  5. package/dist/cjs/classes/worker.js +1 -1
  6. package/dist/cjs/classes/worker.js.map +1 -1
  7. package/dist/cjs/commands/addDelayedJob-6.lua +2 -6
  8. package/dist/cjs/commands/{addPrioritizedJob-7.lua → addPrioritizedJob-8.lua} +9 -7
  9. package/dist/cjs/commands/{addStandardJob-7.lua → addStandardJob-8.lua} +6 -5
  10. package/dist/cjs/commands/changeDelay-4.lua +1 -5
  11. package/dist/cjs/commands/{changePriority-6.lua → changePriority-7.lua} +12 -10
  12. package/dist/cjs/commands/getCountsPerPriority-4.lua +6 -3
  13. package/dist/cjs/commands/includes/addBaseMarkerIfNeeded.lua +2 -2
  14. package/dist/cjs/commands/includes/addJobInTargetList.lua +2 -2
  15. package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -2
  16. package/dist/cjs/commands/includes/getTargetQueueList.lua +14 -4
  17. package/dist/cjs/commands/includes/isQueueMaxed.lua +15 -0
  18. package/dist/cjs/commands/includes/isQueuePaused.lua +1 -1
  19. package/dist/cjs/commands/includes/isQueuePausedOrMaxed.lua +18 -0
  20. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +8 -6
  21. package/dist/cjs/commands/includes/removeParentDependencyKey.lua +3 -3
  22. package/dist/cjs/commands/isMaxed-2.lua +17 -0
  23. package/dist/cjs/commands/moveJobFromActiveToWait-10.lua +2 -2
  24. package/dist/{esm/commands/moveJobsToWait-7.lua → cjs/commands/moveJobsToWait-8.lua} +4 -3
  25. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +3 -3
  26. package/dist/cjs/commands/moveToActive-11.lua +4 -4
  27. package/dist/cjs/commands/moveToDelayed-8.lua +2 -6
  28. package/dist/cjs/commands/moveToFinished-14.lua +4 -4
  29. package/dist/cjs/commands/moveToWaitingChildren-5.lua +4 -3
  30. package/dist/{esm/commands/promote-8.lua → cjs/commands/promote-9.lua} +9 -7
  31. package/dist/{esm/commands/removeJob-1.lua → cjs/commands/removeJob-2.lua} +2 -1
  32. package/dist/{esm/commands/reprocessJob-7.lua → cjs/commands/reprocessJob-8.lua} +4 -3
  33. package/dist/cjs/commands/retryJob-11.lua +8 -4
  34. package/dist/cjs/scripts/addDelayedJob-6.js +41 -23
  35. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  36. package/dist/cjs/scripts/addParentJob-4.js +39 -18
  37. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  38. package/dist/cjs/scripts/{addPrioritizedJob-7.js → addPrioritizedJob-8.js} +49 -26
  39. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -0
  40. package/dist/cjs/scripts/{addStandardJob-7.js → addStandardJob-8.js} +47 -25
  41. package/dist/cjs/scripts/addStandardJob-8.js.map +1 -0
  42. package/dist/cjs/scripts/changeDelay-4.js +1 -11
  43. package/dist/cjs/scripts/changeDelay-4.js.map +1 -1
  44. package/dist/cjs/scripts/{changePriority-6.js → changePriority-7.js} +33 -21
  45. package/dist/cjs/scripts/changePriority-7.js.map +1 -0
  46. package/dist/cjs/scripts/cleanJobsInSet-2.js +20 -11
  47. package/dist/cjs/scripts/cleanJobsInSet-2.js.map +1 -1
  48. package/dist/cjs/scripts/drain-4.js +20 -11
  49. package/dist/cjs/scripts/drain-4.js.map +1 -1
  50. package/dist/cjs/scripts/getCountsPerPriority-4.js +7 -8
  51. package/dist/cjs/scripts/getCountsPerPriority-4.js.map +1 -1
  52. package/dist/cjs/scripts/index.js +8 -7
  53. package/dist/cjs/scripts/index.js.map +1 -1
  54. package/dist/cjs/scripts/isMaxed-2.js +34 -0
  55. package/dist/cjs/scripts/isMaxed-2.js.map +1 -0
  56. package/dist/cjs/scripts/moveJobFromActiveToWait-10.js +19 -10
  57. package/dist/cjs/scripts/moveJobFromActiveToWait-10.js.map +1 -1
  58. package/dist/cjs/scripts/{moveJobsToWait-7.js → moveJobsToWait-8.js} +21 -11
  59. package/dist/cjs/scripts/moveJobsToWait-8.js.map +1 -0
  60. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +45 -24
  61. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  62. package/dist/cjs/scripts/moveToActive-11.js +24 -14
  63. package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
  64. package/dist/cjs/scripts/moveToDelayed-8.js +2 -12
  65. package/dist/cjs/scripts/moveToDelayed-8.js.map +1 -1
  66. package/dist/cjs/scripts/moveToFinished-14.js +46 -25
  67. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  68. package/dist/cjs/scripts/moveToWaitingChildren-5.js +4 -3
  69. package/dist/cjs/scripts/moveToWaitingChildren-5.js.map +1 -1
  70. package/dist/cjs/scripts/obliterate-2.js +20 -11
  71. package/dist/cjs/scripts/obliterate-2.js.map +1 -1
  72. package/dist/cjs/scripts/{promote-8.js → promote-9.js} +31 -19
  73. package/dist/cjs/scripts/promote-9.js.map +1 -0
  74. package/dist/cjs/scripts/removeChildDependency-1.js +20 -11
  75. package/dist/cjs/scripts/removeChildDependency-1.js.map +1 -1
  76. package/dist/cjs/scripts/{removeJob-1.js → removeJob-2.js} +24 -14
  77. package/dist/cjs/scripts/{removeJob-1.js.map → removeJob-2.js.map} +1 -1
  78. package/dist/cjs/scripts/{reprocessJob-7.js → reprocessJob-8.js} +23 -13
  79. package/dist/cjs/scripts/reprocessJob-8.js.map +1 -0
  80. package/dist/cjs/scripts/retryJob-11.js +42 -14
  81. package/dist/cjs/scripts/retryJob-11.js.map +1 -1
  82. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  83. package/dist/esm/classes/queue.d.ts +17 -0
  84. package/dist/esm/classes/queue.js +29 -0
  85. package/dist/esm/classes/queue.js.map +1 -1
  86. package/dist/esm/classes/scripts.d.ts +2 -0
  87. package/dist/esm/classes/scripts.js +17 -1
  88. package/dist/esm/classes/scripts.js.map +1 -1
  89. package/dist/esm/classes/worker.js +1 -1
  90. package/dist/esm/classes/worker.js.map +1 -1
  91. package/dist/esm/commands/addDelayedJob-6.lua +2 -6
  92. package/dist/esm/commands/{addPrioritizedJob-7.lua → addPrioritizedJob-8.lua} +9 -7
  93. package/dist/esm/commands/{addStandardJob-7.lua → addStandardJob-8.lua} +6 -5
  94. package/dist/esm/commands/changeDelay-4.lua +1 -5
  95. package/dist/esm/commands/{changePriority-6.lua → changePriority-7.lua} +12 -10
  96. package/dist/esm/commands/getCountsPerPriority-4.lua +6 -3
  97. package/dist/esm/commands/includes/addBaseMarkerIfNeeded.lua +2 -2
  98. package/dist/esm/commands/includes/addJobInTargetList.lua +2 -2
  99. package/dist/esm/commands/includes/addJobWithPriority.lua +3 -2
  100. package/dist/esm/commands/includes/getTargetQueueList.lua +14 -4
  101. package/dist/esm/commands/includes/isQueueMaxed.lua +15 -0
  102. package/dist/esm/commands/includes/isQueuePaused.lua +1 -1
  103. package/dist/esm/commands/includes/isQueuePausedOrMaxed.lua +18 -0
  104. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +8 -6
  105. package/dist/esm/commands/includes/removeParentDependencyKey.lua +3 -3
  106. package/dist/esm/commands/isMaxed-2.lua +17 -0
  107. package/dist/esm/commands/moveJobFromActiveToWait-10.lua +2 -2
  108. package/dist/{cjs/commands/moveJobsToWait-7.lua → esm/commands/moveJobsToWait-8.lua} +4 -3
  109. package/dist/esm/commands/moveStalledJobsToWait-9.lua +3 -3
  110. package/dist/esm/commands/moveToActive-11.lua +4 -4
  111. package/dist/esm/commands/moveToDelayed-8.lua +2 -6
  112. package/dist/esm/commands/moveToFinished-14.lua +4 -4
  113. package/dist/esm/commands/moveToWaitingChildren-5.lua +4 -3
  114. package/dist/{cjs/commands/promote-8.lua → esm/commands/promote-9.lua} +9 -7
  115. package/dist/{cjs/commands/removeJob-1.lua → esm/commands/removeJob-2.lua} +2 -1
  116. package/dist/{cjs/commands/reprocessJob-7.lua → esm/commands/reprocessJob-8.lua} +4 -3
  117. package/dist/esm/commands/retryJob-11.lua +8 -4
  118. package/dist/esm/scripts/addDelayedJob-6.js +41 -23
  119. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  120. package/dist/esm/scripts/addParentJob-4.js +39 -18
  121. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  122. package/dist/esm/scripts/{addPrioritizedJob-7.js → addPrioritizedJob-8.js} +49 -26
  123. package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -0
  124. package/dist/esm/scripts/{addStandardJob-7.js → addStandardJob-8.js} +47 -25
  125. package/dist/esm/scripts/addStandardJob-8.js.map +1 -0
  126. package/dist/esm/scripts/changeDelay-4.js +1 -11
  127. package/dist/esm/scripts/changeDelay-4.js.map +1 -1
  128. package/dist/esm/scripts/{changePriority-6.js → changePriority-7.js} +33 -21
  129. package/dist/esm/scripts/changePriority-7.js.map +1 -0
  130. package/dist/esm/scripts/cleanJobsInSet-2.js +20 -11
  131. package/dist/esm/scripts/cleanJobsInSet-2.js.map +1 -1
  132. package/dist/esm/scripts/drain-4.js +20 -11
  133. package/dist/esm/scripts/drain-4.js.map +1 -1
  134. package/dist/esm/scripts/getCountsPerPriority-4.js +7 -8
  135. package/dist/esm/scripts/getCountsPerPriority-4.js.map +1 -1
  136. package/dist/esm/scripts/index.d.ts +8 -7
  137. package/dist/esm/scripts/index.js +8 -7
  138. package/dist/esm/scripts/index.js.map +1 -1
  139. package/dist/esm/scripts/isMaxed-2.d.ts +5 -0
  140. package/dist/esm/scripts/isMaxed-2.js +31 -0
  141. package/dist/esm/scripts/isMaxed-2.js.map +1 -0
  142. package/dist/esm/scripts/moveJobFromActiveToWait-10.js +19 -10
  143. package/dist/esm/scripts/moveJobFromActiveToWait-10.js.map +1 -1
  144. package/dist/esm/scripts/{moveJobsToWait-7.js → moveJobsToWait-8.js} +21 -11
  145. package/dist/esm/scripts/moveJobsToWait-8.js.map +1 -0
  146. package/dist/esm/scripts/moveStalledJobsToWait-9.js +45 -24
  147. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  148. package/dist/esm/scripts/moveToActive-11.js +24 -14
  149. package/dist/esm/scripts/moveToActive-11.js.map +1 -1
  150. package/dist/esm/scripts/moveToDelayed-8.js +2 -12
  151. package/dist/esm/scripts/moveToDelayed-8.js.map +1 -1
  152. package/dist/esm/scripts/moveToFinished-14.js +46 -25
  153. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  154. package/dist/esm/scripts/moveToWaitingChildren-5.js +4 -3
  155. package/dist/esm/scripts/moveToWaitingChildren-5.js.map +1 -1
  156. package/dist/esm/scripts/obliterate-2.js +20 -11
  157. package/dist/esm/scripts/obliterate-2.js.map +1 -1
  158. package/dist/esm/scripts/{promote-8.js → promote-9.js} +31 -19
  159. package/dist/esm/scripts/promote-9.js.map +1 -0
  160. package/dist/esm/scripts/removeChildDependency-1.js +20 -11
  161. package/dist/esm/scripts/removeChildDependency-1.js.map +1 -1
  162. package/dist/esm/scripts/{removeJob-1.js → removeJob-2.js} +24 -14
  163. package/dist/esm/scripts/{removeJob-1.js.map → removeJob-2.js.map} +1 -1
  164. package/dist/esm/scripts/{reprocessJob-7.js → reprocessJob-8.js} +23 -13
  165. package/dist/esm/scripts/reprocessJob-8.js.map +1 -0
  166. package/dist/esm/scripts/retryJob-11.js +42 -14
  167. package/dist/esm/scripts/retryJob-11.js.map +1 -1
  168. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  169. package/package.json +2 -1
  170. package/dist/cjs/scripts/addPrioritizedJob-7.js.map +0 -1
  171. package/dist/cjs/scripts/addStandardJob-7.js.map +0 -1
  172. package/dist/cjs/scripts/changePriority-6.js.map +0 -1
  173. package/dist/cjs/scripts/moveJobsToWait-7.js.map +0 -1
  174. package/dist/cjs/scripts/promote-8.js.map +0 -1
  175. package/dist/cjs/scripts/reprocessJob-7.js.map +0 -1
  176. package/dist/esm/scripts/addPrioritizedJob-7.js.map +0 -1
  177. package/dist/esm/scripts/addStandardJob-7.js.map +0 -1
  178. package/dist/esm/scripts/changePriority-6.js.map +0 -1
  179. package/dist/esm/scripts/moveJobsToWait-7.js.map +0 -1
  180. package/dist/esm/scripts/promote-8.js.map +0 -1
  181. package/dist/esm/scripts/reprocessJob-7.js.map +0 -1
  182. /package/dist/esm/scripts/{addPrioritizedJob-7.d.ts → addPrioritizedJob-8.d.ts} +0 -0
  183. /package/dist/esm/scripts/{addStandardJob-7.d.ts → addStandardJob-8.d.ts} +0 -0
  184. /package/dist/esm/scripts/{changePriority-6.d.ts → changePriority-7.d.ts} +0 -0
  185. /package/dist/esm/scripts/{moveJobsToWait-7.d.ts → moveJobsToWait-8.d.ts} +0 -0
  186. /package/dist/esm/scripts/{promote-8.d.ts → promote-9.d.ts} +0 -0
  187. /package/dist/esm/scripts/{removeJob-1.d.ts → removeJob-2.d.ts} +0 -0
  188. /package/dist/esm/scripts/{reprocessJob-7.d.ts → reprocessJob-8.d.ts} +0 -0
@@ -3,10 +3,20 @@
3
3
  (since an empty list and !EXISTS are not really the same).
4
4
  ]]
5
5
 
6
- local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
7
- if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
8
- return waitKey, false
9
- else
6
+ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
7
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
8
+
9
+ if queueAttributes[1] then
10
10
  return pausedKey, true
11
+ else
12
+ if queueAttributes[2] then
13
+ local activeCount = rcall("LLEN", activeKey)
14
+ if activeCount >= tonumber(queueAttributes[2]) then
15
+ return waitKey, true
16
+ else
17
+ return waitKey, false
18
+ end
19
+ end
11
20
  end
21
+ return waitKey, false
12
22
  end
@@ -0,0 +1,15 @@
1
+ --[[
2
+ Function to check if queue is maxed or not.
3
+ ]]
4
+ local function isQueueMaxed(queueMetaKey, activeKey)
5
+ local maxConcurrency = rcall("HGET", queueMetaKey, "concurrency")
6
+
7
+ if maxConcurrency then
8
+ local activeCount = rcall("LLEN", activeKey)
9
+ if activeCount >= tonumber(maxConcurrency) then
10
+ return true
11
+ end
12
+ end
13
+
14
+ return false
15
+ end
@@ -3,5 +3,5 @@
3
3
  (since an empty list and !EXISTS are not really the same).
4
4
  ]]
5
5
  local function isQueuePaused(queueMetaKey)
6
- return rcall("HEXISTS", queueMetaKey, "paused") == 1
6
+ return rcall("HEXISTS", queueMetaKey, "paused") == 1
7
7
  end
@@ -0,0 +1,18 @@
1
+ --[[
2
+ Function to check if queue is paused or maxed
3
+ (since an empty list and !EXISTS are not really the same).
4
+ ]]
5
+
6
+ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
7
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
8
+
9
+ if queueAttributes[1] then
10
+ return true
11
+ else
12
+ if queueAttributes[2] then
13
+ local activeCount = rcall("LLEN", activeKey)
14
+ return activeCount >= tonumber(queueAttributes[2])
15
+ end
16
+ end
17
+ return false
18
+ end
@@ -6,7 +6,7 @@
6
6
  --- @include "addDelayMarkerIfNeeded"
7
7
  --- @include "addJobInTargetList"
8
8
  --- @include "addJobWithPriority"
9
- --- @include "isQueuePaused"
9
+ --- @include "isQueuePausedOrMaxed"
10
10
  --- @include "getTargetQueueList"
11
11
 
12
12
  local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
@@ -17,6 +17,7 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
17
17
  rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
18
18
  local parentWaitKey = parentQueueKey .. ":wait"
19
19
  local parentPausedKey = parentQueueKey .. ":paused"
20
+ local parentActiveKey = parentQueueKey .. ":active"
20
21
  local parentMetaKey = parentQueueKey .. ":meta"
21
22
 
22
23
  local parentMarkerKey = parentQueueKey .. ":marker"
@@ -35,15 +36,16 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
35
36
  addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
36
37
  else
37
38
  if priority == 0 then
38
- local parentTarget, isParentPaused =
39
- getTargetQueueList(parentMetaKey, parentWaitKey,
39
+ local parentTarget, isParentPausedOrMaxed =
40
+ getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
40
41
  parentPausedKey)
41
- addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPaused, parentId)
42
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
43
+ parentId)
42
44
  else
43
- local isPaused = isQueuePaused(parentMetaKey)
45
+ local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
44
46
  addJobWithPriority(parentMarkerKey,
45
47
  parentQueueKey .. ":prioritized", priority,
46
- parentId, parentQueueKey .. ":pc", isPaused)
48
+ parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
47
49
  end
48
50
 
49
51
  rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
@@ -11,9 +11,9 @@
11
11
  --- @include "removeJobKeys"
12
12
 
13
13
  local function moveParentToWait(parentPrefix, parentId, emitEvent)
14
- local parentTarget, isPaused = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait",
15
- parentPrefix .. "paused")
16
- addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPaused, parentId)
14
+ local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
15
+ parentPrefix .. "wait", parentPrefix .. "paused")
16
+ addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
17
17
 
18
18
  if emitEvent then
19
19
  local parentEventStream = parentPrefix .. "events"
@@ -0,0 +1,17 @@
1
+ --[[
2
+ Checks if queue is maxed.
3
+
4
+ Input:
5
+ KEYS[1] meta key
6
+ KEYS[2] active key
7
+
8
+ Output:
9
+ 1 if element found in the list.
10
+ ]]
11
+
12
+ local rcall = redis.call
13
+
14
+ -- Includes
15
+ --- @include "includes/isQueueMaxed"
16
+
17
+ return isQueueMaxed(KEYS[1], KEYS[2])
@@ -35,7 +35,7 @@ if lockToken == token then
35
35
  local metaKey = KEYS[6]
36
36
  local removed = rcall("LREM", KEYS[1], 1, jobId)
37
37
  if removed > 0 then
38
- local target, isPaused = getTargetQueueList(metaKey, KEYS[2], KEYS[5])
38
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[1], KEYS[2], KEYS[5])
39
39
 
40
40
  rcall("SREM", KEYS[3], jobId)
41
41
 
@@ -44,7 +44,7 @@ if lockToken == token then
44
44
  if priority > 0 then
45
45
  pushBackJobWithPriority(KEYS[8], priority, jobId)
46
46
  else
47
- addJobInTargetList(target, KEYS[9], "RPUSH", isPaused, jobId)
47
+ addJobInTargetList(target, KEYS[9], "RPUSH", isPausedOrMaxed, jobId)
48
48
  end
49
49
 
50
50
  rcall("DEL", lockKey)
@@ -10,7 +10,8 @@
10
10
  KEYS[4] 'wait'
11
11
  KEYS[5] 'paused'
12
12
  KEYS[6] 'meta'
13
- KEYS[7] 'marker'
13
+ KEYS[7] 'active'
14
+ KEYS[8] 'marker'
14
15
 
15
16
  ARGV[1] count
16
17
  ARGV[2] timestamp
@@ -32,7 +33,7 @@ local rcall = redis.call;
32
33
  --- @include "includes/getTargetQueueList"
33
34
 
34
35
  local metaKey = KEYS[6]
35
- local target, paused = getTargetQueueList(metaKey, KEYS[4], KEYS[5])
36
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[7], KEYS[4], KEYS[5])
36
37
 
37
38
  local jobs = rcall('ZRANGEBYSCORE', KEYS[3], 0, timestamp, 'LIMIT', 0, maxCount)
38
39
  if (#jobs > 0) then
@@ -62,7 +63,7 @@ if (#jobs > 0) then
62
63
  rcall("LPUSH", target, unpack(jobs, from, to))
63
64
  end
64
65
 
65
- addBaseMarkerIfNeeded(KEYS[7], paused)
66
+ addBaseMarkerIfNeeded(KEYS[8], isPausedOrMaxed)
66
67
  end
67
68
 
68
69
  maxCount = maxCount - #jobs
@@ -142,11 +142,11 @@ if (#stalling > 0) then
142
142
 
143
143
  table.insert(failed, jobId)
144
144
  else
145
- local target, isPaused=
146
- getTargetQueueList(metaKey, waitKey, pausedKey)
145
+ local target, isPausedOrMaxed=
146
+ getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
147
147
 
148
148
  -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
149
- addJobInTargetList(target, markerKey, "RPUSH", isPaused, jobId)
149
+ addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
150
150
 
151
151
  rcall("XADD", eventStreamKey, "*", "event",
152
152
  "waiting", "jobId", jobId, 'prev', 'active')
@@ -50,12 +50,12 @@ local opts = cmsgpack.unpack(ARGV[3])
50
50
  --- @include "includes/prepareJobForProcessing"
51
51
  --- @include "includes/promoteDelayedJobs"
52
52
 
53
- local target, paused = getTargetQueueList(KEYS[9], waitKey, KEYS[8])
53
+ local target, isPausedOrMaxed = getTargetQueueList(KEYS[9], activeKey, waitKey, KEYS[8])
54
54
 
55
55
  -- Check if there are delayed jobs that we can move to wait.
56
56
  local markerKey = KEYS[11]
57
57
  promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
58
- ARGV[2], KEYS[10], paused)
58
+ ARGV[2], KEYS[10], isPausedOrMaxed)
59
59
 
60
60
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
61
61
  local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
@@ -63,8 +63,8 @@ local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
63
63
  -- Check if we are rate limited first.
64
64
  if expireTime > 0 then return {0, 0, expireTime, 0} end
65
65
 
66
- -- paused queue
67
- if paused then return {0, 0, 0, 0} end
66
+ -- paused or maxed queue
67
+ if isPausedOrMaxed then return {0, 0, 0, 0} end
68
68
 
69
69
  -- no job ID, try non-blocking move from wait to active
70
70
  local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
@@ -32,7 +32,6 @@ local rcall = redis.call
32
32
  --- @include "includes/addDelayMarkerIfNeeded"
33
33
  --- @include "includes/getDelayedScore"
34
34
  --- @include "includes/getOrSetMaxEvents"
35
- --- @include "includes/isQueuePaused"
36
35
  --- @include "includes/removeLock"
37
36
 
38
37
  local jobKey = KEYS[5]
@@ -65,11 +64,8 @@ if rcall("EXISTS", jobKey) == 1 then
65
64
  "jobId", jobId, "delay", delayedTimestamp)
66
65
 
67
66
  -- Check if we need to push a marker job to wake up sleeping workers.
68
- local isPaused = isQueuePaused(metaKey)
69
- if not isPaused then
70
- local markerKey = KEYS[1]
71
- addDelayMarkerIfNeeded(markerKey, delayedKey)
72
- end
67
+ local markerKey = KEYS[1]
68
+ addDelayMarkerIfNeeded(markerKey, delayedKey)
73
69
 
74
70
  return 0
75
71
  else
@@ -202,11 +202,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
202
202
  -- and not rate limited.
203
203
  if (ARGV[6] == "1") then
204
204
 
205
- local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[8])
205
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
206
206
 
207
207
  -- Check if there are delayed jobs that can be promoted
208
208
  promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], eventStreamKey, ARGV[7],
209
- timestamp, KEYS[10], paused)
209
+ timestamp, KEYS[10], isPausedOrMaxed)
210
210
 
211
211
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
212
212
  -- Check if we are rate limited first.
@@ -214,8 +214,8 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
214
214
 
215
215
  if expireTime > 0 then return {0, 0, expireTime, 0} end
216
216
 
217
- -- paused queue
218
- if paused then return {0, 0, 0, 0} end
217
+ -- paused or maxed queue
218
+ if isPausedOrMaxed then return {0, 0, 0, 0} end
219
219
 
220
220
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
221
221
 
@@ -21,8 +21,9 @@
21
21
  -3 - Job not in active set
22
22
  ]]
23
23
  local rcall = redis.call
24
+ local stalledKey = KEYS[5]
24
25
 
25
- -- Includes
26
+ --- Includes
26
27
  --- @include "includes/removeLock"
27
28
 
28
29
  local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId,
@@ -43,7 +44,7 @@ end
43
44
  if rcall("EXISTS", KEYS[4]) == 1 then
44
45
  if ARGV[2] ~= "" then
45
46
  if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
46
- local errorCode = removeLock(KEYS[4], KEYS[5], ARGV[1], ARGV[4])
47
+ local errorCode = removeLock(KEYS[4], stalledKey, ARGV[1], ARGV[4])
47
48
  if errorCode < 0 then
48
49
  return errorCode
49
50
  end
@@ -53,7 +54,7 @@ if rcall("EXISTS", KEYS[4]) == 1 then
53
54
  return 1
54
55
  else
55
56
  if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
56
- local errorCode = removeLock(KEYS[4], KEYS[5], ARGV[1], ARGV[4])
57
+ local errorCode = removeLock(KEYS[4], stalledKey, ARGV[1], ARGV[4])
57
58
  if errorCode < 0 then
58
59
  return errorCode
59
60
  end
@@ -7,9 +7,10 @@
7
7
  KEYS[3] 'paused'
8
8
  KEYS[4] 'meta'
9
9
  KEYS[5] 'prioritized'
10
- KEYS[6] 'pc' priority counter
11
- KEYS[7] 'event stream'
12
- KEYS[8] 'marker'
10
+ KEYS[6] 'active'
11
+ KEYS[7] 'pc' priority counter
12
+ KEYS[8] 'event stream'
13
+ KEYS[9] 'marker'
13
14
 
14
15
  ARGV[1] queue.toKey('')
15
16
  ARGV[2] jobId
@@ -33,23 +34,24 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then
33
34
  local jobKey = ARGV[1] .. jobId
34
35
  local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
35
36
  local metaKey = KEYS[4]
37
+ local markerKey = KEYS[9]
36
38
 
37
39
  -- Remove delayed "marker" from the wait list if there is any.
38
40
  -- Since we are adding a job we do not need the marker anymore.
39
41
  -- Markers in waitlist DEPRECATED in v5: Remove in v6.
40
- local target, paused = getTargetQueueList(metaKey, KEYS[2], KEYS[3])
42
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[6], KEYS[2], KEYS[3])
41
43
  local marker = rcall("LINDEX", target, 0)
42
44
  if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
43
45
 
44
46
  if priority == 0 then
45
47
  -- LIFO or FIFO
46
- addJobInTargetList(target, KEYS[8], "LPUSH", paused, jobId)
48
+ addJobInTargetList(target, markerKey, "LPUSH", isPausedOrMaxed, jobId)
47
49
  else
48
- addJobWithPriority(KEYS[8], KEYS[5], priority, jobId, KEYS[6], paused)
50
+ addJobWithPriority(markerKey, KEYS[5], priority, jobId, KEYS[7], isPausedOrMaxed)
49
51
  end
50
52
 
51
53
  -- Emit waiting event (wait..ing@token)
52
- rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev",
54
+ rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId, "prev",
53
55
  "delayed");
54
56
 
55
57
  rcall("HSET", jobKey, "delay", 0)
@@ -4,6 +4,7 @@
4
4
 
5
5
  Input:
6
6
  KEYS[1] queue prefix
7
+ KEYS[2] meta key
7
8
 
8
9
  ARGV[1] jobId
9
10
  ARGV[2] remove children
@@ -66,7 +67,7 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
66
67
  local prev = removeJobFromAnyState(prefix, jobId)
67
68
 
68
69
  if removeJobKeys(jobKey) > 0 then
69
- local maxEvents = getOrSetMaxEvents(prefix .. "meta")
70
+ local maxEvents = getOrSetMaxEvents(KEYS[2])
70
71
  rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
71
72
  "jobId", jobId, "prev", prev)
72
73
  end
@@ -8,7 +8,8 @@
8
8
  KEYS[4] wait key
9
9
  KEYS[5] meta
10
10
  KEYS[6] paused key
11
- KEYS[7] marker key
11
+ KEYS[7] active key
12
+ KEYS[8] marker key
12
13
 
13
14
  ARGV[1] job.id
14
15
  ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
@@ -32,8 +33,8 @@ if rcall("EXISTS", KEYS[1]) == 1 then
32
33
  if (rcall("ZREM", KEYS[3], jobId) == 1) then
33
34
  rcall("HDEL", KEYS[1], "finishedOn", "processedOn", ARGV[3])
34
35
 
35
- local target, isPaused = getTargetQueueList(KEYS[5], KEYS[4], KEYS[6])
36
- addJobInTargetList(target, KEYS[7], ARGV[2], isPaused, jobId)
36
+ local target, isPausedOrMaxed = getTargetQueueList(KEYS[5], KEYS[7], KEYS[4], KEYS[6])
37
+ addJobInTargetList(target, KEYS[8], ARGV[2], isPausedOrMaxed, jobId)
37
38
 
38
39
  local maxEvents = getOrSetMaxEvents(KEYS[5])
39
40
  -- Emit waiting event
@@ -38,13 +38,14 @@ local rcall = redis.call
38
38
  --- @include "includes/getTargetQueueList"
39
39
  --- @include "includes/promoteDelayedJobs"
40
40
  --- @include "includes/removeLock"
41
+ --- @include "includes/isQueuePausedOrMaxed"
41
42
 
42
- local target, paused = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
43
+ local target, isPausedOrMaxed = getTargetQueueList(KEYS[5], KEYS[1], KEYS[2], KEYS[3])
43
44
  local markerKey = KEYS[10]
44
45
 
45
46
  -- Check if there are delayed jobs that we can move to wait.
46
47
  -- test example: when there are delayed jobs between retries
47
- promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], paused)
48
+ promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], isPausedOrMaxed)
48
49
 
49
50
  if rcall("EXISTS", KEYS[4]) == 1 then
50
51
  local errorCode = removeLock(KEYS[4], KEYS[11], ARGV[5], ARGV[4])
@@ -57,11 +58,14 @@ if rcall("EXISTS", KEYS[4]) == 1 then
57
58
 
58
59
  local priority = tonumber(rcall("HGET", KEYS[4], "priority")) or 0
59
60
 
61
+ --need to re-evaluate after removing job from active
62
+ isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[5], KEYS[1])
63
+
60
64
  -- Standard or priority add
61
65
  if priority == 0 then
62
- addJobInTargetList(target, KEYS[10], ARGV[3], paused, ARGV[4])
66
+ addJobInTargetList(target, markerKey, ARGV[3], isPausedOrMaxed, ARGV[4])
63
67
  else
64
- addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], paused)
68
+ addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], isPausedOrMaxed)
65
69
  end
66
70
 
67
71
  rcall("HINCRBY", KEYS[4], "atm", 1)
@@ -131,42 +131,61 @@ end
131
131
  --[[
132
132
  Add marker if needed when a job is available.
133
133
  ]]
134
- local function addBaseMarkerIfNeeded(markerKey, isPaused)
135
- if not isPaused then
134
+ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
135
+ if not isPausedOrMaxed then
136
136
  rcall("ZADD", markerKey, 0, "0")
137
137
  end
138
138
  end
139
- local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
139
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
140
140
  rcall(pushCmd, targetKey, jobId)
141
- addBaseMarkerIfNeeded(markerKey, isPaused)
141
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
142
142
  end
143
143
  --[[
144
144
  Function to add job considering priority.
145
145
  ]]
146
146
  -- Includes
147
- local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
147
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
148
+ isPausedOrMaxed)
148
149
  local prioCounter = rcall("INCR", priorityCounterKey)
149
150
  local score = priority * 0x100000000 + prioCounter % 0x100000000
150
151
  rcall("ZADD", prioritizedKey, score, jobId)
151
- addBaseMarkerIfNeeded(markerKey, isPaused)
152
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
152
153
  end
153
154
  --[[
154
- Function to check for the meta.paused key to decide if we are paused or not
155
+ Function to check if queue is paused or maxed
155
156
  (since an empty list and !EXISTS are not really the same).
156
157
  ]]
157
- local function isQueuePaused(queueMetaKey)
158
- return rcall("HEXISTS", queueMetaKey, "paused") == 1
158
+ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
159
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
160
+ if queueAttributes[1] then
161
+ return true
162
+ else
163
+ if queueAttributes[2] then
164
+ local activeCount = rcall("LLEN", activeKey)
165
+ return activeCount >= tonumber(queueAttributes[2])
166
+ end
167
+ end
168
+ return false
159
169
  end
160
170
  --[[
161
171
  Function to check for the meta.paused key to decide if we are paused or not
162
172
  (since an empty list and !EXISTS are not really the same).
163
173
  ]]
164
- local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
165
- if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
166
- return waitKey, false
167
- else
174
+ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
175
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
176
+ if queueAttributes[1] then
168
177
  return pausedKey, true
178
+ else
179
+ if queueAttributes[2] then
180
+ local activeCount = rcall("LLEN", activeKey)
181
+ if activeCount >= tonumber(queueAttributes[2]) then
182
+ return waitKey, true
183
+ else
184
+ return waitKey, false
185
+ end
186
+ end
169
187
  end
188
+ return waitKey, false
170
189
  end
171
190
  local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
172
191
  parentKey, parentId, timestamp)
@@ -176,6 +195,7 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
176
195
  rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
177
196
  local parentWaitKey = parentQueueKey .. ":wait"
178
197
  local parentPausedKey = parentQueueKey .. ":paused"
198
+ local parentActiveKey = parentQueueKey .. ":active"
179
199
  local parentMetaKey = parentQueueKey .. ":meta"
180
200
  local parentMarkerKey = parentQueueKey .. ":marker"
181
201
  local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
@@ -191,15 +211,16 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
191
211
  addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
192
212
  else
193
213
  if priority == 0 then
194
- local parentTarget, isParentPaused =
195
- getTargetQueueList(parentMetaKey, parentWaitKey,
214
+ local parentTarget, isParentPausedOrMaxed =
215
+ getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
196
216
  parentPausedKey)
197
- addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPaused, parentId)
217
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
218
+ parentId)
198
219
  else
199
- local isPaused = isQueuePaused(parentMetaKey)
220
+ local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
200
221
  addJobWithPriority(parentMarkerKey,
201
222
  parentQueueKey .. ":prioritized", priority,
202
- parentId, parentQueueKey .. ":pc", isPaused)
223
+ parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
203
224
  end
204
225
  rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
205
226
  "jobId", parentId, "prev", "waiting-children")
@@ -301,11 +322,8 @@ rcall("ZADD", delayedKey, score, jobId)
301
322
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
302
323
  "jobId", jobId, "delay", delayedTimestamp)
303
324
  -- mark that a delayed job is available
304
- local isPaused = isQueuePaused(metaKey)
305
- if not isPaused then
306
- local markerKey = KEYS[1]
307
- addDelayMarkerIfNeeded(markerKey, delayedKey)
308
- end
325
+ local markerKey = KEYS[1]
326
+ addDelayMarkerIfNeeded(markerKey, delayedKey)
309
327
  -- Check if this job is a child of another job, if so add it to the parents dependencies
310
328
  if parentDependenciesKey ~= nil then
311
329
  rcall("SADD", parentDependenciesKey, jobIdKey)
@@ -1 +1 @@
1
- {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsTf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwUf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}