bullmq 5.8.7 → 5.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/dist/cjs/classes/queue.js +29 -0
  2. package/dist/cjs/classes/queue.js.map +1 -1
  3. package/dist/cjs/classes/repeat.js +44 -30
  4. package/dist/cjs/classes/repeat.js.map +1 -1
  5. package/dist/cjs/classes/scripts.js +44 -5
  6. package/dist/cjs/classes/scripts.js.map +1 -1
  7. package/dist/cjs/classes/worker.js +1 -1
  8. package/dist/cjs/classes/worker.js.map +1 -1
  9. package/dist/cjs/commands/{addPrioritizedJob-7.lua → addPrioritizedJob-8.lua} +9 -7
  10. package/dist/cjs/commands/addRepeatableJob-2.lua +69 -0
  11. package/dist/cjs/commands/{addStandardJob-7.lua → addStandardJob-8.lua} +6 -5
  12. package/dist/cjs/commands/{changePriority-6.lua → changePriority-7.lua} +12 -10
  13. package/dist/cjs/commands/getCountsPerPriority-4.lua +6 -3
  14. package/dist/cjs/commands/includes/addBaseMarkerIfNeeded.lua +2 -2
  15. package/dist/cjs/commands/includes/addJobInTargetList.lua +2 -2
  16. package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -2
  17. package/dist/cjs/commands/includes/getTargetQueueList.lua +14 -4
  18. package/dist/cjs/commands/includes/isQueueMaxed.lua +15 -0
  19. package/dist/cjs/commands/includes/isQueuePaused.lua +1 -1
  20. package/dist/cjs/commands/includes/isQueuePausedOrMaxed.lua +18 -0
  21. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +8 -6
  22. package/dist/cjs/commands/includes/removeParentDependencyKey.lua +3 -3
  23. package/dist/cjs/commands/isMaxed-2.lua +17 -0
  24. package/dist/cjs/commands/moveJobFromActiveToWait-10.lua +2 -2
  25. package/dist/{esm/commands/moveJobsToWait-7.lua → cjs/commands/moveJobsToWait-8.lua} +4 -3
  26. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +3 -3
  27. package/dist/cjs/commands/moveToActive-11.lua +4 -4
  28. package/dist/cjs/commands/moveToFinished-14.lua +4 -4
  29. package/dist/cjs/commands/moveToWaitingChildren-5.lua +4 -3
  30. package/dist/cjs/commands/obliterate-2.lua +23 -8
  31. package/dist/{esm/commands/promote-8.lua → cjs/commands/promote-9.lua} +9 -7
  32. package/dist/{esm/commands/removeJob-1.lua → cjs/commands/removeJob-2.lua} +2 -1
  33. package/dist/cjs/commands/removeRepeatable-2.lua +27 -6
  34. package/dist/{esm/commands/reprocessJob-7.lua → cjs/commands/reprocessJob-8.lua} +4 -3
  35. package/dist/cjs/commands/retryJob-11.lua +8 -4
  36. package/dist/cjs/interfaces/index.js +1 -0
  37. package/dist/cjs/interfaces/index.js.map +1 -1
  38. package/dist/cjs/interfaces/repeatable-options.js +3 -0
  39. package/dist/cjs/interfaces/repeatable-options.js.map +1 -0
  40. package/dist/cjs/scripts/addDelayedJob-6.js +39 -18
  41. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  42. package/dist/cjs/scripts/addParentJob-4.js +39 -18
  43. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  44. package/dist/cjs/scripts/{addPrioritizedJob-7.js → addPrioritizedJob-8.js} +49 -26
  45. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -0
  46. package/dist/cjs/scripts/addRepeatableJob-2.js +66 -0
  47. package/dist/cjs/scripts/addRepeatableJob-2.js.map +1 -0
  48. package/dist/cjs/scripts/{addStandardJob-7.js → addStandardJob-8.js} +47 -25
  49. package/dist/cjs/scripts/addStandardJob-8.js.map +1 -0
  50. package/dist/cjs/scripts/{changePriority-6.js → changePriority-7.js} +33 -21
  51. package/dist/cjs/scripts/changePriority-7.js.map +1 -0
  52. package/dist/cjs/scripts/cleanJobsInSet-2.js +20 -11
  53. package/dist/cjs/scripts/cleanJobsInSet-2.js.map +1 -1
  54. package/dist/cjs/scripts/drain-4.js +20 -11
  55. package/dist/cjs/scripts/drain-4.js.map +1 -1
  56. package/dist/cjs/scripts/getCountsPerPriority-4.js +7 -8
  57. package/dist/cjs/scripts/getCountsPerPriority-4.js.map +1 -1
  58. package/dist/cjs/scripts/index.js +9 -7
  59. package/dist/cjs/scripts/index.js.map +1 -1
  60. package/dist/cjs/scripts/isMaxed-2.js +34 -0
  61. package/dist/cjs/scripts/isMaxed-2.js.map +1 -0
  62. package/dist/cjs/scripts/moveJobFromActiveToWait-10.js +19 -10
  63. package/dist/cjs/scripts/moveJobFromActiveToWait-10.js.map +1 -1
  64. package/dist/cjs/scripts/{moveJobsToWait-7.js → moveJobsToWait-8.js} +21 -11
  65. package/dist/cjs/scripts/moveJobsToWait-8.js.map +1 -0
  66. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +45 -24
  67. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  68. package/dist/cjs/scripts/moveToActive-11.js +24 -14
  69. package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
  70. package/dist/cjs/scripts/moveToFinished-14.js +46 -25
  71. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  72. package/dist/cjs/scripts/moveToWaitingChildren-5.js +4 -3
  73. package/dist/cjs/scripts/moveToWaitingChildren-5.js.map +1 -1
  74. package/dist/cjs/scripts/obliterate-2.js +41 -18
  75. package/dist/cjs/scripts/obliterate-2.js.map +1 -1
  76. package/dist/cjs/scripts/{promote-8.js → promote-9.js} +31 -19
  77. package/dist/cjs/scripts/promote-9.js.map +1 -0
  78. package/dist/cjs/scripts/removeChildDependency-1.js +20 -11
  79. package/dist/cjs/scripts/removeChildDependency-1.js.map +1 -1
  80. package/dist/cjs/scripts/{removeJob-1.js → removeJob-2.js} +24 -14
  81. package/dist/cjs/scripts/{removeJob-1.js.map → removeJob-2.js.map} +1 -1
  82. package/dist/cjs/scripts/removeRepeatable-2.js +29 -6
  83. package/dist/cjs/scripts/removeRepeatable-2.js.map +1 -1
  84. package/dist/cjs/scripts/{reprocessJob-7.js → reprocessJob-8.js} +23 -13
  85. package/dist/cjs/scripts/reprocessJob-8.js.map +1 -0
  86. package/dist/cjs/scripts/retryJob-11.js +42 -14
  87. package/dist/cjs/scripts/retryJob-11.js.map +1 -1
  88. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  89. package/dist/esm/classes/queue.d.ts +17 -0
  90. package/dist/esm/classes/queue.js +29 -0
  91. package/dist/esm/classes/queue.js.map +1 -1
  92. package/dist/esm/classes/repeat.d.ts +2 -0
  93. package/dist/esm/classes/repeat.js +44 -30
  94. package/dist/esm/classes/repeat.js.map +1 -1
  95. package/dist/esm/classes/scripts.d.ts +6 -2
  96. package/dist/esm/classes/scripts.js +44 -5
  97. package/dist/esm/classes/scripts.js.map +1 -1
  98. package/dist/esm/classes/worker.js +1 -1
  99. package/dist/esm/classes/worker.js.map +1 -1
  100. package/dist/esm/commands/{addPrioritizedJob-7.lua → addPrioritizedJob-8.lua} +9 -7
  101. package/dist/esm/commands/addRepeatableJob-2.lua +69 -0
  102. package/dist/esm/commands/{addStandardJob-7.lua → addStandardJob-8.lua} +6 -5
  103. package/dist/esm/commands/{changePriority-6.lua → changePriority-7.lua} +12 -10
  104. package/dist/esm/commands/getCountsPerPriority-4.lua +6 -3
  105. package/dist/esm/commands/includes/addBaseMarkerIfNeeded.lua +2 -2
  106. package/dist/esm/commands/includes/addJobInTargetList.lua +2 -2
  107. package/dist/esm/commands/includes/addJobWithPriority.lua +3 -2
  108. package/dist/esm/commands/includes/getTargetQueueList.lua +14 -4
  109. package/dist/esm/commands/includes/isQueueMaxed.lua +15 -0
  110. package/dist/esm/commands/includes/isQueuePaused.lua +1 -1
  111. package/dist/esm/commands/includes/isQueuePausedOrMaxed.lua +18 -0
  112. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +8 -6
  113. package/dist/esm/commands/includes/removeParentDependencyKey.lua +3 -3
  114. package/dist/esm/commands/isMaxed-2.lua +17 -0
  115. package/dist/esm/commands/moveJobFromActiveToWait-10.lua +2 -2
  116. package/dist/{cjs/commands/moveJobsToWait-7.lua → esm/commands/moveJobsToWait-8.lua} +4 -3
  117. package/dist/esm/commands/moveStalledJobsToWait-9.lua +3 -3
  118. package/dist/esm/commands/moveToActive-11.lua +4 -4
  119. package/dist/esm/commands/moveToFinished-14.lua +4 -4
  120. package/dist/esm/commands/moveToWaitingChildren-5.lua +4 -3
  121. package/dist/esm/commands/obliterate-2.lua +23 -8
  122. package/dist/{cjs/commands/promote-8.lua → esm/commands/promote-9.lua} +9 -7
  123. package/dist/{cjs/commands/removeJob-1.lua → esm/commands/removeJob-2.lua} +2 -1
  124. package/dist/esm/commands/removeRepeatable-2.lua +27 -6
  125. package/dist/{cjs/commands/reprocessJob-7.lua → esm/commands/reprocessJob-8.lua} +4 -3
  126. package/dist/esm/commands/retryJob-11.lua +8 -4
  127. package/dist/esm/interfaces/index.d.ts +1 -0
  128. package/dist/esm/interfaces/index.js +1 -0
  129. package/dist/esm/interfaces/index.js.map +1 -1
  130. package/dist/esm/interfaces/repeatable-job.d.ts +3 -2
  131. package/dist/esm/interfaces/repeatable-options.d.ts +7 -0
  132. package/dist/esm/interfaces/repeatable-options.js +2 -0
  133. package/dist/esm/interfaces/repeatable-options.js.map +1 -0
  134. package/dist/esm/scripts/addDelayedJob-6.js +39 -18
  135. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  136. package/dist/esm/scripts/addParentJob-4.js +39 -18
  137. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  138. package/dist/esm/scripts/{addPrioritizedJob-7.js → addPrioritizedJob-8.js} +49 -26
  139. package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -0
  140. package/dist/esm/scripts/addRepeatableJob-2.d.ts +5 -0
  141. package/dist/esm/scripts/addRepeatableJob-2.js +63 -0
  142. package/dist/esm/scripts/addRepeatableJob-2.js.map +1 -0
  143. package/dist/esm/scripts/{addStandardJob-7.js → addStandardJob-8.js} +47 -25
  144. package/dist/esm/scripts/addStandardJob-8.js.map +1 -0
  145. package/dist/esm/scripts/{changePriority-6.js → changePriority-7.js} +33 -21
  146. package/dist/esm/scripts/changePriority-7.js.map +1 -0
  147. package/dist/esm/scripts/cleanJobsInSet-2.js +20 -11
  148. package/dist/esm/scripts/cleanJobsInSet-2.js.map +1 -1
  149. package/dist/esm/scripts/drain-4.js +20 -11
  150. package/dist/esm/scripts/drain-4.js.map +1 -1
  151. package/dist/esm/scripts/getCountsPerPriority-4.js +7 -8
  152. package/dist/esm/scripts/getCountsPerPriority-4.js.map +1 -1
  153. package/dist/esm/scripts/index.d.ts +9 -7
  154. package/dist/esm/scripts/index.js +9 -7
  155. package/dist/esm/scripts/index.js.map +1 -1
  156. package/dist/esm/scripts/isMaxed-2.d.ts +5 -0
  157. package/dist/esm/scripts/isMaxed-2.js +31 -0
  158. package/dist/esm/scripts/isMaxed-2.js.map +1 -0
  159. package/dist/esm/scripts/moveJobFromActiveToWait-10.js +19 -10
  160. package/dist/esm/scripts/moveJobFromActiveToWait-10.js.map +1 -1
  161. package/dist/esm/scripts/{moveJobsToWait-7.js → moveJobsToWait-8.js} +21 -11
  162. package/dist/esm/scripts/moveJobsToWait-8.js.map +1 -0
  163. package/dist/esm/scripts/moveStalledJobsToWait-9.js +45 -24
  164. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  165. package/dist/esm/scripts/moveToActive-11.js +24 -14
  166. package/dist/esm/scripts/moveToActive-11.js.map +1 -1
  167. package/dist/esm/scripts/moveToFinished-14.js +46 -25
  168. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  169. package/dist/esm/scripts/moveToWaitingChildren-5.js +4 -3
  170. package/dist/esm/scripts/moveToWaitingChildren-5.js.map +1 -1
  171. package/dist/esm/scripts/obliterate-2.js +41 -18
  172. package/dist/esm/scripts/obliterate-2.js.map +1 -1
  173. package/dist/esm/scripts/{promote-8.js → promote-9.js} +31 -19
  174. package/dist/esm/scripts/promote-9.js.map +1 -0
  175. package/dist/esm/scripts/removeChildDependency-1.js +20 -11
  176. package/dist/esm/scripts/removeChildDependency-1.js.map +1 -1
  177. package/dist/esm/scripts/{removeJob-1.js → removeJob-2.js} +24 -14
  178. package/dist/esm/scripts/{removeJob-1.js.map → removeJob-2.js.map} +1 -1
  179. package/dist/esm/scripts/removeRepeatable-2.js +29 -6
  180. package/dist/esm/scripts/removeRepeatable-2.js.map +1 -1
  181. package/dist/esm/scripts/{reprocessJob-7.js → reprocessJob-8.js} +23 -13
  182. package/dist/esm/scripts/reprocessJob-8.js.map +1 -0
  183. package/dist/esm/scripts/retryJob-11.js +42 -14
  184. package/dist/esm/scripts/retryJob-11.js.map +1 -1
  185. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  186. package/package.json +2 -1
  187. package/dist/cjs/scripts/addPrioritizedJob-7.js.map +0 -1
  188. package/dist/cjs/scripts/addStandardJob-7.js.map +0 -1
  189. package/dist/cjs/scripts/changePriority-6.js.map +0 -1
  190. package/dist/cjs/scripts/moveJobsToWait-7.js.map +0 -1
  191. package/dist/cjs/scripts/promote-8.js.map +0 -1
  192. package/dist/cjs/scripts/reprocessJob-7.js.map +0 -1
  193. package/dist/esm/scripts/addPrioritizedJob-7.js.map +0 -1
  194. package/dist/esm/scripts/addStandardJob-7.js.map +0 -1
  195. package/dist/esm/scripts/changePriority-6.js.map +0 -1
  196. package/dist/esm/scripts/moveJobsToWait-7.js.map +0 -1
  197. package/dist/esm/scripts/promote-8.js.map +0 -1
  198. package/dist/esm/scripts/reprocessJob-7.js.map +0 -1
  199. /package/dist/esm/scripts/{addPrioritizedJob-7.d.ts → addPrioritizedJob-8.d.ts} +0 -0
  200. /package/dist/esm/scripts/{addStandardJob-7.d.ts → addStandardJob-8.d.ts} +0 -0
  201. /package/dist/esm/scripts/{changePriority-6.d.ts → changePriority-7.d.ts} +0 -0
  202. /package/dist/esm/scripts/{moveJobsToWait-7.d.ts → moveJobsToWait-8.d.ts} +0 -0
  203. /package/dist/esm/scripts/{promote-8.d.ts → promote-9.d.ts} +0 -0
  204. /package/dist/esm/scripts/{removeJob-1.d.ts → removeJob-2.d.ts} +0 -0
  205. /package/dist/esm/scripts/{reprocessJob-7.d.ts → reprocessJob-8.d.ts} +0 -0
@@ -10,8 +10,9 @@
10
10
  KEYS[3] 'id'
11
11
  KEYS[4] 'prioritized'
12
12
  KEYS[5] 'completed'
13
- KEYS[6] events stream key
14
- KEYS[7] 'pc' priority counter
13
+ KEYS[6] 'active'
14
+ KEYS[7] events stream key
15
+ KEYS[8] 'pc' priority counter
15
16
 
16
17
  ARGV[1] msgpacked arguments array
17
18
  [1] key prefix,
@@ -36,8 +37,9 @@ local idKey = KEYS[3]
36
37
  local priorityKey = KEYS[4]
37
38
 
38
39
  local completedKey = KEYS[5]
39
- local eventsKey = KEYS[6]
40
- local priorityCounterKey = KEYS[7]
40
+ local activeKey = KEYS[6]
41
+ local eventsKey = KEYS[7]
42
+ local priorityCounterKey = KEYS[8]
41
43
 
42
44
  local jobId
43
45
  local jobIdKey
@@ -58,7 +60,7 @@ local parentData
58
60
  --- @include "includes/storeJob"
59
61
  --- @include "includes/getOrSetMaxEvents"
60
62
  --- @include "includes/handleDuplicatedJob"
61
- --- @include "includes/isQueuePaused"
63
+ --- @include "includes/isQueuePausedOrMaxed"
62
64
 
63
65
  if parentKey ~= nil then
64
66
  if rcall("EXISTS", parentKey) ~= 1 then return -5 end
@@ -91,8 +93,8 @@ local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
91
93
  repeatJobKey)
92
94
 
93
95
  -- Add the job to the prioritized set
94
- local isPause = isQueuePaused(metaKey)
95
- addJobWithPriority( KEYS[1], priorityKey, priority, jobId, priorityCounterKey, isPause)
96
+ local isPausedOrMaxed = isQueuePausedOrMaxed(metaKey, activeKey)
97
+ addJobWithPriority( KEYS[1], priorityKey, priority, jobId, priorityCounterKey, isPausedOrMaxed)
96
98
 
97
99
  -- Emit waiting event
98
100
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
@@ -0,0 +1,69 @@
1
+ --[[
2
+ Adds a repeatable job
3
+
4
+ Input:
5
+ KEYS[1] 'repeat' key
6
+ KEYS[2] custom key
7
+
8
+ ARGV[1] next milliseconds
9
+ ARGV[2] msgpacked options
10
+ [1] name
11
+ [2] tz?
12
+ [3] patten?
13
+ [4] endDate?
14
+ [5] every?
15
+ ARGV[3] legacy custom key TODO: remove this logic in next breaking change
16
+ ARGV[4] skipCheckExists
17
+
18
+ Output:
19
+ repeatableKey - OK
20
+ ]]
21
+ local rcall = redis.call
22
+ local repeatKey = KEYS[1]
23
+ local customKey = KEYS[2]
24
+ local legacyCustomKey = ARGV[3]
25
+ local nextMilli = ARGV[1]
26
+
27
+ local function storeRepeatableJob(repeatKey, customKey, nextMilli, rawOpts)
28
+ rcall("ZADD", repeatKey, nextMilli, customKey)
29
+ local opts = cmsgpack.unpack(rawOpts)
30
+
31
+ local optionalValues = {}
32
+ if opts['tz'] then
33
+ table.insert(optionalValues, "tz")
34
+ table.insert(optionalValues, opts['tz'])
35
+ end
36
+
37
+ if opts['pattern'] then
38
+ table.insert(optionalValues, "pattern")
39
+ table.insert(optionalValues, opts['pattern'])
40
+ end
41
+
42
+ if opts['endDate'] then
43
+ table.insert(optionalValues, "endDate")
44
+ table.insert(optionalValues, opts['endDate'])
45
+ end
46
+
47
+ if opts['every'] then
48
+ table.insert(optionalValues, "every")
49
+ table.insert(optionalValues, opts['every'])
50
+ end
51
+
52
+ rcall("HMSET", repeatKey .. ":" .. customKey, "name", opts['name'],
53
+ unpack(optionalValues))
54
+
55
+ return customKey
56
+ end
57
+
58
+ if ARGV[4] == '0' then
59
+ if rcall("ZSCORE", repeatKey, legacyCustomKey) ~= false then
60
+ rcall("ZADD", repeatKey, nextMilli, legacyCustomKey)
61
+ return legacyCustomKey
62
+ elseif rcall("ZSCORE", repeatKey, customKey) ~= false then
63
+ return storeRepeatableJob(repeatKey, customKey, nextMilli, ARGV[2])
64
+ end
65
+ else
66
+ return storeRepeatableJob(repeatKey, customKey, nextMilli, ARGV[2])
67
+ end
68
+
69
+ return ''
@@ -20,8 +20,9 @@
20
20
  KEYS[3] 'meta'
21
21
  KEYS[4] 'id'
22
22
  KEYS[5] 'completed'
23
- KEYS[6] events stream key
24
- KEYS[7] marker key
23
+ KEYS[6] 'active'
24
+ KEYS[7] events stream key
25
+ KEYS[8] marker key
25
26
 
26
27
  ARGV[1] msgpacked arguments array
27
28
  [1] key prefix,
@@ -41,7 +42,7 @@
41
42
  jobId - OK
42
43
  -5 - Missing parent key
43
44
  ]]
44
- local eventsKey = KEYS[6]
45
+ local eventsKey = KEYS[7]
45
46
 
46
47
  local jobId
47
48
  local jobIdKey
@@ -94,11 +95,11 @@ end
94
95
  storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
95
96
  parentKey, parentData, repeatJobKey)
96
97
 
97
- local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
98
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[6], KEYS[1], KEYS[2])
98
99
 
99
100
  -- LIFO or FIFO
100
101
  local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
101
- addJobInTargetList(target, KEYS[7], pushCmd, paused, jobId)
102
+ addJobInTargetList(target, KEYS[8], pushCmd, isPausedOrMaxed, jobId)
102
103
 
103
104
  -- Emit waiting event
104
105
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
@@ -5,8 +5,9 @@
5
5
  KEYS[2] 'paused'
6
6
  KEYS[3] 'meta'
7
7
  KEYS[4] 'prioritized'
8
- KEYS[5] 'pc' priority counter
9
- KEYS[6] 'marker'
8
+ KEYS[5] 'active'
9
+ KEYS[6] 'pc' priority counter
10
+ KEYS[7] 'marker'
10
11
 
11
12
  ARGV[1] priority value
12
13
  ARGV[2] job key
@@ -29,33 +30,34 @@ local rcall = redis.call
29
30
  --- @include "includes/pushBackJobWithPriority"
30
31
 
31
32
  local function reAddJobWithNewPriority( prioritizedKey, markerKey, targetKey,
32
- priorityCounter, lifo, priority, jobId, paused)
33
+ priorityCounter, lifo, priority, jobId, isPausedOrMaxed)
33
34
  if priority == 0 then
34
35
  local pushCmd = lifo and 'RPUSH' or 'LPUSH'
35
- addJobInTargetList(targetKey, markerKey, pushCmd, paused, jobId)
36
+ addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
36
37
  else
37
38
  if lifo then
38
39
  pushBackJobWithPriority(prioritizedKey, priority, jobId)
39
40
  else
40
41
  addJobWithPriority(markerKey, prioritizedKey, priority, jobId,
41
- priorityCounter, paused)
42
+ priorityCounter, isPausedOrMaxed)
42
43
  end
43
44
  end
44
45
  end
45
46
 
46
47
  if rcall("EXISTS", jobKey) == 1 then
47
48
  local metaKey = KEYS[3]
48
- local target, isPaused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
49
- local markerKey = KEYS[6]
49
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[5], KEYS[1], KEYS[2])
50
50
  local prioritizedKey = KEYS[4]
51
-
51
+ local priorityCounterKey = KEYS[6]
52
+ local markerKey = KEYS[7]
53
+
52
54
  -- Re-add with the new priority
53
55
  if rcall("ZREM", KEYS[4], jobId) > 0 then
54
56
  reAddJobWithNewPriority( prioritizedKey, markerKey, target,
55
- KEYS[5], ARGV[4] == '1', priority, jobId, isPaused)
57
+ priorityCounterKey, ARGV[4] == '1', priority, jobId, isPausedOrMaxed)
56
58
  elseif rcall("LREM", target, -1, jobId) > 0 then
57
59
  reAddJobWithNewPriority( prioritizedKey, markerKey, target,
58
- KEYS[5], ARGV[4] == '1', priority, jobId, isPaused)
60
+ priorityCounterKey, ARGV[4] == '1', priority, jobId, isPausedOrMaxed)
59
61
  end
60
62
 
61
63
  rcall("HSET", jobKey, "priority", priority)
@@ -16,13 +16,16 @@ local pausedKey = KEYS[2]
16
16
  local prioritizedKey = KEYS[4]
17
17
 
18
18
  -- Includes
19
- --- @include "includes/getTargetQueueList"
19
+ --- @include "includes/isQueuePaused"
20
20
 
21
21
  for i = 1, #ARGV do
22
22
  local priority = tonumber(ARGV[i])
23
23
  if priority == 0 then
24
- local target = getTargetQueueList(KEYS[3], waitKey, pausedKey)
25
- results[#results+1] = rcall("LLEN", target)
24
+ if isQueuePaused(KEYS[3]) then
25
+ results[#results+1] = rcall("LLEN", pausedKey)
26
+ else
27
+ results[#results+1] = rcall("LLEN", waitKey)
28
+ end
26
29
  else
27
30
  results[#results+1] = rcall("ZCOUNT", prioritizedKey,
28
31
  priority * 0x100000000, (priority + 1) * 0x100000000 - 1)
@@ -2,8 +2,8 @@
2
2
  Add marker if needed when a job is available.
3
3
  ]]
4
4
 
5
- local function addBaseMarkerIfNeeded(markerKey, isPaused)
6
- if not isPaused then
5
+ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
6
+ if not isPausedOrMaxed then
7
7
  rcall("ZADD", markerKey, 0, "0")
8
8
  end
9
9
  end
@@ -5,7 +5,7 @@
5
5
  -- Includes
6
6
  --- @include "addBaseMarkerIfNeeded"
7
7
 
8
- local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
8
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
9
9
  rcall(pushCmd, targetKey, jobId)
10
- addBaseMarkerIfNeeded(markerKey, isPaused)
10
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
11
11
  end
@@ -5,9 +5,10 @@
5
5
  -- Includes
6
6
  --- @include "addBaseMarkerIfNeeded"
7
7
 
8
- local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
8
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
9
+ isPausedOrMaxed)
9
10
  local prioCounter = rcall("INCR", priorityCounterKey)
10
11
  local score = priority * 0x100000000 + prioCounter % 0x100000000
11
12
  rcall("ZADD", prioritizedKey, score, jobId)
12
- addBaseMarkerIfNeeded(markerKey, isPaused)
13
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
13
14
  end
@@ -3,10 +3,20 @@
3
3
  (since an empty list and !EXISTS are not really the same).
4
4
  ]]
5
5
 
6
- local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
7
- if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
8
- return waitKey, false
9
- else
6
+ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
7
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
8
+
9
+ if queueAttributes[1] then
10
10
  return pausedKey, true
11
+ else
12
+ if queueAttributes[2] then
13
+ local activeCount = rcall("LLEN", activeKey)
14
+ if activeCount >= tonumber(queueAttributes[2]) then
15
+ return waitKey, true
16
+ else
17
+ return waitKey, false
18
+ end
19
+ end
11
20
  end
21
+ return waitKey, false
12
22
  end
@@ -0,0 +1,15 @@
1
+ --[[
2
+ Function to check if queue is maxed or not.
3
+ ]]
4
+ local function isQueueMaxed(queueMetaKey, activeKey)
5
+ local maxConcurrency = rcall("HGET", queueMetaKey, "concurrency")
6
+
7
+ if maxConcurrency then
8
+ local activeCount = rcall("LLEN", activeKey)
9
+ if activeCount >= tonumber(maxConcurrency) then
10
+ return true
11
+ end
12
+ end
13
+
14
+ return false
15
+ end
@@ -3,5 +3,5 @@
3
3
  (since an empty list and !EXISTS are not really the same).
4
4
  ]]
5
5
  local function isQueuePaused(queueMetaKey)
6
- return rcall("HEXISTS", queueMetaKey, "paused") == 1
6
+ return rcall("HEXISTS", queueMetaKey, "paused") == 1
7
7
  end
@@ -0,0 +1,18 @@
1
+ --[[
2
+ Function to check if queue is paused or maxed
3
+ (since an empty list and !EXISTS are not really the same).
4
+ ]]
5
+
6
+ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
7
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
8
+
9
+ if queueAttributes[1] then
10
+ return true
11
+ else
12
+ if queueAttributes[2] then
13
+ local activeCount = rcall("LLEN", activeKey)
14
+ return activeCount >= tonumber(queueAttributes[2])
15
+ end
16
+ end
17
+ return false
18
+ end
@@ -6,7 +6,7 @@
6
6
  --- @include "addDelayMarkerIfNeeded"
7
7
  --- @include "addJobInTargetList"
8
8
  --- @include "addJobWithPriority"
9
- --- @include "isQueuePaused"
9
+ --- @include "isQueuePausedOrMaxed"
10
10
  --- @include "getTargetQueueList"
11
11
 
12
12
  local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
@@ -17,6 +17,7 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
17
17
  rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
18
18
  local parentWaitKey = parentQueueKey .. ":wait"
19
19
  local parentPausedKey = parentQueueKey .. ":paused"
20
+ local parentActiveKey = parentQueueKey .. ":active"
20
21
  local parentMetaKey = parentQueueKey .. ":meta"
21
22
 
22
23
  local parentMarkerKey = parentQueueKey .. ":marker"
@@ -35,15 +36,16 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
35
36
  addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
36
37
  else
37
38
  if priority == 0 then
38
- local parentTarget, isParentPaused =
39
- getTargetQueueList(parentMetaKey, parentWaitKey,
39
+ local parentTarget, isParentPausedOrMaxed =
40
+ getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
40
41
  parentPausedKey)
41
- addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPaused, parentId)
42
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
43
+ parentId)
42
44
  else
43
- local isPaused = isQueuePaused(parentMetaKey)
45
+ local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
44
46
  addJobWithPriority(parentMarkerKey,
45
47
  parentQueueKey .. ":prioritized", priority,
46
- parentId, parentQueueKey .. ":pc", isPaused)
48
+ parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
47
49
  end
48
50
 
49
51
  rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
@@ -11,9 +11,9 @@
11
11
  --- @include "removeJobKeys"
12
12
 
13
13
  local function moveParentToWait(parentPrefix, parentId, emitEvent)
14
- local parentTarget, isPaused = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait",
15
- parentPrefix .. "paused")
16
- addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPaused, parentId)
14
+ local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
15
+ parentPrefix .. "wait", parentPrefix .. "paused")
16
+ addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
17
17
 
18
18
  if emitEvent then
19
19
  local parentEventStream = parentPrefix .. "events"
@@ -0,0 +1,17 @@
1
+ --[[
2
+ Checks if queue is maxed.
3
+
4
+ Input:
5
+ KEYS[1] meta key
6
+ KEYS[2] active key
7
+
8
+ Output:
9
+ 1 if element found in the list.
10
+ ]]
11
+
12
+ local rcall = redis.call
13
+
14
+ -- Includes
15
+ --- @include "includes/isQueueMaxed"
16
+
17
+ return isQueueMaxed(KEYS[1], KEYS[2])
@@ -35,7 +35,7 @@ if lockToken == token then
35
35
  local metaKey = KEYS[6]
36
36
  local removed = rcall("LREM", KEYS[1], 1, jobId)
37
37
  if removed > 0 then
38
- local target, isPaused = getTargetQueueList(metaKey, KEYS[2], KEYS[5])
38
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[1], KEYS[2], KEYS[5])
39
39
 
40
40
  rcall("SREM", KEYS[3], jobId)
41
41
 
@@ -44,7 +44,7 @@ if lockToken == token then
44
44
  if priority > 0 then
45
45
  pushBackJobWithPriority(KEYS[8], priority, jobId)
46
46
  else
47
- addJobInTargetList(target, KEYS[9], "RPUSH", isPaused, jobId)
47
+ addJobInTargetList(target, KEYS[9], "RPUSH", isPausedOrMaxed, jobId)
48
48
  end
49
49
 
50
50
  rcall("DEL", lockKey)
@@ -10,7 +10,8 @@
10
10
  KEYS[4] 'wait'
11
11
  KEYS[5] 'paused'
12
12
  KEYS[6] 'meta'
13
- KEYS[7] 'marker'
13
+ KEYS[7] 'active'
14
+ KEYS[8] 'marker'
14
15
 
15
16
  ARGV[1] count
16
17
  ARGV[2] timestamp
@@ -32,7 +33,7 @@ local rcall = redis.call;
32
33
  --- @include "includes/getTargetQueueList"
33
34
 
34
35
  local metaKey = KEYS[6]
35
- local target, paused = getTargetQueueList(metaKey, KEYS[4], KEYS[5])
36
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[7], KEYS[4], KEYS[5])
36
37
 
37
38
  local jobs = rcall('ZRANGEBYSCORE', KEYS[3], 0, timestamp, 'LIMIT', 0, maxCount)
38
39
  if (#jobs > 0) then
@@ -62,7 +63,7 @@ if (#jobs > 0) then
62
63
  rcall("LPUSH", target, unpack(jobs, from, to))
63
64
  end
64
65
 
65
- addBaseMarkerIfNeeded(KEYS[7], paused)
66
+ addBaseMarkerIfNeeded(KEYS[8], isPausedOrMaxed)
66
67
  end
67
68
 
68
69
  maxCount = maxCount - #jobs
@@ -142,11 +142,11 @@ if (#stalling > 0) then
142
142
 
143
143
  table.insert(failed, jobId)
144
144
  else
145
- local target, isPaused=
146
- getTargetQueueList(metaKey, waitKey, pausedKey)
145
+ local target, isPausedOrMaxed=
146
+ getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
147
147
 
148
148
  -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
149
- addJobInTargetList(target, markerKey, "RPUSH", isPaused, jobId)
149
+ addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
150
150
 
151
151
  rcall("XADD", eventStreamKey, "*", "event",
152
152
  "waiting", "jobId", jobId, 'prev', 'active')
@@ -50,12 +50,12 @@ local opts = cmsgpack.unpack(ARGV[3])
50
50
  --- @include "includes/prepareJobForProcessing"
51
51
  --- @include "includes/promoteDelayedJobs"
52
52
 
53
- local target, paused = getTargetQueueList(KEYS[9], waitKey, KEYS[8])
53
+ local target, isPausedOrMaxed = getTargetQueueList(KEYS[9], activeKey, waitKey, KEYS[8])
54
54
 
55
55
  -- Check if there are delayed jobs that we can move to wait.
56
56
  local markerKey = KEYS[11]
57
57
  promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
58
- ARGV[2], KEYS[10], paused)
58
+ ARGV[2], KEYS[10], isPausedOrMaxed)
59
59
 
60
60
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
61
61
  local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
@@ -63,8 +63,8 @@ local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
63
63
  -- Check if we are rate limited first.
64
64
  if expireTime > 0 then return {0, 0, expireTime, 0} end
65
65
 
66
- -- paused queue
67
- if paused then return {0, 0, 0, 0} end
66
+ -- paused or maxed queue
67
+ if isPausedOrMaxed then return {0, 0, 0, 0} end
68
68
 
69
69
  -- no job ID, try non-blocking move from wait to active
70
70
  local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
@@ -202,11 +202,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
202
202
  -- and not rate limited.
203
203
  if (ARGV[6] == "1") then
204
204
 
205
- local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[8])
205
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
206
206
 
207
207
  -- Check if there are delayed jobs that can be promoted
208
208
  promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], eventStreamKey, ARGV[7],
209
- timestamp, KEYS[10], paused)
209
+ timestamp, KEYS[10], isPausedOrMaxed)
210
210
 
211
211
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
212
212
  -- Check if we are rate limited first.
@@ -214,8 +214,8 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
214
214
 
215
215
  if expireTime > 0 then return {0, 0, expireTime, 0} end
216
216
 
217
- -- paused queue
218
- if paused then return {0, 0, 0, 0} end
217
+ -- paused or maxed queue
218
+ if isPausedOrMaxed then return {0, 0, 0, 0} end
219
219
 
220
220
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
221
221
 
@@ -21,8 +21,9 @@
21
21
  -3 - Job not in active set
22
22
  ]]
23
23
  local rcall = redis.call
24
+ local stalledKey = KEYS[5]
24
25
 
25
- -- Includes
26
+ --- Includes
26
27
  --- @include "includes/removeLock"
27
28
 
28
29
  local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId,
@@ -43,7 +44,7 @@ end
43
44
  if rcall("EXISTS", KEYS[4]) == 1 then
44
45
  if ARGV[2] ~= "" then
45
46
  if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
46
- local errorCode = removeLock(KEYS[4], KEYS[5], ARGV[1], ARGV[4])
47
+ local errorCode = removeLock(KEYS[4], stalledKey, ARGV[1], ARGV[4])
47
48
  if errorCode < 0 then
48
49
  return errorCode
49
50
  end
@@ -53,7 +54,7 @@ if rcall("EXISTS", KEYS[4]) == 1 then
53
54
  return 1
54
55
  else
55
56
  if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
56
- local errorCode = removeLock(KEYS[4], KEYS[5], ARGV[1], ARGV[4])
57
+ local errorCode = removeLock(KEYS[4], stalledKey, ARGV[1], ARGV[4])
57
58
  if errorCode < 0 then
58
59
  return errorCode
59
60
  end
@@ -1,5 +1,12 @@
1
1
  --[[
2
2
  Completely obliterates a queue and all of its contents
3
+ This command completely destroys a queue including all of its jobs, current or past
4
+ leaving no trace of its existence. Since this script needs to iterate to find all the job
5
+ keys, consider that this call may be slow for very large queues.
6
+
7
+ The queue needs to be "paused" or it will return an error
8
+ If the queue has currently active jobs then the script by default will return error,
9
+ however this behaviour can be overrided using the 'force' option.
3
10
 
4
11
  Input:
5
12
  KEYS[1] meta
@@ -9,13 +16,6 @@
9
16
  ARGV[2] force
10
17
  ]]
11
18
 
12
- -- This command completely destroys a queue including all of its jobs, current or past
13
- -- leaving no trace of its existence. Since this script needs to iterate to find all the job
14
- -- keys, consider that this call may be slow for very large queues.
15
-
16
- -- The queue needs to be "paused" or it will return an error
17
- -- If the queue has currently active jobs then the script by default will return error,
18
- -- however this behaviour can be overrided using the 'force' option.
19
19
  local maxCount = tonumber(ARGV[1])
20
20
  local baseKey = KEYS[2]
21
21
 
@@ -59,6 +59,22 @@ if(maxCount <= 0) then
59
59
  return 1
60
60
  end
61
61
 
62
+ local repeatKey = baseKey .. 'repeat'
63
+ local repeatJobsIds = getZSetItems(repeatKey, maxCount)
64
+ for i, key in ipairs(repeatJobsIds) do
65
+ local jobKey = repeatKey .. ":" .. key
66
+ rcall("DEL", jobKey)
67
+ end
68
+ if(#repeatJobsIds > 0) then
69
+ for from, to in batches(#repeatJobsIds, 7000) do
70
+ rcall("ZREM", repeatKey, unpack(repeatJobsIds, from, to))
71
+ end
72
+ end
73
+ maxCount = maxCount - #repeatJobsIds
74
+ if(maxCount <= 0) then
75
+ return 1
76
+ end
77
+
62
78
  local completedKey = baseKey .. 'completed'
63
79
  maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
64
80
  if(maxCount <= 0) then
@@ -92,7 +108,6 @@ if(maxCount > 0) then
92
108
  baseKey .. 'id',
93
109
  baseKey .. 'pc',
94
110
  baseKey .. 'meta',
95
- baseKey .. 'repeat',
96
111
  baseKey .. 'metrics:completed',
97
112
  baseKey .. 'metrics:completed:data',
98
113
  baseKey .. 'metrics:failed',
@@ -7,9 +7,10 @@
7
7
  KEYS[3] 'paused'
8
8
  KEYS[4] 'meta'
9
9
  KEYS[5] 'prioritized'
10
- KEYS[6] 'pc' priority counter
11
- KEYS[7] 'event stream'
12
- KEYS[8] 'marker'
10
+ KEYS[6] 'active'
11
+ KEYS[7] 'pc' priority counter
12
+ KEYS[8] 'event stream'
13
+ KEYS[9] 'marker'
13
14
 
14
15
  ARGV[1] queue.toKey('')
15
16
  ARGV[2] jobId
@@ -33,23 +34,24 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then
33
34
  local jobKey = ARGV[1] .. jobId
34
35
  local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
35
36
  local metaKey = KEYS[4]
37
+ local markerKey = KEYS[9]
36
38
 
37
39
  -- Remove delayed "marker" from the wait list if there is any.
38
40
  -- Since we are adding a job we do not need the marker anymore.
39
41
  -- Markers in waitlist DEPRECATED in v5: Remove in v6.
40
- local target, paused = getTargetQueueList(metaKey, KEYS[2], KEYS[3])
42
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[6], KEYS[2], KEYS[3])
41
43
  local marker = rcall("LINDEX", target, 0)
42
44
  if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
43
45
 
44
46
  if priority == 0 then
45
47
  -- LIFO or FIFO
46
- addJobInTargetList(target, KEYS[8], "LPUSH", paused, jobId)
48
+ addJobInTargetList(target, markerKey, "LPUSH", isPausedOrMaxed, jobId)
47
49
  else
48
- addJobWithPriority(KEYS[8], KEYS[5], priority, jobId, KEYS[6], paused)
50
+ addJobWithPriority(markerKey, KEYS[5], priority, jobId, KEYS[7], isPausedOrMaxed)
49
51
  end
50
52
 
51
53
  -- Emit waiting event (wait..ing@token)
52
- rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev",
54
+ rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId, "prev",
53
55
  "delayed");
54
56
 
55
57
  rcall("HSET", jobKey, "delay", 0)