bullmq 4.16.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/dist/cjs/classes/flow-producer.js +3 -10
  2. package/dist/cjs/classes/flow-producer.js.map +1 -1
  3. package/dist/cjs/classes/job.js +21 -11
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/queue-base.js +2 -9
  6. package/dist/cjs/classes/queue-base.js.map +1 -1
  7. package/dist/cjs/classes/queue-events.js +3 -1
  8. package/dist/cjs/classes/queue-events.js.map +1 -1
  9. package/dist/cjs/classes/queue-keys.js +1 -0
  10. package/dist/cjs/classes/queue-keys.js.map +1 -1
  11. package/dist/cjs/classes/redis-connection.js +9 -7
  12. package/dist/cjs/classes/redis-connection.js.map +1 -1
  13. package/dist/cjs/classes/scripts.js +34 -34
  14. package/dist/cjs/classes/scripts.js.map +1 -1
  15. package/dist/cjs/classes/worker.js +41 -37
  16. package/dist/cjs/classes/worker.js.map +1 -1
  17. package/dist/cjs/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
  18. package/dist/cjs/commands/addParentJob-4.lua +0 -2
  19. package/dist/{esm/commands/addPrioritizedJob-8.lua → cjs/commands/addPrioritizedJob-7.lua} +18 -24
  20. package/dist/cjs/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
  21. package/dist/cjs/commands/changePriority-6.lua +57 -0
  22. package/dist/cjs/commands/getCounts-1.lua +1 -0
  23. package/dist/cjs/commands/getRanges-1.lua +1 -0
  24. package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
  25. package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -7
  26. package/dist/cjs/commands/includes/checkStalledJobs.lua +1 -1
  27. package/dist/cjs/commands/includes/isQueuePaused.lua +7 -0
  28. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
  29. package/dist/cjs/commands/includes/prepareJobForProcessing.lua +1 -1
  30. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +7 -4
  31. package/dist/{esm/commands/moveToActive-10.lua → cjs/commands/moveToActive-11.lua} +18 -29
  32. package/dist/cjs/commands/moveToDelayed-7.lua +77 -0
  33. package/dist/cjs/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
  34. package/dist/cjs/commands/moveToWaitingChildren-4.lua +7 -4
  35. package/dist/cjs/commands/pause-7.lua +41 -0
  36. package/dist/cjs/commands/promote-8.lua +60 -0
  37. package/dist/cjs/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
  38. package/dist/cjs/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
  39. package/dist/cjs/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
  40. package/dist/cjs/scripts/addParentJob-4.js +51 -61
  41. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  42. package/dist/cjs/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
  43. package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -0
  44. package/dist/cjs/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
  45. package/dist/cjs/scripts/addStandardJob-7.js.map +1 -0
  46. package/dist/cjs/scripts/changePriority-6.js +85 -0
  47. package/dist/cjs/scripts/changePriority-6.js.map +1 -0
  48. package/dist/cjs/scripts/getCounts-1.js +1 -0
  49. package/dist/cjs/scripts/getCounts-1.js.map +1 -1
  50. package/dist/cjs/scripts/getRanges-1.js +1 -0
  51. package/dist/cjs/scripts/getRanges-1.js.map +1 -1
  52. package/dist/cjs/scripts/index.js +10 -10
  53. package/dist/cjs/scripts/index.js.map +1 -1
  54. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +1 -0
  55. package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
  56. package/dist/cjs/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
  57. package/dist/cjs/scripts/moveToActive-11.js.map +1 -0
  58. package/dist/cjs/scripts/moveToDelayed-7.js +103 -0
  59. package/dist/cjs/scripts/moveToDelayed-7.js.map +1 -0
  60. package/dist/cjs/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
  61. package/dist/cjs/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
  62. package/dist/cjs/scripts/moveToWaitingChildren-4.js +6 -3
  63. package/dist/cjs/scripts/moveToWaitingChildren-4.js.map +1 -1
  64. package/dist/cjs/scripts/pause-7.js +67 -0
  65. package/dist/cjs/scripts/pause-7.js.map +1 -0
  66. package/dist/cjs/scripts/promote-8.js +79 -0
  67. package/dist/cjs/scripts/promote-8.js.map +1 -0
  68. package/dist/cjs/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
  69. package/dist/cjs/scripts/retryJob-10.js.map +1 -0
  70. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  71. package/dist/esm/classes/flow-producer.js +3 -10
  72. package/dist/esm/classes/flow-producer.js.map +1 -1
  73. package/dist/esm/classes/job.d.ts +5 -0
  74. package/dist/esm/classes/job.js +21 -11
  75. package/dist/esm/classes/job.js.map +1 -1
  76. package/dist/esm/classes/queue-base.js +2 -9
  77. package/dist/esm/classes/queue-base.js.map +1 -1
  78. package/dist/esm/classes/queue-events.js +3 -1
  79. package/dist/esm/classes/queue-events.js.map +1 -1
  80. package/dist/esm/classes/queue-keys.js +1 -0
  81. package/dist/esm/classes/queue-keys.js.map +1 -1
  82. package/dist/esm/classes/redis-connection.js +9 -7
  83. package/dist/esm/classes/redis-connection.js.map +1 -1
  84. package/dist/esm/classes/scripts.d.ts +4 -4
  85. package/dist/esm/classes/scripts.js +34 -34
  86. package/dist/esm/classes/scripts.js.map +1 -1
  87. package/dist/esm/classes/worker.d.ts +4 -2
  88. package/dist/esm/classes/worker.js +41 -37
  89. package/dist/esm/classes/worker.js.map +1 -1
  90. package/dist/esm/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
  91. package/dist/esm/commands/addParentJob-4.lua +0 -2
  92. package/dist/{cjs/commands/addPrioritizedJob-8.lua → esm/commands/addPrioritizedJob-7.lua} +18 -24
  93. package/dist/esm/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
  94. package/dist/esm/commands/changePriority-6.lua +57 -0
  95. package/dist/esm/commands/getCounts-1.lua +1 -0
  96. package/dist/esm/commands/getRanges-1.lua +1 -0
  97. package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
  98. package/dist/esm/commands/includes/addJobWithPriority.lua +3 -7
  99. package/dist/esm/commands/includes/checkStalledJobs.lua +1 -1
  100. package/dist/esm/commands/includes/isQueuePaused.lua +7 -0
  101. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
  102. package/dist/esm/commands/includes/prepareJobForProcessing.lua +1 -1
  103. package/dist/esm/commands/includes/promoteDelayedJobs.lua +7 -4
  104. package/dist/{cjs/commands/moveToActive-10.lua → esm/commands/moveToActive-11.lua} +18 -29
  105. package/dist/esm/commands/moveToDelayed-7.lua +77 -0
  106. package/dist/esm/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
  107. package/dist/esm/commands/moveToWaitingChildren-4.lua +7 -4
  108. package/dist/esm/commands/pause-7.lua +41 -0
  109. package/dist/esm/commands/promote-8.lua +60 -0
  110. package/dist/esm/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
  111. package/dist/esm/interfaces/job-json.d.ts +4 -1
  112. package/dist/esm/interfaces/minimal-job.d.ts +3 -0
  113. package/dist/esm/interfaces/queue-options.d.ts +1 -1
  114. package/dist/esm/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
  115. package/dist/esm/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
  116. package/dist/esm/scripts/addParentJob-4.js +51 -61
  117. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  118. package/dist/esm/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
  119. package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -0
  120. package/dist/esm/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
  121. package/dist/esm/scripts/addStandardJob-7.js.map +1 -0
  122. package/dist/esm/scripts/changePriority-6.js +82 -0
  123. package/dist/esm/scripts/changePriority-6.js.map +1 -0
  124. package/dist/esm/scripts/getCounts-1.js +1 -0
  125. package/dist/esm/scripts/getCounts-1.js.map +1 -1
  126. package/dist/esm/scripts/getRanges-1.js +1 -0
  127. package/dist/esm/scripts/getRanges-1.js.map +1 -1
  128. package/dist/esm/scripts/index.d.ts +10 -10
  129. package/dist/esm/scripts/index.js +10 -10
  130. package/dist/esm/scripts/index.js.map +1 -1
  131. package/dist/esm/scripts/moveStalledJobsToWait-8.js +1 -0
  132. package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
  133. package/dist/esm/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
  134. package/dist/esm/scripts/moveToActive-11.js.map +1 -0
  135. package/dist/esm/scripts/moveToDelayed-7.js +100 -0
  136. package/dist/esm/scripts/moveToDelayed-7.js.map +1 -0
  137. package/dist/esm/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
  138. package/dist/esm/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
  139. package/dist/esm/scripts/moveToWaitingChildren-4.js +6 -3
  140. package/dist/esm/scripts/moveToWaitingChildren-4.js.map +1 -1
  141. package/dist/esm/scripts/pause-7.js +64 -0
  142. package/dist/esm/scripts/pause-7.js.map +1 -0
  143. package/dist/esm/scripts/promote-8.js +76 -0
  144. package/dist/esm/scripts/promote-8.js.map +1 -0
  145. package/dist/esm/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
  146. package/dist/esm/scripts/retryJob-10.js.map +1 -0
  147. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  148. package/package.json +1 -1
  149. package/dist/cjs/commands/changePriority-5.lua +0 -52
  150. package/dist/cjs/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
  151. package/dist/cjs/commands/moveToDelayed-8.lua +0 -72
  152. package/dist/cjs/commands/pause-5.lua +0 -36
  153. package/dist/cjs/commands/promote-7.lua +0 -57
  154. package/dist/cjs/scripts/addDelayedJob-7.js.map +0 -1
  155. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +0 -1
  156. package/dist/cjs/scripts/addStandardJob-6.js.map +0 -1
  157. package/dist/cjs/scripts/changePriority-5.js +0 -87
  158. package/dist/cjs/scripts/changePriority-5.js.map +0 -1
  159. package/dist/cjs/scripts/moveToDelayed-8.js +0 -168
  160. package/dist/cjs/scripts/moveToDelayed-8.js.map +0 -1
  161. package/dist/cjs/scripts/pause-5.js +0 -48
  162. package/dist/cjs/scripts/pause-5.js.map +0 -1
  163. package/dist/cjs/scripts/promote-7.js +0 -87
  164. package/dist/cjs/scripts/promote-7.js.map +0 -1
  165. package/dist/cjs/scripts/retryJob-9.js.map +0 -1
  166. package/dist/esm/commands/changePriority-5.lua +0 -52
  167. package/dist/esm/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
  168. package/dist/esm/commands/moveToDelayed-8.lua +0 -72
  169. package/dist/esm/commands/pause-5.lua +0 -36
  170. package/dist/esm/commands/promote-7.lua +0 -57
  171. package/dist/esm/scripts/addDelayedJob-7.js.map +0 -1
  172. package/dist/esm/scripts/addPrioritizedJob-8.js.map +0 -1
  173. package/dist/esm/scripts/addStandardJob-6.js.map +0 -1
  174. package/dist/esm/scripts/changePriority-5.js +0 -84
  175. package/dist/esm/scripts/changePriority-5.js.map +0 -1
  176. package/dist/esm/scripts/moveToDelayed-8.js +0 -165
  177. package/dist/esm/scripts/moveToDelayed-8.js.map +0 -1
  178. package/dist/esm/scripts/pause-5.js +0 -45
  179. package/dist/esm/scripts/pause-5.js.map +0 -1
  180. package/dist/esm/scripts/promote-7.js +0 -84
  181. package/dist/esm/scripts/promote-7.js.map +0 -1
  182. package/dist/esm/scripts/retryJob-9.js.map +0 -1
  183. /package/dist/esm/scripts/{addDelayedJob-7.d.ts → addDelayedJob-6.d.ts} +0 -0
  184. /package/dist/esm/scripts/{addPrioritizedJob-8.d.ts → addPrioritizedJob-7.d.ts} +0 -0
  185. /package/dist/esm/scripts/{addStandardJob-6.d.ts → addStandardJob-7.d.ts} +0 -0
  186. /package/dist/esm/scripts/{changePriority-5.d.ts → changePriority-6.d.ts} +0 -0
  187. /package/dist/esm/scripts/{moveToActive-10.d.ts → moveToActive-11.d.ts} +0 -0
  188. /package/dist/esm/scripts/{moveToDelayed-8.d.ts → moveToDelayed-7.d.ts} +0 -0
  189. /package/dist/esm/scripts/{moveToFinished-13.d.ts → moveToFinished-14.d.ts} +0 -0
  190. /package/dist/esm/scripts/{pause-5.d.ts → pause-7.d.ts} +0 -0
  191. /package/dist/esm/scripts/{promote-7.d.ts → promote-8.d.ts} +0 -0
  192. /package/dist/esm/scripts/{retryJob-9.d.ts → retryJob-10.d.ts} +0 -0
@@ -8,13 +8,12 @@
8
8
  - Emits a global event 'delayed' if the job is delayed.
9
9
 
10
10
  Input:
11
- KEYS[1] 'wait',
12
- KEYS[2] 'paused'
13
- KEYS[3] 'meta'
14
- KEYS[4] 'id'
15
- KEYS[5] 'delayed'
16
- KEYS[6] 'completed'
17
- KEYS[7] events stream key
11
+ KEYS[1] 'marker',
12
+ KEYS[2] 'meta'
13
+ KEYS[3] 'id'
14
+ KEYS[4] 'delayed'
15
+ KEYS[5] 'completed'
16
+ KEYS[6] events stream key
18
17
 
19
18
  ARGV[1] msgpacked arguments array
20
19
  [1] key prefix,
@@ -34,15 +33,12 @@
34
33
  jobId - OK
35
34
  -5 - Missing parent key
36
35
  ]]
37
- local waitKey = KEYS[1]
38
- local pausedKey = KEYS[2]
36
+ local metaKey = KEYS[2]
37
+ local idKey = KEYS[3]
38
+ local delayedKey = KEYS[4]
39
39
 
40
- local metaKey = KEYS[3]
41
- local idKey = KEYS[4]
42
- local delayedKey = KEYS[5]
43
-
44
- local completedKey = KEYS[6]
45
- local eventsKey = KEYS[7]
40
+ local completedKey = KEYS[5]
41
+ local eventsKey = KEYS[6]
46
42
 
47
43
  local jobId
48
44
  local jobIdKey
@@ -61,7 +57,7 @@ local parentData
61
57
  -- Includes
62
58
  --- @include "includes/storeJob"
63
59
  --- @include "includes/addDelayMarkerIfNeeded"
64
- --- @include "includes/getTargetQueueList"
60
+ --- @include "includes/isQueuePaused"
65
61
  --- @include "includes/getNextDelayedTimestamp"
66
62
  --- @include "includes/updateExistingJobsParent"
67
63
  --- @include "includes/getOrSetMaxEvents"
@@ -109,14 +105,14 @@ rcall("ZADD", delayedKey, score, jobId)
109
105
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
110
106
  "jobId", jobId, "delay", delayedTimestamp)
111
107
 
112
- -- If wait list is empty, and this delayed job is the next one to be processed,
113
- -- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
114
- local target = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
115
- addDelayMarkerIfNeeded(target, delayedKey)
108
+ -- mark that a delayed job is available
109
+ local isPaused = isQueuePaused(metaKey)
110
+ if not isPaused then
111
+ local markerKey = KEYS[1]
112
+ addDelayMarkerIfNeeded(markerKey, delayedKey)
113
+ end
116
114
 
117
115
  -- Check if this job is a child of another job, if so add it to the parents dependencies
118
- -- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
119
- -- fail in this case.
120
116
  if parentDependenciesKey ~= nil then
121
117
  rcall("SADD", parentDependenciesKey, jobIdKey)
122
118
  end
@@ -92,8 +92,6 @@ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
92
92
  "waiting-children", "jobId", jobId)
93
93
 
94
94
  -- Check if this job is a child of another job, if so add it to the parents dependencies
95
- -- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
96
- -- fail in this case.
97
95
  if parentDependenciesKey ~= nil then
98
96
  rcall("SADD", parentDependenciesKey, jobIdKey)
99
97
  end
@@ -5,14 +5,13 @@
5
5
  - Adds the job to the "added" list so that workers gets notified.
6
6
 
7
7
  Input:
8
- KEYS[1] 'wait',
9
- KEYS[2] 'paused'
10
- KEYS[3] 'meta'
11
- KEYS[4] 'id'
12
- KEYS[5] 'prioritized'
13
- KEYS[6] 'completed'
14
- KEYS[7] events stream key
15
- KEYS[8] 'pc' priority counter
8
+ KEYS[1] 'marker',
9
+ KEYS[2] 'meta'
10
+ KEYS[3] 'id'
11
+ KEYS[4] 'prioritized'
12
+ KEYS[5] 'completed'
13
+ KEYS[6] events stream key
14
+ KEYS[7] 'pc' priority counter
16
15
 
17
16
  ARGV[1] msgpacked arguments array
18
17
  [1] key prefix,
@@ -31,17 +30,14 @@
31
30
  Output:
32
31
  jobId - OK
33
32
  -5 - Missing parent key
34
- ]]
35
- local waitKey = KEYS[1]
36
- local pausedKey = KEYS[2]
33
+ ]]
34
+ local metaKey = KEYS[2]
35
+ local idKey = KEYS[3]
36
+ local priorityKey = KEYS[4]
37
37
 
38
- local metaKey = KEYS[3]
39
- local idKey = KEYS[4]
40
- local priorityKey = KEYS[5]
41
-
42
- local completedKey = KEYS[6]
43
- local eventsKey = KEYS[7]
44
- local priorityCounterKey = KEYS[8]
38
+ local completedKey = KEYS[5]
39
+ local eventsKey = KEYS[6]
40
+ local priorityCounterKey = KEYS[7]
45
41
 
46
42
  local jobId
47
43
  local jobIdKey
@@ -59,8 +55,8 @@ local parentData
59
55
 
60
56
  -- Includes
61
57
  --- @include "includes/storeJob"
58
+ --- @include "includes/isQueuePaused"
62
59
  --- @include "includes/addJobWithPriority"
63
- --- @include "includes/getTargetQueueList"
64
60
  --- @include "includes/updateExistingJobsParent"
65
61
  --- @include "includes/getOrSetMaxEvents"
66
62
 
@@ -99,17 +95,15 @@ local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
99
95
  opts, timestamp, parentKey, parentData,
100
96
  repeatJobKey)
101
97
 
102
- local target, paused = getTargetQueueList(metaKey, waitKey, pausedKey)
98
+ -- Add the job to the prioritized set
99
+ local isPause = isQueuePaused(metaKey)
100
+ addJobWithPriority( KEYS[1], priorityKey, priority, jobId, priorityCounterKey, isPause)
103
101
 
104
- addJobWithPriority(waitKey, priorityKey, priority, paused, jobId,
105
- priorityCounterKey)
106
102
  -- Emit waiting event
107
103
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
108
104
  "jobId", jobId)
109
105
 
110
106
  -- Check if this job is a child of another job, if so add it to the parents dependencies
111
- -- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
112
- -- fail in this case.
113
107
  if parentDependenciesKey ~= nil then
114
108
  rcall("SADD", parentDependenciesKey, jobIdKey)
115
109
  end
@@ -21,6 +21,7 @@
21
21
  KEYS[4] 'id'
22
22
  KEYS[5] 'completed'
23
23
  KEYS[6] events stream key
24
+ KEYS[7] marker key
24
25
 
25
26
  ARGV[1] msgpacked arguments array
26
27
  [1] key prefix,
@@ -99,6 +100,11 @@ storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
99
100
 
100
101
  local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
101
102
 
103
+ if not paused then
104
+ -- mark that a job is available
105
+ rcall("ZADD", KEYS[7], 0, "0")
106
+ end
107
+
102
108
  -- LIFO or FIFO
103
109
  local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
104
110
  rcall(pushCmd, target, jobId)
@@ -108,8 +114,6 @@ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
108
114
  "jobId", jobId)
109
115
 
110
116
  -- Check if this job is a child of another job, if so add it to the parents dependencies
111
- -- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
112
- -- fail in this case.
113
117
  if parentDependenciesKey ~= nil then
114
118
  rcall("SADD", parentDependenciesKey, jobIdKey)
115
119
  end
@@ -0,0 +1,57 @@
1
+ --[[
2
+ Change job priority
3
+ Input:
4
+ KEYS[1] 'wait',
5
+ KEYS[2] 'paused'
6
+ KEYS[3] 'meta'
7
+ KEYS[4] 'prioritized'
8
+ KEYS[5] 'pc' priority counter
9
+ KEYS[6] 'marker'
10
+
11
+ ARGV[1] priority value
12
+ ARGV[2] job key
13
+ ARGV[3] job id
14
+ ARGV[4] lifo
15
+
16
+ Output:
17
+ 0 - OK
18
+ -1 - Missing job
19
+ ]]
20
+ local jobKey = ARGV[2]
21
+ local jobId = ARGV[3]
22
+ local priority = tonumber(ARGV[1])
23
+ local rcall = redis.call
24
+
25
+ -- Includes
26
+ --- @include "includes/isQueuePaused"
27
+ --- @include "includes/addJobWithPriority"
28
+ --- @include "includes/getTargetQueueList"
29
+
30
+ if rcall("EXISTS", jobKey) == 1 then
31
+ local metaKey = KEYS[3]
32
+ local isPaused = isQueuePaused(metaKey)
33
+ local markerKey = KEYS[6]
34
+ local prioritizedKey = KEYS[4]
35
+
36
+ -- Re-add with the new priority
37
+ if rcall("ZREM", KEYS[4], jobId) > 0 then
38
+ addJobWithPriority(markerKey, prioritizedKey, priority, jobId, KEYS[5],
39
+ isPaused)
40
+ -- If the new priority is 0, then just leave the job where it is in the wait list.
41
+ elseif priority > 0 then
42
+ -- Job is already in the wait list, we need to re-add it with the new priority.
43
+ local target = isPaused and KEYS[2] or KEYS[1]
44
+
45
+ local numRemovedElements = rcall("LREM", target, -1, jobId)
46
+ if numRemovedElements > 0 then
47
+ addJobWithPriority(markerKey, prioritizedKey, priority, jobId,
48
+ KEYS[5], isPaused)
49
+ end
50
+ end
51
+
52
+ rcall("HSET", jobKey, "priority", priority)
53
+
54
+ return 0
55
+ else
56
+ return -1
57
+ end
@@ -13,6 +13,7 @@ local results = {}
13
13
  for i = 1, #ARGV do
14
14
  local stateKey = prefix .. ARGV[i]
15
15
  if ARGV[i] == "wait" or ARGV[i] == "paused" then
16
+ -- Markers in waitlist DEPRECATED in v5: Remove in v6.
16
17
  local marker = rcall("LINDEX", stateKey, -1)
17
18
  if marker and string.sub(marker, 1, 2) == "0:" then
18
19
  local count = rcall("LLEN", stateKey)
@@ -43,6 +43,7 @@ end
43
43
  for i = 4, #ARGV do
44
44
  local stateKey = prefix .. ARGV[i]
45
45
  if ARGV[i] == "wait" or ARGV[i] == "paused" then
46
+ -- Markers in waitlist DEPRECATED in v5: Remove in v6.
46
47
  local marker = rcall("LINDEX", stateKey, -1)
47
48
  if marker and string.sub(marker, 1, 2) == "0:" then
48
49
  local count = rcall("LLEN", stateKey)
@@ -1,27 +1,13 @@
1
1
  --[[
2
2
  Add delay marker if needed.
3
- ]]
4
-
3
+ ]]
5
4
  -- Includes
6
5
  --- @include "getNextDelayedTimestamp"
7
-
8
- local function addDelayMarkerIfNeeded(targetKey, delayedKey)
9
- local waitLen = rcall("LLEN", targetKey)
10
- if waitLen <= 1 then
6
+ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
11
7
  local nextTimestamp = getNextDelayedTimestamp(delayedKey)
12
8
  if nextTimestamp ~= nil then
13
- -- Check if there is already a marker with older timestamp
14
- -- if there is, we need to replace it.
15
- if waitLen == 1 then
16
- local marker = rcall("LINDEX", targetKey, 0)
17
- local oldTimestamp = tonumber(marker:sub(3))
18
- if oldTimestamp and oldTimestamp > nextTimestamp then
19
- rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
20
- end
21
- else
22
- -- if there is no marker, then we need to add one
23
- rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
24
- end
9
+ -- Replace the score of the marker with the newest known
10
+ -- next timestamp.
11
+ rcall("ZADD", markerKey, nextTimestamp, "0")
25
12
  end
26
- end
27
13
  end
@@ -1,15 +1,11 @@
1
1
  --[[
2
2
  Function to add job considering priority.
3
3
  ]]
4
-
5
- -- Includes
6
- --- @include "addPriorityMarkerIfNeeded"
7
-
8
- local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
4
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
9
5
  local prioCounter = rcall("INCR", priorityCounterKey)
10
6
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
11
7
  rcall("ZADD", prioritizedKey, score, jobId)
12
- if not paused then
13
- addPriorityMarkerIfNeeded(waitKey)
8
+ if not isPaused then
9
+ rcall("ZADD", markerKey, 0, "0")
14
10
  end
15
11
  end
@@ -53,7 +53,7 @@ local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey,
53
53
 
54
54
  -- Remove from active list
55
55
  for i, jobId in ipairs(stalling) do
56
-
56
+ -- Markers in waitlist DEPRECATED in v5: Remove in v6.
57
57
  if string.sub(jobId, 1, 2) == "0:" then
58
58
  -- If the jobId is a delay marker ID we just remove it.
59
59
  rcall("LREM", activeKey, 1, jobId)
@@ -0,0 +1,7 @@
1
+ --[[
2
+ Function to check for the meta.paused key to decide if we are paused or not
3
+ (since an empty list and !EXISTS are not really the same).
4
+ ]]
5
+ local function isQueuePaused(queueMetaKey)
6
+ return rcall("HEXISTS", queueMetaKey, "paused") == 1
7
+ end
@@ -1,42 +1,51 @@
1
1
  --[[
2
2
  Validate and move parent to active if needed.
3
3
  ]]
4
-
5
4
  -- Includes
6
5
  --- @include "addDelayMarkerIfNeeded"
6
+ --- @include "isQueuePaused"
7
7
  --- @include "addJobWithPriority"
8
8
  --- @include "getTargetQueueList"
9
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
10
+ parentKey, parentId, timestamp)
11
+ local isParentActive = rcall("ZSCORE",
12
+ parentQueueKey .. ":waiting-children", parentId)
13
+ if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
14
+ rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
15
+ local parentWaitKey = parentQueueKey .. ":wait"
16
+ local parentPausedKey = parentQueueKey .. ":paused"
17
+ local parentMetaKey = parentQueueKey .. ":meta"
18
+
19
+ local parentMarkerKey = parentQueueKey .. ":marker"
20
+ local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
21
+ local priority = tonumber(jobAttributes[1]) or 0
22
+ local delay = tonumber(jobAttributes[2]) or 0
9
23
 
10
- local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
11
- local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
12
- if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
13
- rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
14
- local parentWaitKey = parentQueueKey .. ":wait"
15
- local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
16
- parentQueueKey .. ":paused")
17
- local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
18
- local priority = tonumber(jobAttributes[1]) or 0
19
- local delay = tonumber(jobAttributes[2]) or 0
24
+ if delay > 0 then
25
+ local delayedTimestamp = tonumber(timestamp) + delay
26
+ local score = delayedTimestamp * 0x1000
27
+ local parentDelayedKey = parentQueueKey .. ":delayed"
28
+ rcall("ZADD", parentDelayedKey, score, parentId)
29
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
30
+ "jobId", parentId, "delay", delayedTimestamp)
20
31
 
21
- if delay > 0 then
22
- local delayedTimestamp = tonumber(timestamp) + delay
23
- local score = delayedTimestamp * 0x1000
24
- local parentDelayedKey = parentQueueKey .. ":delayed"
25
- rcall("ZADD", parentDelayedKey, score, parentId)
26
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
27
- "delay", delayedTimestamp)
32
+ addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
33
+ else
34
+ if priority == 0 then
35
+ local parentTarget, _paused =
36
+ getTargetQueueList(parentMetaKey, parentWaitKey,
37
+ parentPausedKey)
38
+ rcall("RPUSH", parentTarget, parentId)
39
+ rcall("ZADD", parentMarkerKey, 0, "0")
40
+ else
41
+ local isPaused = isQueuePaused(parentMetaKey)
42
+ addJobWithPriority(parentMarkerKey,
43
+ parentQueueKey .. ":prioritized", priority,
44
+ parentId, parentQueueKey .. ":pc", isPaused)
45
+ end
28
46
 
29
- addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
30
- else
31
- if priority == 0 then
32
- rcall("RPUSH", parentTarget, parentId)
33
- else
34
- addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
35
- parentId, parentQueueKey .. ":pc")
36
- end
37
-
38
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
39
- "prev", "waiting-children")
47
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
48
+ "jobId", parentId, "prev", "waiting-children")
49
+ end
40
50
  end
41
- end
42
51
  end
@@ -67,7 +67,7 @@ local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, proces
67
67
 
68
68
  rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
69
69
  rcall("HSET", jobKey, "processedOn", processedOn)
70
- rcall("HINCRBY", jobKey, "attemptsMade", 1)
70
+ rcall("HINCRBY", jobKey, "ats", 1)
71
71
 
72
72
  return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
73
73
  end
@@ -10,8 +10,8 @@
10
10
  --- @include "addJobWithPriority"
11
11
 
12
12
  -- Try to get as much as 1000 jobs at once
13
- local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey,
14
- eventStreamKey, prefix, timestamp, paused, priorityCounterKey)
13
+ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
14
+ eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
15
15
  local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
16
16
 
17
17
  if (#jobs > 0) then
@@ -25,9 +25,12 @@ local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey
25
25
  if priority == 0 then
26
26
  -- LIFO or FIFO
27
27
  rcall("LPUSH", targetKey, jobId)
28
+ if not isPaused then
29
+ rcall("ZADD", markerKey, 0, "0")
30
+ end
28
31
  else
29
- addJobWithPriority(waitKey, prioritizedKey, priority, paused,
30
- jobId, priorityCounterKey)
32
+ addJobWithPriority(markerKey, prioritizedKey, priority,
33
+ jobId, priorityCounterKey, isPaused)
31
34
  end
32
35
 
33
36
  -- Emit waiting event
@@ -17,16 +17,18 @@
17
17
  KEYS[6] rate limiter key
18
18
  KEYS[7] delayed key
19
19
 
20
- -- Promote delayed jobs
20
+ -- Delayed jobs
21
21
  KEYS[8] paused key
22
22
  KEYS[9] meta key
23
23
  KEYS[10] pc priority counter
24
24
 
25
+ -- Marker
26
+ KEYS[11] marker key
27
+
25
28
  -- Arguments
26
29
  ARGV[1] key prefix
27
30
  ARGV[2] timestamp
28
- ARGV[3] optional job ID
29
- ARGV[4] opts
31
+ ARGV[3] opts
30
32
 
31
33
  opts - token - lock token
32
34
  opts - lockDuration
@@ -37,7 +39,7 @@ local waitKey = KEYS[1]
37
39
  local activeKey = KEYS[2]
38
40
  local rateLimiterKey = KEYS[6]
39
41
  local delayedKey = KEYS[7]
40
- local opts = cmsgpack.unpack(ARGV[4])
42
+ local opts = cmsgpack.unpack(ARGV[3])
41
43
 
42
44
  -- Includes
43
45
  --- @include "includes/getNextDelayedTimestamp"
@@ -50,39 +52,26 @@ local opts = cmsgpack.unpack(ARGV[4])
50
52
  local target, paused = getTargetQueueList(KEYS[9], waitKey, KEYS[8])
51
53
 
52
54
  -- Check if there are delayed jobs that we can move to wait.
53
- promoteDelayedJobs(delayedKey, waitKey, target, KEYS[3], KEYS[4], ARGV[1],
54
- ARGV[2], paused, KEYS[10])
55
+ local markerKey = KEYS[11]
56
+ promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], KEYS[4], ARGV[1],
57
+ ARGV[2], KEYS[10], paused)
55
58
 
56
59
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
57
60
  local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
58
61
 
59
- local jobId = nil
60
- if ARGV[3] ~= "" then
61
- jobId = ARGV[3]
62
-
63
- -- clean stalled key
64
- rcall("SREM", KEYS[5], jobId)
65
- end
66
-
67
- if not jobId or (jobId and string.sub(jobId, 1, 2) == "0:") then
68
- -- If jobId is special ID 0:delay, then there is no job to process
69
- if jobId then rcall("LREM", activeKey, 1, jobId) end
62
+ -- Check if we are rate limited first.
63
+ if expireTime > 0 then return {0, 0, expireTime, 0} end
70
64
 
71
- -- Check if we are rate limited first.
72
- if expireTime > 0 then return {0, 0, expireTime, 0} end
65
+ -- paused queue
66
+ if paused then return {0, 0, 0, 0} end
73
67
 
74
- -- paused queue
75
- if paused then return {0, 0, 0, 0} end
68
+ -- no job ID, try non-blocking move from wait to active
69
+ local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
76
70
 
77
- -- no job ID, try non-blocking move from wait to active
71
+ -- Markers in waitlist DEPRECATED in v5: Will be completely removed in v6.
72
+ if jobId and string.sub(jobId, 1, 2) == "0:" then
73
+ rcall("LREM", activeKey, 1, jobId)
78
74
  jobId = rcall("RPOPLPUSH", waitKey, activeKey)
79
-
80
- -- Since it is possible that between a call to BRPOPLPUSH and moveToActive
81
- -- another script puts a new maker in wait, we need to check again.
82
- if jobId and string.sub(jobId, 1, 2) == "0:" then
83
- rcall("LREM", activeKey, 1, jobId)
84
- jobId = rcall("RPOPLPUSH", waitKey, activeKey)
85
- end
86
75
  end
87
76
 
88
77
  if jobId then
@@ -0,0 +1,77 @@
1
+ --[[
2
+ Moves job from active to delayed set.
3
+
4
+ Input:
5
+ KEYS[1] marker key
6
+ KEYS[2] active key
7
+ KEYS[3] prioritized key
8
+ KEYS[4] delayed key
9
+ KEYS[5] job key
10
+ KEYS[6] events stream
11
+ KEYS[7] meta key
12
+
13
+ ARGV[1] key prefix
14
+ ARGV[2] timestamp
15
+ ARGV[3] delayedTimestamp
16
+ ARGV[4] the id of the job
17
+ ARGV[5] queue token
18
+ ARGV[6] delay value
19
+ ARGV[7] skip attempt
20
+
21
+ Output:
22
+ 0 - OK
23
+ -1 - Missing job.
24
+ -3 - Job not in active set.
25
+
26
+ Events:
27
+ - delayed key.
28
+ ]]
29
+ local rcall = redis.call
30
+
31
+ -- Includes
32
+ --- @include "includes/addDelayMarkerIfNeeded"
33
+ --- @include "includes/isQueuePaused"
34
+
35
+ local jobKey = KEYS[5]
36
+ local metaKey = KEYS[7]
37
+ if rcall("EXISTS", jobKey) == 1 then
38
+ local delayedKey = KEYS[4]
39
+ if ARGV[5] ~= "0" then
40
+ local lockKey = jobKey .. ':lock'
41
+ if rcall("GET", lockKey) == ARGV[5] then
42
+ rcall("DEL", lockKey)
43
+ else
44
+ return -2
45
+ end
46
+ end
47
+
48
+ local jobId = ARGV[4]
49
+ local score = tonumber(ARGV[3])
50
+ local delayedTimestamp = (score / 0x1000)
51
+
52
+ local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
53
+ if numRemovedElements < 1 then return -3 end
54
+
55
+ if ARGV[7] == "0" then
56
+ rcall("HINCRBY", jobKey, "atm", 1)
57
+ end
58
+
59
+ rcall("HSET", jobKey, "delay", ARGV[6])
60
+
61
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents") or 10000
62
+
63
+ rcall("ZADD", delayedKey, score, jobId)
64
+ rcall("XADD", KEYS[6], "MAXLEN", "~", maxEvents, "*", "event", "delayed",
65
+ "jobId", jobId, "delay", delayedTimestamp)
66
+
67
+ -- Check if we need to push a marker job to wake up sleeping workers.
68
+ local isPaused = isQueuePaused(metaKey)
69
+ if not isPaused then
70
+ local markerKey = KEYS[1]
71
+ addDelayMarkerIfNeeded(markerKey, delayedKey)
72
+ end
73
+
74
+ return 0
75
+ else
76
+ return -1
77
+ end