bullmq 4.16.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/dist/cjs/classes/flow-producer.js +3 -10
  2. package/dist/cjs/classes/flow-producer.js.map +1 -1
  3. package/dist/cjs/classes/job.js +21 -11
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/queue-base.js +2 -9
  6. package/dist/cjs/classes/queue-base.js.map +1 -1
  7. package/dist/cjs/classes/queue-events.js +3 -1
  8. package/dist/cjs/classes/queue-events.js.map +1 -1
  9. package/dist/cjs/classes/queue-keys.js +1 -0
  10. package/dist/cjs/classes/queue-keys.js.map +1 -1
  11. package/dist/cjs/classes/redis-connection.js +9 -7
  12. package/dist/cjs/classes/redis-connection.js.map +1 -1
  13. package/dist/cjs/classes/scripts.js +34 -34
  14. package/dist/cjs/classes/scripts.js.map +1 -1
  15. package/dist/cjs/classes/worker.js +41 -37
  16. package/dist/cjs/classes/worker.js.map +1 -1
  17. package/dist/cjs/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
  18. package/dist/cjs/commands/addParentJob-4.lua +0 -2
  19. package/dist/{esm/commands/addPrioritizedJob-8.lua → cjs/commands/addPrioritizedJob-7.lua} +18 -24
  20. package/dist/cjs/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
  21. package/dist/cjs/commands/changePriority-6.lua +57 -0
  22. package/dist/cjs/commands/getCounts-1.lua +1 -0
  23. package/dist/cjs/commands/getRanges-1.lua +1 -0
  24. package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
  25. package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -7
  26. package/dist/cjs/commands/includes/checkStalledJobs.lua +1 -1
  27. package/dist/cjs/commands/includes/isQueuePaused.lua +7 -0
  28. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
  29. package/dist/cjs/commands/includes/prepareJobForProcessing.lua +1 -1
  30. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +7 -4
  31. package/dist/{esm/commands/moveToActive-10.lua → cjs/commands/moveToActive-11.lua} +18 -29
  32. package/dist/cjs/commands/moveToDelayed-7.lua +77 -0
  33. package/dist/cjs/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
  34. package/dist/cjs/commands/moveToWaitingChildren-4.lua +7 -4
  35. package/dist/cjs/commands/pause-7.lua +41 -0
  36. package/dist/cjs/commands/promote-8.lua +60 -0
  37. package/dist/cjs/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
  38. package/dist/cjs/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
  39. package/dist/cjs/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
  40. package/dist/cjs/scripts/addParentJob-4.js +51 -61
  41. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  42. package/dist/cjs/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
  43. package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -0
  44. package/dist/cjs/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
  45. package/dist/cjs/scripts/addStandardJob-7.js.map +1 -0
  46. package/dist/cjs/scripts/changePriority-6.js +85 -0
  47. package/dist/cjs/scripts/changePriority-6.js.map +1 -0
  48. package/dist/cjs/scripts/getCounts-1.js +1 -0
  49. package/dist/cjs/scripts/getCounts-1.js.map +1 -1
  50. package/dist/cjs/scripts/getRanges-1.js +1 -0
  51. package/dist/cjs/scripts/getRanges-1.js.map +1 -1
  52. package/dist/cjs/scripts/index.js +10 -10
  53. package/dist/cjs/scripts/index.js.map +1 -1
  54. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +1 -0
  55. package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
  56. package/dist/cjs/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
  57. package/dist/cjs/scripts/moveToActive-11.js.map +1 -0
  58. package/dist/cjs/scripts/moveToDelayed-7.js +103 -0
  59. package/dist/cjs/scripts/moveToDelayed-7.js.map +1 -0
  60. package/dist/cjs/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
  61. package/dist/cjs/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
  62. package/dist/cjs/scripts/moveToWaitingChildren-4.js +6 -3
  63. package/dist/cjs/scripts/moveToWaitingChildren-4.js.map +1 -1
  64. package/dist/cjs/scripts/pause-7.js +67 -0
  65. package/dist/cjs/scripts/pause-7.js.map +1 -0
  66. package/dist/cjs/scripts/promote-8.js +79 -0
  67. package/dist/cjs/scripts/promote-8.js.map +1 -0
  68. package/dist/cjs/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
  69. package/dist/cjs/scripts/retryJob-10.js.map +1 -0
  70. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  71. package/dist/esm/classes/flow-producer.js +3 -10
  72. package/dist/esm/classes/flow-producer.js.map +1 -1
  73. package/dist/esm/classes/job.d.ts +5 -0
  74. package/dist/esm/classes/job.js +21 -11
  75. package/dist/esm/classes/job.js.map +1 -1
  76. package/dist/esm/classes/queue-base.js +2 -9
  77. package/dist/esm/classes/queue-base.js.map +1 -1
  78. package/dist/esm/classes/queue-events.js +3 -1
  79. package/dist/esm/classes/queue-events.js.map +1 -1
  80. package/dist/esm/classes/queue-keys.js +1 -0
  81. package/dist/esm/classes/queue-keys.js.map +1 -1
  82. package/dist/esm/classes/redis-connection.js +9 -7
  83. package/dist/esm/classes/redis-connection.js.map +1 -1
  84. package/dist/esm/classes/scripts.d.ts +4 -4
  85. package/dist/esm/classes/scripts.js +34 -34
  86. package/dist/esm/classes/scripts.js.map +1 -1
  87. package/dist/esm/classes/worker.d.ts +4 -2
  88. package/dist/esm/classes/worker.js +41 -37
  89. package/dist/esm/classes/worker.js.map +1 -1
  90. package/dist/esm/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
  91. package/dist/esm/commands/addParentJob-4.lua +0 -2
  92. package/dist/{cjs/commands/addPrioritizedJob-8.lua → esm/commands/addPrioritizedJob-7.lua} +18 -24
  93. package/dist/esm/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
  94. package/dist/esm/commands/changePriority-6.lua +57 -0
  95. package/dist/esm/commands/getCounts-1.lua +1 -0
  96. package/dist/esm/commands/getRanges-1.lua +1 -0
  97. package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
  98. package/dist/esm/commands/includes/addJobWithPriority.lua +3 -7
  99. package/dist/esm/commands/includes/checkStalledJobs.lua +1 -1
  100. package/dist/esm/commands/includes/isQueuePaused.lua +7 -0
  101. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
  102. package/dist/esm/commands/includes/prepareJobForProcessing.lua +1 -1
  103. package/dist/esm/commands/includes/promoteDelayedJobs.lua +7 -4
  104. package/dist/{cjs/commands/moveToActive-10.lua → esm/commands/moveToActive-11.lua} +18 -29
  105. package/dist/esm/commands/moveToDelayed-7.lua +77 -0
  106. package/dist/esm/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
  107. package/dist/esm/commands/moveToWaitingChildren-4.lua +7 -4
  108. package/dist/esm/commands/pause-7.lua +41 -0
  109. package/dist/esm/commands/promote-8.lua +60 -0
  110. package/dist/esm/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
  111. package/dist/esm/interfaces/job-json.d.ts +4 -1
  112. package/dist/esm/interfaces/minimal-job.d.ts +3 -0
  113. package/dist/esm/interfaces/queue-options.d.ts +1 -1
  114. package/dist/esm/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
  115. package/dist/esm/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
  116. package/dist/esm/scripts/addParentJob-4.js +51 -61
  117. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  118. package/dist/esm/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
  119. package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -0
  120. package/dist/esm/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
  121. package/dist/esm/scripts/addStandardJob-7.js.map +1 -0
  122. package/dist/esm/scripts/changePriority-6.js +82 -0
  123. package/dist/esm/scripts/changePriority-6.js.map +1 -0
  124. package/dist/esm/scripts/getCounts-1.js +1 -0
  125. package/dist/esm/scripts/getCounts-1.js.map +1 -1
  126. package/dist/esm/scripts/getRanges-1.js +1 -0
  127. package/dist/esm/scripts/getRanges-1.js.map +1 -1
  128. package/dist/esm/scripts/index.d.ts +10 -10
  129. package/dist/esm/scripts/index.js +10 -10
  130. package/dist/esm/scripts/index.js.map +1 -1
  131. package/dist/esm/scripts/moveStalledJobsToWait-8.js +1 -0
  132. package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
  133. package/dist/esm/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
  134. package/dist/esm/scripts/moveToActive-11.js.map +1 -0
  135. package/dist/esm/scripts/moveToDelayed-7.js +100 -0
  136. package/dist/esm/scripts/moveToDelayed-7.js.map +1 -0
  137. package/dist/esm/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
  138. package/dist/esm/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
  139. package/dist/esm/scripts/moveToWaitingChildren-4.js +6 -3
  140. package/dist/esm/scripts/moveToWaitingChildren-4.js.map +1 -1
  141. package/dist/esm/scripts/pause-7.js +64 -0
  142. package/dist/esm/scripts/pause-7.js.map +1 -0
  143. package/dist/esm/scripts/promote-8.js +76 -0
  144. package/dist/esm/scripts/promote-8.js.map +1 -0
  145. package/dist/esm/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
  146. package/dist/esm/scripts/retryJob-10.js.map +1 -0
  147. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  148. package/package.json +1 -1
  149. package/dist/cjs/commands/changePriority-5.lua +0 -52
  150. package/dist/cjs/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
  151. package/dist/cjs/commands/moveToDelayed-8.lua +0 -72
  152. package/dist/cjs/commands/pause-5.lua +0 -36
  153. package/dist/cjs/commands/promote-7.lua +0 -57
  154. package/dist/cjs/scripts/addDelayedJob-7.js.map +0 -1
  155. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +0 -1
  156. package/dist/cjs/scripts/addStandardJob-6.js.map +0 -1
  157. package/dist/cjs/scripts/changePriority-5.js +0 -87
  158. package/dist/cjs/scripts/changePriority-5.js.map +0 -1
  159. package/dist/cjs/scripts/moveToDelayed-8.js +0 -168
  160. package/dist/cjs/scripts/moveToDelayed-8.js.map +0 -1
  161. package/dist/cjs/scripts/pause-5.js +0 -48
  162. package/dist/cjs/scripts/pause-5.js.map +0 -1
  163. package/dist/cjs/scripts/promote-7.js +0 -87
  164. package/dist/cjs/scripts/promote-7.js.map +0 -1
  165. package/dist/cjs/scripts/retryJob-9.js.map +0 -1
  166. package/dist/esm/commands/changePriority-5.lua +0 -52
  167. package/dist/esm/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
  168. package/dist/esm/commands/moveToDelayed-8.lua +0 -72
  169. package/dist/esm/commands/pause-5.lua +0 -36
  170. package/dist/esm/commands/promote-7.lua +0 -57
  171. package/dist/esm/scripts/addDelayedJob-7.js.map +0 -1
  172. package/dist/esm/scripts/addPrioritizedJob-8.js.map +0 -1
  173. package/dist/esm/scripts/addStandardJob-6.js.map +0 -1
  174. package/dist/esm/scripts/changePriority-5.js +0 -84
  175. package/dist/esm/scripts/changePriority-5.js.map +0 -1
  176. package/dist/esm/scripts/moveToDelayed-8.js +0 -165
  177. package/dist/esm/scripts/moveToDelayed-8.js.map +0 -1
  178. package/dist/esm/scripts/pause-5.js +0 -45
  179. package/dist/esm/scripts/pause-5.js.map +0 -1
  180. package/dist/esm/scripts/promote-7.js +0 -84
  181. package/dist/esm/scripts/promote-7.js.map +0 -1
  182. package/dist/esm/scripts/retryJob-9.js.map +0 -1
  183. /package/dist/esm/scripts/{addDelayedJob-7.d.ts → addDelayedJob-6.d.ts} +0 -0
  184. /package/dist/esm/scripts/{addPrioritizedJob-8.d.ts → addPrioritizedJob-7.d.ts} +0 -0
  185. /package/dist/esm/scripts/{addStandardJob-6.d.ts → addStandardJob-7.d.ts} +0 -0
  186. /package/dist/esm/scripts/{changePriority-5.d.ts → changePriority-6.d.ts} +0 -0
  187. /package/dist/esm/scripts/{moveToActive-10.d.ts → moveToActive-11.d.ts} +0 -0
  188. /package/dist/esm/scripts/{moveToDelayed-8.d.ts → moveToDelayed-7.d.ts} +0 -0
  189. /package/dist/esm/scripts/{moveToFinished-13.d.ts → moveToFinished-14.d.ts} +0 -0
  190. /package/dist/esm/scripts/{pause-5.d.ts → pause-7.d.ts} +0 -0
  191. /package/dist/esm/scripts/{promote-7.d.ts → promote-8.d.ts} +0 -0
  192. /package/dist/esm/scripts/{retryJob-9.d.ts → retryJob-10.d.ts} +0 -0
@@ -1,27 +1,13 @@
1
1
  --[[
2
2
  Add delay marker if needed.
3
- ]]
4
-
3
+ ]]
5
4
  -- Includes
6
5
  --- @include "getNextDelayedTimestamp"
7
-
8
- local function addDelayMarkerIfNeeded(targetKey, delayedKey)
9
- local waitLen = rcall("LLEN", targetKey)
10
- if waitLen <= 1 then
6
+ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
11
7
  local nextTimestamp = getNextDelayedTimestamp(delayedKey)
12
8
  if nextTimestamp ~= nil then
13
- -- Check if there is already a marker with older timestamp
14
- -- if there is, we need to replace it.
15
- if waitLen == 1 then
16
- local marker = rcall("LINDEX", targetKey, 0)
17
- local oldTimestamp = tonumber(marker:sub(3))
18
- if oldTimestamp and oldTimestamp > nextTimestamp then
19
- rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
20
- end
21
- else
22
- -- if there is no marker, then we need to add one
23
- rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
24
- end
9
+ -- Replace the score of the marker with the newest known
10
+ -- next timestamp.
11
+ rcall("ZADD", markerKey, nextTimestamp, "0")
25
12
  end
26
- end
27
13
  end
@@ -1,15 +1,11 @@
1
1
  --[[
2
2
  Function to add job considering priority.
3
3
  ]]
4
-
5
- -- Includes
6
- --- @include "addPriorityMarkerIfNeeded"
7
-
8
- local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
4
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
9
5
  local prioCounter = rcall("INCR", priorityCounterKey)
10
6
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
11
7
  rcall("ZADD", prioritizedKey, score, jobId)
12
- if not paused then
13
- addPriorityMarkerIfNeeded(waitKey)
8
+ if not isPaused then
9
+ rcall("ZADD", markerKey, 0, "0")
14
10
  end
15
11
  end
@@ -53,7 +53,7 @@ local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey,
53
53
 
54
54
  -- Remove from active list
55
55
  for i, jobId in ipairs(stalling) do
56
-
56
+ -- Markers in waitlist DEPRECATED in v5: Remove in v6.
57
57
  if string.sub(jobId, 1, 2) == "0:" then
58
58
  -- If the jobId is a delay marker ID we just remove it.
59
59
  rcall("LREM", activeKey, 1, jobId)
@@ -0,0 +1,7 @@
1
+ --[[
2
+ Function to check for the meta.paused key to decide if we are paused or not
3
+ (since an empty list and !EXISTS are not really the same).
4
+ ]]
5
+ local function isQueuePaused(queueMetaKey)
6
+ return rcall("HEXISTS", queueMetaKey, "paused") == 1
7
+ end
@@ -1,42 +1,51 @@
1
1
  --[[
2
2
  Validate and move parent to active if needed.
3
3
  ]]
4
-
5
4
  -- Includes
6
5
  --- @include "addDelayMarkerIfNeeded"
6
+ --- @include "isQueuePaused"
7
7
  --- @include "addJobWithPriority"
8
8
  --- @include "getTargetQueueList"
9
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
10
+ parentKey, parentId, timestamp)
11
+ local isParentActive = rcall("ZSCORE",
12
+ parentQueueKey .. ":waiting-children", parentId)
13
+ if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
14
+ rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
15
+ local parentWaitKey = parentQueueKey .. ":wait"
16
+ local parentPausedKey = parentQueueKey .. ":paused"
17
+ local parentMetaKey = parentQueueKey .. ":meta"
18
+
19
+ local parentMarkerKey = parentQueueKey .. ":marker"
20
+ local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
21
+ local priority = tonumber(jobAttributes[1]) or 0
22
+ local delay = tonumber(jobAttributes[2]) or 0
9
23
 
10
- local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
11
- local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
12
- if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
13
- rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
14
- local parentWaitKey = parentQueueKey .. ":wait"
15
- local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
16
- parentQueueKey .. ":paused")
17
- local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
18
- local priority = tonumber(jobAttributes[1]) or 0
19
- local delay = tonumber(jobAttributes[2]) or 0
24
+ if delay > 0 then
25
+ local delayedTimestamp = tonumber(timestamp) + delay
26
+ local score = delayedTimestamp * 0x1000
27
+ local parentDelayedKey = parentQueueKey .. ":delayed"
28
+ rcall("ZADD", parentDelayedKey, score, parentId)
29
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
30
+ "jobId", parentId, "delay", delayedTimestamp)
20
31
 
21
- if delay > 0 then
22
- local delayedTimestamp = tonumber(timestamp) + delay
23
- local score = delayedTimestamp * 0x1000
24
- local parentDelayedKey = parentQueueKey .. ":delayed"
25
- rcall("ZADD", parentDelayedKey, score, parentId)
26
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
27
- "delay", delayedTimestamp)
32
+ addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
33
+ else
34
+ if priority == 0 then
35
+ local parentTarget, _paused =
36
+ getTargetQueueList(parentMetaKey, parentWaitKey,
37
+ parentPausedKey)
38
+ rcall("RPUSH", parentTarget, parentId)
39
+ rcall("ZADD", parentMarkerKey, 0, "0")
40
+ else
41
+ local isPaused = isQueuePaused(parentMetaKey)
42
+ addJobWithPriority(parentMarkerKey,
43
+ parentQueueKey .. ":prioritized", priority,
44
+ parentId, parentQueueKey .. ":pc", isPaused)
45
+ end
28
46
 
29
- addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
30
- else
31
- if priority == 0 then
32
- rcall("RPUSH", parentTarget, parentId)
33
- else
34
- addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
35
- parentId, parentQueueKey .. ":pc")
36
- end
37
-
38
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
39
- "prev", "waiting-children")
47
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
48
+ "jobId", parentId, "prev", "waiting-children")
49
+ end
40
50
  end
41
- end
42
51
  end
@@ -67,7 +67,7 @@ local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, proces
67
67
 
68
68
  rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
69
69
  rcall("HSET", jobKey, "processedOn", processedOn)
70
- rcall("HINCRBY", jobKey, "attemptsMade", 1)
70
+ rcall("HINCRBY", jobKey, "ats", 1)
71
71
 
72
72
  return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
73
73
  end
@@ -10,8 +10,8 @@
10
10
  --- @include "addJobWithPriority"
11
11
 
12
12
  -- Try to get as much as 1000 jobs at once
13
- local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey,
14
- eventStreamKey, prefix, timestamp, paused, priorityCounterKey)
13
+ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
14
+ eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
15
15
  local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
16
16
 
17
17
  if (#jobs > 0) then
@@ -25,9 +25,12 @@ local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey
25
25
  if priority == 0 then
26
26
  -- LIFO or FIFO
27
27
  rcall("LPUSH", targetKey, jobId)
28
+ if not isPaused then
29
+ rcall("ZADD", markerKey, 0, "0")
30
+ end
28
31
  else
29
- addJobWithPriority(waitKey, prioritizedKey, priority, paused,
30
- jobId, priorityCounterKey)
32
+ addJobWithPriority(markerKey, prioritizedKey, priority,
33
+ jobId, priorityCounterKey, isPaused)
31
34
  end
32
35
 
33
36
  -- Emit waiting event
@@ -17,16 +17,18 @@
17
17
  KEYS[6] rate limiter key
18
18
  KEYS[7] delayed key
19
19
 
20
- -- Promote delayed jobs
20
+ -- Delayed jobs
21
21
  KEYS[8] paused key
22
22
  KEYS[9] meta key
23
23
  KEYS[10] pc priority counter
24
24
 
25
+ -- Marker
26
+ KEYS[11] marker key
27
+
25
28
  -- Arguments
26
29
  ARGV[1] key prefix
27
30
  ARGV[2] timestamp
28
- ARGV[3] optional job ID
29
- ARGV[4] opts
31
+ ARGV[3] opts
30
32
 
31
33
  opts - token - lock token
32
34
  opts - lockDuration
@@ -37,7 +39,7 @@ local waitKey = KEYS[1]
37
39
  local activeKey = KEYS[2]
38
40
  local rateLimiterKey = KEYS[6]
39
41
  local delayedKey = KEYS[7]
40
- local opts = cmsgpack.unpack(ARGV[4])
42
+ local opts = cmsgpack.unpack(ARGV[3])
41
43
 
42
44
  -- Includes
43
45
  --- @include "includes/getNextDelayedTimestamp"
@@ -50,39 +52,26 @@ local opts = cmsgpack.unpack(ARGV[4])
50
52
  local target, paused = getTargetQueueList(KEYS[9], waitKey, KEYS[8])
51
53
 
52
54
  -- Check if there are delayed jobs that we can move to wait.
53
- promoteDelayedJobs(delayedKey, waitKey, target, KEYS[3], KEYS[4], ARGV[1],
54
- ARGV[2], paused, KEYS[10])
55
+ local markerKey = KEYS[11]
56
+ promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], KEYS[4], ARGV[1],
57
+ ARGV[2], KEYS[10], paused)
55
58
 
56
59
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
57
60
  local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
58
61
 
59
- local jobId = nil
60
- if ARGV[3] ~= "" then
61
- jobId = ARGV[3]
62
-
63
- -- clean stalled key
64
- rcall("SREM", KEYS[5], jobId)
65
- end
66
-
67
- if not jobId or (jobId and string.sub(jobId, 1, 2) == "0:") then
68
- -- If jobId is special ID 0:delay, then there is no job to process
69
- if jobId then rcall("LREM", activeKey, 1, jobId) end
62
+ -- Check if we are rate limited first.
63
+ if expireTime > 0 then return {0, 0, expireTime, 0} end
70
64
 
71
- -- Check if we are rate limited first.
72
- if expireTime > 0 then return {0, 0, expireTime, 0} end
65
+ -- paused queue
66
+ if paused then return {0, 0, 0, 0} end
73
67
 
74
- -- paused queue
75
- if paused then return {0, 0, 0, 0} end
68
+ -- no job ID, try non-blocking move from wait to active
69
+ local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
76
70
 
77
- -- no job ID, try non-blocking move from wait to active
71
+ -- Markers in waitlist DEPRECATED in v5: Will be completely removed in v6.
72
+ if jobId and string.sub(jobId, 1, 2) == "0:" then
73
+ rcall("LREM", activeKey, 1, jobId)
78
74
  jobId = rcall("RPOPLPUSH", waitKey, activeKey)
79
-
80
- -- Since it is possible that between a call to BRPOPLPUSH and moveToActive
81
- -- another script puts a new maker in wait, we need to check again.
82
- if jobId and string.sub(jobId, 1, 2) == "0:" then
83
- rcall("LREM", activeKey, 1, jobId)
84
- jobId = rcall("RPOPLPUSH", waitKey, activeKey)
85
- end
86
75
  end
87
76
 
88
77
  if jobId then
@@ -0,0 +1,77 @@
1
+ --[[
2
+ Moves job from active to delayed set.
3
+
4
+ Input:
5
+ KEYS[1] marker key
6
+ KEYS[2] active key
7
+ KEYS[3] prioritized key
8
+ KEYS[4] delayed key
9
+ KEYS[5] job key
10
+ KEYS[6] events stream
11
+ KEYS[7] meta key
12
+
13
+ ARGV[1] key prefix
14
+ ARGV[2] timestamp
15
+ ARGV[3] delayedTimestamp
16
+ ARGV[4] the id of the job
17
+ ARGV[5] queue token
18
+ ARGV[6] delay value
19
+ ARGV[7] skip attempt
20
+
21
+ Output:
22
+ 0 - OK
23
+ -1 - Missing job.
24
+ -3 - Job not in active set.
25
+
26
+ Events:
27
+ - delayed key.
28
+ ]]
29
+ local rcall = redis.call
30
+
31
+ -- Includes
32
+ --- @include "includes/addDelayMarkerIfNeeded"
33
+ --- @include "includes/isQueuePaused"
34
+
35
+ local jobKey = KEYS[5]
36
+ local metaKey = KEYS[7]
37
+ if rcall("EXISTS", jobKey) == 1 then
38
+ local delayedKey = KEYS[4]
39
+ if ARGV[5] ~= "0" then
40
+ local lockKey = jobKey .. ':lock'
41
+ if rcall("GET", lockKey) == ARGV[5] then
42
+ rcall("DEL", lockKey)
43
+ else
44
+ return -2
45
+ end
46
+ end
47
+
48
+ local jobId = ARGV[4]
49
+ local score = tonumber(ARGV[3])
50
+ local delayedTimestamp = (score / 0x1000)
51
+
52
+ local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
53
+ if numRemovedElements < 1 then return -3 end
54
+
55
+ if ARGV[7] == "0" then
56
+ rcall("HINCRBY", jobKey, "atm", 1)
57
+ end
58
+
59
+ rcall("HSET", jobKey, "delay", ARGV[6])
60
+
61
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents") or 10000
62
+
63
+ rcall("ZADD", delayedKey, score, jobId)
64
+ rcall("XADD", KEYS[6], "MAXLEN", "~", maxEvents, "*", "event", "delayed",
65
+ "jobId", jobId, "delay", delayedTimestamp)
66
+
67
+ -- Check if we need to push a marker job to wake up sleeping workers.
68
+ local isPaused = isQueuePaused(metaKey)
69
+ if not isPaused then
70
+ local markerKey = KEYS[1]
71
+ addDelayMarkerIfNeeded(markerKey, delayedKey)
72
+ end
73
+
74
+ return 0
75
+ else
76
+ return -1
77
+ end
@@ -22,22 +22,21 @@
22
22
  KEYS[11] completed/failed key
23
23
  KEYS[12] jobId key
24
24
  KEYS[13] metrics key
25
+ KEYS[14] marker key
25
26
 
26
27
  ARGV[1] jobId
27
28
  ARGV[2] timestamp
28
29
  ARGV[3] msg property returnvalue / failedReason
29
30
  ARGV[4] return value / failed reason
30
31
  ARGV[5] target (completed/failed)
31
- ARGV[6] event data (? maybe just send jobid).
32
- ARGV[7] fetch next?
33
- ARGV[8] keys prefix
34
- ARGV[9] opts
32
+ ARGV[6] fetch next?
33
+ ARGV[7] keys prefix
34
+ ARGV[8] opts
35
35
 
36
36
  opts - token - lock token
37
37
  opts - keepJobs
38
38
  opts - lockDuration - lock duration in milliseconds
39
39
  opts - attempts max attempts
40
- opts - attemptsMade
41
40
  opts - maxMetricsSize
42
41
  opts - fpof - fail parent on fail
43
42
  opts - rdof - remove dependency on fail
@@ -72,11 +71,10 @@ local rcall = redis.call
72
71
 
73
72
  local jobIdKey = KEYS[12]
74
73
  if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
75
- local opts = cmsgpack.unpack(ARGV[9])
74
+ local opts = cmsgpack.unpack(ARGV[8])
76
75
 
77
76
  local token = opts['token']
78
77
  local attempts = opts['attempts']
79
- local attemptsMade = opts['attemptsMade']
80
78
  local maxMetricsSize = opts['maxMetricsSize']
81
79
  local maxCount = opts['keepJobs']['count']
82
80
  local maxAge = opts['keepJobs']['age']
@@ -120,8 +118,9 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
120
118
 
121
119
  if (numRemovedElements < 1) then return -3 end
122
120
 
121
+ local metaKey = KEYS[9]
123
122
  -- Trim events before emiting them to avoid trimming events emitted in this script
124
- trimEvents(KEYS[9], KEYS[4])
123
+ trimEvents(metaKey, KEYS[4])
125
124
 
126
125
  -- If job has a parent we need to
127
126
  -- 1) remove this job id from parents dependencies
@@ -144,17 +143,20 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
144
143
  else
145
144
  if opts['fpof'] then
146
145
  moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
147
- parentId, jobIdKey, timestamp)
146
+ parentId, jobIdKey,
147
+ timestamp)
148
148
  elseif opts['rdof'] then
149
149
  local dependenciesSet = parentKey .. ":dependencies"
150
150
  if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
151
151
  moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
152
- parentKey, parentId, timestamp)
152
+ parentKey, parentId, timestamp)
153
153
  end
154
154
  end
155
155
  end
156
156
  end
157
157
 
158
+ local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
159
+
158
160
  -- Remove job?
159
161
  if maxCount ~= 0 then
160
162
  local targetSet = KEYS[11]
@@ -164,7 +166,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
164
166
  -- "returnvalue" / "failedReason" and "finishedOn"
165
167
 
166
168
  -- Remove old jobs?
167
- local prefix = ARGV[8]
169
+ local prefix = ARGV[7]
168
170
 
169
171
  if maxAge ~= nil then
170
172
  removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
@@ -197,13 +199,13 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
197
199
 
198
200
  -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
199
201
  -- and not rate limited.
200
- if (ARGV[7] == "1") then
202
+ if (ARGV[6] == "1") then
201
203
 
202
- local target, paused = getTargetQueueList(KEYS[9], KEYS[1], KEYS[8])
204
+ local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[8])
203
205
 
204
206
  -- Check if there are delayed jobs that can be promoted
205
- promoteDelayedJobs(KEYS[7], KEYS[1], target, KEYS[3],
206
- KEYS[4], ARGV[8], timestamp, paused, KEYS[10])
207
+ promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], KEYS[4], ARGV[7],
208
+ timestamp, KEYS[10], paused)
207
209
 
208
210
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
209
211
  -- Check if we are rate limited first.
@@ -217,25 +219,30 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
217
219
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
218
220
 
219
221
  if jobId then
222
+ -- Markers in waitlist DEPRECATED in v5: Remove in v6.
220
223
  if string.sub(jobId, 1, 2) == "0:" then
221
224
  rcall("LREM", KEYS[2], 1, jobId)
222
225
 
223
226
  -- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
224
227
  -- but if ID is 0:0, then there is at least 1 prioritized job to process
225
228
  if jobId == "0:0" then
226
- jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
227
- return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp,
228
- maxJobs, expireTime, opts)
229
+ jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
230
+ KEYS[10])
231
+ return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
232
+ timestamp, maxJobs,
233
+ expireTime, opts)
229
234
  end
230
235
  else
231
- return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
232
- expireTime, opts)
236
+ return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
237
+ timestamp, maxJobs, expireTime,
238
+ opts)
233
239
  end
234
240
  else
235
241
  jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
236
242
  if jobId then
237
- return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
238
- expireTime, opts)
243
+ return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
244
+ timestamp, maxJobs, expireTime,
245
+ opts)
239
246
  end
240
247
  end
241
248
 
@@ -11,7 +11,7 @@
11
11
  ARGV[2] child key
12
12
  ARGV[3] timestamp
13
13
  ARGV[4] the id of the job
14
-
14
+
15
15
  Output:
16
16
  0 - OK
17
17
  1 - There are not pending dependencies.
@@ -21,7 +21,8 @@
21
21
  ]]
22
22
  local rcall = redis.call
23
23
 
24
- local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp, lockKey, token)
24
+ local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp,
25
+ lockKey, jobKey, token)
25
26
  if token ~= "0" then
26
27
  if rcall("GET", lockKey) == token then
27
28
  rcall("DEL", lockKey)
@@ -46,13 +47,15 @@ end
46
47
  if rcall("EXISTS", KEYS[4]) == 1 then
47
48
  if ARGV[2] ~= "" then
48
49
  if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
49
- return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
50
+ return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], KEYS[4],
51
+ ARGV[1])
50
52
  end
51
53
 
52
54
  return 1
53
55
  else
54
56
  if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
55
- return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
57
+ return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], KEYS[4],
58
+ ARGV[1])
56
59
  end
57
60
 
58
61
  return 1
@@ -0,0 +1,41 @@
1
+ --[[
2
+ Pauses or resumes a queue globably.
3
+
4
+ Input:
5
+ KEYS[1] 'wait' or 'paused''
6
+ KEYS[2] 'paused' or 'wait'
7
+ KEYS[3] 'meta'
8
+ KEYS[4] 'prioritized'
9
+ KEYS[5] events stream key
10
+ KEYS[6] 'delayed'
11
+ KEYS|7] 'marker'
12
+
13
+ ARGV[1] 'paused' or 'resumed'
14
+
15
+ Event:
16
+ publish paused or resumed event.
17
+ ]]
18
+ local rcall = redis.call
19
+
20
+ -- Includes
21
+ --- @include "includes/addDelayMarkerIfNeeded"
22
+
23
+ local markerKey = KEYS[7]
24
+ local hasJobs = rcall("EXISTS", KEYS[1]) == 1
25
+ if hasJobs then rcall("RENAME", KEYS[1], KEYS[2]) end
26
+
27
+ if ARGV[1] == "paused" then
28
+ rcall("HSET", KEYS[3], "paused", 1)
29
+ rcall("DEL", markerKey)
30
+ else
31
+ rcall("HDEL", KEYS[3], "paused")
32
+
33
+ if hasJobs or rcall("ZCARD", KEYS[4]) > 0 then
34
+ -- Add marker if there are waiting or priority jobs
35
+ rcall("ZADD", markerKey, 0, "0")
36
+ else
37
+ addDelayMarkerIfNeeded(markerKey, KEYS[6])
38
+ end
39
+ end
40
+
41
+ rcall("XADD", KEYS[5], "*", "event", ARGV[1]);
@@ -0,0 +1,60 @@
1
+ --[[
2
+ Promotes a job that is currently "delayed" to the "waiting" state
3
+
4
+ Input:
5
+ KEYS[1] 'delayed'
6
+ KEYS[2] 'wait'
7
+ KEYS[3] 'paused'
8
+ KEYS[4] 'meta'
9
+ KEYS[5] 'prioritized'
10
+ KEYS[6] 'pc' priority counter
11
+ KEYS[7] 'event stream'
12
+ KEYS[8] 'marker'
13
+
14
+ ARGV[1] queue.toKey('')
15
+ ARGV[2] jobId
16
+
17
+ Output:
18
+ 0 - OK
19
+ -3 - Job not in delayed zset.
20
+
21
+ Events:
22
+ 'waiting'
23
+ ]]
24
+ local rcall = redis.call
25
+ local jobId = ARGV[2]
26
+
27
+ -- Includes
28
+ --- @include "includes/addJobWithPriority"
29
+ --- @include "includes/getTargetQueueList"
30
+
31
+ if rcall("ZREM", KEYS[1], jobId) == 1 then
32
+ local jobKey = ARGV[1] .. jobId
33
+ local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
34
+ local metaKey = KEYS[4]
35
+
36
+ -- Remove delayed "marker" from the wait list if there is any.
37
+ -- Since we are adding a job we do not need the marker anymore.
38
+ -- Markers in waitlist DEPRECATED in v5: Remove in v6.
39
+ local target, paused = getTargetQueueList(metaKey, KEYS[2], KEYS[3])
40
+ local marker = rcall("LINDEX", target, 0)
41
+ if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
42
+
43
+ if priority == 0 then
44
+ -- LIFO or FIFO
45
+ rcall("LPUSH", target, jobId)
46
+ if not paused then rcall("ZADD", KEYS[8], 0, "0") end
47
+ else
48
+ addJobWithPriority(KEYS[8], KEYS[5], priority, jobId, KEYS[6], paused)
49
+ end
50
+
51
+ -- Emit waiting event (wait..ing@token)
52
+ rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev",
53
+ "delayed");
54
+
55
+ rcall("HSET", jobKey, "delay", 0)
56
+
57
+ return 0
58
+ else
59
+ return -3
60
+ end