bullmq 4.16.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/flow-producer.js +3 -10
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job.js +21 -11
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-base.js +2 -9
- package/dist/cjs/classes/queue-base.js.map +1 -1
- package/dist/cjs/classes/queue-events.js +3 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js +1 -0
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/redis-connection.js +9 -7
- package/dist/cjs/classes/redis-connection.js.map +1 -1
- package/dist/cjs/classes/scripts.js +34 -34
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +41 -37
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
- package/dist/cjs/commands/addParentJob-4.lua +0 -2
- package/dist/{esm/commands/addPrioritizedJob-8.lua → cjs/commands/addPrioritizedJob-7.lua} +18 -24
- package/dist/cjs/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
- package/dist/cjs/commands/changePriority-6.lua +57 -0
- package/dist/cjs/commands/getCounts-1.lua +1 -0
- package/dist/cjs/commands/getRanges-1.lua +1 -0
- package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
- package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -7
- package/dist/cjs/commands/includes/checkStalledJobs.lua +1 -1
- package/dist/cjs/commands/includes/isQueuePaused.lua +7 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
- package/dist/cjs/commands/includes/prepareJobForProcessing.lua +1 -1
- package/dist/cjs/commands/includes/promoteDelayedJobs.lua +7 -4
- package/dist/{esm/commands/moveToActive-10.lua → cjs/commands/moveToActive-11.lua} +18 -29
- package/dist/cjs/commands/moveToDelayed-7.lua +77 -0
- package/dist/cjs/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
- package/dist/cjs/commands/moveToWaitingChildren-4.lua +7 -4
- package/dist/cjs/commands/pause-7.lua +41 -0
- package/dist/cjs/commands/promote-8.lua +60 -0
- package/dist/cjs/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
- package/dist/cjs/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
- package/dist/cjs/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +51 -61
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
- package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -0
- package/dist/cjs/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
- package/dist/cjs/scripts/addStandardJob-7.js.map +1 -0
- package/dist/cjs/scripts/changePriority-6.js +85 -0
- package/dist/cjs/scripts/changePriority-6.js.map +1 -0
- package/dist/cjs/scripts/getCounts-1.js +1 -0
- package/dist/cjs/scripts/getCounts-1.js.map +1 -1
- package/dist/cjs/scripts/getRanges-1.js +1 -0
- package/dist/cjs/scripts/getRanges-1.js.map +1 -1
- package/dist/cjs/scripts/index.js +10 -10
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js +1 -0
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/cjs/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
- package/dist/cjs/scripts/moveToActive-11.js.map +1 -0
- package/dist/cjs/scripts/moveToDelayed-7.js +103 -0
- package/dist/cjs/scripts/moveToDelayed-7.js.map +1 -0
- package/dist/cjs/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
- package/dist/cjs/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-4.js +6 -3
- package/dist/cjs/scripts/moveToWaitingChildren-4.js.map +1 -1
- package/dist/cjs/scripts/pause-7.js +67 -0
- package/dist/cjs/scripts/pause-7.js.map +1 -0
- package/dist/cjs/scripts/promote-8.js +79 -0
- package/dist/cjs/scripts/promote-8.js.map +1 -0
- package/dist/cjs/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
- package/dist/cjs/scripts/retryJob-10.js.map +1 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/esm/classes/flow-producer.js +3 -10
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job.d.ts +5 -0
- package/dist/esm/classes/job.js +21 -11
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-base.js +2 -9
- package/dist/esm/classes/queue-base.js.map +1 -1
- package/dist/esm/classes/queue-events.js +3 -1
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-keys.js +1 -0
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/redis-connection.js +9 -7
- package/dist/esm/classes/redis-connection.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +4 -4
- package/dist/esm/classes/scripts.js +34 -34
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +4 -2
- package/dist/esm/classes/worker.js +41 -37
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
- package/dist/esm/commands/addParentJob-4.lua +0 -2
- package/dist/{cjs/commands/addPrioritizedJob-8.lua → esm/commands/addPrioritizedJob-7.lua} +18 -24
- package/dist/esm/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
- package/dist/esm/commands/changePriority-6.lua +57 -0
- package/dist/esm/commands/getCounts-1.lua +1 -0
- package/dist/esm/commands/getRanges-1.lua +1 -0
- package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
- package/dist/esm/commands/includes/addJobWithPriority.lua +3 -7
- package/dist/esm/commands/includes/checkStalledJobs.lua +1 -1
- package/dist/esm/commands/includes/isQueuePaused.lua +7 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
- package/dist/esm/commands/includes/prepareJobForProcessing.lua +1 -1
- package/dist/esm/commands/includes/promoteDelayedJobs.lua +7 -4
- package/dist/{cjs/commands/moveToActive-10.lua → esm/commands/moveToActive-11.lua} +18 -29
- package/dist/esm/commands/moveToDelayed-7.lua +77 -0
- package/dist/esm/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
- package/dist/esm/commands/moveToWaitingChildren-4.lua +7 -4
- package/dist/esm/commands/pause-7.lua +41 -0
- package/dist/esm/commands/promote-8.lua +60 -0
- package/dist/esm/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
- package/dist/esm/interfaces/job-json.d.ts +4 -1
- package/dist/esm/interfaces/minimal-job.d.ts +3 -0
- package/dist/esm/interfaces/queue-options.d.ts +1 -1
- package/dist/esm/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
- package/dist/esm/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
- package/dist/esm/scripts/addParentJob-4.js +51 -61
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
- package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -0
- package/dist/esm/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
- package/dist/esm/scripts/addStandardJob-7.js.map +1 -0
- package/dist/esm/scripts/changePriority-6.js +82 -0
- package/dist/esm/scripts/changePriority-6.js.map +1 -0
- package/dist/esm/scripts/getCounts-1.js +1 -0
- package/dist/esm/scripts/getCounts-1.js.map +1 -1
- package/dist/esm/scripts/getRanges-1.js +1 -0
- package/dist/esm/scripts/getRanges-1.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +10 -10
- package/dist/esm/scripts/index.js +10 -10
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-8.js +1 -0
- package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/esm/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
- package/dist/esm/scripts/moveToActive-11.js.map +1 -0
- package/dist/esm/scripts/moveToDelayed-7.js +100 -0
- package/dist/esm/scripts/moveToDelayed-7.js.map +1 -0
- package/dist/esm/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
- package/dist/esm/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-4.js +6 -3
- package/dist/esm/scripts/moveToWaitingChildren-4.js.map +1 -1
- package/dist/esm/scripts/pause-7.js +64 -0
- package/dist/esm/scripts/pause-7.js.map +1 -0
- package/dist/esm/scripts/promote-8.js +76 -0
- package/dist/esm/scripts/promote-8.js.map +1 -0
- package/dist/esm/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
- package/dist/esm/scripts/retryJob-10.js.map +1 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/changePriority-5.lua +0 -52
- package/dist/cjs/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
- package/dist/cjs/commands/moveToDelayed-8.lua +0 -72
- package/dist/cjs/commands/pause-5.lua +0 -36
- package/dist/cjs/commands/promote-7.lua +0 -57
- package/dist/cjs/scripts/addDelayedJob-7.js.map +0 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +0 -1
- package/dist/cjs/scripts/addStandardJob-6.js.map +0 -1
- package/dist/cjs/scripts/changePriority-5.js +0 -87
- package/dist/cjs/scripts/changePriority-5.js.map +0 -1
- package/dist/cjs/scripts/moveToDelayed-8.js +0 -168
- package/dist/cjs/scripts/moveToDelayed-8.js.map +0 -1
- package/dist/cjs/scripts/pause-5.js +0 -48
- package/dist/cjs/scripts/pause-5.js.map +0 -1
- package/dist/cjs/scripts/promote-7.js +0 -87
- package/dist/cjs/scripts/promote-7.js.map +0 -1
- package/dist/cjs/scripts/retryJob-9.js.map +0 -1
- package/dist/esm/commands/changePriority-5.lua +0 -52
- package/dist/esm/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
- package/dist/esm/commands/moveToDelayed-8.lua +0 -72
- package/dist/esm/commands/pause-5.lua +0 -36
- package/dist/esm/commands/promote-7.lua +0 -57
- package/dist/esm/scripts/addDelayedJob-7.js.map +0 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +0 -1
- package/dist/esm/scripts/addStandardJob-6.js.map +0 -1
- package/dist/esm/scripts/changePriority-5.js +0 -84
- package/dist/esm/scripts/changePriority-5.js.map +0 -1
- package/dist/esm/scripts/moveToDelayed-8.js +0 -165
- package/dist/esm/scripts/moveToDelayed-8.js.map +0 -1
- package/dist/esm/scripts/pause-5.js +0 -45
- package/dist/esm/scripts/pause-5.js.map +0 -1
- package/dist/esm/scripts/promote-7.js +0 -84
- package/dist/esm/scripts/promote-7.js.map +0 -1
- package/dist/esm/scripts/retryJob-9.js.map +0 -1
- /package/dist/esm/scripts/{addDelayedJob-7.d.ts → addDelayedJob-6.d.ts} +0 -0
- /package/dist/esm/scripts/{addPrioritizedJob-8.d.ts → addPrioritizedJob-7.d.ts} +0 -0
- /package/dist/esm/scripts/{addStandardJob-6.d.ts → addStandardJob-7.d.ts} +0 -0
- /package/dist/esm/scripts/{changePriority-5.d.ts → changePriority-6.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToActive-10.d.ts → moveToActive-11.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToDelayed-8.d.ts → moveToDelayed-7.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToFinished-13.d.ts → moveToFinished-14.d.ts} +0 -0
- /package/dist/esm/scripts/{pause-5.d.ts → pause-7.d.ts} +0 -0
- /package/dist/esm/scripts/{promote-7.d.ts → promote-8.d.ts} +0 -0
- /package/dist/esm/scripts/{retryJob-9.d.ts → retryJob-10.d.ts} +0 -0
@@ -8,13 +8,12 @@
|
|
8
8
|
- Emits a global event 'delayed' if the job is delayed.
|
9
9
|
|
10
10
|
Input:
|
11
|
-
KEYS[1] '
|
12
|
-
KEYS[2] '
|
13
|
-
KEYS[3] '
|
14
|
-
KEYS[4] '
|
15
|
-
KEYS[5] '
|
16
|
-
KEYS[6]
|
17
|
-
KEYS[7] events stream key
|
11
|
+
KEYS[1] 'marker',
|
12
|
+
KEYS[2] 'meta'
|
13
|
+
KEYS[3] 'id'
|
14
|
+
KEYS[4] 'delayed'
|
15
|
+
KEYS[5] 'completed'
|
16
|
+
KEYS[6] events stream key
|
18
17
|
|
19
18
|
ARGV[1] msgpacked arguments array
|
20
19
|
[1] key prefix,
|
@@ -34,15 +33,12 @@
|
|
34
33
|
jobId - OK
|
35
34
|
-5 - Missing parent key
|
36
35
|
]]
|
37
|
-
local
|
38
|
-
local
|
36
|
+
local metaKey = KEYS[2]
|
37
|
+
local idKey = KEYS[3]
|
38
|
+
local delayedKey = KEYS[4]
|
39
39
|
|
40
|
-
local
|
41
|
-
local
|
42
|
-
local delayedKey = KEYS[5]
|
43
|
-
|
44
|
-
local completedKey = KEYS[6]
|
45
|
-
local eventsKey = KEYS[7]
|
40
|
+
local completedKey = KEYS[5]
|
41
|
+
local eventsKey = KEYS[6]
|
46
42
|
|
47
43
|
local jobId
|
48
44
|
local jobIdKey
|
@@ -61,7 +57,7 @@ local parentData
|
|
61
57
|
-- Includes
|
62
58
|
--- @include "includes/storeJob"
|
63
59
|
--- @include "includes/addDelayMarkerIfNeeded"
|
64
|
-
--- @include "includes/
|
60
|
+
--- @include "includes/isQueuePaused"
|
65
61
|
--- @include "includes/getNextDelayedTimestamp"
|
66
62
|
--- @include "includes/updateExistingJobsParent"
|
67
63
|
--- @include "includes/getOrSetMaxEvents"
|
@@ -109,14 +105,14 @@ rcall("ZADD", delayedKey, score, jobId)
|
|
109
105
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
110
106
|
"jobId", jobId, "delay", delayedTimestamp)
|
111
107
|
|
112
|
-
--
|
113
|
-
|
114
|
-
|
115
|
-
|
108
|
+
-- mark that a delayed job is available
|
109
|
+
local isPaused = isQueuePaused(metaKey)
|
110
|
+
if not isPaused then
|
111
|
+
local markerKey = KEYS[1]
|
112
|
+
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
113
|
+
end
|
116
114
|
|
117
115
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
118
|
-
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
119
|
-
-- fail in this case.
|
120
116
|
if parentDependenciesKey ~= nil then
|
121
117
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
122
118
|
end
|
@@ -92,8 +92,6 @@ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
|
92
92
|
"waiting-children", "jobId", jobId)
|
93
93
|
|
94
94
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
95
|
-
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
96
|
-
-- fail in this case.
|
97
95
|
if parentDependenciesKey ~= nil then
|
98
96
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
99
97
|
end
|
@@ -5,14 +5,13 @@
|
|
5
5
|
- Adds the job to the "added" list so that workers gets notified.
|
6
6
|
|
7
7
|
Input:
|
8
|
-
KEYS[1] '
|
9
|
-
KEYS[2] '
|
10
|
-
KEYS[3] '
|
11
|
-
KEYS[4] '
|
12
|
-
KEYS[5] '
|
13
|
-
KEYS[6]
|
14
|
-
KEYS[7]
|
15
|
-
KEYS[8] 'pc' priority counter
|
8
|
+
KEYS[1] 'marker',
|
9
|
+
KEYS[2] 'meta'
|
10
|
+
KEYS[3] 'id'
|
11
|
+
KEYS[4] 'prioritized'
|
12
|
+
KEYS[5] 'completed'
|
13
|
+
KEYS[6] events stream key
|
14
|
+
KEYS[7] 'pc' priority counter
|
16
15
|
|
17
16
|
ARGV[1] msgpacked arguments array
|
18
17
|
[1] key prefix,
|
@@ -31,17 +30,14 @@
|
|
31
30
|
Output:
|
32
31
|
jobId - OK
|
33
32
|
-5 - Missing parent key
|
34
|
-
]]
|
35
|
-
local
|
36
|
-
local
|
33
|
+
]]
|
34
|
+
local metaKey = KEYS[2]
|
35
|
+
local idKey = KEYS[3]
|
36
|
+
local priorityKey = KEYS[4]
|
37
37
|
|
38
|
-
local
|
39
|
-
local
|
40
|
-
local
|
41
|
-
|
42
|
-
local completedKey = KEYS[6]
|
43
|
-
local eventsKey = KEYS[7]
|
44
|
-
local priorityCounterKey = KEYS[8]
|
38
|
+
local completedKey = KEYS[5]
|
39
|
+
local eventsKey = KEYS[6]
|
40
|
+
local priorityCounterKey = KEYS[7]
|
45
41
|
|
46
42
|
local jobId
|
47
43
|
local jobIdKey
|
@@ -59,8 +55,8 @@ local parentData
|
|
59
55
|
|
60
56
|
-- Includes
|
61
57
|
--- @include "includes/storeJob"
|
58
|
+
--- @include "includes/isQueuePaused"
|
62
59
|
--- @include "includes/addJobWithPriority"
|
63
|
-
--- @include "includes/getTargetQueueList"
|
64
60
|
--- @include "includes/updateExistingJobsParent"
|
65
61
|
--- @include "includes/getOrSetMaxEvents"
|
66
62
|
|
@@ -99,17 +95,15 @@ local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
|
99
95
|
opts, timestamp, parentKey, parentData,
|
100
96
|
repeatJobKey)
|
101
97
|
|
102
|
-
|
98
|
+
-- Add the job to the prioritized set
|
99
|
+
local isPause = isQueuePaused(metaKey)
|
100
|
+
addJobWithPriority( KEYS[1], priorityKey, priority, jobId, priorityCounterKey, isPause)
|
103
101
|
|
104
|
-
addJobWithPriority(waitKey, priorityKey, priority, paused, jobId,
|
105
|
-
priorityCounterKey)
|
106
102
|
-- Emit waiting event
|
107
103
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
108
104
|
"jobId", jobId)
|
109
105
|
|
110
106
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
111
|
-
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
112
|
-
-- fail in this case.
|
113
107
|
if parentDependenciesKey ~= nil then
|
114
108
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
115
109
|
end
|
@@ -21,6 +21,7 @@
|
|
21
21
|
KEYS[4] 'id'
|
22
22
|
KEYS[5] 'completed'
|
23
23
|
KEYS[6] events stream key
|
24
|
+
KEYS[7] marker key
|
24
25
|
|
25
26
|
ARGV[1] msgpacked arguments array
|
26
27
|
[1] key prefix,
|
@@ -99,6 +100,11 @@ storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
|
99
100
|
|
100
101
|
local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
|
101
102
|
|
103
|
+
if not paused then
|
104
|
+
-- mark that a job is available
|
105
|
+
rcall("ZADD", KEYS[7], 0, "0")
|
106
|
+
end
|
107
|
+
|
102
108
|
-- LIFO or FIFO
|
103
109
|
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
|
104
110
|
rcall(pushCmd, target, jobId)
|
@@ -108,8 +114,6 @@ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
|
108
114
|
"jobId", jobId)
|
109
115
|
|
110
116
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
111
|
-
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
112
|
-
-- fail in this case.
|
113
117
|
if parentDependenciesKey ~= nil then
|
114
118
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
115
119
|
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
--[[
|
2
|
+
Change job priority
|
3
|
+
Input:
|
4
|
+
KEYS[1] 'wait',
|
5
|
+
KEYS[2] 'paused'
|
6
|
+
KEYS[3] 'meta'
|
7
|
+
KEYS[4] 'prioritized'
|
8
|
+
KEYS[5] 'pc' priority counter
|
9
|
+
KEYS[6] 'marker'
|
10
|
+
|
11
|
+
ARGV[1] priority value
|
12
|
+
ARGV[2] job key
|
13
|
+
ARGV[3] job id
|
14
|
+
ARGV[4] lifo
|
15
|
+
|
16
|
+
Output:
|
17
|
+
0 - OK
|
18
|
+
-1 - Missing job
|
19
|
+
]]
|
20
|
+
local jobKey = ARGV[2]
|
21
|
+
local jobId = ARGV[3]
|
22
|
+
local priority = tonumber(ARGV[1])
|
23
|
+
local rcall = redis.call
|
24
|
+
|
25
|
+
-- Includes
|
26
|
+
--- @include "includes/isQueuePaused"
|
27
|
+
--- @include "includes/addJobWithPriority"
|
28
|
+
--- @include "includes/getTargetQueueList"
|
29
|
+
|
30
|
+
if rcall("EXISTS", jobKey) == 1 then
|
31
|
+
local metaKey = KEYS[3]
|
32
|
+
local isPaused = isQueuePaused(metaKey)
|
33
|
+
local markerKey = KEYS[6]
|
34
|
+
local prioritizedKey = KEYS[4]
|
35
|
+
|
36
|
+
-- Re-add with the new priority
|
37
|
+
if rcall("ZREM", KEYS[4], jobId) > 0 then
|
38
|
+
addJobWithPriority(markerKey, prioritizedKey, priority, jobId, KEYS[5],
|
39
|
+
isPaused)
|
40
|
+
-- If the new priority is 0, then just leave the job where it is in the wait list.
|
41
|
+
elseif priority > 0 then
|
42
|
+
-- Job is already in the wait list, we need to re-add it with the new priority.
|
43
|
+
local target = isPaused and KEYS[2] or KEYS[1]
|
44
|
+
|
45
|
+
local numRemovedElements = rcall("LREM", target, -1, jobId)
|
46
|
+
if numRemovedElements > 0 then
|
47
|
+
addJobWithPriority(markerKey, prioritizedKey, priority, jobId,
|
48
|
+
KEYS[5], isPaused)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
rcall("HSET", jobKey, "priority", priority)
|
53
|
+
|
54
|
+
return 0
|
55
|
+
else
|
56
|
+
return -1
|
57
|
+
end
|
@@ -13,6 +13,7 @@ local results = {}
|
|
13
13
|
for i = 1, #ARGV do
|
14
14
|
local stateKey = prefix .. ARGV[i]
|
15
15
|
if ARGV[i] == "wait" or ARGV[i] == "paused" then
|
16
|
+
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
16
17
|
local marker = rcall("LINDEX", stateKey, -1)
|
17
18
|
if marker and string.sub(marker, 1, 2) == "0:" then
|
18
19
|
local count = rcall("LLEN", stateKey)
|
@@ -43,6 +43,7 @@ end
|
|
43
43
|
for i = 4, #ARGV do
|
44
44
|
local stateKey = prefix .. ARGV[i]
|
45
45
|
if ARGV[i] == "wait" or ARGV[i] == "paused" then
|
46
|
+
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
46
47
|
local marker = rcall("LINDEX", stateKey, -1)
|
47
48
|
if marker and string.sub(marker, 1, 2) == "0:" then
|
48
49
|
local count = rcall("LLEN", stateKey)
|
@@ -1,27 +1,13 @@
|
|
1
1
|
--[[
|
2
2
|
Add delay marker if needed.
|
3
|
-
]]
|
4
|
-
|
3
|
+
]]
|
5
4
|
-- Includes
|
6
5
|
--- @include "getNextDelayedTimestamp"
|
7
|
-
|
8
|
-
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
|
9
|
-
local waitLen = rcall("LLEN", targetKey)
|
10
|
-
if waitLen <= 1 then
|
6
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
11
7
|
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
12
8
|
if nextTimestamp ~= nil then
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
local marker = rcall("LINDEX", targetKey, 0)
|
17
|
-
local oldTimestamp = tonumber(marker:sub(3))
|
18
|
-
if oldTimestamp and oldTimestamp > nextTimestamp then
|
19
|
-
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
20
|
-
end
|
21
|
-
else
|
22
|
-
-- if there is no marker, then we need to add one
|
23
|
-
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
24
|
-
end
|
9
|
+
-- Replace the score of the marker with the newest known
|
10
|
+
-- next timestamp.
|
11
|
+
rcall("ZADD", markerKey, nextTimestamp, "0")
|
25
12
|
end
|
26
|
-
end
|
27
13
|
end
|
@@ -1,15 +1,11 @@
|
|
1
1
|
--[[
|
2
2
|
Function to add job considering priority.
|
3
3
|
]]
|
4
|
-
|
5
|
-
-- Includes
|
6
|
-
--- @include "addPriorityMarkerIfNeeded"
|
7
|
-
|
8
|
-
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
4
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
9
5
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
10
6
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
11
7
|
rcall("ZADD", prioritizedKey, score, jobId)
|
12
|
-
if not
|
13
|
-
|
8
|
+
if not isPaused then
|
9
|
+
rcall("ZADD", markerKey, 0, "0")
|
14
10
|
end
|
15
11
|
end
|
@@ -53,7 +53,7 @@ local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey,
|
|
53
53
|
|
54
54
|
-- Remove from active list
|
55
55
|
for i, jobId in ipairs(stalling) do
|
56
|
-
|
56
|
+
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
57
57
|
if string.sub(jobId, 1, 2) == "0:" then
|
58
58
|
-- If the jobId is a delay marker ID we just remove it.
|
59
59
|
rcall("LREM", activeKey, 1, jobId)
|
@@ -1,42 +1,51 @@
|
|
1
1
|
--[[
|
2
2
|
Validate and move parent to active if needed.
|
3
3
|
]]
|
4
|
-
|
5
4
|
-- Includes
|
6
5
|
--- @include "addDelayMarkerIfNeeded"
|
6
|
+
--- @include "isQueuePaused"
|
7
7
|
--- @include "addJobWithPriority"
|
8
8
|
--- @include "getTargetQueueList"
|
9
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
10
|
+
parentKey, parentId, timestamp)
|
11
|
+
local isParentActive = rcall("ZSCORE",
|
12
|
+
parentQueueKey .. ":waiting-children", parentId)
|
13
|
+
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
14
|
+
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
15
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
16
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
17
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
18
|
+
|
19
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
20
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
21
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
22
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
9
23
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
18
|
-
local priority = tonumber(jobAttributes[1]) or 0
|
19
|
-
local delay = tonumber(jobAttributes[2]) or 0
|
24
|
+
if delay > 0 then
|
25
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
26
|
+
local score = delayedTimestamp * 0x1000
|
27
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
28
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
29
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
30
|
+
"jobId", parentId, "delay", delayedTimestamp)
|
20
31
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
32
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
33
|
+
else
|
34
|
+
if priority == 0 then
|
35
|
+
local parentTarget, _paused =
|
36
|
+
getTargetQueueList(parentMetaKey, parentWaitKey,
|
37
|
+
parentPausedKey)
|
38
|
+
rcall("RPUSH", parentTarget, parentId)
|
39
|
+
rcall("ZADD", parentMarkerKey, 0, "0")
|
40
|
+
else
|
41
|
+
local isPaused = isQueuePaused(parentMetaKey)
|
42
|
+
addJobWithPriority(parentMarkerKey,
|
43
|
+
parentQueueKey .. ":prioritized", priority,
|
44
|
+
parentId, parentQueueKey .. ":pc", isPaused)
|
45
|
+
end
|
28
46
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
rcall("RPUSH", parentTarget, parentId)
|
33
|
-
else
|
34
|
-
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
|
35
|
-
parentId, parentQueueKey .. ":pc")
|
36
|
-
end
|
37
|
-
|
38
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
|
39
|
-
"prev", "waiting-children")
|
47
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
48
|
+
"jobId", parentId, "prev", "waiting-children")
|
49
|
+
end
|
40
50
|
end
|
41
|
-
end
|
42
51
|
end
|
@@ -67,7 +67,7 @@ local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, proces
|
|
67
67
|
|
68
68
|
rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
|
69
69
|
rcall("HSET", jobKey, "processedOn", processedOn)
|
70
|
-
rcall("HINCRBY", jobKey, "
|
70
|
+
rcall("HINCRBY", jobKey, "ats", 1)
|
71
71
|
|
72
72
|
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
|
73
73
|
end
|
@@ -10,8 +10,8 @@
|
|
10
10
|
--- @include "addJobWithPriority"
|
11
11
|
|
12
12
|
-- Try to get as much as 1000 jobs at once
|
13
|
-
local function promoteDelayedJobs(delayedKey,
|
14
|
-
eventStreamKey, prefix, timestamp,
|
13
|
+
local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
|
14
|
+
eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
|
15
15
|
local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
|
16
16
|
|
17
17
|
if (#jobs > 0) then
|
@@ -25,9 +25,12 @@ local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey
|
|
25
25
|
if priority == 0 then
|
26
26
|
-- LIFO or FIFO
|
27
27
|
rcall("LPUSH", targetKey, jobId)
|
28
|
+
if not isPaused then
|
29
|
+
rcall("ZADD", markerKey, 0, "0")
|
30
|
+
end
|
28
31
|
else
|
29
|
-
addJobWithPriority(
|
30
|
-
jobId, priorityCounterKey)
|
32
|
+
addJobWithPriority(markerKey, prioritizedKey, priority,
|
33
|
+
jobId, priorityCounterKey, isPaused)
|
31
34
|
end
|
32
35
|
|
33
36
|
-- Emit waiting event
|
@@ -17,16 +17,18 @@
|
|
17
17
|
KEYS[6] rate limiter key
|
18
18
|
KEYS[7] delayed key
|
19
19
|
|
20
|
-
--
|
20
|
+
-- Delayed jobs
|
21
21
|
KEYS[8] paused key
|
22
22
|
KEYS[9] meta key
|
23
23
|
KEYS[10] pc priority counter
|
24
24
|
|
25
|
+
-- Marker
|
26
|
+
KEYS[11] marker key
|
27
|
+
|
25
28
|
-- Arguments
|
26
29
|
ARGV[1] key prefix
|
27
30
|
ARGV[2] timestamp
|
28
|
-
ARGV[3]
|
29
|
-
ARGV[4] opts
|
31
|
+
ARGV[3] opts
|
30
32
|
|
31
33
|
opts - token - lock token
|
32
34
|
opts - lockDuration
|
@@ -37,7 +39,7 @@ local waitKey = KEYS[1]
|
|
37
39
|
local activeKey = KEYS[2]
|
38
40
|
local rateLimiterKey = KEYS[6]
|
39
41
|
local delayedKey = KEYS[7]
|
40
|
-
local opts = cmsgpack.unpack(ARGV[
|
42
|
+
local opts = cmsgpack.unpack(ARGV[3])
|
41
43
|
|
42
44
|
-- Includes
|
43
45
|
--- @include "includes/getNextDelayedTimestamp"
|
@@ -50,39 +52,26 @@ local opts = cmsgpack.unpack(ARGV[4])
|
|
50
52
|
local target, paused = getTargetQueueList(KEYS[9], waitKey, KEYS[8])
|
51
53
|
|
52
54
|
-- Check if there are delayed jobs that we can move to wait.
|
53
|
-
|
54
|
-
|
55
|
+
local markerKey = KEYS[11]
|
56
|
+
promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], KEYS[4], ARGV[1],
|
57
|
+
ARGV[2], KEYS[10], paused)
|
55
58
|
|
56
59
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
57
60
|
local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
|
58
61
|
|
59
|
-
|
60
|
-
if
|
61
|
-
jobId = ARGV[3]
|
62
|
-
|
63
|
-
-- clean stalled key
|
64
|
-
rcall("SREM", KEYS[5], jobId)
|
65
|
-
end
|
66
|
-
|
67
|
-
if not jobId or (jobId and string.sub(jobId, 1, 2) == "0:") then
|
68
|
-
-- If jobId is special ID 0:delay, then there is no job to process
|
69
|
-
if jobId then rcall("LREM", activeKey, 1, jobId) end
|
62
|
+
-- Check if we are rate limited first.
|
63
|
+
if expireTime > 0 then return {0, 0, expireTime, 0} end
|
70
64
|
|
71
|
-
|
72
|
-
|
65
|
+
-- paused queue
|
66
|
+
if paused then return {0, 0, 0, 0} end
|
73
67
|
|
74
|
-
|
75
|
-
|
68
|
+
-- no job ID, try non-blocking move from wait to active
|
69
|
+
local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
76
70
|
|
77
|
-
|
71
|
+
-- Markers in waitlist DEPRECATED in v5: Will be completely removed in v6.
|
72
|
+
if jobId and string.sub(jobId, 1, 2) == "0:" then
|
73
|
+
rcall("LREM", activeKey, 1, jobId)
|
78
74
|
jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
79
|
-
|
80
|
-
-- Since it is possible that between a call to BRPOPLPUSH and moveToActive
|
81
|
-
-- another script puts a new maker in wait, we need to check again.
|
82
|
-
if jobId and string.sub(jobId, 1, 2) == "0:" then
|
83
|
-
rcall("LREM", activeKey, 1, jobId)
|
84
|
-
jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
85
|
-
end
|
86
75
|
end
|
87
76
|
|
88
77
|
if jobId then
|
@@ -0,0 +1,77 @@
|
|
1
|
+
--[[
|
2
|
+
Moves job from active to delayed set.
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] marker key
|
6
|
+
KEYS[2] active key
|
7
|
+
KEYS[3] prioritized key
|
8
|
+
KEYS[4] delayed key
|
9
|
+
KEYS[5] job key
|
10
|
+
KEYS[6] events stream
|
11
|
+
KEYS[7] meta key
|
12
|
+
|
13
|
+
ARGV[1] key prefix
|
14
|
+
ARGV[2] timestamp
|
15
|
+
ARGV[3] delayedTimestamp
|
16
|
+
ARGV[4] the id of the job
|
17
|
+
ARGV[5] queue token
|
18
|
+
ARGV[6] delay value
|
19
|
+
ARGV[7] skip attempt
|
20
|
+
|
21
|
+
Output:
|
22
|
+
0 - OK
|
23
|
+
-1 - Missing job.
|
24
|
+
-3 - Job not in active set.
|
25
|
+
|
26
|
+
Events:
|
27
|
+
- delayed key.
|
28
|
+
]]
|
29
|
+
local rcall = redis.call
|
30
|
+
|
31
|
+
-- Includes
|
32
|
+
--- @include "includes/addDelayMarkerIfNeeded"
|
33
|
+
--- @include "includes/isQueuePaused"
|
34
|
+
|
35
|
+
local jobKey = KEYS[5]
|
36
|
+
local metaKey = KEYS[7]
|
37
|
+
if rcall("EXISTS", jobKey) == 1 then
|
38
|
+
local delayedKey = KEYS[4]
|
39
|
+
if ARGV[5] ~= "0" then
|
40
|
+
local lockKey = jobKey .. ':lock'
|
41
|
+
if rcall("GET", lockKey) == ARGV[5] then
|
42
|
+
rcall("DEL", lockKey)
|
43
|
+
else
|
44
|
+
return -2
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
local jobId = ARGV[4]
|
49
|
+
local score = tonumber(ARGV[3])
|
50
|
+
local delayedTimestamp = (score / 0x1000)
|
51
|
+
|
52
|
+
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
53
|
+
if numRemovedElements < 1 then return -3 end
|
54
|
+
|
55
|
+
if ARGV[7] == "0" then
|
56
|
+
rcall("HINCRBY", jobKey, "atm", 1)
|
57
|
+
end
|
58
|
+
|
59
|
+
rcall("HSET", jobKey, "delay", ARGV[6])
|
60
|
+
|
61
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents") or 10000
|
62
|
+
|
63
|
+
rcall("ZADD", delayedKey, score, jobId)
|
64
|
+
rcall("XADD", KEYS[6], "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
65
|
+
"jobId", jobId, "delay", delayedTimestamp)
|
66
|
+
|
67
|
+
-- Check if we need to push a marker job to wake up sleeping workers.
|
68
|
+
local isPaused = isQueuePaused(metaKey)
|
69
|
+
if not isPaused then
|
70
|
+
local markerKey = KEYS[1]
|
71
|
+
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
72
|
+
end
|
73
|
+
|
74
|
+
return 0
|
75
|
+
else
|
76
|
+
return -1
|
77
|
+
end
|