bullmq 4.16.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/flow-producer.js +3 -10
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job.js +21 -11
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-base.js +2 -9
- package/dist/cjs/classes/queue-base.js.map +1 -1
- package/dist/cjs/classes/queue-events.js +3 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js +1 -0
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/redis-connection.js +9 -7
- package/dist/cjs/classes/redis-connection.js.map +1 -1
- package/dist/cjs/classes/scripts.js +34 -34
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +41 -37
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
- package/dist/cjs/commands/addParentJob-4.lua +0 -2
- package/dist/{esm/commands/addPrioritizedJob-8.lua → cjs/commands/addPrioritizedJob-7.lua} +18 -24
- package/dist/cjs/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
- package/dist/cjs/commands/changePriority-6.lua +57 -0
- package/dist/cjs/commands/getCounts-1.lua +1 -0
- package/dist/cjs/commands/getRanges-1.lua +1 -0
- package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
- package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -7
- package/dist/cjs/commands/includes/checkStalledJobs.lua +1 -1
- package/dist/cjs/commands/includes/isQueuePaused.lua +7 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
- package/dist/cjs/commands/includes/prepareJobForProcessing.lua +1 -1
- package/dist/cjs/commands/includes/promoteDelayedJobs.lua +7 -4
- package/dist/{esm/commands/moveToActive-10.lua → cjs/commands/moveToActive-11.lua} +18 -29
- package/dist/cjs/commands/moveToDelayed-7.lua +77 -0
- package/dist/cjs/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
- package/dist/cjs/commands/moveToWaitingChildren-4.lua +7 -4
- package/dist/cjs/commands/pause-7.lua +41 -0
- package/dist/cjs/commands/promote-8.lua +60 -0
- package/dist/cjs/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
- package/dist/cjs/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
- package/dist/cjs/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +51 -61
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
- package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -0
- package/dist/cjs/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
- package/dist/cjs/scripts/addStandardJob-7.js.map +1 -0
- package/dist/cjs/scripts/changePriority-6.js +85 -0
- package/dist/cjs/scripts/changePriority-6.js.map +1 -0
- package/dist/cjs/scripts/getCounts-1.js +1 -0
- package/dist/cjs/scripts/getCounts-1.js.map +1 -1
- package/dist/cjs/scripts/getRanges-1.js +1 -0
- package/dist/cjs/scripts/getRanges-1.js.map +1 -1
- package/dist/cjs/scripts/index.js +10 -10
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js +1 -0
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/cjs/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
- package/dist/cjs/scripts/moveToActive-11.js.map +1 -0
- package/dist/cjs/scripts/moveToDelayed-7.js +103 -0
- package/dist/cjs/scripts/moveToDelayed-7.js.map +1 -0
- package/dist/cjs/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
- package/dist/cjs/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-4.js +6 -3
- package/dist/cjs/scripts/moveToWaitingChildren-4.js.map +1 -1
- package/dist/cjs/scripts/pause-7.js +67 -0
- package/dist/cjs/scripts/pause-7.js.map +1 -0
- package/dist/cjs/scripts/promote-8.js +79 -0
- package/dist/cjs/scripts/promote-8.js.map +1 -0
- package/dist/cjs/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
- package/dist/cjs/scripts/retryJob-10.js.map +1 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/esm/classes/flow-producer.js +3 -10
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job.d.ts +5 -0
- package/dist/esm/classes/job.js +21 -11
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-base.js +2 -9
- package/dist/esm/classes/queue-base.js.map +1 -1
- package/dist/esm/classes/queue-events.js +3 -1
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-keys.js +1 -0
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/redis-connection.js +9 -7
- package/dist/esm/classes/redis-connection.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +4 -4
- package/dist/esm/classes/scripts.js +34 -34
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +4 -2
- package/dist/esm/classes/worker.js +41 -37
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
- package/dist/esm/commands/addParentJob-4.lua +0 -2
- package/dist/{cjs/commands/addPrioritizedJob-8.lua → esm/commands/addPrioritizedJob-7.lua} +18 -24
- package/dist/esm/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
- package/dist/esm/commands/changePriority-6.lua +57 -0
- package/dist/esm/commands/getCounts-1.lua +1 -0
- package/dist/esm/commands/getRanges-1.lua +1 -0
- package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
- package/dist/esm/commands/includes/addJobWithPriority.lua +3 -7
- package/dist/esm/commands/includes/checkStalledJobs.lua +1 -1
- package/dist/esm/commands/includes/isQueuePaused.lua +7 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
- package/dist/esm/commands/includes/prepareJobForProcessing.lua +1 -1
- package/dist/esm/commands/includes/promoteDelayedJobs.lua +7 -4
- package/dist/{cjs/commands/moveToActive-10.lua → esm/commands/moveToActive-11.lua} +18 -29
- package/dist/esm/commands/moveToDelayed-7.lua +77 -0
- package/dist/esm/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
- package/dist/esm/commands/moveToWaitingChildren-4.lua +7 -4
- package/dist/esm/commands/pause-7.lua +41 -0
- package/dist/esm/commands/promote-8.lua +60 -0
- package/dist/esm/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
- package/dist/esm/interfaces/job-json.d.ts +4 -1
- package/dist/esm/interfaces/minimal-job.d.ts +3 -0
- package/dist/esm/interfaces/queue-options.d.ts +1 -1
- package/dist/esm/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
- package/dist/esm/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
- package/dist/esm/scripts/addParentJob-4.js +51 -61
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
- package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -0
- package/dist/esm/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
- package/dist/esm/scripts/addStandardJob-7.js.map +1 -0
- package/dist/esm/scripts/changePriority-6.js +82 -0
- package/dist/esm/scripts/changePriority-6.js.map +1 -0
- package/dist/esm/scripts/getCounts-1.js +1 -0
- package/dist/esm/scripts/getCounts-1.js.map +1 -1
- package/dist/esm/scripts/getRanges-1.js +1 -0
- package/dist/esm/scripts/getRanges-1.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +10 -10
- package/dist/esm/scripts/index.js +10 -10
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-8.js +1 -0
- package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/esm/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
- package/dist/esm/scripts/moveToActive-11.js.map +1 -0
- package/dist/esm/scripts/moveToDelayed-7.js +100 -0
- package/dist/esm/scripts/moveToDelayed-7.js.map +1 -0
- package/dist/esm/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
- package/dist/esm/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-4.js +6 -3
- package/dist/esm/scripts/moveToWaitingChildren-4.js.map +1 -1
- package/dist/esm/scripts/pause-7.js +64 -0
- package/dist/esm/scripts/pause-7.js.map +1 -0
- package/dist/esm/scripts/promote-8.js +76 -0
- package/dist/esm/scripts/promote-8.js.map +1 -0
- package/dist/esm/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
- package/dist/esm/scripts/retryJob-10.js.map +1 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/changePriority-5.lua +0 -52
- package/dist/cjs/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
- package/dist/cjs/commands/moveToDelayed-8.lua +0 -72
- package/dist/cjs/commands/pause-5.lua +0 -36
- package/dist/cjs/commands/promote-7.lua +0 -57
- package/dist/cjs/scripts/addDelayedJob-7.js.map +0 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +0 -1
- package/dist/cjs/scripts/addStandardJob-6.js.map +0 -1
- package/dist/cjs/scripts/changePriority-5.js +0 -87
- package/dist/cjs/scripts/changePriority-5.js.map +0 -1
- package/dist/cjs/scripts/moveToDelayed-8.js +0 -168
- package/dist/cjs/scripts/moveToDelayed-8.js.map +0 -1
- package/dist/cjs/scripts/pause-5.js +0 -48
- package/dist/cjs/scripts/pause-5.js.map +0 -1
- package/dist/cjs/scripts/promote-7.js +0 -87
- package/dist/cjs/scripts/promote-7.js.map +0 -1
- package/dist/cjs/scripts/retryJob-9.js.map +0 -1
- package/dist/esm/commands/changePriority-5.lua +0 -52
- package/dist/esm/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
- package/dist/esm/commands/moveToDelayed-8.lua +0 -72
- package/dist/esm/commands/pause-5.lua +0 -36
- package/dist/esm/commands/promote-7.lua +0 -57
- package/dist/esm/scripts/addDelayedJob-7.js.map +0 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +0 -1
- package/dist/esm/scripts/addStandardJob-6.js.map +0 -1
- package/dist/esm/scripts/changePriority-5.js +0 -84
- package/dist/esm/scripts/changePriority-5.js.map +0 -1
- package/dist/esm/scripts/moveToDelayed-8.js +0 -165
- package/dist/esm/scripts/moveToDelayed-8.js.map +0 -1
- package/dist/esm/scripts/pause-5.js +0 -45
- package/dist/esm/scripts/pause-5.js.map +0 -1
- package/dist/esm/scripts/promote-7.js +0 -84
- package/dist/esm/scripts/promote-7.js.map +0 -1
- package/dist/esm/scripts/retryJob-9.js.map +0 -1
- /package/dist/esm/scripts/{addDelayedJob-7.d.ts → addDelayedJob-6.d.ts} +0 -0
- /package/dist/esm/scripts/{addPrioritizedJob-8.d.ts → addPrioritizedJob-7.d.ts} +0 -0
- /package/dist/esm/scripts/{addStandardJob-6.d.ts → addStandardJob-7.d.ts} +0 -0
- /package/dist/esm/scripts/{changePriority-5.d.ts → changePriority-6.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToActive-10.d.ts → moveToActive-11.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToDelayed-8.d.ts → moveToDelayed-7.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToFinished-13.d.ts → moveToFinished-14.d.ts} +0 -0
- /package/dist/esm/scripts/{pause-5.d.ts → pause-7.d.ts} +0 -0
- /package/dist/esm/scripts/{promote-7.d.ts → promote-8.d.ts} +0 -0
- /package/dist/esm/scripts/{retryJob-9.d.ts → retryJob-10.d.ts} +0 -0
@@ -1,27 +1,13 @@
|
|
1
1
|
--[[
|
2
2
|
Add delay marker if needed.
|
3
|
-
]]
|
4
|
-
|
3
|
+
]]
|
5
4
|
-- Includes
|
6
5
|
--- @include "getNextDelayedTimestamp"
|
7
|
-
|
8
|
-
local function addDelayMarkerIfNeeded(targetKey, delayedKey)
|
9
|
-
local waitLen = rcall("LLEN", targetKey)
|
10
|
-
if waitLen <= 1 then
|
6
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
11
7
|
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
12
8
|
if nextTimestamp ~= nil then
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
local marker = rcall("LINDEX", targetKey, 0)
|
17
|
-
local oldTimestamp = tonumber(marker:sub(3))
|
18
|
-
if oldTimestamp and oldTimestamp > nextTimestamp then
|
19
|
-
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
20
|
-
end
|
21
|
-
else
|
22
|
-
-- if there is no marker, then we need to add one
|
23
|
-
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
24
|
-
end
|
9
|
+
-- Replace the score of the marker with the newest known
|
10
|
+
-- next timestamp.
|
11
|
+
rcall("ZADD", markerKey, nextTimestamp, "0")
|
25
12
|
end
|
26
|
-
end
|
27
13
|
end
|
@@ -1,15 +1,11 @@
|
|
1
1
|
--[[
|
2
2
|
Function to add job considering priority.
|
3
3
|
]]
|
4
|
-
|
5
|
-
-- Includes
|
6
|
-
--- @include "addPriorityMarkerIfNeeded"
|
7
|
-
|
8
|
-
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
4
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
9
5
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
10
6
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
11
7
|
rcall("ZADD", prioritizedKey, score, jobId)
|
12
|
-
if not
|
13
|
-
|
8
|
+
if not isPaused then
|
9
|
+
rcall("ZADD", markerKey, 0, "0")
|
14
10
|
end
|
15
11
|
end
|
@@ -53,7 +53,7 @@ local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey,
|
|
53
53
|
|
54
54
|
-- Remove from active list
|
55
55
|
for i, jobId in ipairs(stalling) do
|
56
|
-
|
56
|
+
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
57
57
|
if string.sub(jobId, 1, 2) == "0:" then
|
58
58
|
-- If the jobId is a delay marker ID we just remove it.
|
59
59
|
rcall("LREM", activeKey, 1, jobId)
|
@@ -1,42 +1,51 @@
|
|
1
1
|
--[[
|
2
2
|
Validate and move parent to active if needed.
|
3
3
|
]]
|
4
|
-
|
5
4
|
-- Includes
|
6
5
|
--- @include "addDelayMarkerIfNeeded"
|
6
|
+
--- @include "isQueuePaused"
|
7
7
|
--- @include "addJobWithPriority"
|
8
8
|
--- @include "getTargetQueueList"
|
9
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
10
|
+
parentKey, parentId, timestamp)
|
11
|
+
local isParentActive = rcall("ZSCORE",
|
12
|
+
parentQueueKey .. ":waiting-children", parentId)
|
13
|
+
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
14
|
+
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
15
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
16
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
17
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
18
|
+
|
19
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
20
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
21
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
22
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
9
23
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
18
|
-
local priority = tonumber(jobAttributes[1]) or 0
|
19
|
-
local delay = tonumber(jobAttributes[2]) or 0
|
24
|
+
if delay > 0 then
|
25
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
26
|
+
local score = delayedTimestamp * 0x1000
|
27
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
28
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
29
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
30
|
+
"jobId", parentId, "delay", delayedTimestamp)
|
20
31
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
32
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
33
|
+
else
|
34
|
+
if priority == 0 then
|
35
|
+
local parentTarget, _paused =
|
36
|
+
getTargetQueueList(parentMetaKey, parentWaitKey,
|
37
|
+
parentPausedKey)
|
38
|
+
rcall("RPUSH", parentTarget, parentId)
|
39
|
+
rcall("ZADD", parentMarkerKey, 0, "0")
|
40
|
+
else
|
41
|
+
local isPaused = isQueuePaused(parentMetaKey)
|
42
|
+
addJobWithPriority(parentMarkerKey,
|
43
|
+
parentQueueKey .. ":prioritized", priority,
|
44
|
+
parentId, parentQueueKey .. ":pc", isPaused)
|
45
|
+
end
|
28
46
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
rcall("RPUSH", parentTarget, parentId)
|
33
|
-
else
|
34
|
-
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
|
35
|
-
parentId, parentQueueKey .. ":pc")
|
36
|
-
end
|
37
|
-
|
38
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
|
39
|
-
"prev", "waiting-children")
|
47
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
48
|
+
"jobId", parentId, "prev", "waiting-children")
|
49
|
+
end
|
40
50
|
end
|
41
|
-
end
|
42
51
|
end
|
@@ -67,7 +67,7 @@ local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, proces
|
|
67
67
|
|
68
68
|
rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
|
69
69
|
rcall("HSET", jobKey, "processedOn", processedOn)
|
70
|
-
rcall("HINCRBY", jobKey, "
|
70
|
+
rcall("HINCRBY", jobKey, "ats", 1)
|
71
71
|
|
72
72
|
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
|
73
73
|
end
|
@@ -10,8 +10,8 @@
|
|
10
10
|
--- @include "addJobWithPriority"
|
11
11
|
|
12
12
|
-- Try to get as much as 1000 jobs at once
|
13
|
-
local function promoteDelayedJobs(delayedKey,
|
14
|
-
eventStreamKey, prefix, timestamp,
|
13
|
+
local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
|
14
|
+
eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
|
15
15
|
local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
|
16
16
|
|
17
17
|
if (#jobs > 0) then
|
@@ -25,9 +25,12 @@ local function promoteDelayedJobs(delayedKey, waitKey, targetKey, prioritizedKey
|
|
25
25
|
if priority == 0 then
|
26
26
|
-- LIFO or FIFO
|
27
27
|
rcall("LPUSH", targetKey, jobId)
|
28
|
+
if not isPaused then
|
29
|
+
rcall("ZADD", markerKey, 0, "0")
|
30
|
+
end
|
28
31
|
else
|
29
|
-
addJobWithPriority(
|
30
|
-
jobId, priorityCounterKey)
|
32
|
+
addJobWithPriority(markerKey, prioritizedKey, priority,
|
33
|
+
jobId, priorityCounterKey, isPaused)
|
31
34
|
end
|
32
35
|
|
33
36
|
-- Emit waiting event
|
@@ -17,16 +17,18 @@
|
|
17
17
|
KEYS[6] rate limiter key
|
18
18
|
KEYS[7] delayed key
|
19
19
|
|
20
|
-
--
|
20
|
+
-- Delayed jobs
|
21
21
|
KEYS[8] paused key
|
22
22
|
KEYS[9] meta key
|
23
23
|
KEYS[10] pc priority counter
|
24
24
|
|
25
|
+
-- Marker
|
26
|
+
KEYS[11] marker key
|
27
|
+
|
25
28
|
-- Arguments
|
26
29
|
ARGV[1] key prefix
|
27
30
|
ARGV[2] timestamp
|
28
|
-
ARGV[3]
|
29
|
-
ARGV[4] opts
|
31
|
+
ARGV[3] opts
|
30
32
|
|
31
33
|
opts - token - lock token
|
32
34
|
opts - lockDuration
|
@@ -37,7 +39,7 @@ local waitKey = KEYS[1]
|
|
37
39
|
local activeKey = KEYS[2]
|
38
40
|
local rateLimiterKey = KEYS[6]
|
39
41
|
local delayedKey = KEYS[7]
|
40
|
-
local opts = cmsgpack.unpack(ARGV[
|
42
|
+
local opts = cmsgpack.unpack(ARGV[3])
|
41
43
|
|
42
44
|
-- Includes
|
43
45
|
--- @include "includes/getNextDelayedTimestamp"
|
@@ -50,39 +52,26 @@ local opts = cmsgpack.unpack(ARGV[4])
|
|
50
52
|
local target, paused = getTargetQueueList(KEYS[9], waitKey, KEYS[8])
|
51
53
|
|
52
54
|
-- Check if there are delayed jobs that we can move to wait.
|
53
|
-
|
54
|
-
|
55
|
+
local markerKey = KEYS[11]
|
56
|
+
promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], KEYS[4], ARGV[1],
|
57
|
+
ARGV[2], KEYS[10], paused)
|
55
58
|
|
56
59
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
57
60
|
local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
|
58
61
|
|
59
|
-
|
60
|
-
if
|
61
|
-
jobId = ARGV[3]
|
62
|
-
|
63
|
-
-- clean stalled key
|
64
|
-
rcall("SREM", KEYS[5], jobId)
|
65
|
-
end
|
66
|
-
|
67
|
-
if not jobId or (jobId and string.sub(jobId, 1, 2) == "0:") then
|
68
|
-
-- If jobId is special ID 0:delay, then there is no job to process
|
69
|
-
if jobId then rcall("LREM", activeKey, 1, jobId) end
|
62
|
+
-- Check if we are rate limited first.
|
63
|
+
if expireTime > 0 then return {0, 0, expireTime, 0} end
|
70
64
|
|
71
|
-
|
72
|
-
|
65
|
+
-- paused queue
|
66
|
+
if paused then return {0, 0, 0, 0} end
|
73
67
|
|
74
|
-
|
75
|
-
|
68
|
+
-- no job ID, try non-blocking move from wait to active
|
69
|
+
local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
76
70
|
|
77
|
-
|
71
|
+
-- Markers in waitlist DEPRECATED in v5: Will be completely removed in v6.
|
72
|
+
if jobId and string.sub(jobId, 1, 2) == "0:" then
|
73
|
+
rcall("LREM", activeKey, 1, jobId)
|
78
74
|
jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
79
|
-
|
80
|
-
-- Since it is possible that between a call to BRPOPLPUSH and moveToActive
|
81
|
-
-- another script puts a new maker in wait, we need to check again.
|
82
|
-
if jobId and string.sub(jobId, 1, 2) == "0:" then
|
83
|
-
rcall("LREM", activeKey, 1, jobId)
|
84
|
-
jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
85
|
-
end
|
86
75
|
end
|
87
76
|
|
88
77
|
if jobId then
|
@@ -0,0 +1,77 @@
|
|
1
|
+
--[[
|
2
|
+
Moves job from active to delayed set.
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] marker key
|
6
|
+
KEYS[2] active key
|
7
|
+
KEYS[3] prioritized key
|
8
|
+
KEYS[4] delayed key
|
9
|
+
KEYS[5] job key
|
10
|
+
KEYS[6] events stream
|
11
|
+
KEYS[7] meta key
|
12
|
+
|
13
|
+
ARGV[1] key prefix
|
14
|
+
ARGV[2] timestamp
|
15
|
+
ARGV[3] delayedTimestamp
|
16
|
+
ARGV[4] the id of the job
|
17
|
+
ARGV[5] queue token
|
18
|
+
ARGV[6] delay value
|
19
|
+
ARGV[7] skip attempt
|
20
|
+
|
21
|
+
Output:
|
22
|
+
0 - OK
|
23
|
+
-1 - Missing job.
|
24
|
+
-3 - Job not in active set.
|
25
|
+
|
26
|
+
Events:
|
27
|
+
- delayed key.
|
28
|
+
]]
|
29
|
+
local rcall = redis.call
|
30
|
+
|
31
|
+
-- Includes
|
32
|
+
--- @include "includes/addDelayMarkerIfNeeded"
|
33
|
+
--- @include "includes/isQueuePaused"
|
34
|
+
|
35
|
+
local jobKey = KEYS[5]
|
36
|
+
local metaKey = KEYS[7]
|
37
|
+
if rcall("EXISTS", jobKey) == 1 then
|
38
|
+
local delayedKey = KEYS[4]
|
39
|
+
if ARGV[5] ~= "0" then
|
40
|
+
local lockKey = jobKey .. ':lock'
|
41
|
+
if rcall("GET", lockKey) == ARGV[5] then
|
42
|
+
rcall("DEL", lockKey)
|
43
|
+
else
|
44
|
+
return -2
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
local jobId = ARGV[4]
|
49
|
+
local score = tonumber(ARGV[3])
|
50
|
+
local delayedTimestamp = (score / 0x1000)
|
51
|
+
|
52
|
+
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
53
|
+
if numRemovedElements < 1 then return -3 end
|
54
|
+
|
55
|
+
if ARGV[7] == "0" then
|
56
|
+
rcall("HINCRBY", jobKey, "atm", 1)
|
57
|
+
end
|
58
|
+
|
59
|
+
rcall("HSET", jobKey, "delay", ARGV[6])
|
60
|
+
|
61
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents") or 10000
|
62
|
+
|
63
|
+
rcall("ZADD", delayedKey, score, jobId)
|
64
|
+
rcall("XADD", KEYS[6], "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
65
|
+
"jobId", jobId, "delay", delayedTimestamp)
|
66
|
+
|
67
|
+
-- Check if we need to push a marker job to wake up sleeping workers.
|
68
|
+
local isPaused = isQueuePaused(metaKey)
|
69
|
+
if not isPaused then
|
70
|
+
local markerKey = KEYS[1]
|
71
|
+
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
72
|
+
end
|
73
|
+
|
74
|
+
return 0
|
75
|
+
else
|
76
|
+
return -1
|
77
|
+
end
|
@@ -22,22 +22,21 @@
|
|
22
22
|
KEYS[11] completed/failed key
|
23
23
|
KEYS[12] jobId key
|
24
24
|
KEYS[13] metrics key
|
25
|
+
KEYS[14] marker key
|
25
26
|
|
26
27
|
ARGV[1] jobId
|
27
28
|
ARGV[2] timestamp
|
28
29
|
ARGV[3] msg property returnvalue / failedReason
|
29
30
|
ARGV[4] return value / failed reason
|
30
31
|
ARGV[5] target (completed/failed)
|
31
|
-
ARGV[6]
|
32
|
-
ARGV[7]
|
33
|
-
ARGV[8]
|
34
|
-
ARGV[9] opts
|
32
|
+
ARGV[6] fetch next?
|
33
|
+
ARGV[7] keys prefix
|
34
|
+
ARGV[8] opts
|
35
35
|
|
36
36
|
opts - token - lock token
|
37
37
|
opts - keepJobs
|
38
38
|
opts - lockDuration - lock duration in milliseconds
|
39
39
|
opts - attempts max attempts
|
40
|
-
opts - attemptsMade
|
41
40
|
opts - maxMetricsSize
|
42
41
|
opts - fpof - fail parent on fail
|
43
42
|
opts - rdof - remove dependency on fail
|
@@ -72,11 +71,10 @@ local rcall = redis.call
|
|
72
71
|
|
73
72
|
local jobIdKey = KEYS[12]
|
74
73
|
if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
75
|
-
local opts = cmsgpack.unpack(ARGV[
|
74
|
+
local opts = cmsgpack.unpack(ARGV[8])
|
76
75
|
|
77
76
|
local token = opts['token']
|
78
77
|
local attempts = opts['attempts']
|
79
|
-
local attemptsMade = opts['attemptsMade']
|
80
78
|
local maxMetricsSize = opts['maxMetricsSize']
|
81
79
|
local maxCount = opts['keepJobs']['count']
|
82
80
|
local maxAge = opts['keepJobs']['age']
|
@@ -120,8 +118,9 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
120
118
|
|
121
119
|
if (numRemovedElements < 1) then return -3 end
|
122
120
|
|
121
|
+
local metaKey = KEYS[9]
|
123
122
|
-- Trim events before emiting them to avoid trimming events emitted in this script
|
124
|
-
trimEvents(
|
123
|
+
trimEvents(metaKey, KEYS[4])
|
125
124
|
|
126
125
|
-- If job has a parent we need to
|
127
126
|
-- 1) remove this job id from parents dependencies
|
@@ -144,17 +143,20 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
144
143
|
else
|
145
144
|
if opts['fpof'] then
|
146
145
|
moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
|
147
|
-
|
146
|
+
parentId, jobIdKey,
|
147
|
+
timestamp)
|
148
148
|
elseif opts['rdof'] then
|
149
149
|
local dependenciesSet = parentKey .. ":dependencies"
|
150
150
|
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
151
151
|
moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
|
152
|
-
|
152
|
+
parentKey, parentId, timestamp)
|
153
153
|
end
|
154
154
|
end
|
155
155
|
end
|
156
156
|
end
|
157
157
|
|
158
|
+
local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
|
159
|
+
|
158
160
|
-- Remove job?
|
159
161
|
if maxCount ~= 0 then
|
160
162
|
local targetSet = KEYS[11]
|
@@ -164,7 +166,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
164
166
|
-- "returnvalue" / "failedReason" and "finishedOn"
|
165
167
|
|
166
168
|
-- Remove old jobs?
|
167
|
-
local prefix = ARGV[
|
169
|
+
local prefix = ARGV[7]
|
168
170
|
|
169
171
|
if maxAge ~= nil then
|
170
172
|
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
@@ -197,13 +199,13 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
197
199
|
|
198
200
|
-- Try to get next job to avoid an extra roundtrip if the queue is not closing,
|
199
201
|
-- and not rate limited.
|
200
|
-
if (ARGV[
|
202
|
+
if (ARGV[6] == "1") then
|
201
203
|
|
202
|
-
local target, paused = getTargetQueueList(
|
204
|
+
local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[8])
|
203
205
|
|
204
206
|
-- Check if there are delayed jobs that can be promoted
|
205
|
-
promoteDelayedJobs(KEYS[7], KEYS[
|
206
|
-
|
207
|
+
promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], KEYS[4], ARGV[7],
|
208
|
+
timestamp, KEYS[10], paused)
|
207
209
|
|
208
210
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
209
211
|
-- Check if we are rate limited first.
|
@@ -217,25 +219,30 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
217
219
|
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
218
220
|
|
219
221
|
if jobId then
|
222
|
+
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
220
223
|
if string.sub(jobId, 1, 2) == "0:" then
|
221
224
|
rcall("LREM", KEYS[2], 1, jobId)
|
222
225
|
|
223
226
|
-- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
|
224
227
|
-- but if ID is 0:0, then there is at least 1 prioritized job to process
|
225
228
|
if jobId == "0:0" then
|
226
|
-
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
|
227
|
-
|
228
|
-
|
229
|
+
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
|
230
|
+
KEYS[10])
|
231
|
+
return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
|
232
|
+
timestamp, maxJobs,
|
233
|
+
expireTime, opts)
|
229
234
|
end
|
230
235
|
else
|
231
|
-
return prepareJobForProcessing(KEYS, ARGV[
|
232
|
-
|
236
|
+
return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
|
237
|
+
timestamp, maxJobs, expireTime,
|
238
|
+
opts)
|
233
239
|
end
|
234
240
|
else
|
235
241
|
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
236
242
|
if jobId then
|
237
|
-
return prepareJobForProcessing(KEYS, ARGV[
|
238
|
-
|
243
|
+
return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
|
244
|
+
timestamp, maxJobs, expireTime,
|
245
|
+
opts)
|
239
246
|
end
|
240
247
|
end
|
241
248
|
|
@@ -11,7 +11,7 @@
|
|
11
11
|
ARGV[2] child key
|
12
12
|
ARGV[3] timestamp
|
13
13
|
ARGV[4] the id of the job
|
14
|
-
|
14
|
+
|
15
15
|
Output:
|
16
16
|
0 - OK
|
17
17
|
1 - There are not pending dependencies.
|
@@ -21,7 +21,8 @@
|
|
21
21
|
]]
|
22
22
|
local rcall = redis.call
|
23
23
|
|
24
|
-
local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp,
|
24
|
+
local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp,
|
25
|
+
lockKey, jobKey, token)
|
25
26
|
if token ~= "0" then
|
26
27
|
if rcall("GET", lockKey) == token then
|
27
28
|
rcall("DEL", lockKey)
|
@@ -46,13 +47,15 @@ end
|
|
46
47
|
if rcall("EXISTS", KEYS[4]) == 1 then
|
47
48
|
if ARGV[2] ~= "" then
|
48
49
|
if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
|
49
|
-
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1],
|
50
|
+
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], KEYS[4],
|
51
|
+
ARGV[1])
|
50
52
|
end
|
51
53
|
|
52
54
|
return 1
|
53
55
|
else
|
54
56
|
if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
|
55
|
-
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1],
|
57
|
+
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], KEYS[4],
|
58
|
+
ARGV[1])
|
56
59
|
end
|
57
60
|
|
58
61
|
return 1
|
@@ -0,0 +1,41 @@
|
|
1
|
+
--[[
|
2
|
+
Pauses or resumes a queue globably.
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] 'wait' or 'paused''
|
6
|
+
KEYS[2] 'paused' or 'wait'
|
7
|
+
KEYS[3] 'meta'
|
8
|
+
KEYS[4] 'prioritized'
|
9
|
+
KEYS[5] events stream key
|
10
|
+
KEYS[6] 'delayed'
|
11
|
+
KEYS|7] 'marker'
|
12
|
+
|
13
|
+
ARGV[1] 'paused' or 'resumed'
|
14
|
+
|
15
|
+
Event:
|
16
|
+
publish paused or resumed event.
|
17
|
+
]]
|
18
|
+
local rcall = redis.call
|
19
|
+
|
20
|
+
-- Includes
|
21
|
+
--- @include "includes/addDelayMarkerIfNeeded"
|
22
|
+
|
23
|
+
local markerKey = KEYS[7]
|
24
|
+
local hasJobs = rcall("EXISTS", KEYS[1]) == 1
|
25
|
+
if hasJobs then rcall("RENAME", KEYS[1], KEYS[2]) end
|
26
|
+
|
27
|
+
if ARGV[1] == "paused" then
|
28
|
+
rcall("HSET", KEYS[3], "paused", 1)
|
29
|
+
rcall("DEL", markerKey)
|
30
|
+
else
|
31
|
+
rcall("HDEL", KEYS[3], "paused")
|
32
|
+
|
33
|
+
if hasJobs or rcall("ZCARD", KEYS[4]) > 0 then
|
34
|
+
-- Add marker if there are waiting or priority jobs
|
35
|
+
rcall("ZADD", markerKey, 0, "0")
|
36
|
+
else
|
37
|
+
addDelayMarkerIfNeeded(markerKey, KEYS[6])
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
rcall("XADD", KEYS[5], "*", "event", ARGV[1]);
|
@@ -0,0 +1,60 @@
|
|
1
|
+
--[[
|
2
|
+
Promotes a job that is currently "delayed" to the "waiting" state
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] 'delayed'
|
6
|
+
KEYS[2] 'wait'
|
7
|
+
KEYS[3] 'paused'
|
8
|
+
KEYS[4] 'meta'
|
9
|
+
KEYS[5] 'prioritized'
|
10
|
+
KEYS[6] 'pc' priority counter
|
11
|
+
KEYS[7] 'event stream'
|
12
|
+
KEYS[8] 'marker'
|
13
|
+
|
14
|
+
ARGV[1] queue.toKey('')
|
15
|
+
ARGV[2] jobId
|
16
|
+
|
17
|
+
Output:
|
18
|
+
0 - OK
|
19
|
+
-3 - Job not in delayed zset.
|
20
|
+
|
21
|
+
Events:
|
22
|
+
'waiting'
|
23
|
+
]]
|
24
|
+
local rcall = redis.call
|
25
|
+
local jobId = ARGV[2]
|
26
|
+
|
27
|
+
-- Includes
|
28
|
+
--- @include "includes/addJobWithPriority"
|
29
|
+
--- @include "includes/getTargetQueueList"
|
30
|
+
|
31
|
+
if rcall("ZREM", KEYS[1], jobId) == 1 then
|
32
|
+
local jobKey = ARGV[1] .. jobId
|
33
|
+
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
|
34
|
+
local metaKey = KEYS[4]
|
35
|
+
|
36
|
+
-- Remove delayed "marker" from the wait list if there is any.
|
37
|
+
-- Since we are adding a job we do not need the marker anymore.
|
38
|
+
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
39
|
+
local target, paused = getTargetQueueList(metaKey, KEYS[2], KEYS[3])
|
40
|
+
local marker = rcall("LINDEX", target, 0)
|
41
|
+
if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
|
42
|
+
|
43
|
+
if priority == 0 then
|
44
|
+
-- LIFO or FIFO
|
45
|
+
rcall("LPUSH", target, jobId)
|
46
|
+
if not paused then rcall("ZADD", KEYS[8], 0, "0") end
|
47
|
+
else
|
48
|
+
addJobWithPriority(KEYS[8], KEYS[5], priority, jobId, KEYS[6], paused)
|
49
|
+
end
|
50
|
+
|
51
|
+
-- Emit waiting event (wait..ing@token)
|
52
|
+
rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev",
|
53
|
+
"delayed");
|
54
|
+
|
55
|
+
rcall("HSET", jobKey, "delay", 0)
|
56
|
+
|
57
|
+
return 0
|
58
|
+
else
|
59
|
+
return -3
|
60
|
+
end
|