bullmq 5.8.6 → 5.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/queue.js +29 -0
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/scripts.js +17 -1
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +1 -1
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addDelayedJob-6.lua +2 -6
- package/dist/cjs/commands/{addPrioritizedJob-7.lua → addPrioritizedJob-8.lua} +9 -7
- package/dist/cjs/commands/{addStandardJob-7.lua → addStandardJob-8.lua} +6 -5
- package/dist/cjs/commands/changeDelay-4.lua +1 -5
- package/dist/cjs/commands/{changePriority-6.lua → changePriority-7.lua} +12 -10
- package/dist/cjs/commands/getCountsPerPriority-4.lua +6 -3
- package/dist/cjs/commands/includes/addBaseMarkerIfNeeded.lua +2 -2
- package/dist/cjs/commands/includes/addJobInTargetList.lua +2 -2
- package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -2
- package/dist/cjs/commands/includes/getTargetQueueList.lua +14 -4
- package/dist/cjs/commands/includes/isQueueMaxed.lua +15 -0
- package/dist/cjs/commands/includes/isQueuePaused.lua +1 -1
- package/dist/cjs/commands/includes/isQueuePausedOrMaxed.lua +18 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +8 -6
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +3 -3
- package/dist/cjs/commands/isMaxed-2.lua +17 -0
- package/dist/cjs/commands/moveJobFromActiveToWait-10.lua +2 -2
- package/dist/{esm/commands/moveJobsToWait-7.lua → cjs/commands/moveJobsToWait-8.lua} +4 -3
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +3 -3
- package/dist/cjs/commands/moveToActive-11.lua +4 -4
- package/dist/cjs/commands/moveToDelayed-8.lua +2 -6
- package/dist/cjs/commands/moveToFinished-14.lua +4 -4
- package/dist/cjs/commands/moveToWaitingChildren-5.lua +4 -3
- package/dist/{esm/commands/promote-8.lua → cjs/commands/promote-9.lua} +9 -7
- package/dist/{esm/commands/removeJob-1.lua → cjs/commands/removeJob-2.lua} +2 -1
- package/dist/{esm/commands/reprocessJob-7.lua → cjs/commands/reprocessJob-8.lua} +4 -3
- package/dist/cjs/commands/retryJob-11.lua +8 -4
- package/dist/cjs/scripts/addDelayedJob-6.js +41 -23
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +39 -18
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/{addPrioritizedJob-7.js → addPrioritizedJob-8.js} +49 -26
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -0
- package/dist/cjs/scripts/{addStandardJob-7.js → addStandardJob-8.js} +47 -25
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -0
- package/dist/cjs/scripts/changeDelay-4.js +1 -11
- package/dist/cjs/scripts/changeDelay-4.js.map +1 -1
- package/dist/cjs/scripts/{changePriority-6.js → changePriority-7.js} +33 -21
- package/dist/cjs/scripts/changePriority-7.js.map +1 -0
- package/dist/cjs/scripts/cleanJobsInSet-2.js +20 -11
- package/dist/cjs/scripts/cleanJobsInSet-2.js.map +1 -1
- package/dist/cjs/scripts/drain-4.js +20 -11
- package/dist/cjs/scripts/drain-4.js.map +1 -1
- package/dist/cjs/scripts/getCountsPerPriority-4.js +7 -8
- package/dist/cjs/scripts/getCountsPerPriority-4.js.map +1 -1
- package/dist/cjs/scripts/index.js +8 -7
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/isMaxed-2.js +34 -0
- package/dist/cjs/scripts/isMaxed-2.js.map +1 -0
- package/dist/cjs/scripts/moveJobFromActiveToWait-10.js +19 -10
- package/dist/cjs/scripts/moveJobFromActiveToWait-10.js.map +1 -1
- package/dist/cjs/scripts/{moveJobsToWait-7.js → moveJobsToWait-8.js} +21 -11
- package/dist/cjs/scripts/moveJobsToWait-8.js.map +1 -0
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +45 -24
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToActive-11.js +24 -14
- package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
- package/dist/cjs/scripts/moveToDelayed-8.js +2 -12
- package/dist/cjs/scripts/moveToDelayed-8.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +46 -25
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-5.js +4 -3
- package/dist/cjs/scripts/moveToWaitingChildren-5.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +20 -11
- package/dist/cjs/scripts/obliterate-2.js.map +1 -1
- package/dist/cjs/scripts/{promote-8.js → promote-9.js} +31 -19
- package/dist/cjs/scripts/promote-9.js.map +1 -0
- package/dist/cjs/scripts/removeChildDependency-1.js +20 -11
- package/dist/cjs/scripts/removeChildDependency-1.js.map +1 -1
- package/dist/cjs/scripts/{removeJob-1.js → removeJob-2.js} +24 -14
- package/dist/cjs/scripts/{removeJob-1.js.map → removeJob-2.js.map} +1 -1
- package/dist/cjs/scripts/{reprocessJob-7.js → reprocessJob-8.js} +23 -13
- package/dist/cjs/scripts/reprocessJob-8.js.map +1 -0
- package/dist/cjs/scripts/retryJob-11.js +42 -14
- package/dist/cjs/scripts/retryJob-11.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/esm/classes/queue.d.ts +17 -0
- package/dist/esm/classes/queue.js +29 -0
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +2 -0
- package/dist/esm/classes/scripts.js +17 -1
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.js +1 -1
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addDelayedJob-6.lua +2 -6
- package/dist/esm/commands/{addPrioritizedJob-7.lua → addPrioritizedJob-8.lua} +9 -7
- package/dist/esm/commands/{addStandardJob-7.lua → addStandardJob-8.lua} +6 -5
- package/dist/esm/commands/changeDelay-4.lua +1 -5
- package/dist/esm/commands/{changePriority-6.lua → changePriority-7.lua} +12 -10
- package/dist/esm/commands/getCountsPerPriority-4.lua +6 -3
- package/dist/esm/commands/includes/addBaseMarkerIfNeeded.lua +2 -2
- package/dist/esm/commands/includes/addJobInTargetList.lua +2 -2
- package/dist/esm/commands/includes/addJobWithPriority.lua +3 -2
- package/dist/esm/commands/includes/getTargetQueueList.lua +14 -4
- package/dist/esm/commands/includes/isQueueMaxed.lua +15 -0
- package/dist/esm/commands/includes/isQueuePaused.lua +1 -1
- package/dist/esm/commands/includes/isQueuePausedOrMaxed.lua +18 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +8 -6
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +3 -3
- package/dist/esm/commands/isMaxed-2.lua +17 -0
- package/dist/esm/commands/moveJobFromActiveToWait-10.lua +2 -2
- package/dist/{cjs/commands/moveJobsToWait-7.lua → esm/commands/moveJobsToWait-8.lua} +4 -3
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +3 -3
- package/dist/esm/commands/moveToActive-11.lua +4 -4
- package/dist/esm/commands/moveToDelayed-8.lua +2 -6
- package/dist/esm/commands/moveToFinished-14.lua +4 -4
- package/dist/esm/commands/moveToWaitingChildren-5.lua +4 -3
- package/dist/{cjs/commands/promote-8.lua → esm/commands/promote-9.lua} +9 -7
- package/dist/{cjs/commands/removeJob-1.lua → esm/commands/removeJob-2.lua} +2 -1
- package/dist/{cjs/commands/reprocessJob-7.lua → esm/commands/reprocessJob-8.lua} +4 -3
- package/dist/esm/commands/retryJob-11.lua +8 -4
- package/dist/esm/scripts/addDelayedJob-6.js +41 -23
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addParentJob-4.js +39 -18
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/{addPrioritizedJob-7.js → addPrioritizedJob-8.js} +49 -26
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -0
- package/dist/esm/scripts/{addStandardJob-7.js → addStandardJob-8.js} +47 -25
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -0
- package/dist/esm/scripts/changeDelay-4.js +1 -11
- package/dist/esm/scripts/changeDelay-4.js.map +1 -1
- package/dist/esm/scripts/{changePriority-6.js → changePriority-7.js} +33 -21
- package/dist/esm/scripts/changePriority-7.js.map +1 -0
- package/dist/esm/scripts/cleanJobsInSet-2.js +20 -11
- package/dist/esm/scripts/cleanJobsInSet-2.js.map +1 -1
- package/dist/esm/scripts/drain-4.js +20 -11
- package/dist/esm/scripts/drain-4.js.map +1 -1
- package/dist/esm/scripts/getCountsPerPriority-4.js +7 -8
- package/dist/esm/scripts/getCountsPerPriority-4.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +8 -7
- package/dist/esm/scripts/index.js +8 -7
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/isMaxed-2.d.ts +5 -0
- package/dist/esm/scripts/isMaxed-2.js +31 -0
- package/dist/esm/scripts/isMaxed-2.js.map +1 -0
- package/dist/esm/scripts/moveJobFromActiveToWait-10.js +19 -10
- package/dist/esm/scripts/moveJobFromActiveToWait-10.js.map +1 -1
- package/dist/esm/scripts/{moveJobsToWait-7.js → moveJobsToWait-8.js} +21 -11
- package/dist/esm/scripts/moveJobsToWait-8.js.map +1 -0
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +45 -24
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToActive-11.js +24 -14
- package/dist/esm/scripts/moveToActive-11.js.map +1 -1
- package/dist/esm/scripts/moveToDelayed-8.js +2 -12
- package/dist/esm/scripts/moveToDelayed-8.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +46 -25
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-5.js +4 -3
- package/dist/esm/scripts/moveToWaitingChildren-5.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +20 -11
- package/dist/esm/scripts/obliterate-2.js.map +1 -1
- package/dist/esm/scripts/{promote-8.js → promote-9.js} +31 -19
- package/dist/esm/scripts/promote-9.js.map +1 -0
- package/dist/esm/scripts/removeChildDependency-1.js +20 -11
- package/dist/esm/scripts/removeChildDependency-1.js.map +1 -1
- package/dist/esm/scripts/{removeJob-1.js → removeJob-2.js} +24 -14
- package/dist/esm/scripts/{removeJob-1.js.map → removeJob-2.js.map} +1 -1
- package/dist/esm/scripts/{reprocessJob-7.js → reprocessJob-8.js} +23 -13
- package/dist/esm/scripts/reprocessJob-8.js.map +1 -0
- package/dist/esm/scripts/retryJob-11.js +42 -14
- package/dist/esm/scripts/retryJob-11.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -1
- package/dist/cjs/scripts/addPrioritizedJob-7.js.map +0 -1
- package/dist/cjs/scripts/addStandardJob-7.js.map +0 -1
- package/dist/cjs/scripts/changePriority-6.js.map +0 -1
- package/dist/cjs/scripts/moveJobsToWait-7.js.map +0 -1
- package/dist/cjs/scripts/promote-8.js.map +0 -1
- package/dist/cjs/scripts/reprocessJob-7.js.map +0 -1
- package/dist/esm/scripts/addPrioritizedJob-7.js.map +0 -1
- package/dist/esm/scripts/addStandardJob-7.js.map +0 -1
- package/dist/esm/scripts/changePriority-6.js.map +0 -1
- package/dist/esm/scripts/moveJobsToWait-7.js.map +0 -1
- package/dist/esm/scripts/promote-8.js.map +0 -1
- package/dist/esm/scripts/reprocessJob-7.js.map +0 -1
- /package/dist/esm/scripts/{addPrioritizedJob-7.d.ts → addPrioritizedJob-8.d.ts} +0 -0
- /package/dist/esm/scripts/{addStandardJob-7.d.ts → addStandardJob-8.d.ts} +0 -0
- /package/dist/esm/scripts/{changePriority-6.d.ts → changePriority-7.d.ts} +0 -0
- /package/dist/esm/scripts/{moveJobsToWait-7.d.ts → moveJobsToWait-8.d.ts} +0 -0
- /package/dist/esm/scripts/{promote-8.d.ts → promote-9.d.ts} +0 -0
- /package/dist/esm/scripts/{removeJob-1.d.ts → removeJob-2.d.ts} +0 -0
- /package/dist/esm/scripts/{reprocessJob-7.d.ts → reprocessJob-8.d.ts} +0 -0
@@ -3,10 +3,20 @@
|
|
3
3
|
(since an empty list and !EXISTS are not really the same).
|
4
4
|
]]
|
5
5
|
|
6
|
-
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
7
|
-
|
8
|
-
|
9
|
-
|
6
|
+
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
7
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
8
|
+
|
9
|
+
if queueAttributes[1] then
|
10
10
|
return pausedKey, true
|
11
|
+
else
|
12
|
+
if queueAttributes[2] then
|
13
|
+
local activeCount = rcall("LLEN", activeKey)
|
14
|
+
if activeCount >= tonumber(queueAttributes[2]) then
|
15
|
+
return waitKey, true
|
16
|
+
else
|
17
|
+
return waitKey, false
|
18
|
+
end
|
19
|
+
end
|
11
20
|
end
|
21
|
+
return waitKey, false
|
12
22
|
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
--[[
|
2
|
+
Function to check if queue is maxed or not.
|
3
|
+
]]
|
4
|
+
local function isQueueMaxed(queueMetaKey, activeKey)
|
5
|
+
local maxConcurrency = rcall("HGET", queueMetaKey, "concurrency")
|
6
|
+
|
7
|
+
if maxConcurrency then
|
8
|
+
local activeCount = rcall("LLEN", activeKey)
|
9
|
+
if activeCount >= tonumber(maxConcurrency) then
|
10
|
+
return true
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
return false
|
15
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
--[[
|
2
|
+
Function to check if queue is paused or maxed
|
3
|
+
(since an empty list and !EXISTS are not really the same).
|
4
|
+
]]
|
5
|
+
|
6
|
+
local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
7
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
8
|
+
|
9
|
+
if queueAttributes[1] then
|
10
|
+
return true
|
11
|
+
else
|
12
|
+
if queueAttributes[2] then
|
13
|
+
local activeCount = rcall("LLEN", activeKey)
|
14
|
+
return activeCount >= tonumber(queueAttributes[2])
|
15
|
+
end
|
16
|
+
end
|
17
|
+
return false
|
18
|
+
end
|
@@ -6,7 +6,7 @@
|
|
6
6
|
--- @include "addDelayMarkerIfNeeded"
|
7
7
|
--- @include "addJobInTargetList"
|
8
8
|
--- @include "addJobWithPriority"
|
9
|
-
--- @include "
|
9
|
+
--- @include "isQueuePausedOrMaxed"
|
10
10
|
--- @include "getTargetQueueList"
|
11
11
|
|
12
12
|
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
@@ -17,6 +17,7 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
17
17
|
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
18
18
|
local parentWaitKey = parentQueueKey .. ":wait"
|
19
19
|
local parentPausedKey = parentQueueKey .. ":paused"
|
20
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
20
21
|
local parentMetaKey = parentQueueKey .. ":meta"
|
21
22
|
|
22
23
|
local parentMarkerKey = parentQueueKey .. ":marker"
|
@@ -35,15 +36,16 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
35
36
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
36
37
|
else
|
37
38
|
if priority == 0 then
|
38
|
-
local parentTarget,
|
39
|
-
getTargetQueueList(parentMetaKey, parentWaitKey,
|
39
|
+
local parentTarget, isParentPausedOrMaxed =
|
40
|
+
getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
|
40
41
|
parentPausedKey)
|
41
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH",
|
42
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
43
|
+
parentId)
|
42
44
|
else
|
43
|
-
local
|
45
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
44
46
|
addJobWithPriority(parentMarkerKey,
|
45
47
|
parentQueueKey .. ":prioritized", priority,
|
46
|
-
parentId, parentQueueKey .. ":pc",
|
48
|
+
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
47
49
|
end
|
48
50
|
|
49
51
|
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
@@ -11,9 +11,9 @@
|
|
11
11
|
--- @include "removeJobKeys"
|
12
12
|
|
13
13
|
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
14
|
-
local parentTarget,
|
15
|
-
parentPrefix .. "paused")
|
16
|
-
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH",
|
14
|
+
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
15
|
+
parentPrefix .. "wait", parentPrefix .. "paused")
|
16
|
+
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
17
17
|
|
18
18
|
if emitEvent then
|
19
19
|
local parentEventStream = parentPrefix .. "events"
|
@@ -0,0 +1,17 @@
|
|
1
|
+
--[[
|
2
|
+
Checks if queue is maxed.
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] meta key
|
6
|
+
KEYS[2] active key
|
7
|
+
|
8
|
+
Output:
|
9
|
+
1 if element found in the list.
|
10
|
+
]]
|
11
|
+
|
12
|
+
local rcall = redis.call
|
13
|
+
|
14
|
+
-- Includes
|
15
|
+
--- @include "includes/isQueueMaxed"
|
16
|
+
|
17
|
+
return isQueueMaxed(KEYS[1], KEYS[2])
|
@@ -35,7 +35,7 @@ if lockToken == token then
|
|
35
35
|
local metaKey = KEYS[6]
|
36
36
|
local removed = rcall("LREM", KEYS[1], 1, jobId)
|
37
37
|
if removed > 0 then
|
38
|
-
local target,
|
38
|
+
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[1], KEYS[2], KEYS[5])
|
39
39
|
|
40
40
|
rcall("SREM", KEYS[3], jobId)
|
41
41
|
|
@@ -44,7 +44,7 @@ if lockToken == token then
|
|
44
44
|
if priority > 0 then
|
45
45
|
pushBackJobWithPriority(KEYS[8], priority, jobId)
|
46
46
|
else
|
47
|
-
addJobInTargetList(target, KEYS[9], "RPUSH",
|
47
|
+
addJobInTargetList(target, KEYS[9], "RPUSH", isPausedOrMaxed, jobId)
|
48
48
|
end
|
49
49
|
|
50
50
|
rcall("DEL", lockKey)
|
@@ -10,7 +10,8 @@
|
|
10
10
|
KEYS[4] 'wait'
|
11
11
|
KEYS[5] 'paused'
|
12
12
|
KEYS[6] 'meta'
|
13
|
-
KEYS[7] '
|
13
|
+
KEYS[7] 'active'
|
14
|
+
KEYS[8] 'marker'
|
14
15
|
|
15
16
|
ARGV[1] count
|
16
17
|
ARGV[2] timestamp
|
@@ -32,7 +33,7 @@ local rcall = redis.call;
|
|
32
33
|
--- @include "includes/getTargetQueueList"
|
33
34
|
|
34
35
|
local metaKey = KEYS[6]
|
35
|
-
local target,
|
36
|
+
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[7], KEYS[4], KEYS[5])
|
36
37
|
|
37
38
|
local jobs = rcall('ZRANGEBYSCORE', KEYS[3], 0, timestamp, 'LIMIT', 0, maxCount)
|
38
39
|
if (#jobs > 0) then
|
@@ -62,7 +63,7 @@ if (#jobs > 0) then
|
|
62
63
|
rcall("LPUSH", target, unpack(jobs, from, to))
|
63
64
|
end
|
64
65
|
|
65
|
-
addBaseMarkerIfNeeded(KEYS[
|
66
|
+
addBaseMarkerIfNeeded(KEYS[8], isPausedOrMaxed)
|
66
67
|
end
|
67
68
|
|
68
69
|
maxCount = maxCount - #jobs
|
@@ -142,11 +142,11 @@ if (#stalling > 0) then
|
|
142
142
|
|
143
143
|
table.insert(failed, jobId)
|
144
144
|
else
|
145
|
-
local target,
|
146
|
-
getTargetQueueList(metaKey, waitKey, pausedKey)
|
145
|
+
local target, isPausedOrMaxed=
|
146
|
+
getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
|
147
147
|
|
148
148
|
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
|
149
|
-
addJobInTargetList(target, markerKey, "RPUSH",
|
149
|
+
addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
|
150
150
|
|
151
151
|
rcall("XADD", eventStreamKey, "*", "event",
|
152
152
|
"waiting", "jobId", jobId, 'prev', 'active')
|
@@ -50,12 +50,12 @@ local opts = cmsgpack.unpack(ARGV[3])
|
|
50
50
|
--- @include "includes/prepareJobForProcessing"
|
51
51
|
--- @include "includes/promoteDelayedJobs"
|
52
52
|
|
53
|
-
local target,
|
53
|
+
local target, isPausedOrMaxed = getTargetQueueList(KEYS[9], activeKey, waitKey, KEYS[8])
|
54
54
|
|
55
55
|
-- Check if there are delayed jobs that we can move to wait.
|
56
56
|
local markerKey = KEYS[11]
|
57
57
|
promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
|
58
|
-
ARGV[2], KEYS[10],
|
58
|
+
ARGV[2], KEYS[10], isPausedOrMaxed)
|
59
59
|
|
60
60
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
61
61
|
local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
|
@@ -63,8 +63,8 @@ local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
|
|
63
63
|
-- Check if we are rate limited first.
|
64
64
|
if expireTime > 0 then return {0, 0, expireTime, 0} end
|
65
65
|
|
66
|
-
-- paused queue
|
67
|
-
if
|
66
|
+
-- paused or maxed queue
|
67
|
+
if isPausedOrMaxed then return {0, 0, 0, 0} end
|
68
68
|
|
69
69
|
-- no job ID, try non-blocking move from wait to active
|
70
70
|
local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
@@ -32,7 +32,6 @@ local rcall = redis.call
|
|
32
32
|
--- @include "includes/addDelayMarkerIfNeeded"
|
33
33
|
--- @include "includes/getDelayedScore"
|
34
34
|
--- @include "includes/getOrSetMaxEvents"
|
35
|
-
--- @include "includes/isQueuePaused"
|
36
35
|
--- @include "includes/removeLock"
|
37
36
|
|
38
37
|
local jobKey = KEYS[5]
|
@@ -65,11 +64,8 @@ if rcall("EXISTS", jobKey) == 1 then
|
|
65
64
|
"jobId", jobId, "delay", delayedTimestamp)
|
66
65
|
|
67
66
|
-- Check if we need to push a marker job to wake up sleeping workers.
|
68
|
-
local
|
69
|
-
|
70
|
-
local markerKey = KEYS[1]
|
71
|
-
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
72
|
-
end
|
67
|
+
local markerKey = KEYS[1]
|
68
|
+
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
73
69
|
|
74
70
|
return 0
|
75
71
|
else
|
@@ -202,11 +202,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
202
202
|
-- and not rate limited.
|
203
203
|
if (ARGV[6] == "1") then
|
204
204
|
|
205
|
-
local target,
|
205
|
+
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
|
206
206
|
|
207
207
|
-- Check if there are delayed jobs that can be promoted
|
208
208
|
promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], eventStreamKey, ARGV[7],
|
209
|
-
timestamp, KEYS[10],
|
209
|
+
timestamp, KEYS[10], isPausedOrMaxed)
|
210
210
|
|
211
211
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
212
212
|
-- Check if we are rate limited first.
|
@@ -214,8 +214,8 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
214
214
|
|
215
215
|
if expireTime > 0 then return {0, 0, expireTime, 0} end
|
216
216
|
|
217
|
-
-- paused queue
|
218
|
-
if
|
217
|
+
-- paused or maxed queue
|
218
|
+
if isPausedOrMaxed then return {0, 0, 0, 0} end
|
219
219
|
|
220
220
|
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
221
221
|
|
@@ -21,8 +21,9 @@
|
|
21
21
|
-3 - Job not in active set
|
22
22
|
]]
|
23
23
|
local rcall = redis.call
|
24
|
+
local stalledKey = KEYS[5]
|
24
25
|
|
25
|
-
|
26
|
+
--- Includes
|
26
27
|
--- @include "includes/removeLock"
|
27
28
|
|
28
29
|
local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId,
|
@@ -43,7 +44,7 @@ end
|
|
43
44
|
if rcall("EXISTS", KEYS[4]) == 1 then
|
44
45
|
if ARGV[2] ~= "" then
|
45
46
|
if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
|
46
|
-
local errorCode = removeLock(KEYS[4],
|
47
|
+
local errorCode = removeLock(KEYS[4], stalledKey, ARGV[1], ARGV[4])
|
47
48
|
if errorCode < 0 then
|
48
49
|
return errorCode
|
49
50
|
end
|
@@ -53,7 +54,7 @@ if rcall("EXISTS", KEYS[4]) == 1 then
|
|
53
54
|
return 1
|
54
55
|
else
|
55
56
|
if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
|
56
|
-
local errorCode = removeLock(KEYS[4],
|
57
|
+
local errorCode = removeLock(KEYS[4], stalledKey, ARGV[1], ARGV[4])
|
57
58
|
if errorCode < 0 then
|
58
59
|
return errorCode
|
59
60
|
end
|
@@ -7,9 +7,10 @@
|
|
7
7
|
KEYS[3] 'paused'
|
8
8
|
KEYS[4] 'meta'
|
9
9
|
KEYS[5] 'prioritized'
|
10
|
-
KEYS[6] '
|
11
|
-
KEYS[7] '
|
12
|
-
KEYS[8] '
|
10
|
+
KEYS[6] 'active'
|
11
|
+
KEYS[7] 'pc' priority counter
|
12
|
+
KEYS[8] 'event stream'
|
13
|
+
KEYS[9] 'marker'
|
13
14
|
|
14
15
|
ARGV[1] queue.toKey('')
|
15
16
|
ARGV[2] jobId
|
@@ -33,23 +34,24 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then
|
|
33
34
|
local jobKey = ARGV[1] .. jobId
|
34
35
|
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
|
35
36
|
local metaKey = KEYS[4]
|
37
|
+
local markerKey = KEYS[9]
|
36
38
|
|
37
39
|
-- Remove delayed "marker" from the wait list if there is any.
|
38
40
|
-- Since we are adding a job we do not need the marker anymore.
|
39
41
|
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
40
|
-
local target,
|
42
|
+
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[6], KEYS[2], KEYS[3])
|
41
43
|
local marker = rcall("LINDEX", target, 0)
|
42
44
|
if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
|
43
45
|
|
44
46
|
if priority == 0 then
|
45
47
|
-- LIFO or FIFO
|
46
|
-
addJobInTargetList(target,
|
48
|
+
addJobInTargetList(target, markerKey, "LPUSH", isPausedOrMaxed, jobId)
|
47
49
|
else
|
48
|
-
addJobWithPriority(
|
50
|
+
addJobWithPriority(markerKey, KEYS[5], priority, jobId, KEYS[7], isPausedOrMaxed)
|
49
51
|
end
|
50
52
|
|
51
53
|
-- Emit waiting event (wait..ing@token)
|
52
|
-
rcall("XADD", KEYS[
|
54
|
+
rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId, "prev",
|
53
55
|
"delayed");
|
54
56
|
|
55
57
|
rcall("HSET", jobKey, "delay", 0)
|
@@ -4,6 +4,7 @@
|
|
4
4
|
|
5
5
|
Input:
|
6
6
|
KEYS[1] queue prefix
|
7
|
+
KEYS[2] meta key
|
7
8
|
|
8
9
|
ARGV[1] jobId
|
9
10
|
ARGV[2] remove children
|
@@ -66,7 +67,7 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
|
|
66
67
|
local prev = removeJobFromAnyState(prefix, jobId)
|
67
68
|
|
68
69
|
if removeJobKeys(jobKey) > 0 then
|
69
|
-
local maxEvents = getOrSetMaxEvents(
|
70
|
+
local maxEvents = getOrSetMaxEvents(KEYS[2])
|
70
71
|
rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
|
71
72
|
"jobId", jobId, "prev", prev)
|
72
73
|
end
|
@@ -8,7 +8,8 @@
|
|
8
8
|
KEYS[4] wait key
|
9
9
|
KEYS[5] meta
|
10
10
|
KEYS[6] paused key
|
11
|
-
KEYS[7]
|
11
|
+
KEYS[7] active key
|
12
|
+
KEYS[8] marker key
|
12
13
|
|
13
14
|
ARGV[1] job.id
|
14
15
|
ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
|
@@ -32,8 +33,8 @@ if rcall("EXISTS", KEYS[1]) == 1 then
|
|
32
33
|
if (rcall("ZREM", KEYS[3], jobId) == 1) then
|
33
34
|
rcall("HDEL", KEYS[1], "finishedOn", "processedOn", ARGV[3])
|
34
35
|
|
35
|
-
local target,
|
36
|
-
addJobInTargetList(target, KEYS[
|
36
|
+
local target, isPausedOrMaxed = getTargetQueueList(KEYS[5], KEYS[7], KEYS[4], KEYS[6])
|
37
|
+
addJobInTargetList(target, KEYS[8], ARGV[2], isPausedOrMaxed, jobId)
|
37
38
|
|
38
39
|
local maxEvents = getOrSetMaxEvents(KEYS[5])
|
39
40
|
-- Emit waiting event
|
@@ -38,13 +38,14 @@ local rcall = redis.call
|
|
38
38
|
--- @include "includes/getTargetQueueList"
|
39
39
|
--- @include "includes/promoteDelayedJobs"
|
40
40
|
--- @include "includes/removeLock"
|
41
|
+
--- @include "includes/isQueuePausedOrMaxed"
|
41
42
|
|
42
|
-
local target,
|
43
|
+
local target, isPausedOrMaxed = getTargetQueueList(KEYS[5], KEYS[1], KEYS[2], KEYS[3])
|
43
44
|
local markerKey = KEYS[10]
|
44
45
|
|
45
46
|
-- Check if there are delayed jobs that we can move to wait.
|
46
47
|
-- test example: when there are delayed jobs between retries
|
47
|
-
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9],
|
48
|
+
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], isPausedOrMaxed)
|
48
49
|
|
49
50
|
if rcall("EXISTS", KEYS[4]) == 1 then
|
50
51
|
local errorCode = removeLock(KEYS[4], KEYS[11], ARGV[5], ARGV[4])
|
@@ -57,11 +58,14 @@ if rcall("EXISTS", KEYS[4]) == 1 then
|
|
57
58
|
|
58
59
|
local priority = tonumber(rcall("HGET", KEYS[4], "priority")) or 0
|
59
60
|
|
61
|
+
--need to re-evaluate after removing job from active
|
62
|
+
isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[5], KEYS[1])
|
63
|
+
|
60
64
|
-- Standard or priority add
|
61
65
|
if priority == 0 then
|
62
|
-
addJobInTargetList(target,
|
66
|
+
addJobInTargetList(target, markerKey, ARGV[3], isPausedOrMaxed, ARGV[4])
|
63
67
|
else
|
64
|
-
addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9],
|
68
|
+
addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], isPausedOrMaxed)
|
65
69
|
end
|
66
70
|
|
67
71
|
rcall("HINCRBY", KEYS[4], "atm", 1)
|
@@ -131,42 +131,61 @@ end
|
|
131
131
|
--[[
|
132
132
|
Add marker if needed when a job is available.
|
133
133
|
]]
|
134
|
-
local function addBaseMarkerIfNeeded(markerKey,
|
135
|
-
if not
|
134
|
+
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
135
|
+
if not isPausedOrMaxed then
|
136
136
|
rcall("ZADD", markerKey, 0, "0")
|
137
137
|
end
|
138
138
|
end
|
139
|
-
local function addJobInTargetList(targetKey, markerKey, pushCmd,
|
139
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
140
140
|
rcall(pushCmd, targetKey, jobId)
|
141
|
-
addBaseMarkerIfNeeded(markerKey,
|
141
|
+
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
142
142
|
end
|
143
143
|
--[[
|
144
144
|
Function to add job considering priority.
|
145
145
|
]]
|
146
146
|
-- Includes
|
147
|
-
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
|
147
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
|
148
|
+
isPausedOrMaxed)
|
148
149
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
149
150
|
local score = priority * 0x100000000 + prioCounter % 0x100000000
|
150
151
|
rcall("ZADD", prioritizedKey, score, jobId)
|
151
|
-
addBaseMarkerIfNeeded(markerKey,
|
152
|
+
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
152
153
|
end
|
153
154
|
--[[
|
154
|
-
Function to check
|
155
|
+
Function to check if queue is paused or maxed
|
155
156
|
(since an empty list and !EXISTS are not really the same).
|
156
157
|
]]
|
157
|
-
local function
|
158
|
-
|
158
|
+
local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
159
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
160
|
+
if queueAttributes[1] then
|
161
|
+
return true
|
162
|
+
else
|
163
|
+
if queueAttributes[2] then
|
164
|
+
local activeCount = rcall("LLEN", activeKey)
|
165
|
+
return activeCount >= tonumber(queueAttributes[2])
|
166
|
+
end
|
167
|
+
end
|
168
|
+
return false
|
159
169
|
end
|
160
170
|
--[[
|
161
171
|
Function to check for the meta.paused key to decide if we are paused or not
|
162
172
|
(since an empty list and !EXISTS are not really the same).
|
163
173
|
]]
|
164
|
-
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
165
|
-
|
166
|
-
|
167
|
-
else
|
174
|
+
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
175
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
176
|
+
if queueAttributes[1] then
|
168
177
|
return pausedKey, true
|
178
|
+
else
|
179
|
+
if queueAttributes[2] then
|
180
|
+
local activeCount = rcall("LLEN", activeKey)
|
181
|
+
if activeCount >= tonumber(queueAttributes[2]) then
|
182
|
+
return waitKey, true
|
183
|
+
else
|
184
|
+
return waitKey, false
|
185
|
+
end
|
186
|
+
end
|
169
187
|
end
|
188
|
+
return waitKey, false
|
170
189
|
end
|
171
190
|
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
172
191
|
parentKey, parentId, timestamp)
|
@@ -176,6 +195,7 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
176
195
|
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
177
196
|
local parentWaitKey = parentQueueKey .. ":wait"
|
178
197
|
local parentPausedKey = parentQueueKey .. ":paused"
|
198
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
179
199
|
local parentMetaKey = parentQueueKey .. ":meta"
|
180
200
|
local parentMarkerKey = parentQueueKey .. ":marker"
|
181
201
|
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
@@ -191,15 +211,16 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
191
211
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
192
212
|
else
|
193
213
|
if priority == 0 then
|
194
|
-
local parentTarget,
|
195
|
-
getTargetQueueList(parentMetaKey, parentWaitKey,
|
214
|
+
local parentTarget, isParentPausedOrMaxed =
|
215
|
+
getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
|
196
216
|
parentPausedKey)
|
197
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH",
|
217
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
218
|
+
parentId)
|
198
219
|
else
|
199
|
-
local
|
220
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
200
221
|
addJobWithPriority(parentMarkerKey,
|
201
222
|
parentQueueKey .. ":prioritized", priority,
|
202
|
-
parentId, parentQueueKey .. ":pc",
|
223
|
+
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
203
224
|
end
|
204
225
|
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
205
226
|
"jobId", parentId, "prev", "waiting-children")
|
@@ -301,11 +322,8 @@ rcall("ZADD", delayedKey, score, jobId)
|
|
301
322
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
302
323
|
"jobId", jobId, "delay", delayedTimestamp)
|
303
324
|
-- mark that a delayed job is available
|
304
|
-
local
|
305
|
-
|
306
|
-
local markerKey = KEYS[1]
|
307
|
-
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
308
|
-
end
|
325
|
+
local markerKey = KEYS[1]
|
326
|
+
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
309
327
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
310
328
|
if parentDependenciesKey ~= nil then
|
311
329
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwUf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|