bullmq 2.3.1 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/flow-producer.js +16 -1
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/redis-connection.js +1 -1
- package/dist/cjs/classes/redis-connection.js.map +1 -1
- package/dist/cjs/commands/addJob-8.lua +174 -0
- package/dist/cjs/commands/changeDelay-4.lua +43 -0
- package/dist/cjs/commands/cleanJobsInSet-2.lua +46 -0
- package/dist/cjs/commands/drain-4.lua +25 -0
- package/dist/cjs/commands/extendLock-2.lua +23 -0
- package/dist/cjs/commands/getState-7.lua +57 -0
- package/dist/cjs/commands/getStateV2-7.lua +51 -0
- package/dist/cjs/commands/includes/addJobWithPriority.lua +16 -0
- package/dist/cjs/commands/includes/batches.lua +18 -0
- package/dist/cjs/commands/includes/checkItemInList.lua +12 -0
- package/dist/cjs/commands/includes/checkStalledJobs.lua +137 -0
- package/dist/cjs/commands/includes/cleanList.lua +50 -0
- package/dist/cjs/commands/includes/cleanSet.lua +50 -0
- package/dist/cjs/commands/includes/collectMetrics.lua +46 -0
- package/dist/cjs/commands/includes/destructureJobKey.lua +12 -0
- package/dist/cjs/commands/includes/getNextDelayedTimestamp.lua +13 -0
- package/dist/cjs/commands/includes/getTargetQueueList.lua +12 -0
- package/dist/cjs/commands/includes/getTimestamp.lua +19 -0
- package/dist/cjs/commands/includes/getZSetItems.lua +7 -0
- package/dist/cjs/commands/includes/isLocked.lua +31 -0
- package/dist/cjs/commands/includes/moveJobFromWaitToActive.lua +84 -0
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +28 -0
- package/dist/cjs/commands/includes/promoteDelayedJobs.lua +49 -0
- package/dist/cjs/commands/includes/removeJob.lua +13 -0
- package/dist/cjs/commands/includes/removeJobFromAnyState.lua +26 -0
- package/dist/cjs/commands/includes/removeJobs.lua +38 -0
- package/dist/cjs/commands/includes/removeJobsByMaxAge.lua +15 -0
- package/dist/cjs/commands/includes/removeJobsByMaxCount.lua +15 -0
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +81 -0
- package/dist/cjs/commands/includes/trimEvents.lua +12 -0
- package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +28 -0
- package/dist/cjs/commands/isFinished-3.lua +48 -0
- package/dist/cjs/commands/isJobInList-1.lua +16 -0
- package/dist/cjs/commands/moveStalledJobsToWait-8.lua +24 -0
- package/dist/cjs/commands/moveToActive-9.lua +67 -0
- package/dist/cjs/commands/moveToDelayed-5.lua +54 -0
- package/dist/cjs/commands/moveToFinished-12.lua +201 -0
- package/dist/cjs/commands/moveToWaitingChildren-4.lua +62 -0
- package/dist/cjs/commands/obliterate-2.lua +94 -0
- package/dist/cjs/commands/pause-4.lua +27 -0
- package/dist/cjs/commands/promote-6.lua +51 -0
- package/dist/cjs/commands/releaseLock-1.lua +19 -0
- package/dist/cjs/commands/removeJob-1.lua +72 -0
- package/dist/cjs/commands/removeRepeatable-2.lua +32 -0
- package/dist/cjs/commands/reprocessJob-4.lua +35 -0
- package/dist/cjs/commands/retryJob-6.lua +51 -0
- package/dist/cjs/commands/retryJobs-6.lua +53 -0
- package/dist/cjs/commands/takeLock-1.lua +17 -0
- package/dist/cjs/commands/updateData-1.lua +16 -0
- package/dist/cjs/commands/updateProgress-2.lua +22 -0
- package/dist/cjs/scripts/moveToFinished-12.js +5 -2
- package/dist/cjs/scripts/moveToFinished-12.js.map +1 -1
- package/dist/cjs/scripts/promote-6.js +15 -9
- package/dist/cjs/scripts/promote-6.js.map +1 -1
- package/dist/esm/classes/flow-producer.js +16 -1
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/redis-connection.js +1 -1
- package/dist/esm/classes/redis-connection.js.map +1 -1
- package/dist/esm/commands/addJob-8.lua +174 -0
- package/dist/esm/commands/changeDelay-4.lua +43 -0
- package/dist/esm/commands/cleanJobsInSet-2.lua +46 -0
- package/dist/esm/commands/drain-4.lua +25 -0
- package/dist/esm/commands/extendLock-2.lua +23 -0
- package/dist/esm/commands/getState-7.lua +57 -0
- package/dist/esm/commands/getStateV2-7.lua +51 -0
- package/dist/esm/commands/includes/addJobWithPriority.lua +16 -0
- package/dist/esm/commands/includes/batches.lua +18 -0
- package/dist/esm/commands/includes/checkItemInList.lua +12 -0
- package/dist/esm/commands/includes/checkStalledJobs.lua +137 -0
- package/dist/esm/commands/includes/cleanList.lua +50 -0
- package/dist/esm/commands/includes/cleanSet.lua +50 -0
- package/dist/esm/commands/includes/collectMetrics.lua +46 -0
- package/dist/esm/commands/includes/destructureJobKey.lua +12 -0
- package/dist/esm/commands/includes/getNextDelayedTimestamp.lua +13 -0
- package/dist/esm/commands/includes/getTargetQueueList.lua +12 -0
- package/dist/esm/commands/includes/getTimestamp.lua +19 -0
- package/dist/esm/commands/includes/getZSetItems.lua +7 -0
- package/dist/esm/commands/includes/isLocked.lua +31 -0
- package/dist/esm/commands/includes/moveJobFromWaitToActive.lua +84 -0
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +28 -0
- package/dist/esm/commands/includes/promoteDelayedJobs.lua +49 -0
- package/dist/esm/commands/includes/removeJob.lua +13 -0
- package/dist/esm/commands/includes/removeJobFromAnyState.lua +26 -0
- package/dist/esm/commands/includes/removeJobs.lua +38 -0
- package/dist/esm/commands/includes/removeJobsByMaxAge.lua +15 -0
- package/dist/esm/commands/includes/removeJobsByMaxCount.lua +15 -0
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +81 -0
- package/dist/esm/commands/includes/trimEvents.lua +12 -0
- package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +28 -0
- package/dist/esm/commands/isFinished-3.lua +48 -0
- package/dist/esm/commands/isJobInList-1.lua +16 -0
- package/dist/esm/commands/moveStalledJobsToWait-8.lua +24 -0
- package/dist/esm/commands/moveToActive-9.lua +67 -0
- package/dist/esm/commands/moveToDelayed-5.lua +54 -0
- package/dist/esm/commands/moveToFinished-12.lua +201 -0
- package/dist/esm/commands/moveToWaitingChildren-4.lua +62 -0
- package/dist/esm/commands/obliterate-2.lua +94 -0
- package/dist/esm/commands/pause-4.lua +27 -0
- package/dist/esm/commands/promote-6.lua +51 -0
- package/dist/esm/commands/releaseLock-1.lua +19 -0
- package/dist/esm/commands/removeJob-1.lua +72 -0
- package/dist/esm/commands/removeRepeatable-2.lua +32 -0
- package/dist/esm/commands/reprocessJob-4.lua +35 -0
- package/dist/esm/commands/retryJob-6.lua +51 -0
- package/dist/esm/commands/retryJobs-6.lua +53 -0
- package/dist/esm/commands/takeLock-1.lua +17 -0
- package/dist/esm/commands/updateData-1.lua +16 -0
- package/dist/esm/commands/updateProgress-2.lua +22 -0
- package/dist/esm/scripts/moveToFinished-12.js +5 -2
- package/dist/esm/scripts/moveToFinished-12.js.map +1 -1
- package/dist/esm/scripts/promote-6.js +15 -9
- package/dist/esm/scripts/promote-6.js.map +1 -1
- package/package.json +2 -2
@@ -0,0 +1,81 @@
|
|
1
|
+
--[[
|
2
|
+
Check if this job has a parent. If so we will just remove it from
|
3
|
+
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
4
|
+
which requires code from "moveToFinished"
|
5
|
+
]]
|
6
|
+
|
7
|
+
--- @include "destructureJobKey"
|
8
|
+
--- @include "getTargetQueueList"
|
9
|
+
|
10
|
+
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
11
|
+
local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
|
12
|
+
rcall("RPUSH", parentTarget, parentId)
|
13
|
+
|
14
|
+
if emitEvent then
|
15
|
+
local parentEventStream = parentPrefix .. "events"
|
16
|
+
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
21
|
+
if parentKey then
|
22
|
+
local parentProcessedKey = parentKey .. ":processed"
|
23
|
+
rcall("HDEL", parentProcessedKey, jobKey)
|
24
|
+
local parentDependenciesKey = parentKey .. ":dependencies"
|
25
|
+
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
26
|
+
if result > 0 then
|
27
|
+
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
28
|
+
if pendingDependencies == 0 then
|
29
|
+
local parentId = getJobIdFromKey(parentKey)
|
30
|
+
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
31
|
+
|
32
|
+
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
33
|
+
|
34
|
+
if numRemovedElements == 1 then
|
35
|
+
if hard then
|
36
|
+
if parentPrefix == baseKey then
|
37
|
+
removeParentDependencyKey(parentKey, hard, nil, baseKey)
|
38
|
+
rcall("DEL", parentKey, parentKey .. ':logs',
|
39
|
+
parentKey .. ':dependencies', parentKey .. ':processed')
|
40
|
+
else
|
41
|
+
moveParentToWait(parentPrefix, parentId)
|
42
|
+
end
|
43
|
+
else
|
44
|
+
moveParentToWait(parentPrefix, parentId, true)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
else
|
50
|
+
local missedParentKey = rcall("HGET", jobKey, "parentKey")
|
51
|
+
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
|
52
|
+
local parentProcessedKey = missedParentKey .. ":processed"
|
53
|
+
rcall("HDEL", parentProcessedKey, jobKey)
|
54
|
+
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
55
|
+
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
56
|
+
if result > 0 then
|
57
|
+
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
58
|
+
if pendingDependencies == 0 then
|
59
|
+
local parentId = getJobIdFromKey(missedParentKey)
|
60
|
+
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
61
|
+
|
62
|
+
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
63
|
+
|
64
|
+
if numRemovedElements == 1 then
|
65
|
+
if hard then
|
66
|
+
if parentPrefix == baseKey then
|
67
|
+
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
|
68
|
+
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
|
69
|
+
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
|
70
|
+
else
|
71
|
+
moveParentToWait(parentPrefix, parentId)
|
72
|
+
end
|
73
|
+
else
|
74
|
+
moveParentToWait(parentPrefix, parentId, true)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
--[[
|
2
|
+
Function to trim events, default 10000.
|
3
|
+
]]
|
4
|
+
|
5
|
+
local function trimEvents(metaKey, eventStreamKey)
|
6
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
7
|
+
if maxEvents ~= false then
|
8
|
+
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
|
9
|
+
else
|
10
|
+
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
--[[
|
2
|
+
Validate and move or add dependencies to parent.
|
3
|
+
]]
|
4
|
+
|
5
|
+
-- Includes
|
6
|
+
--- @include "addJobWithPriority"
|
7
|
+
--- @include "getTargetQueueList"
|
8
|
+
|
9
|
+
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
10
|
+
parentId, jobIdKey, returnvalue )
|
11
|
+
local processedSet = parentKey .. ":processed"
|
12
|
+
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
13
|
+
local activeParent = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
|
14
|
+
if rcall("SCARD", parentDependenciesKey) == 0 and activeParent then
|
15
|
+
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
16
|
+
local parentTarget = getTargetQueueList(parentQueueKey .. ":meta", parentQueueKey .. ":wait",
|
17
|
+
parentQueueKey .. ":paused")
|
18
|
+
local priority = tonumber(rcall("HGET", parentKey, "priority"))
|
19
|
+
-- Standard or priority add
|
20
|
+
if priority == 0 then
|
21
|
+
rcall("RPUSH", parentTarget, parentId)
|
22
|
+
else
|
23
|
+
addJobWithPriority(parentQueueKey .. ":priority", priority, parentTarget, parentId)
|
24
|
+
end
|
25
|
+
|
26
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
--[[
|
2
|
+
Checks if a job is finished (.i.e. is in the completed or failed set)
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] completed key
|
6
|
+
KEYS[2] failed key
|
7
|
+
KEYS[3] job key
|
8
|
+
|
9
|
+
ARGV[1] job id
|
10
|
+
ARGV[2] return value?
|
11
|
+
Output:
|
12
|
+
0 - Not finished.
|
13
|
+
1 - Completed.
|
14
|
+
2 - Failed.
|
15
|
+
-5 - Missing job.
|
16
|
+
]]
|
17
|
+
local rcall = redis.call
|
18
|
+
if rcall("EXISTS", KEYS[3]) ~= 1 then
|
19
|
+
if ARGV[2] == "1" then
|
20
|
+
|
21
|
+
return {-5,"Missing key for job " .. KEYS[3] .. ". isFinished"}
|
22
|
+
end
|
23
|
+
return -5
|
24
|
+
end
|
25
|
+
|
26
|
+
if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
|
27
|
+
if ARGV[2] == "1" then
|
28
|
+
local returnValue = rcall("HGET", KEYS[3], "returnvalue")
|
29
|
+
|
30
|
+
return {1,returnValue}
|
31
|
+
end
|
32
|
+
return 1
|
33
|
+
end
|
34
|
+
|
35
|
+
if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
|
36
|
+
if ARGV[2] == "1" then
|
37
|
+
local failedReason = rcall("HGET", KEYS[3], "failedReason")
|
38
|
+
|
39
|
+
return {2,failedReason}
|
40
|
+
end
|
41
|
+
return 2
|
42
|
+
end
|
43
|
+
|
44
|
+
if ARGV[2] == "1" then
|
45
|
+
return {0}
|
46
|
+
end
|
47
|
+
|
48
|
+
return 0
|
@@ -0,0 +1,16 @@
|
|
1
|
+
--[[
|
2
|
+
Checks if job is in a given list.
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1]
|
6
|
+
ARGV[1]
|
7
|
+
|
8
|
+
Output:
|
9
|
+
1 if element found in the list.
|
10
|
+
]]
|
11
|
+
|
12
|
+
-- Includes
|
13
|
+
--- @include "includes/checkItemInList"
|
14
|
+
|
15
|
+
local items = redis.call("LRANGE", KEYS[1] , 0, -1)
|
16
|
+
return checkItemInList(items, ARGV[1])
|
@@ -0,0 +1,24 @@
|
|
1
|
+
--[[
|
2
|
+
Move stalled jobs to wait.
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] 'stalled' (SET)
|
6
|
+
KEYS[2] 'wait', (LIST)
|
7
|
+
KEYS[3] 'active', (LIST)
|
8
|
+
KEYS[4] 'failed', (ZSET)
|
9
|
+
KEYS[5] 'stalled-check', (KEY)
|
10
|
+
KEYS[6] 'meta', (KEY)
|
11
|
+
KEYS[7] 'paused', (LIST)
|
12
|
+
KEYS[8] 'event stream' (STREAM)
|
13
|
+
|
14
|
+
ARGV[1] Max stalled job count
|
15
|
+
ARGV[2] queue.toKey('')
|
16
|
+
ARGV[3] timestamp
|
17
|
+
ARGV[4] max check time
|
18
|
+
|
19
|
+
Events:
|
20
|
+
'stalled' with stalled job id.
|
21
|
+
]] -- Includes
|
22
|
+
--- @include "includes/checkStalledJobs"
|
23
|
+
return checkStalledJobs(KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6],
|
24
|
+
KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])
|
@@ -0,0 +1,67 @@
|
|
1
|
+
--[[
|
2
|
+
Move next job to be processed to active, lock it and fetch its data. The job
|
3
|
+
may be delayed, in that case we need to move it to the delayed set instead.
|
4
|
+
|
5
|
+
This operation guarantees that the worker owns the job during the lock
|
6
|
+
expiration time. The worker is responsible of keeping the lock fresh
|
7
|
+
so that no other worker picks this job again.
|
8
|
+
|
9
|
+
Input:
|
10
|
+
KEYS[1] wait key
|
11
|
+
KEYS[2] active key
|
12
|
+
KEYS[3] priority key
|
13
|
+
KEYS[4] stream events key
|
14
|
+
KEYS[5] stalled key
|
15
|
+
|
16
|
+
-- Rate limiting
|
17
|
+
KEYS[6] rate limiter key
|
18
|
+
KEYS[7] delayed key
|
19
|
+
|
20
|
+
-- Promote delayed jobs
|
21
|
+
KEYS[8] paused key
|
22
|
+
KEYS[9] meta key
|
23
|
+
|
24
|
+
-- Arguments
|
25
|
+
ARGV[1] key prefix
|
26
|
+
ARGV[2] timestamp
|
27
|
+
ARGV[3] optional job ID
|
28
|
+
ARGV[4] opts
|
29
|
+
|
30
|
+
opts - token - lock token
|
31
|
+
opts - lockDuration
|
32
|
+
opts - limiter
|
33
|
+
]]
|
34
|
+
local jobId
|
35
|
+
local rcall = redis.call
|
36
|
+
|
37
|
+
-- Includes
|
38
|
+
--- @include "includes/moveJobFromWaitToActive"
|
39
|
+
--- @include "includes/getNextDelayedTimestamp"
|
40
|
+
--- @include "includes/promoteDelayedJobs"
|
41
|
+
|
42
|
+
-- Check if there are delayed jobs that we can move to wait.
|
43
|
+
promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[9], KEYS[4], ARGV[1], ARGV[2])
|
44
|
+
|
45
|
+
if (ARGV[3] ~= "") then
|
46
|
+
jobId = ARGV[3]
|
47
|
+
-- clean stalled key
|
48
|
+
rcall("SREM", KEYS[5], jobId)
|
49
|
+
else
|
50
|
+
-- no job ID, try non-blocking move from wait to active
|
51
|
+
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
52
|
+
end
|
53
|
+
|
54
|
+
-- If jobId is special ID 0, then there is no job to process
|
55
|
+
if jobId == "0" then
|
56
|
+
rcall("LREM", KEYS[2], 1, 0)
|
57
|
+
elseif jobId then
|
58
|
+
local opts = cmsgpack.unpack(ARGV[4])
|
59
|
+
-- this script is not really moving, it is preparing the job for processing
|
60
|
+
return moveJobFromWaitToActive(KEYS, ARGV[1], jobId, ARGV[2], opts)
|
61
|
+
end
|
62
|
+
|
63
|
+
-- Return the timestamp for the next delayed job if any.
|
64
|
+
local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
|
65
|
+
if (nextTimestamp ~= nil) then
|
66
|
+
return nextTimestamp - tonumber(ARGV[2])
|
67
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
--[[
|
2
|
+
Moves job from active to delayed set.
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] active key
|
6
|
+
KEYS[2] delayed key
|
7
|
+
KEYS[3] job key
|
8
|
+
KEYS[4] events stream
|
9
|
+
KEYS[5] delayed stream
|
10
|
+
|
11
|
+
ARGV[1] delayedTimestamp
|
12
|
+
ARGV[2] the id of the job
|
13
|
+
ARGV[3] queue token
|
14
|
+
|
15
|
+
Output:
|
16
|
+
0 - OK
|
17
|
+
-1 - Missing job.
|
18
|
+
-3 - Job not in active set.
|
19
|
+
|
20
|
+
Events:
|
21
|
+
- delayed key.
|
22
|
+
]]
|
23
|
+
local rcall = redis.call
|
24
|
+
|
25
|
+
if rcall("EXISTS", KEYS[3]) == 1 then
|
26
|
+
|
27
|
+
if ARGV[3] ~= "0" then
|
28
|
+
local lockKey = KEYS[3] .. ':lock'
|
29
|
+
if rcall("GET", lockKey) == ARGV[3] then
|
30
|
+
rcall("DEL", lockKey)
|
31
|
+
else
|
32
|
+
return -2
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
local jobId = ARGV[2]
|
37
|
+
local score = tonumber(ARGV[1])
|
38
|
+
local delayedTimestamp = (score / 0x1000)
|
39
|
+
|
40
|
+
local numRemovedElements = rcall("LREM", KEYS[1], -1, jobId)
|
41
|
+
|
42
|
+
if(numRemovedElements < 1) then
|
43
|
+
return -3
|
44
|
+
end
|
45
|
+
|
46
|
+
rcall("ZADD", KEYS[2], score, jobId)
|
47
|
+
|
48
|
+
rcall("XADD", KEYS[4], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
|
49
|
+
rcall("XADD", KEYS[5], "*", "nextTimestamp", delayedTimestamp);
|
50
|
+
|
51
|
+
return 0
|
52
|
+
else
|
53
|
+
return -1
|
54
|
+
end
|
@@ -0,0 +1,201 @@
|
|
1
|
+
--[[
|
2
|
+
Move job from active to a finished status (completed o failed)
|
3
|
+
A job can only be moved to completed if it was active.
|
4
|
+
The job must be locked before it can be moved to a finished status,
|
5
|
+
and the lock must be released in this script.
|
6
|
+
|
7
|
+
Input:
|
8
|
+
KEYS[1] wait key
|
9
|
+
KEYS[2] active key
|
10
|
+
KEYS[3] priority key
|
11
|
+
KEYS[4] event stream key
|
12
|
+
KEYS[5] stalled key
|
13
|
+
|
14
|
+
-- Rate limiting
|
15
|
+
KEYS[6] rate limiter key
|
16
|
+
KEYS[7] delayed key
|
17
|
+
|
18
|
+
KEYS[8] paused key
|
19
|
+
|
20
|
+
KEYS[9] completed/failed key
|
21
|
+
KEYS[10] jobId key
|
22
|
+
KEYS[11] meta key
|
23
|
+
KEYS[12] metrics key
|
24
|
+
|
25
|
+
ARGV[1] jobId
|
26
|
+
ARGV[2] timestamp
|
27
|
+
ARGV[3] msg property returnvalue / failedReason
|
28
|
+
ARGV[4] return value / failed reason
|
29
|
+
ARGV[5] target (completed/failed)
|
30
|
+
ARGV[6] event data (? maybe just send jobid).
|
31
|
+
ARGV[7] fetch next?
|
32
|
+
ARGV[8] keys prefix
|
33
|
+
ARGV[9] opts
|
34
|
+
|
35
|
+
opts - token - lock token
|
36
|
+
opts - keepJobs
|
37
|
+
opts - lockDuration - lock duration in milliseconds
|
38
|
+
opts - parent - parent data
|
39
|
+
opts - parentKey
|
40
|
+
opts - attempts max attempts
|
41
|
+
opts - attemptsMade
|
42
|
+
opts - maxMetricsSize
|
43
|
+
opts - fpof - fail parent on fail
|
44
|
+
|
45
|
+
Output:
|
46
|
+
0 OK
|
47
|
+
-1 Missing key.
|
48
|
+
-2 Missing lock.
|
49
|
+
-3 Job not in active set
|
50
|
+
-4 Job has pending dependencies
|
51
|
+
|
52
|
+
Events:
|
53
|
+
'completed/failed'
|
54
|
+
]]
|
55
|
+
local rcall = redis.call
|
56
|
+
|
57
|
+
--- Includes
|
58
|
+
--- @include "includes/collectMetrics"
|
59
|
+
--- @include "includes/getNextDelayedTimestamp"
|
60
|
+
--- @include "includes/moveJobFromWaitToActive"
|
61
|
+
--- @include "includes/moveParentFromWaitingChildrenToFailed"
|
62
|
+
--- @include "includes/promoteDelayedJobs"
|
63
|
+
--- @include "includes/removeJobsByMaxAge"
|
64
|
+
--- @include "includes/removeJobsByMaxCount"
|
65
|
+
--- @include "includes/trimEvents"
|
66
|
+
--- @include "includes/updateParentDepsIfNeeded"
|
67
|
+
|
68
|
+
local jobIdKey = KEYS[10]
|
69
|
+
if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
70
|
+
local opts = cmsgpack.unpack(ARGV[9])
|
71
|
+
|
72
|
+
local token = opts['token']
|
73
|
+
local parentId = opts['parent'] and opts['parent']['id'] or ""
|
74
|
+
local parentQueueKey = opts['parent'] and opts['parent']['queue'] or ""
|
75
|
+
local parentKey = opts['parentKey'] or ""
|
76
|
+
local attempts = opts['attempts']
|
77
|
+
local attemptsMade = opts['attemptsMade']
|
78
|
+
local maxMetricsSize = opts['maxMetricsSize']
|
79
|
+
local maxCount = opts['keepJobs']['count']
|
80
|
+
local maxAge = opts['keepJobs']['age']
|
81
|
+
|
82
|
+
if token ~= "0" then
|
83
|
+
local lockKey = jobIdKey .. ':lock'
|
84
|
+
if rcall("GET", lockKey) == token then
|
85
|
+
rcall("DEL", lockKey)
|
86
|
+
rcall("SREM", KEYS[5], ARGV[1])
|
87
|
+
else
|
88
|
+
return -2
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
|
93
|
+
return -4
|
94
|
+
end
|
95
|
+
|
96
|
+
local jobId = ARGV[1]
|
97
|
+
local timestamp = ARGV[2]
|
98
|
+
|
99
|
+
-- Remove from active list (if not active we shall return error)
|
100
|
+
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
101
|
+
|
102
|
+
if (numRemovedElements < 1) then return -3 end
|
103
|
+
|
104
|
+
-- Trim events before emiting them to avoid trimming events emitted in this script
|
105
|
+
trimEvents(KEYS[11], KEYS[4])
|
106
|
+
|
107
|
+
-- If job has a parent we need to
|
108
|
+
-- 1) remove this job id from parents dependencies
|
109
|
+
-- 2) move the job Id to parent "processed" set
|
110
|
+
-- 3) push the results into parent "results" list
|
111
|
+
-- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
|
112
|
+
if parentId == "" and parentKey ~= "" then
|
113
|
+
parentId = getJobIdFromKey(parentKey)
|
114
|
+
parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
|
115
|
+
end
|
116
|
+
if parentId ~= "" then
|
117
|
+
if ARGV[5] == "completed" then
|
118
|
+
local dependenciesSet = parentKey .. ":dependencies"
|
119
|
+
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
120
|
+
updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet,
|
121
|
+
parentId, jobIdKey, ARGV[4])
|
122
|
+
end
|
123
|
+
elseif opts['fpof'] then
|
124
|
+
moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
-- Remove job?
|
129
|
+
if maxCount ~= 0 then
|
130
|
+
local targetSet = KEYS[9]
|
131
|
+
-- Add to complete/failed set
|
132
|
+
rcall("ZADD", targetSet, timestamp, jobId)
|
133
|
+
rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
|
134
|
+
-- "returnvalue" / "failedReason" and "finishedOn"
|
135
|
+
|
136
|
+
-- Remove old jobs?
|
137
|
+
local prefix = ARGV[8]
|
138
|
+
|
139
|
+
if maxAge ~= nil then
|
140
|
+
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
141
|
+
end
|
142
|
+
|
143
|
+
if maxCount ~= nil and maxCount > 0 then
|
144
|
+
removeJobsByMaxCount(maxCount, targetSet, prefix)
|
145
|
+
end
|
146
|
+
else
|
147
|
+
rcall("DEL", jobIdKey, jobIdKey .. ':logs', jobIdKey .. ':processed')
|
148
|
+
end
|
149
|
+
|
150
|
+
rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
|
151
|
+
ARGV[4])
|
152
|
+
|
153
|
+
if ARGV[5] == "failed" then
|
154
|
+
if tonumber(attemptsMade) >= tonumber(attempts) then
|
155
|
+
rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
|
156
|
+
jobId, "attemptsMade", attemptsMade)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
-- Collect metrics
|
161
|
+
if maxMetricsSize ~= "" then
|
162
|
+
collectMetrics(KEYS[12], KEYS[12]..':data', maxMetricsSize, timestamp)
|
163
|
+
end
|
164
|
+
|
165
|
+
-- Try to get next job to avoid an extra roundtrip if the queue is not closing,
|
166
|
+
-- and not rate limited.
|
167
|
+
if (ARGV[7] == "1") then
|
168
|
+
|
169
|
+
-- Check if there are delayed jobs that can be promoted
|
170
|
+
promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[11], KEYS[4], ARGV[8], timestamp)
|
171
|
+
|
172
|
+
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
173
|
+
|
174
|
+
if jobId == "0" then
|
175
|
+
rcall("LREM", KEYS[2], 1, 0)
|
176
|
+
elseif jobId then
|
177
|
+
return moveJobFromWaitToActive(KEYS, ARGV[8], jobId, timestamp, opts)
|
178
|
+
end
|
179
|
+
|
180
|
+
-- Return the timestamp for the next delayed job if any.
|
181
|
+
local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
|
182
|
+
if (nextTimestamp ~= nil) then
|
183
|
+
-- The result is guaranteed to be positive, since the
|
184
|
+
-- ZRANGEBYSCORE command would have return a job otherwise.
|
185
|
+
return nextTimestamp - timestamp
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
local waitLen = rcall("LLEN", KEYS[1])
|
190
|
+
if waitLen == 0 then
|
191
|
+
local activeLen = rcall("LLEN", KEYS[2])
|
192
|
+
|
193
|
+
if activeLen == 0 then
|
194
|
+
rcall("XADD", KEYS[4], "*", "event", "drained")
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
return 0
|
199
|
+
else
|
200
|
+
return -1
|
201
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
--[[
|
2
|
+
Moves job from active to waiting children set.
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] lock key
|
6
|
+
KEYS[2] active key
|
7
|
+
KEYS[3] waitChildrenKey key
|
8
|
+
KEYS[4] job key
|
9
|
+
|
10
|
+
ARGV[1] token
|
11
|
+
ARGV[2] child key
|
12
|
+
ARGV[3] timestamp
|
13
|
+
ARGV[4] the id of the job
|
14
|
+
|
15
|
+
Output:
|
16
|
+
0 - OK
|
17
|
+
1 - There are not pending dependencies.
|
18
|
+
-1 - Missing job.
|
19
|
+
-2 - Missing lock
|
20
|
+
-3 - Job not in active set
|
21
|
+
]]
|
22
|
+
local rcall = redis.call
|
23
|
+
|
24
|
+
local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp, lockKey, token)
|
25
|
+
if token ~= "0" then
|
26
|
+
if rcall("GET", lockKey) == token then
|
27
|
+
rcall("DEL", lockKey)
|
28
|
+
else
|
29
|
+
return -2
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
local score = tonumber(timestamp)
|
34
|
+
|
35
|
+
local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
|
36
|
+
|
37
|
+
if(numRemovedElements < 1) then
|
38
|
+
return -3
|
39
|
+
end
|
40
|
+
|
41
|
+
rcall("ZADD", waitingChildrenKey, score, jobId)
|
42
|
+
|
43
|
+
return 0
|
44
|
+
end
|
45
|
+
|
46
|
+
if rcall("EXISTS", KEYS[4]) == 1 then
|
47
|
+
if ARGV[2] ~= "" then
|
48
|
+
if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
|
49
|
+
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
|
50
|
+
end
|
51
|
+
|
52
|
+
return 1
|
53
|
+
else
|
54
|
+
if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
|
55
|
+
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
|
56
|
+
end
|
57
|
+
|
58
|
+
return 1
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
return -1
|
@@ -0,0 +1,94 @@
|
|
1
|
+
--[[
|
2
|
+
Completely obliterates a queue and all of its contents
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] meta
|
6
|
+
KEYS[2] base
|
7
|
+
|
8
|
+
ARGV[1] count
|
9
|
+
ARGV[2] force
|
10
|
+
]]
|
11
|
+
|
12
|
+
-- This command completely destroys a queue including all of its jobs, current or past
|
13
|
+
-- leaving no trace of its existence. Since this script needs to iterate to find all the job
|
14
|
+
-- keys, consider that this call may be slow for very large queues.
|
15
|
+
|
16
|
+
-- The queue needs to be "paused" or it will return an error
|
17
|
+
-- If the queue has currently active jobs then the script by default will return error,
|
18
|
+
-- however this behaviour can be overrided using the 'force' option.
|
19
|
+
local maxCount = tonumber(ARGV[1])
|
20
|
+
local baseKey = KEYS[2]
|
21
|
+
|
22
|
+
local rcall = redis.call
|
23
|
+
|
24
|
+
--- @include "includes/removeJobs"
|
25
|
+
|
26
|
+
local function removeLockKeys(keys)
|
27
|
+
for i, key in ipairs(keys) do
|
28
|
+
rcall("DEL", baseKey .. key .. ':lock')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
-- 1) Check if paused, if not return with error.
|
33
|
+
if rcall("HEXISTS", KEYS[1], "paused") ~= 1 then
|
34
|
+
return -1 -- Error, NotPaused
|
35
|
+
end
|
36
|
+
|
37
|
+
-- 2) Check if there are active jobs, if there are and not "force" return error.
|
38
|
+
local activeKey = baseKey .. 'active'
|
39
|
+
local activeJobs = getListItems(activeKey, maxCount)
|
40
|
+
if (#activeJobs > 0) then
|
41
|
+
if(ARGV[2] == "") then
|
42
|
+
return -2 -- Error, ExistActiveJobs
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
removeLockKeys(activeJobs)
|
47
|
+
maxCount = removeJobs(activeJobs, true, baseKey, maxCount)
|
48
|
+
rcall("LTRIM", activeKey, #activeJobs, -1)
|
49
|
+
if(maxCount <= 0) then
|
50
|
+
return 1
|
51
|
+
end
|
52
|
+
|
53
|
+
local delayedKey = baseKey .. 'delayed'
|
54
|
+
maxCount = removeZSetJobs(delayedKey, true, baseKey, maxCount)
|
55
|
+
if(maxCount <= 0) then
|
56
|
+
return 1
|
57
|
+
end
|
58
|
+
|
59
|
+
local completedKey = baseKey .. 'completed'
|
60
|
+
maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
|
61
|
+
if(maxCount <= 0) then
|
62
|
+
return 1
|
63
|
+
end
|
64
|
+
|
65
|
+
local waitKey = baseKey .. 'paused'
|
66
|
+
maxCount = removeListJobs(waitKey, true, baseKey, maxCount)
|
67
|
+
if(maxCount <= 0) then
|
68
|
+
return 1
|
69
|
+
end
|
70
|
+
|
71
|
+
local failedKey = baseKey .. 'failed'
|
72
|
+
maxCount = removeZSetJobs(failedKey, true, baseKey, maxCount)
|
73
|
+
if(maxCount <= 0) then
|
74
|
+
return 1
|
75
|
+
end
|
76
|
+
|
77
|
+
if(maxCount > 0) then
|
78
|
+
rcall("DEL",
|
79
|
+
baseKey .. 'priority',
|
80
|
+
baseKey .. 'events',
|
81
|
+
baseKey .. 'delay',
|
82
|
+
baseKey .. 'stalled-check',
|
83
|
+
baseKey .. 'stalled',
|
84
|
+
baseKey .. 'id',
|
85
|
+
baseKey .. 'meta',
|
86
|
+
baseKey .. 'repeat',
|
87
|
+
baseKey .. 'metrics:completed',
|
88
|
+
baseKey .. 'metrics:completed:data',
|
89
|
+
baseKey .. 'metrics:failed',
|
90
|
+
baseKey .. 'metrics:failed:data')
|
91
|
+
return 0
|
92
|
+
else
|
93
|
+
return 1
|
94
|
+
end
|