bullmq 5.15.0 → 5.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/child-pool.js +2 -2
- package/dist/cjs/classes/child-pool.js.map +1 -1
- package/dist/cjs/classes/job-scheduler.js +140 -0
- package/dist/cjs/classes/job-scheduler.js.map +1 -0
- package/dist/cjs/classes/job.js +6 -2
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-getters.js +10 -0
- package/dist/cjs/classes/queue-getters.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js +1 -1
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/queue.js +74 -0
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/repeat.js +9 -10
- package/dist/cjs/classes/repeat.js.map +1 -1
- package/dist/cjs/classes/sandbox.js +57 -47
- package/dist/cjs/classes/sandbox.js.map +1 -1
- package/dist/cjs/classes/scripts.js +46 -40
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +23 -4
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addDelayedJob-6.lua +7 -7
- package/dist/cjs/commands/addJobScheduler-2.lua +75 -0
- package/dist/cjs/commands/addParentJob-4.lua +7 -7
- package/dist/cjs/commands/addPrioritizedJob-8.lua +7 -7
- package/dist/cjs/commands/addStandardJob-8.lua +7 -7
- package/dist/cjs/commands/{cleanJobsInSet-2.lua → cleanJobsInSet-3.lua} +6 -5
- package/dist/cjs/commands/drain-5.lua +41 -0
- package/dist/cjs/commands/includes/cleanSet.lua +50 -29
- package/dist/cjs/commands/includes/deduplicateJob.lua +25 -0
- package/dist/cjs/commands/includes/removeZSetJobs.lua +13 -1
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +2 -4
- package/dist/cjs/commands/removeJob-2.lua +17 -9
- package/dist/cjs/commands/removeJobScheduler-3.lua +43 -0
- package/dist/cjs/enums/error-code.js +1 -0
- package/dist/cjs/enums/error-code.js.map +1 -1
- package/dist/cjs/scripts/addDelayedJob-6.js +18 -16
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-2.js +223 -0
- package/dist/cjs/scripts/addJobScheduler-2.js.map +1 -0
- package/dist/cjs/scripts/addParentJob-4.js +18 -16
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js +18 -16
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/cjs/scripts/addStandardJob-8.js +18 -16
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
- package/dist/cjs/scripts/{cleanJobsInSet-2.js → cleanJobsInSet-3.js} +57 -35
- package/dist/cjs/scripts/{cleanJobsInSet-2.js.map → cleanJobsInSet-3.js.map} +1 -1
- package/dist/cjs/scripts/{drain-4.js → drain-5.js} +28 -7
- package/dist/cjs/scripts/drain-5.js.map +1 -0
- package/dist/cjs/scripts/index.js +4 -2
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +2 -3
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +11 -1
- package/dist/cjs/scripts/obliterate-2.js.map +1 -1
- package/dist/cjs/scripts/removeJob-2.js +16 -9
- package/dist/cjs/scripts/removeJob-2.js.map +1 -1
- package/dist/cjs/scripts/removeJobScheduler-3.js +49 -0
- package/dist/cjs/scripts/removeJobScheduler-3.js.map +1 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/utils.js +25 -1
- package/dist/cjs/utils.js.map +1 -1
- package/dist/esm/classes/child-pool.d.ts +1 -1
- package/dist/esm/classes/child-pool.js +2 -2
- package/dist/esm/classes/child-pool.js.map +1 -1
- package/dist/esm/classes/job-scheduler.d.ts +30 -0
- package/dist/esm/classes/job-scheduler.js +135 -0
- package/dist/esm/classes/job-scheduler.js.map +1 -0
- package/dist/esm/classes/job.d.ts +6 -1
- package/dist/esm/classes/job.js +7 -3
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-events.d.ts +10 -0
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-getters.d.ts +7 -0
- package/dist/esm/classes/queue-getters.js +10 -0
- package/dist/esm/classes/queue-getters.js.map +1 -1
- package/dist/esm/classes/queue-keys.js +1 -1
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +54 -0
- package/dist/esm/classes/queue.js +74 -0
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/repeat.js +9 -10
- package/dist/esm/classes/repeat.js.map +1 -1
- package/dist/esm/classes/sandbox.js +57 -47
- package/dist/esm/classes/sandbox.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +3 -7
- package/dist/esm/classes/scripts.js +47 -41
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +3 -0
- package/dist/esm/classes/worker.js +23 -4
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addDelayedJob-6.lua +7 -7
- package/dist/esm/commands/addJobScheduler-2.lua +75 -0
- package/dist/esm/commands/addParentJob-4.lua +7 -7
- package/dist/esm/commands/addPrioritizedJob-8.lua +7 -7
- package/dist/esm/commands/addStandardJob-8.lua +7 -7
- package/dist/esm/commands/{cleanJobsInSet-2.lua → cleanJobsInSet-3.lua} +6 -5
- package/dist/esm/commands/drain-5.lua +41 -0
- package/dist/esm/commands/includes/cleanSet.lua +50 -29
- package/dist/esm/commands/includes/deduplicateJob.lua +25 -0
- package/dist/esm/commands/includes/removeZSetJobs.lua +13 -1
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +2 -4
- package/dist/esm/commands/removeJob-2.lua +17 -9
- package/dist/esm/commands/removeJobScheduler-3.lua +43 -0
- package/dist/esm/enums/error-code.d.ts +2 -1
- package/dist/esm/enums/error-code.js +1 -0
- package/dist/esm/enums/error-code.js.map +1 -1
- package/dist/esm/interfaces/job-json.d.ts +1 -0
- package/dist/esm/scripts/addDelayedJob-6.js +18 -16
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addJobScheduler-2.d.ts +5 -0
- package/dist/esm/scripts/addJobScheduler-2.js +220 -0
- package/dist/esm/scripts/addJobScheduler-2.js.map +1 -0
- package/dist/esm/scripts/addParentJob-4.js +18 -16
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js +18 -16
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/esm/scripts/addStandardJob-8.js +18 -16
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
- package/dist/esm/scripts/{cleanJobsInSet-2.js → cleanJobsInSet-3.js} +57 -35
- package/dist/esm/scripts/{cleanJobsInSet-2.js.map → cleanJobsInSet-3.js.map} +1 -1
- package/dist/esm/scripts/{drain-4.js → drain-5.js} +28 -7
- package/dist/esm/scripts/drain-5.js.map +1 -0
- package/dist/esm/scripts/index.d.ts +4 -2
- package/dist/esm/scripts/index.js +4 -2
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +2 -3
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +11 -1
- package/dist/esm/scripts/obliterate-2.js.map +1 -1
- package/dist/esm/scripts/removeJob-2.js +16 -9
- package/dist/esm/scripts/removeJob-2.js.map +1 -1
- package/dist/esm/scripts/removeJobScheduler-3.d.ts +5 -0
- package/dist/esm/scripts/removeJobScheduler-3.js +46 -0
- package/dist/esm/scripts/removeJobScheduler-3.js.map +1 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/job-options.d.ts +5 -0
- package/dist/esm/utils.d.ts +7 -0
- package/dist/esm/utils.js +23 -0
- package/dist/esm/utils.js.map +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/drain-4.lua +0 -26
- package/dist/cjs/commands/includes/debounceJob.lua +0 -23
- package/dist/cjs/scripts/drain-4.js.map +0 -1
- package/dist/esm/commands/drain-4.lua +0 -26
- package/dist/esm/commands/includes/debounceJob.lua +0 -23
- package/dist/esm/scripts/drain-4.js.map +0 -1
- /package/dist/esm/scripts/{cleanJobsInSet-2.d.ts → cleanJobsInSet-3.d.ts} +0 -0
- /package/dist/esm/scripts/{drain-4.d.ts → drain-5.d.ts} +0 -0
@@ -0,0 +1,75 @@
|
|
1
|
+
--[[
|
2
|
+
Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] 'repeat' key
|
6
|
+
KEYS[2] 'delayed' key
|
7
|
+
|
8
|
+
ARGV[1] next milliseconds
|
9
|
+
ARGV[2] msgpacked options
|
10
|
+
[1] name
|
11
|
+
[2] tz?
|
12
|
+
[3] patten?
|
13
|
+
[4] endDate?
|
14
|
+
[5] every?
|
15
|
+
ARGV[3] jobs scheduler id
|
16
|
+
ARGV[4] prefix key
|
17
|
+
|
18
|
+
Output:
|
19
|
+
repeatableKey - OK
|
20
|
+
]]
|
21
|
+
local rcall = redis.call
|
22
|
+
local repeatKey = KEYS[1]
|
23
|
+
local delayedKey = KEYS[2]
|
24
|
+
|
25
|
+
local nextMillis = ARGV[1]
|
26
|
+
local jobSchedulerId = ARGV[3]
|
27
|
+
local prefixKey = ARGV[4]
|
28
|
+
|
29
|
+
-- Includes
|
30
|
+
--- @include "includes/removeJob"
|
31
|
+
|
32
|
+
local function storeRepeatableJob(repeatKey, nextMillis, rawOpts)
|
33
|
+
rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
|
34
|
+
local opts = cmsgpack.unpack(rawOpts)
|
35
|
+
|
36
|
+
local optionalValues = {}
|
37
|
+
if opts['tz'] then
|
38
|
+
table.insert(optionalValues, "tz")
|
39
|
+
table.insert(optionalValues, opts['tz'])
|
40
|
+
end
|
41
|
+
|
42
|
+
if opts['pattern'] then
|
43
|
+
table.insert(optionalValues, "pattern")
|
44
|
+
table.insert(optionalValues, opts['pattern'])
|
45
|
+
end
|
46
|
+
|
47
|
+
if opts['endDate'] then
|
48
|
+
table.insert(optionalValues, "endDate")
|
49
|
+
table.insert(optionalValues, opts['endDate'])
|
50
|
+
end
|
51
|
+
|
52
|
+
if opts['every'] then
|
53
|
+
table.insert(optionalValues, "every")
|
54
|
+
table.insert(optionalValues, opts['every'])
|
55
|
+
end
|
56
|
+
|
57
|
+
rcall("HMSET", repeatKey .. ":" .. jobSchedulerId, "name", opts['name'],
|
58
|
+
unpack(optionalValues))
|
59
|
+
end
|
60
|
+
|
61
|
+
-- If we are overriding a repeatable job we must delete the delayed job for
|
62
|
+
-- the next iteration.
|
63
|
+
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
64
|
+
if prevMillis ~= false then
|
65
|
+
local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
66
|
+
local nextDelayedJobId = repeatKey .. ":" .. jobSchedulerId .. ":" .. nextMillis
|
67
|
+
|
68
|
+
if rcall("ZSCORE", delayedKey, delayedJobId) ~= false
|
69
|
+
and rcall("EXISTS", nextDelayedJobId) ~= 1 then
|
70
|
+
removeJob(delayedJobId, true, prefixKey, true --[[remove debounce key]])
|
71
|
+
rcall("ZREM", delayedKey, delayedJobId)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
return storeRepeatableJob(repeatKey, nextMillis, ARGV[2])
|
@@ -20,7 +20,7 @@
|
|
20
20
|
[7] parent dependencies key.
|
21
21
|
[8] parent? {id, queueKey}
|
22
22
|
[9] repeat job key
|
23
|
-
[10]
|
23
|
+
[10] deduplication key
|
24
24
|
|
25
25
|
ARGV[2] Json stringified job data
|
26
26
|
ARGV[3] msgpacked options
|
@@ -47,11 +47,11 @@ local opts = cmsgpack.unpack(ARGV[3])
|
|
47
47
|
local parentKey = args[5]
|
48
48
|
local parent = args[8]
|
49
49
|
local repeatJobKey = args[9]
|
50
|
-
local
|
50
|
+
local deduplicationKey = args[10]
|
51
51
|
local parentData
|
52
52
|
|
53
53
|
-- Includes
|
54
|
-
--- @include "includes/
|
54
|
+
--- @include "includes/deduplicateJob"
|
55
55
|
--- @include "includes/getOrSetMaxEvents"
|
56
56
|
--- @include "includes/handleDuplicatedJob"
|
57
57
|
--- @include "includes/storeJob"
|
@@ -81,10 +81,10 @@ else
|
|
81
81
|
end
|
82
82
|
end
|
83
83
|
|
84
|
-
local
|
85
|
-
jobId,
|
86
|
-
if
|
87
|
-
return
|
84
|
+
local deduplicationJobId = deduplicateJob(args[1], opts['de'],
|
85
|
+
jobId, deduplicationKey, eventsKey, maxEvents)
|
86
|
+
if deduplicationJobId then
|
87
|
+
return deduplicationJobId
|
88
88
|
end
|
89
89
|
|
90
90
|
-- Store the job.
|
@@ -24,7 +24,7 @@
|
|
24
24
|
[7] parent dependencies key.
|
25
25
|
[8] parent? {id, queueKey}
|
26
26
|
[9] repeat job key
|
27
|
-
[10]
|
27
|
+
[10] deduplication key
|
28
28
|
|
29
29
|
ARGV[2] Json stringified job data
|
30
30
|
ARGV[3] msgpacked options
|
@@ -54,12 +54,12 @@ local opts = cmsgpack.unpack(ARGV[3])
|
|
54
54
|
local parentKey = args[5]
|
55
55
|
local parent = args[8]
|
56
56
|
local repeatJobKey = args[9]
|
57
|
-
local
|
57
|
+
local deduplicationKey = args[10]
|
58
58
|
local parentData
|
59
59
|
|
60
60
|
-- Includes
|
61
61
|
--- @include "includes/addJobWithPriority"
|
62
|
-
--- @include "includes/
|
62
|
+
--- @include "includes/deduplicateJob"
|
63
63
|
--- @include "includes/storeJob"
|
64
64
|
--- @include "includes/getOrSetMaxEvents"
|
65
65
|
--- @include "includes/handleDuplicatedJob"
|
@@ -90,10 +90,10 @@ else
|
|
90
90
|
end
|
91
91
|
end
|
92
92
|
|
93
|
-
local
|
94
|
-
jobId,
|
95
|
-
if
|
96
|
-
return
|
93
|
+
local deduplicationJobId = deduplicateJob(args[1], opts['de'],
|
94
|
+
jobId, deduplicationKey, eventsKey, maxEvents)
|
95
|
+
if deduplicationJobId then
|
96
|
+
return deduplicationJobId
|
97
97
|
end
|
98
98
|
|
99
99
|
-- Store the job.
|
@@ -34,7 +34,7 @@
|
|
34
34
|
[7] parent dependencies key.
|
35
35
|
[8] parent? {id, queueKey}
|
36
36
|
[9] repeat job key
|
37
|
-
[10]
|
37
|
+
[10] deduplication key
|
38
38
|
|
39
39
|
ARGV[2] Json stringified job data
|
40
40
|
ARGV[3] msgpacked options
|
@@ -57,12 +57,12 @@ local opts = cmsgpack.unpack(ARGV[3])
|
|
57
57
|
local parentKey = args[5]
|
58
58
|
local parent = args[8]
|
59
59
|
local repeatJobKey = args[9]
|
60
|
-
local
|
60
|
+
local deduplicationKey = args[10]
|
61
61
|
local parentData
|
62
62
|
|
63
63
|
-- Includes
|
64
64
|
--- @include "includes/addJobInTargetList"
|
65
|
-
--- @include "includes/
|
65
|
+
--- @include "includes/deduplicateJob"
|
66
66
|
--- @include "includes/getOrSetMaxEvents"
|
67
67
|
--- @include "includes/getTargetQueueList"
|
68
68
|
--- @include "includes/handleDuplicatedJob"
|
@@ -94,10 +94,10 @@ else
|
|
94
94
|
end
|
95
95
|
end
|
96
96
|
|
97
|
-
local
|
98
|
-
jobId,
|
99
|
-
if
|
100
|
-
return
|
97
|
+
local deduplicationJobId = deduplicateJob(args[1], opts['de'],
|
98
|
+
jobId, deduplicationKey, eventsKey, maxEvents)
|
99
|
+
if deduplicationJobId then
|
100
|
+
return deduplicationJobId
|
101
101
|
end
|
102
102
|
|
103
103
|
-- Store the job.
|
@@ -4,6 +4,7 @@
|
|
4
4
|
Input:
|
5
5
|
KEYS[1] set key,
|
6
6
|
KEYS[2] events stream key
|
7
|
+
KEYS[3] job schedulers key
|
7
8
|
|
8
9
|
ARGV[1] jobKey prefix
|
9
10
|
ARGV[2] timestamp
|
@@ -32,21 +33,21 @@ end
|
|
32
33
|
|
33
34
|
local result
|
34
35
|
if ARGV[4] == "active" then
|
35
|
-
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
|
36
|
+
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false --[[ hasFinished ]])
|
36
37
|
elseif ARGV[4] == "delayed" then
|
37
38
|
rangeEnd = "+inf"
|
38
39
|
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
39
|
-
{"processedOn", "timestamp"}, false)
|
40
|
+
{"processedOn", "timestamp"}, false --[[ hasFinished ]], KEYS[3])
|
40
41
|
elseif ARGV[4] == "prioritized" then
|
41
42
|
rangeEnd = "+inf"
|
42
43
|
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
43
|
-
{"timestamp"}, false)
|
44
|
+
{"timestamp"}, false --[[ hasFinished ]])
|
44
45
|
elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
|
45
|
-
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
|
46
|
+
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true --[[ hasFinished ]])
|
46
47
|
else
|
47
48
|
rangeEnd = ARGV[2]
|
48
49
|
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
49
|
-
{"finishedOn"}, true)
|
50
|
+
{"finishedOn"}, true --[[ hasFinished ]])
|
50
51
|
end
|
51
52
|
|
52
53
|
rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
|
@@ -0,0 +1,41 @@
|
|
1
|
+
--[[
|
2
|
+
Drains the queue, removes all jobs that are waiting
|
3
|
+
or delayed, but not active, completed or failed
|
4
|
+
|
5
|
+
Input:
|
6
|
+
KEYS[1] 'wait',
|
7
|
+
KEYS[2] 'paused'
|
8
|
+
KEYS[3] 'delayed'
|
9
|
+
KEYS[4] 'prioritized'
|
10
|
+
KEYS[5] 'jobschedulers' (repeat)
|
11
|
+
|
12
|
+
ARGV[1] queue key prefix
|
13
|
+
]]
|
14
|
+
local rcall = redis.call
|
15
|
+
local queueBaseKey = ARGV[1]
|
16
|
+
|
17
|
+
--- @include "includes/removeListJobs"
|
18
|
+
--- @include "includes/removeZSetJobs"
|
19
|
+
|
20
|
+
removeListJobs(KEYS[1], true, queueBaseKey, 0) -- wait
|
21
|
+
removeListJobs(KEYS[2], true, queueBaseKey, 0) -- paused
|
22
|
+
|
23
|
+
if KEYS[3] ~= "" then
|
24
|
+
|
25
|
+
-- We must not remove delayed jobs if they are associated to a job scheduler.
|
26
|
+
local scheduledJobs = {}
|
27
|
+
local jobSchedulers = rcall("ZRANGE", KEYS[5], 0, -1, "WITHSCORES")
|
28
|
+
|
29
|
+
-- For every job scheduler, get the current delayed job id.
|
30
|
+
for i = 1, #jobSchedulers, 2 do
|
31
|
+
local jobSchedulerId = jobSchedulers[i]
|
32
|
+
local jobSchedulerMillis = jobSchedulers[i + 1]
|
33
|
+
|
34
|
+
local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. jobSchedulerMillis
|
35
|
+
scheduledJobs[delayedJobId] = true
|
36
|
+
end
|
37
|
+
|
38
|
+
removeZSetJobs(KEYS[3], true, queueBaseKey, 0, scheduledJobs) -- delayed
|
39
|
+
end
|
40
|
+
|
41
|
+
removeZSetJobs(KEYS[4], true, queueBaseKey, 0) -- prioritized
|
@@ -1,45 +1,66 @@
|
|
1
1
|
--[[
|
2
2
|
Function to clean job set.
|
3
3
|
Returns jobIds and deleted count number.
|
4
|
-
]]
|
4
|
+
]]
|
5
5
|
|
6
6
|
-- Includes
|
7
7
|
--- @include "batches"
|
8
8
|
--- @include "getJobsInZset"
|
9
9
|
--- @include "getTimestamp"
|
10
10
|
--- @include "removeJob"
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
for i, job in ipairs(jobs) do
|
18
|
-
if limit > 0 and deletedCount >= limit then
|
19
|
-
break
|
11
|
+
local function isJobSchedulerJob(jobId, jobSchedulersKey)
|
12
|
+
if jobSchedulersKey then
|
13
|
+
local jobSchedulerId = jobId:match("repeat:(.*):%d+")
|
14
|
+
if jobSchedulerId then
|
15
|
+
return rcall("ZSCORE", jobSchedulersKey, jobSchedulerId)
|
16
|
+
end
|
20
17
|
end
|
18
|
+
return false
|
19
|
+
end
|
20
|
+
|
21
|
+
local function cleanSet(
|
22
|
+
setKey,
|
23
|
+
jobKeyPrefix,
|
24
|
+
rangeEnd,
|
25
|
+
timestamp,
|
26
|
+
limit,
|
27
|
+
attributes,
|
28
|
+
isFinished,
|
29
|
+
jobSchedulersKey)
|
30
|
+
local jobs = getJobsInZset(setKey, rangeEnd, limit)
|
31
|
+
local deleted = {}
|
32
|
+
local deletedCount = 0
|
33
|
+
local jobTS
|
34
|
+
for i, job in ipairs(jobs) do
|
35
|
+
if limit > 0 and deletedCount >= limit then
|
36
|
+
break
|
37
|
+
end
|
21
38
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
39
|
+
-- Extract a Job Scheduler Id from jobId ("repeat:job-scheduler-id:millis")
|
40
|
+
-- and check if it is in the scheduled jobs
|
41
|
+
if not isJobSchedulerJob(job, jobSchedulersKey) then
|
42
|
+
local jobKey = jobKeyPrefix .. job
|
43
|
+
if isFinished then
|
44
|
+
removeJob(job, true, jobKeyPrefix, true --[[remove debounce key]] )
|
45
|
+
deletedCount = deletedCount + 1
|
46
|
+
table.insert(deleted, job)
|
47
|
+
else
|
48
|
+
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
49
|
+
jobTS = getTimestamp(jobKey, attributes)
|
50
|
+
if (not jobTS or jobTS <= timestamp) then
|
51
|
+
removeJob(job, true, jobKeyPrefix, true --[[remove debounce key]] )
|
52
|
+
deletedCount = deletedCount + 1
|
53
|
+
table.insert(deleted, job)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
35
57
|
end
|
36
|
-
end
|
37
58
|
|
38
|
-
|
39
|
-
|
40
|
-
|
59
|
+
if (#deleted > 0) then
|
60
|
+
for from, to in batches(#deleted, 7000) do
|
61
|
+
rcall("ZREM", setKey, unpack(deleted, from, to))
|
62
|
+
end
|
41
63
|
end
|
42
|
-
end
|
43
64
|
|
44
|
-
|
65
|
+
return {deleted, deletedCount}
|
45
66
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
--[[
|
2
|
+
Function to debounce a job.
|
3
|
+
]]
|
4
|
+
|
5
|
+
local function deduplicateJob(prefixKey, deduplicationOpts, jobId, deduplicationKey, eventsKey, maxEvents)
|
6
|
+
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
7
|
+
if deduplicationId then
|
8
|
+
local ttl = deduplicationOpts['ttl']
|
9
|
+
local deduplicationKeyExists
|
10
|
+
if ttl then
|
11
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
|
12
|
+
else
|
13
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
|
14
|
+
end
|
15
|
+
if deduplicationKeyExists then
|
16
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
17
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
18
|
+
"debounced", "jobId", currentDebounceJobId, "debounceId", deduplicationId)
|
19
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
20
|
+
"deduplicated", "jobId", currentDebounceJobId, "deduplicationId", deduplicationId)
|
21
|
+
return currentDebounceJobId
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
@@ -3,8 +3,20 @@
|
|
3
3
|
--- @include "getZSetItems"
|
4
4
|
--- @include "removeJobs"
|
5
5
|
|
6
|
-
local function removeZSetJobs(keyName, hard, baseKey, max)
|
6
|
+
local function removeZSetJobs(keyName, hard, baseKey, max, jobsToIgnore)
|
7
7
|
local jobs = getZSetItems(keyName, max)
|
8
|
+
|
9
|
+
-- filter out jobs to ignore
|
10
|
+
if jobsToIgnore then
|
11
|
+
local filteredJobs = {}
|
12
|
+
for i = 1, #jobs do
|
13
|
+
if not jobsToIgnore[jobs[i]] then
|
14
|
+
table.insert(filteredJobs, jobs[i])
|
15
|
+
end
|
16
|
+
end
|
17
|
+
jobs = filteredJobs
|
18
|
+
end
|
19
|
+
|
8
20
|
local count = removeJobs(jobs, hard, baseKey, max)
|
9
21
|
if(#jobs > 0) then
|
10
22
|
for from, to in batches(#jobs, 7000) do
|
@@ -44,7 +44,7 @@ local metaKey = KEYS[6]
|
|
44
44
|
local pausedKey = KEYS[7]
|
45
45
|
local markerKey = KEYS[8]
|
46
46
|
local eventStreamKey = KEYS[9]
|
47
|
-
local maxStalledJobCount = ARGV[1]
|
47
|
+
local maxStalledJobCount = tonumber(ARGV[1])
|
48
48
|
local queueKeyPrefix = ARGV[2]
|
49
49
|
local timestamp = ARGV[3]
|
50
50
|
local maxCheckTime = ARGV[4]
|
@@ -63,8 +63,6 @@ local failed = {}
|
|
63
63
|
if (#stalling > 0) then
|
64
64
|
rcall('DEL', stalledKey)
|
65
65
|
|
66
|
-
local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
|
67
|
-
|
68
66
|
-- Remove from active list
|
69
67
|
for i, jobId in ipairs(stalling) do
|
70
68
|
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
@@ -83,7 +81,7 @@ if (#stalling > 0) then
|
|
83
81
|
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
84
82
|
local stalledCount =
|
85
83
|
rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
86
|
-
if (stalledCount >
|
84
|
+
if (stalledCount > maxStalledJobCount) then
|
87
85
|
local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
|
88
86
|
local rawOpts = jobAttributes[1]
|
89
87
|
local rawParentData = jobAttributes[2]
|
@@ -24,7 +24,7 @@ local rcall = redis.call
|
|
24
24
|
--- @include "includes/removeJobKeys"
|
25
25
|
--- @include "includes/removeParentDependencyKey"
|
26
26
|
|
27
|
-
local function removeJob(
|
27
|
+
local function removeJob(prefix, jobId, parentKey, removeChildren)
|
28
28
|
local jobKey = prefix .. jobId;
|
29
29
|
|
30
30
|
removeParentDependencyKey(jobKey, false, parentKey, nil)
|
@@ -33,14 +33,14 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
|
|
33
33
|
-- Check if this job has children
|
34
34
|
-- If so, we are going to try to remove the children recursively in deep first way because
|
35
35
|
-- if some job is locked we must exit with and error.
|
36
|
-
--local countProcessed = rcall("HLEN", jobKey .. ":processed")
|
36
|
+
-- local countProcessed = rcall("HLEN", jobKey .. ":processed")
|
37
37
|
local processed = rcall("HGETALL", jobKey .. ":processed")
|
38
38
|
|
39
39
|
if (#processed > 0) then
|
40
40
|
for i = 1, #processed, 2 do
|
41
41
|
local childJobId = getJobIdFromKey(processed[i])
|
42
42
|
local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
|
43
|
-
removeJob(
|
43
|
+
removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
@@ -50,7 +50,7 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
|
|
50
50
|
-- We need to get the jobId for this job.
|
51
51
|
local childJobId = getJobIdFromKey(childJobKey)
|
52
52
|
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
53
|
-
removeJob(
|
53
|
+
removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
|
54
54
|
end
|
55
55
|
end
|
56
56
|
|
@@ -60,7 +60,7 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
|
|
60
60
|
for i = 1, #failed, 2 do
|
61
61
|
local childJobId = getJobIdFromKey(failed[i])
|
62
62
|
local childJobPrefix = getJobKeyPrefix(failed[i], childJobId)
|
63
|
-
removeJob(
|
63
|
+
removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
|
64
64
|
end
|
65
65
|
end
|
66
66
|
end
|
@@ -70,15 +70,23 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
|
|
70
70
|
removeDebounceKey(prefix, jobKey)
|
71
71
|
if removeJobKeys(jobKey) > 0 then
|
72
72
|
local maxEvents = getOrSetMaxEvents(KEYS[2])
|
73
|
-
rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
|
74
|
-
|
73
|
+
rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed", "jobId", jobId, "prev",
|
74
|
+
prev)
|
75
75
|
end
|
76
76
|
end
|
77
77
|
|
78
78
|
local prefix = KEYS[1]
|
79
|
+
local jobId = ARGV[1]
|
80
|
+
local shouldRemoveChildren = ARGV[2]
|
81
|
+
local jobKey = prefix .. jobId
|
79
82
|
|
80
|
-
if
|
81
|
-
|
83
|
+
-- Check if the job belongs to a job scheduler and it is in delayed state.
|
84
|
+
if rcall("ZSCORE", prefix .. "delayed", jobId) and rcall("HGET", jobKey, "rjk") then
|
85
|
+
return -8 -- Return error code as the job is part of a job scheduler and is in delayed state.
|
86
|
+
end
|
87
|
+
|
88
|
+
if not isLocked(prefix, jobId, shouldRemoveChildren) then
|
89
|
+
removeJob(prefix, jobId, nil, shouldRemoveChildren)
|
82
90
|
return 1
|
83
91
|
end
|
84
92
|
return 0
|
@@ -0,0 +1,43 @@
|
|
1
|
+
|
2
|
+
--[[
|
3
|
+
Removes a repeatable job
|
4
|
+
Input:
|
5
|
+
KEYS[1] job schedulers key
|
6
|
+
KEYS[2] delayed jobs key
|
7
|
+
KEYS[3] events key
|
8
|
+
|
9
|
+
ARGV[1] job scheduler id
|
10
|
+
ARGV[2] prefix key
|
11
|
+
|
12
|
+
Output:
|
13
|
+
0 - OK
|
14
|
+
1 - Missing repeat job
|
15
|
+
|
16
|
+
Events:
|
17
|
+
'removed'
|
18
|
+
]]
|
19
|
+
local rcall = redis.call
|
20
|
+
|
21
|
+
-- Includes
|
22
|
+
--- @include "includes/removeJobKeys"
|
23
|
+
|
24
|
+
local jobSchedulerId = ARGV[1]
|
25
|
+
local prefix = ARGV[2]
|
26
|
+
|
27
|
+
local millis = rcall("ZSCORE", KEYS[1], jobSchedulerId)
|
28
|
+
|
29
|
+
if millis then
|
30
|
+
-- Delete next programmed job.
|
31
|
+
local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. millis
|
32
|
+
if(rcall("ZREM", KEYS[2], delayedJobId) == 1) then
|
33
|
+
removeJobKeys(prefix .. delayedJobId)
|
34
|
+
rcall("XADD", KEYS[3], "*", "event", "removed", "jobId", delayedJobId, "prev", "delayed")
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
if(rcall("ZREM", KEYS[1], jobSchedulerId) == 1) then
|
39
|
+
rcall("DEL", KEYS[1] .. ":" .. jobSchedulerId)
|
40
|
+
return 0
|
41
|
+
end
|
42
|
+
|
43
|
+
return 1
|
@@ -7,5 +7,6 @@ export var ErrorCode;
|
|
7
7
|
ErrorCode[ErrorCode["ParentJobNotExist"] = -5] = "ParentJobNotExist";
|
8
8
|
ErrorCode[ErrorCode["JobLockMismatch"] = -6] = "JobLockMismatch";
|
9
9
|
ErrorCode[ErrorCode["ParentJobCannotBeReplaced"] = -7] = "ParentJobCannotBeReplaced";
|
10
|
+
ErrorCode[ErrorCode["JobBelongsToJobScheduler"] = -8] = "JobBelongsToJobScheduler";
|
10
11
|
})(ErrorCode || (ErrorCode = {}));
|
11
12
|
//# sourceMappingURL=error-code.js.map
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,
|
1
|
+
{"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,SASX;AATD,WAAY,SAAS;IACnB,wDAAgB,CAAA;IAChB,gEAAoB,CAAA;IACpB,4DAAkB,CAAA;IAClB,8EAA2B,CAAA;IAC3B,oEAAsB,CAAA;IACtB,gEAAoB,CAAA;IACpB,oFAA8B,CAAA;IAC9B,kFAA6B,CAAA;AAC/B,CAAC,EATW,SAAS,KAAT,SAAS,QASpB"}
|
@@ -22,7 +22,7 @@ const content = `--[[
|
|
22
22
|
[7] parent dependencies key.
|
23
23
|
[8] parent? {id, queueKey}
|
24
24
|
[9] repeat job key
|
25
|
-
[10]
|
25
|
+
[10] deduplication key
|
26
26
|
ARGV[2] Json stringified job data
|
27
27
|
ARGV[3] msgpacked options
|
28
28
|
Output:
|
@@ -42,7 +42,7 @@ local data = ARGV[2]
|
|
42
42
|
local parentKey = args[5]
|
43
43
|
local parent = args[8]
|
44
44
|
local repeatJobKey = args[9]
|
45
|
-
local
|
45
|
+
local deduplicationKey = args[10]
|
46
46
|
local parentData
|
47
47
|
-- Includes
|
48
48
|
--[[
|
@@ -72,20 +72,22 @@ end
|
|
72
72
|
--[[
|
73
73
|
Function to debounce a job.
|
74
74
|
]]
|
75
|
-
local function
|
76
|
-
local
|
77
|
-
if
|
78
|
-
local ttl =
|
79
|
-
local
|
75
|
+
local function deduplicateJob(prefixKey, deduplicationOpts, jobId, deduplicationKey, eventsKey, maxEvents)
|
76
|
+
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
77
|
+
if deduplicationId then
|
78
|
+
local ttl = deduplicationOpts['ttl']
|
79
|
+
local deduplicationKeyExists
|
80
80
|
if ttl then
|
81
|
-
|
81
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
|
82
82
|
else
|
83
|
-
|
83
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
|
84
84
|
end
|
85
|
-
if
|
86
|
-
local currentDebounceJobId = rcall('GET',
|
85
|
+
if deduplicationKeyExists then
|
86
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
87
87
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
88
|
-
"debounced", "jobId", currentDebounceJobId, "debounceId",
|
88
|
+
"debounced", "jobId", currentDebounceJobId, "debounceId", deduplicationId)
|
89
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
90
|
+
"deduplicated", "jobId", currentDebounceJobId, "deduplicationId", deduplicationId)
|
89
91
|
return currentDebounceJobId
|
90
92
|
end
|
91
93
|
end
|
@@ -338,10 +340,10 @@ else
|
|
338
340
|
maxEvents, timestamp)
|
339
341
|
end
|
340
342
|
end
|
341
|
-
local
|
342
|
-
jobId,
|
343
|
-
if
|
344
|
-
return
|
343
|
+
local deduplicationJobId = deduplicateJob(args[1], opts['de'],
|
344
|
+
jobId, deduplicationKey, eventsKey, maxEvents)
|
345
|
+
if deduplicationJobId then
|
346
|
+
return deduplicationJobId
|
345
347
|
end
|
346
348
|
-- Store the job.
|
347
349
|
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2Wf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|