bullmq 5.14.0 → 5.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/child-pool.js +9 -2
- package/dist/cjs/classes/child-pool.js.map +1 -1
- package/dist/cjs/classes/child.js +5 -6
- package/dist/cjs/classes/child.js.map +1 -1
- package/dist/cjs/classes/job-scheduler.js +140 -0
- package/dist/cjs/classes/job-scheduler.js.map +1 -0
- package/dist/cjs/classes/job.js +1 -1
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue.js +64 -0
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/repeat.js +9 -10
- package/dist/cjs/classes/repeat.js.map +1 -1
- package/dist/cjs/classes/scripts.js +45 -39
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +25 -4
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addJobScheduler-2.lua +75 -0
- package/dist/cjs/commands/{cleanJobsInSet-2.lua → cleanJobsInSet-3.lua} +6 -5
- package/dist/cjs/commands/drain-5.lua +41 -0
- package/dist/cjs/commands/includes/cleanSet.lua +50 -29
- package/dist/cjs/commands/includes/removeZSetJobs.lua +13 -1
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +2 -4
- package/dist/cjs/commands/removeJob-2.lua +17 -9
- package/dist/cjs/commands/removeJobScheduler-3.lua +43 -0
- package/dist/cjs/enums/error-code.js +1 -0
- package/dist/cjs/enums/error-code.js.map +1 -1
- package/dist/cjs/interfaces/index.js +1 -0
- package/dist/cjs/interfaces/index.js.map +1 -1
- package/dist/cjs/interfaces/sandboxed-options.js +3 -0
- package/dist/cjs/interfaces/sandboxed-options.js.map +1 -0
- package/dist/cjs/scripts/addJobScheduler-2.js +223 -0
- package/dist/cjs/scripts/addJobScheduler-2.js.map +1 -0
- package/dist/cjs/scripts/{cleanJobsInSet-2.js → cleanJobsInSet-3.js} +57 -35
- package/dist/cjs/scripts/{cleanJobsInSet-2.js.map → cleanJobsInSet-3.js.map} +1 -1
- package/dist/cjs/scripts/{drain-4.js → drain-5.js} +28 -7
- package/dist/cjs/scripts/drain-5.js.map +1 -0
- package/dist/cjs/scripts/index.js +4 -2
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +2 -3
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +11 -1
- package/dist/cjs/scripts/obliterate-2.js.map +1 -1
- package/dist/cjs/scripts/removeJob-2.js +16 -9
- package/dist/cjs/scripts/removeJob-2.js.map +1 -1
- package/dist/cjs/scripts/removeJobScheduler-3.js +49 -0
- package/dist/cjs/scripts/removeJobScheduler-3.js.map +1 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/utils.js +25 -1
- package/dist/cjs/utils.js.map +1 -1
- package/dist/esm/classes/child-pool.d.ts +3 -3
- package/dist/esm/classes/child-pool.js +9 -2
- package/dist/esm/classes/child-pool.js.map +1 -1
- package/dist/esm/classes/child.d.ts +3 -5
- package/dist/esm/classes/child.js +5 -6
- package/dist/esm/classes/child.js.map +1 -1
- package/dist/esm/classes/job-scheduler.d.ts +30 -0
- package/dist/esm/classes/job-scheduler.js +135 -0
- package/dist/esm/classes/job-scheduler.js.map +1 -0
- package/dist/esm/classes/job.d.ts +1 -1
- package/dist/esm/classes/job.js +2 -2
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +47 -0
- package/dist/esm/classes/queue.js +64 -0
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/repeat.js +9 -10
- package/dist/esm/classes/repeat.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +3 -7
- package/dist/esm/classes/scripts.js +46 -40
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +3 -0
- package/dist/esm/classes/worker.js +25 -4
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addJobScheduler-2.lua +75 -0
- package/dist/esm/commands/{cleanJobsInSet-2.lua → cleanJobsInSet-3.lua} +6 -5
- package/dist/esm/commands/drain-5.lua +41 -0
- package/dist/esm/commands/includes/cleanSet.lua +50 -29
- package/dist/esm/commands/includes/removeZSetJobs.lua +13 -1
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +2 -4
- package/dist/esm/commands/removeJob-2.lua +17 -9
- package/dist/esm/commands/removeJobScheduler-3.lua +43 -0
- package/dist/esm/enums/error-code.d.ts +2 -1
- package/dist/esm/enums/error-code.js +1 -0
- package/dist/esm/enums/error-code.js.map +1 -1
- package/dist/esm/interfaces/index.d.ts +1 -0
- package/dist/esm/interfaces/index.js +1 -0
- package/dist/esm/interfaces/index.js.map +1 -1
- package/dist/esm/interfaces/sandboxed-options.d.ts +28 -0
- package/dist/esm/interfaces/sandboxed-options.js +2 -0
- package/dist/esm/interfaces/sandboxed-options.js.map +1 -0
- package/dist/esm/interfaces/worker-options.d.ts +2 -18
- package/dist/esm/scripts/addJobScheduler-2.d.ts +5 -0
- package/dist/esm/scripts/addJobScheduler-2.js +220 -0
- package/dist/esm/scripts/addJobScheduler-2.js.map +1 -0
- package/dist/esm/scripts/{cleanJobsInSet-2.js → cleanJobsInSet-3.js} +57 -35
- package/dist/esm/scripts/{cleanJobsInSet-2.js.map → cleanJobsInSet-3.js.map} +1 -1
- package/dist/esm/scripts/{drain-4.js → drain-5.js} +28 -7
- package/dist/esm/scripts/drain-5.js.map +1 -0
- package/dist/esm/scripts/index.d.ts +4 -2
- package/dist/esm/scripts/index.js +4 -2
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +2 -3
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +11 -1
- package/dist/esm/scripts/obliterate-2.js.map +1 -1
- package/dist/esm/scripts/removeJob-2.js +16 -9
- package/dist/esm/scripts/removeJob-2.js.map +1 -1
- package/dist/esm/scripts/removeJobScheduler-3.d.ts +5 -0
- package/dist/esm/scripts/removeJobScheduler-3.js +46 -0
- package/dist/esm/scripts/removeJobScheduler-3.js.map +1 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/utils.d.ts +7 -0
- package/dist/esm/utils.js +23 -0
- package/dist/esm/utils.js.map +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/drain-4.lua +0 -26
- package/dist/cjs/scripts/drain-4.js.map +0 -1
- package/dist/esm/commands/drain-4.lua +0 -26
- package/dist/esm/scripts/drain-4.js.map +0 -1
- /package/dist/esm/scripts/{cleanJobsInSet-2.d.ts → cleanJobsInSet-3.d.ts} +0 -0
- /package/dist/esm/scripts/{drain-4.d.ts → drain-5.d.ts} +0 -0
@@ -0,0 +1,75 @@
|
|
1
|
+
--[[
|
2
|
+
Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] 'repeat' key
|
6
|
+
KEYS[2] 'delayed' key
|
7
|
+
|
8
|
+
ARGV[1] next milliseconds
|
9
|
+
ARGV[2] msgpacked options
|
10
|
+
[1] name
|
11
|
+
[2] tz?
|
12
|
+
[3] patten?
|
13
|
+
[4] endDate?
|
14
|
+
[5] every?
|
15
|
+
ARGV[3] jobs scheduler id
|
16
|
+
ARGV[4] prefix key
|
17
|
+
|
18
|
+
Output:
|
19
|
+
repeatableKey - OK
|
20
|
+
]]
|
21
|
+
local rcall = redis.call
|
22
|
+
local repeatKey = KEYS[1]
|
23
|
+
local delayedKey = KEYS[2]
|
24
|
+
|
25
|
+
local nextMillis = ARGV[1]
|
26
|
+
local jobSchedulerId = ARGV[3]
|
27
|
+
local prefixKey = ARGV[4]
|
28
|
+
|
29
|
+
-- Includes
|
30
|
+
--- @include "includes/removeJob"
|
31
|
+
|
32
|
+
local function storeRepeatableJob(repeatKey, nextMillis, rawOpts)
|
33
|
+
rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
|
34
|
+
local opts = cmsgpack.unpack(rawOpts)
|
35
|
+
|
36
|
+
local optionalValues = {}
|
37
|
+
if opts['tz'] then
|
38
|
+
table.insert(optionalValues, "tz")
|
39
|
+
table.insert(optionalValues, opts['tz'])
|
40
|
+
end
|
41
|
+
|
42
|
+
if opts['pattern'] then
|
43
|
+
table.insert(optionalValues, "pattern")
|
44
|
+
table.insert(optionalValues, opts['pattern'])
|
45
|
+
end
|
46
|
+
|
47
|
+
if opts['endDate'] then
|
48
|
+
table.insert(optionalValues, "endDate")
|
49
|
+
table.insert(optionalValues, opts['endDate'])
|
50
|
+
end
|
51
|
+
|
52
|
+
if opts['every'] then
|
53
|
+
table.insert(optionalValues, "every")
|
54
|
+
table.insert(optionalValues, opts['every'])
|
55
|
+
end
|
56
|
+
|
57
|
+
rcall("HMSET", repeatKey .. ":" .. jobSchedulerId, "name", opts['name'],
|
58
|
+
unpack(optionalValues))
|
59
|
+
end
|
60
|
+
|
61
|
+
-- If we are overriding a repeatable job we must delete the delayed job for
|
62
|
+
-- the next iteration.
|
63
|
+
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
64
|
+
if prevMillis ~= false then
|
65
|
+
local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
66
|
+
local nextDelayedJobId = repeatKey .. ":" .. jobSchedulerId .. ":" .. nextMillis
|
67
|
+
|
68
|
+
if rcall("ZSCORE", delayedKey, delayedJobId) ~= false
|
69
|
+
and rcall("EXISTS", nextDelayedJobId) ~= 1 then
|
70
|
+
removeJob(delayedJobId, true, prefixKey, true --[[remove debounce key]])
|
71
|
+
rcall("ZREM", delayedKey, delayedJobId)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
return storeRepeatableJob(repeatKey, nextMillis, ARGV[2])
|
@@ -4,6 +4,7 @@
|
|
4
4
|
Input:
|
5
5
|
KEYS[1] set key,
|
6
6
|
KEYS[2] events stream key
|
7
|
+
KEYS[3] job schedulers key
|
7
8
|
|
8
9
|
ARGV[1] jobKey prefix
|
9
10
|
ARGV[2] timestamp
|
@@ -32,21 +33,21 @@ end
|
|
32
33
|
|
33
34
|
local result
|
34
35
|
if ARGV[4] == "active" then
|
35
|
-
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
|
36
|
+
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false --[[ hasFinished ]])
|
36
37
|
elseif ARGV[4] == "delayed" then
|
37
38
|
rangeEnd = "+inf"
|
38
39
|
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
39
|
-
{"processedOn", "timestamp"}, false)
|
40
|
+
{"processedOn", "timestamp"}, false --[[ hasFinished ]], KEYS[3])
|
40
41
|
elseif ARGV[4] == "prioritized" then
|
41
42
|
rangeEnd = "+inf"
|
42
43
|
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
43
|
-
{"timestamp"}, false)
|
44
|
+
{"timestamp"}, false --[[ hasFinished ]])
|
44
45
|
elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
|
45
|
-
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
|
46
|
+
result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true --[[ hasFinished ]])
|
46
47
|
else
|
47
48
|
rangeEnd = ARGV[2]
|
48
49
|
result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
|
49
|
-
{"finishedOn"}, true)
|
50
|
+
{"finishedOn"}, true --[[ hasFinished ]])
|
50
51
|
end
|
51
52
|
|
52
53
|
rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
|
@@ -0,0 +1,41 @@
|
|
1
|
+
--[[
|
2
|
+
Drains the queue, removes all jobs that are waiting
|
3
|
+
or delayed, but not active, completed or failed
|
4
|
+
|
5
|
+
Input:
|
6
|
+
KEYS[1] 'wait',
|
7
|
+
KEYS[2] 'paused'
|
8
|
+
KEYS[3] 'delayed'
|
9
|
+
KEYS[4] 'prioritized'
|
10
|
+
KEYS[5] 'jobschedulers' (repeat)
|
11
|
+
|
12
|
+
ARGV[1] queue key prefix
|
13
|
+
]]
|
14
|
+
local rcall = redis.call
|
15
|
+
local queueBaseKey = ARGV[1]
|
16
|
+
|
17
|
+
--- @include "includes/removeListJobs"
|
18
|
+
--- @include "includes/removeZSetJobs"
|
19
|
+
|
20
|
+
removeListJobs(KEYS[1], true, queueBaseKey, 0) -- wait
|
21
|
+
removeListJobs(KEYS[2], true, queueBaseKey, 0) -- paused
|
22
|
+
|
23
|
+
if KEYS[3] ~= "" then
|
24
|
+
|
25
|
+
-- We must not remove delayed jobs if they are associated to a job scheduler.
|
26
|
+
local scheduledJobs = {}
|
27
|
+
local jobSchedulers = rcall("ZRANGE", KEYS[5], 0, -1, "WITHSCORES")
|
28
|
+
|
29
|
+
-- For every job scheduler, get the current delayed job id.
|
30
|
+
for i = 1, #jobSchedulers, 2 do
|
31
|
+
local jobSchedulerId = jobSchedulers[i]
|
32
|
+
local jobSchedulerMillis = jobSchedulers[i + 1]
|
33
|
+
|
34
|
+
local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. jobSchedulerMillis
|
35
|
+
scheduledJobs[delayedJobId] = true
|
36
|
+
end
|
37
|
+
|
38
|
+
removeZSetJobs(KEYS[3], true, queueBaseKey, 0, scheduledJobs) -- delayed
|
39
|
+
end
|
40
|
+
|
41
|
+
removeZSetJobs(KEYS[4], true, queueBaseKey, 0) -- prioritized
|
@@ -1,45 +1,66 @@
|
|
1
1
|
--[[
|
2
2
|
Function to clean job set.
|
3
3
|
Returns jobIds and deleted count number.
|
4
|
-
]]
|
4
|
+
]]
|
5
5
|
|
6
6
|
-- Includes
|
7
7
|
--- @include "batches"
|
8
8
|
--- @include "getJobsInZset"
|
9
9
|
--- @include "getTimestamp"
|
10
10
|
--- @include "removeJob"
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
for i, job in ipairs(jobs) do
|
18
|
-
if limit > 0 and deletedCount >= limit then
|
19
|
-
break
|
11
|
+
local function isJobSchedulerJob(jobId, jobSchedulersKey)
|
12
|
+
if jobSchedulersKey then
|
13
|
+
local jobSchedulerId = jobId:match("repeat:(.*):%d+")
|
14
|
+
if jobSchedulerId then
|
15
|
+
return rcall("ZSCORE", jobSchedulersKey, jobSchedulerId)
|
16
|
+
end
|
20
17
|
end
|
18
|
+
return false
|
19
|
+
end
|
20
|
+
|
21
|
+
local function cleanSet(
|
22
|
+
setKey,
|
23
|
+
jobKeyPrefix,
|
24
|
+
rangeEnd,
|
25
|
+
timestamp,
|
26
|
+
limit,
|
27
|
+
attributes,
|
28
|
+
isFinished,
|
29
|
+
jobSchedulersKey)
|
30
|
+
local jobs = getJobsInZset(setKey, rangeEnd, limit)
|
31
|
+
local deleted = {}
|
32
|
+
local deletedCount = 0
|
33
|
+
local jobTS
|
34
|
+
for i, job in ipairs(jobs) do
|
35
|
+
if limit > 0 and deletedCount >= limit then
|
36
|
+
break
|
37
|
+
end
|
21
38
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
39
|
+
-- Extract a Job Scheduler Id from jobId ("repeat:job-scheduler-id:millis")
|
40
|
+
-- and check if it is in the scheduled jobs
|
41
|
+
if not isJobSchedulerJob(job, jobSchedulersKey) then
|
42
|
+
local jobKey = jobKeyPrefix .. job
|
43
|
+
if isFinished then
|
44
|
+
removeJob(job, true, jobKeyPrefix, true --[[remove debounce key]] )
|
45
|
+
deletedCount = deletedCount + 1
|
46
|
+
table.insert(deleted, job)
|
47
|
+
else
|
48
|
+
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
49
|
+
jobTS = getTimestamp(jobKey, attributes)
|
50
|
+
if (not jobTS or jobTS <= timestamp) then
|
51
|
+
removeJob(job, true, jobKeyPrefix, true --[[remove debounce key]] )
|
52
|
+
deletedCount = deletedCount + 1
|
53
|
+
table.insert(deleted, job)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
35
57
|
end
|
36
|
-
end
|
37
58
|
|
38
|
-
|
39
|
-
|
40
|
-
|
59
|
+
if (#deleted > 0) then
|
60
|
+
for from, to in batches(#deleted, 7000) do
|
61
|
+
rcall("ZREM", setKey, unpack(deleted, from, to))
|
62
|
+
end
|
41
63
|
end
|
42
|
-
end
|
43
64
|
|
44
|
-
|
65
|
+
return {deleted, deletedCount}
|
45
66
|
end
|
@@ -3,8 +3,20 @@
|
|
3
3
|
--- @include "getZSetItems"
|
4
4
|
--- @include "removeJobs"
|
5
5
|
|
6
|
-
local function removeZSetJobs(keyName, hard, baseKey, max)
|
6
|
+
local function removeZSetJobs(keyName, hard, baseKey, max, jobsToIgnore)
|
7
7
|
local jobs = getZSetItems(keyName, max)
|
8
|
+
|
9
|
+
-- filter out jobs to ignore
|
10
|
+
if jobsToIgnore then
|
11
|
+
local filteredJobs = {}
|
12
|
+
for i = 1, #jobs do
|
13
|
+
if not jobsToIgnore[jobs[i]] then
|
14
|
+
table.insert(filteredJobs, jobs[i])
|
15
|
+
end
|
16
|
+
end
|
17
|
+
jobs = filteredJobs
|
18
|
+
end
|
19
|
+
|
8
20
|
local count = removeJobs(jobs, hard, baseKey, max)
|
9
21
|
if(#jobs > 0) then
|
10
22
|
for from, to in batches(#jobs, 7000) do
|
@@ -44,7 +44,7 @@ local metaKey = KEYS[6]
|
|
44
44
|
local pausedKey = KEYS[7]
|
45
45
|
local markerKey = KEYS[8]
|
46
46
|
local eventStreamKey = KEYS[9]
|
47
|
-
local maxStalledJobCount = ARGV[1]
|
47
|
+
local maxStalledJobCount = tonumber(ARGV[1])
|
48
48
|
local queueKeyPrefix = ARGV[2]
|
49
49
|
local timestamp = ARGV[3]
|
50
50
|
local maxCheckTime = ARGV[4]
|
@@ -63,8 +63,6 @@ local failed = {}
|
|
63
63
|
if (#stalling > 0) then
|
64
64
|
rcall('DEL', stalledKey)
|
65
65
|
|
66
|
-
local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
|
67
|
-
|
68
66
|
-- Remove from active list
|
69
67
|
for i, jobId in ipairs(stalling) do
|
70
68
|
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
@@ -83,7 +81,7 @@ if (#stalling > 0) then
|
|
83
81
|
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
84
82
|
local stalledCount =
|
85
83
|
rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
86
|
-
if (stalledCount >
|
84
|
+
if (stalledCount > maxStalledJobCount) then
|
87
85
|
local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
|
88
86
|
local rawOpts = jobAttributes[1]
|
89
87
|
local rawParentData = jobAttributes[2]
|
@@ -24,7 +24,7 @@ local rcall = redis.call
|
|
24
24
|
--- @include "includes/removeJobKeys"
|
25
25
|
--- @include "includes/removeParentDependencyKey"
|
26
26
|
|
27
|
-
local function removeJob(
|
27
|
+
local function removeJob(prefix, jobId, parentKey, removeChildren)
|
28
28
|
local jobKey = prefix .. jobId;
|
29
29
|
|
30
30
|
removeParentDependencyKey(jobKey, false, parentKey, nil)
|
@@ -33,14 +33,14 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
|
|
33
33
|
-- Check if this job has children
|
34
34
|
-- If so, we are going to try to remove the children recursively in deep first way because
|
35
35
|
-- if some job is locked we must exit with and error.
|
36
|
-
--local countProcessed = rcall("HLEN", jobKey .. ":processed")
|
36
|
+
-- local countProcessed = rcall("HLEN", jobKey .. ":processed")
|
37
37
|
local processed = rcall("HGETALL", jobKey .. ":processed")
|
38
38
|
|
39
39
|
if (#processed > 0) then
|
40
40
|
for i = 1, #processed, 2 do
|
41
41
|
local childJobId = getJobIdFromKey(processed[i])
|
42
42
|
local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
|
43
|
-
removeJob(
|
43
|
+
removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
@@ -50,7 +50,7 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
|
|
50
50
|
-- We need to get the jobId for this job.
|
51
51
|
local childJobId = getJobIdFromKey(childJobKey)
|
52
52
|
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
53
|
-
removeJob(
|
53
|
+
removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
|
54
54
|
end
|
55
55
|
end
|
56
56
|
|
@@ -60,7 +60,7 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
|
|
60
60
|
for i = 1, #failed, 2 do
|
61
61
|
local childJobId = getJobIdFromKey(failed[i])
|
62
62
|
local childJobPrefix = getJobKeyPrefix(failed[i], childJobId)
|
63
|
-
removeJob(
|
63
|
+
removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
|
64
64
|
end
|
65
65
|
end
|
66
66
|
end
|
@@ -70,15 +70,23 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
|
|
70
70
|
removeDebounceKey(prefix, jobKey)
|
71
71
|
if removeJobKeys(jobKey) > 0 then
|
72
72
|
local maxEvents = getOrSetMaxEvents(KEYS[2])
|
73
|
-
rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
|
74
|
-
|
73
|
+
rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed", "jobId", jobId, "prev",
|
74
|
+
prev)
|
75
75
|
end
|
76
76
|
end
|
77
77
|
|
78
78
|
local prefix = KEYS[1]
|
79
|
+
local jobId = ARGV[1]
|
80
|
+
local shouldRemoveChildren = ARGV[2]
|
81
|
+
local jobKey = prefix .. jobId
|
79
82
|
|
80
|
-
if
|
81
|
-
|
83
|
+
-- Check if the job belongs to a job scheduler and it is in delayed state.
|
84
|
+
if rcall("ZSCORE", prefix .. "delayed", jobId) and rcall("HGET", jobKey, "rjk") then
|
85
|
+
return -8 -- Return error code as the job is part of a job scheduler and is in delayed state.
|
86
|
+
end
|
87
|
+
|
88
|
+
if not isLocked(prefix, jobId, shouldRemoveChildren) then
|
89
|
+
removeJob(prefix, jobId, nil, shouldRemoveChildren)
|
82
90
|
return 1
|
83
91
|
end
|
84
92
|
return 0
|
@@ -0,0 +1,43 @@
|
|
1
|
+
|
2
|
+
--[[
|
3
|
+
Removes a repeatable job
|
4
|
+
Input:
|
5
|
+
KEYS[1] job schedulers key
|
6
|
+
KEYS[2] delayed jobs key
|
7
|
+
KEYS[3] events key
|
8
|
+
|
9
|
+
ARGV[1] job scheduler id
|
10
|
+
ARGV[2] prefix key
|
11
|
+
|
12
|
+
Output:
|
13
|
+
0 - OK
|
14
|
+
1 - Missing repeat job
|
15
|
+
|
16
|
+
Events:
|
17
|
+
'removed'
|
18
|
+
]]
|
19
|
+
local rcall = redis.call
|
20
|
+
|
21
|
+
-- Includes
|
22
|
+
--- @include "includes/removeJobKeys"
|
23
|
+
|
24
|
+
local jobSchedulerId = ARGV[1]
|
25
|
+
local prefix = ARGV[2]
|
26
|
+
|
27
|
+
local millis = rcall("ZSCORE", KEYS[1], jobSchedulerId)
|
28
|
+
|
29
|
+
if millis then
|
30
|
+
-- Delete next programmed job.
|
31
|
+
local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. millis
|
32
|
+
if(rcall("ZREM", KEYS[2], delayedJobId) == 1) then
|
33
|
+
removeJobKeys(prefix .. delayedJobId)
|
34
|
+
rcall("XADD", KEYS[3], "*", "event", "removed", "jobId", delayedJobId, "prev", "delayed")
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
if(rcall("ZREM", KEYS[1], jobSchedulerId) == 1) then
|
39
|
+
rcall("DEL", KEYS[1] .. ":" .. jobSchedulerId)
|
40
|
+
return 0
|
41
|
+
end
|
42
|
+
|
43
|
+
return 1
|
@@ -7,5 +7,6 @@ export var ErrorCode;
|
|
7
7
|
ErrorCode[ErrorCode["ParentJobNotExist"] = -5] = "ParentJobNotExist";
|
8
8
|
ErrorCode[ErrorCode["JobLockMismatch"] = -6] = "JobLockMismatch";
|
9
9
|
ErrorCode[ErrorCode["ParentJobCannotBeReplaced"] = -7] = "ParentJobCannotBeReplaced";
|
10
|
+
ErrorCode[ErrorCode["JobBelongsToJobScheduler"] = -8] = "JobBelongsToJobScheduler";
|
10
11
|
})(ErrorCode || (ErrorCode = {}));
|
11
12
|
//# sourceMappingURL=error-code.js.map
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,
|
1
|
+
{"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,SASX;AATD,WAAY,SAAS;IACnB,wDAAgB,CAAA;IAChB,gEAAoB,CAAA;IACpB,4DAAkB,CAAA;IAClB,8EAA2B,CAAA;IAC3B,oEAAsB,CAAA;IACtB,gEAAoB,CAAA;IACpB,oFAA8B,CAAA;IAC9B,kFAA6B,CAAA;AAC/B,CAAC,EATW,SAAS,KAAT,SAAS,QASpB"}
|
@@ -22,5 +22,6 @@ export * from './repeatable-options';
|
|
22
22
|
export * from './repeat-options';
|
23
23
|
export * from './sandboxed-job-processor';
|
24
24
|
export * from './sandboxed-job';
|
25
|
+
export * from './sandboxed-options';
|
25
26
|
export * from './worker-options';
|
26
27
|
//# sourceMappingURL=index.js.map
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC"}
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,kBAAkB,CAAC"}
|
@@ -0,0 +1,28 @@
|
|
1
|
+
/// <reference types="node" />
|
2
|
+
/// <reference types="node" />
|
3
|
+
import { ForkOptions } from 'child_process';
|
4
|
+
import { WorkerOptions as WorkerThreadsOptions } from 'worker_threads';
|
5
|
+
export interface SandboxedOptions {
|
6
|
+
/**
|
7
|
+
* Use Worker Threads instead of Child Processes.
|
8
|
+
* Note: This option can only be used when specifying
|
9
|
+
* a file for the processor argument.
|
10
|
+
*
|
11
|
+
* @default false
|
12
|
+
*/
|
13
|
+
useWorkerThreads?: boolean;
|
14
|
+
/**
|
15
|
+
* Support passing Worker Fork Options.
|
16
|
+
* Note: This option can only be used when specifying
|
17
|
+
* a file for the processor argument and useWorkerThreads is passed as false (default value).
|
18
|
+
* @see {@link https://nodejs.org/api/child_process.html#child_processforkmodulepath-args-options}
|
19
|
+
*/
|
20
|
+
workerForkOptions?: ForkOptions;
|
21
|
+
/**
|
22
|
+
* Support passing Worker Threads Options.
|
23
|
+
* Note: This option can only be used when specifying
|
24
|
+
* a file for the processor argument and useWorkerThreads is passed as true.
|
25
|
+
* @see {@link https://nodejs.org/api/worker_threads.html#new-workerfilename-options}
|
26
|
+
*/
|
27
|
+
workerThreadsOptions?: WorkerThreadsOptions;
|
28
|
+
}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"sandboxed-options.js","sourceRoot":"","sources":["../../../src/interfaces/sandboxed-options.ts"],"names":[],"mappings":""}
|
@@ -1,16 +1,15 @@
|
|
1
|
-
/// <reference types="node" />
|
2
|
-
import { WorkerOptions as WorkerThreadsOptions } from 'worker_threads';
|
3
1
|
import { Job } from '../classes/job';
|
4
2
|
import { AdvancedOptions } from './advanced-options';
|
5
3
|
import { QueueBaseOptions } from './queue-options';
|
6
4
|
import { RateLimiterOptions } from './rate-limiter-options';
|
7
5
|
import { MetricsOptions } from './metrics-options';
|
8
6
|
import { KeepJobs } from './keep-jobs';
|
7
|
+
import { SandboxedOptions } from './sandboxed-options';
|
9
8
|
/**
|
10
9
|
* An async function that receives `Job`s and handles them.
|
11
10
|
*/
|
12
11
|
export type Processor<T = any, R = any, N extends string = string> = (job: Job<T, R, N>, token?: string) => Promise<R>;
|
13
|
-
export interface WorkerOptions extends QueueBaseOptions {
|
12
|
+
export interface WorkerOptions extends QueueBaseOptions, SandboxedOptions {
|
14
13
|
/**
|
15
14
|
* Optional worker name. The name will be stored on every job
|
16
15
|
* processed by this worker instance, and can be used to monitor
|
@@ -117,21 +116,6 @@ export interface WorkerOptions extends QueueBaseOptions {
|
|
117
116
|
* More advanced options.
|
118
117
|
*/
|
119
118
|
settings?: AdvancedOptions;
|
120
|
-
/**
|
121
|
-
* Use Worker Threads instead of Child Processes.
|
122
|
-
* Note: This option can only be used when specifying
|
123
|
-
* a file for the processor argument.
|
124
|
-
*
|
125
|
-
* @default false
|
126
|
-
*/
|
127
|
-
useWorkerThreads?: boolean;
|
128
|
-
/**
|
129
|
-
* Support passing Worker Threads Options.
|
130
|
-
* Note: This option can only be used when specifying
|
131
|
-
* a file for the processor argument and useWorkerThreads is passed as true.
|
132
|
-
* @see {@link https://nodejs.org/api/worker_threads.html#new-workerfilename-options}
|
133
|
-
*/
|
134
|
-
workerThreadsOptions?: WorkerThreadsOptions;
|
135
119
|
}
|
136
120
|
export interface GetNextJobOptions {
|
137
121
|
block?: boolean;
|