bullmq 4.16.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/flow-producer.js +3 -10
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job.js +21 -11
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-base.js +2 -9
- package/dist/cjs/classes/queue-base.js.map +1 -1
- package/dist/cjs/classes/queue-events.js +3 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js +1 -0
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/redis-connection.js +9 -7
- package/dist/cjs/classes/redis-connection.js.map +1 -1
- package/dist/cjs/classes/scripts.js +34 -34
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +41 -37
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
- package/dist/cjs/commands/addParentJob-4.lua +0 -2
- package/dist/{esm/commands/addPrioritizedJob-8.lua → cjs/commands/addPrioritizedJob-7.lua} +18 -24
- package/dist/cjs/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
- package/dist/cjs/commands/changePriority-6.lua +57 -0
- package/dist/cjs/commands/getCounts-1.lua +1 -0
- package/dist/cjs/commands/getRanges-1.lua +1 -0
- package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
- package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -7
- package/dist/cjs/commands/includes/checkStalledJobs.lua +1 -1
- package/dist/cjs/commands/includes/isQueuePaused.lua +7 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
- package/dist/cjs/commands/includes/prepareJobForProcessing.lua +1 -1
- package/dist/cjs/commands/includes/promoteDelayedJobs.lua +7 -4
- package/dist/{esm/commands/moveToActive-10.lua → cjs/commands/moveToActive-11.lua} +18 -29
- package/dist/cjs/commands/moveToDelayed-7.lua +77 -0
- package/dist/cjs/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
- package/dist/cjs/commands/moveToWaitingChildren-4.lua +7 -4
- package/dist/cjs/commands/pause-7.lua +41 -0
- package/dist/cjs/commands/promote-8.lua +60 -0
- package/dist/cjs/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
- package/dist/cjs/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
- package/dist/cjs/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +51 -61
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
- package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -0
- package/dist/cjs/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
- package/dist/cjs/scripts/addStandardJob-7.js.map +1 -0
- package/dist/cjs/scripts/changePriority-6.js +85 -0
- package/dist/cjs/scripts/changePriority-6.js.map +1 -0
- package/dist/cjs/scripts/getCounts-1.js +1 -0
- package/dist/cjs/scripts/getCounts-1.js.map +1 -1
- package/dist/cjs/scripts/getRanges-1.js +1 -0
- package/dist/cjs/scripts/getRanges-1.js.map +1 -1
- package/dist/cjs/scripts/index.js +10 -10
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js +1 -0
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/cjs/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
- package/dist/cjs/scripts/moveToActive-11.js.map +1 -0
- package/dist/cjs/scripts/moveToDelayed-7.js +103 -0
- package/dist/cjs/scripts/moveToDelayed-7.js.map +1 -0
- package/dist/cjs/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
- package/dist/cjs/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-4.js +6 -3
- package/dist/cjs/scripts/moveToWaitingChildren-4.js.map +1 -1
- package/dist/cjs/scripts/pause-7.js +67 -0
- package/dist/cjs/scripts/pause-7.js.map +1 -0
- package/dist/cjs/scripts/promote-8.js +79 -0
- package/dist/cjs/scripts/promote-8.js.map +1 -0
- package/dist/cjs/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
- package/dist/cjs/scripts/retryJob-10.js.map +1 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/esm/classes/flow-producer.js +3 -10
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job.d.ts +5 -0
- package/dist/esm/classes/job.js +21 -11
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-base.js +2 -9
- package/dist/esm/classes/queue-base.js.map +1 -1
- package/dist/esm/classes/queue-events.js +3 -1
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-keys.js +1 -0
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/redis-connection.js +9 -7
- package/dist/esm/classes/redis-connection.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +4 -4
- package/dist/esm/classes/scripts.js +34 -34
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +4 -2
- package/dist/esm/classes/worker.js +41 -37
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
- package/dist/esm/commands/addParentJob-4.lua +0 -2
- package/dist/{cjs/commands/addPrioritizedJob-8.lua → esm/commands/addPrioritizedJob-7.lua} +18 -24
- package/dist/esm/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
- package/dist/esm/commands/changePriority-6.lua +57 -0
- package/dist/esm/commands/getCounts-1.lua +1 -0
- package/dist/esm/commands/getRanges-1.lua +1 -0
- package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
- package/dist/esm/commands/includes/addJobWithPriority.lua +3 -7
- package/dist/esm/commands/includes/checkStalledJobs.lua +1 -1
- package/dist/esm/commands/includes/isQueuePaused.lua +7 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
- package/dist/esm/commands/includes/prepareJobForProcessing.lua +1 -1
- package/dist/esm/commands/includes/promoteDelayedJobs.lua +7 -4
- package/dist/{cjs/commands/moveToActive-10.lua → esm/commands/moveToActive-11.lua} +18 -29
- package/dist/esm/commands/moveToDelayed-7.lua +77 -0
- package/dist/esm/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
- package/dist/esm/commands/moveToWaitingChildren-4.lua +7 -4
- package/dist/esm/commands/pause-7.lua +41 -0
- package/dist/esm/commands/promote-8.lua +60 -0
- package/dist/esm/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
- package/dist/esm/interfaces/job-json.d.ts +4 -1
- package/dist/esm/interfaces/minimal-job.d.ts +3 -0
- package/dist/esm/interfaces/queue-options.d.ts +1 -1
- package/dist/esm/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
- package/dist/esm/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
- package/dist/esm/scripts/addParentJob-4.js +51 -61
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
- package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -0
- package/dist/esm/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
- package/dist/esm/scripts/addStandardJob-7.js.map +1 -0
- package/dist/esm/scripts/changePriority-6.js +82 -0
- package/dist/esm/scripts/changePriority-6.js.map +1 -0
- package/dist/esm/scripts/getCounts-1.js +1 -0
- package/dist/esm/scripts/getCounts-1.js.map +1 -1
- package/dist/esm/scripts/getRanges-1.js +1 -0
- package/dist/esm/scripts/getRanges-1.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +10 -10
- package/dist/esm/scripts/index.js +10 -10
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-8.js +1 -0
- package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/esm/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
- package/dist/esm/scripts/moveToActive-11.js.map +1 -0
- package/dist/esm/scripts/moveToDelayed-7.js +100 -0
- package/dist/esm/scripts/moveToDelayed-7.js.map +1 -0
- package/dist/esm/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
- package/dist/esm/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-4.js +6 -3
- package/dist/esm/scripts/moveToWaitingChildren-4.js.map +1 -1
- package/dist/esm/scripts/pause-7.js +64 -0
- package/dist/esm/scripts/pause-7.js.map +1 -0
- package/dist/esm/scripts/promote-8.js +76 -0
- package/dist/esm/scripts/promote-8.js.map +1 -0
- package/dist/esm/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
- package/dist/esm/scripts/retryJob-10.js.map +1 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/changePriority-5.lua +0 -52
- package/dist/cjs/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
- package/dist/cjs/commands/moveToDelayed-8.lua +0 -72
- package/dist/cjs/commands/pause-5.lua +0 -36
- package/dist/cjs/commands/promote-7.lua +0 -57
- package/dist/cjs/scripts/addDelayedJob-7.js.map +0 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +0 -1
- package/dist/cjs/scripts/addStandardJob-6.js.map +0 -1
- package/dist/cjs/scripts/changePriority-5.js +0 -87
- package/dist/cjs/scripts/changePriority-5.js.map +0 -1
- package/dist/cjs/scripts/moveToDelayed-8.js +0 -168
- package/dist/cjs/scripts/moveToDelayed-8.js.map +0 -1
- package/dist/cjs/scripts/pause-5.js +0 -48
- package/dist/cjs/scripts/pause-5.js.map +0 -1
- package/dist/cjs/scripts/promote-7.js +0 -87
- package/dist/cjs/scripts/promote-7.js.map +0 -1
- package/dist/cjs/scripts/retryJob-9.js.map +0 -1
- package/dist/esm/commands/changePriority-5.lua +0 -52
- package/dist/esm/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
- package/dist/esm/commands/moveToDelayed-8.lua +0 -72
- package/dist/esm/commands/pause-5.lua +0 -36
- package/dist/esm/commands/promote-7.lua +0 -57
- package/dist/esm/scripts/addDelayedJob-7.js.map +0 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +0 -1
- package/dist/esm/scripts/addStandardJob-6.js.map +0 -1
- package/dist/esm/scripts/changePriority-5.js +0 -84
- package/dist/esm/scripts/changePriority-5.js.map +0 -1
- package/dist/esm/scripts/moveToDelayed-8.js +0 -165
- package/dist/esm/scripts/moveToDelayed-8.js.map +0 -1
- package/dist/esm/scripts/pause-5.js +0 -45
- package/dist/esm/scripts/pause-5.js.map +0 -1
- package/dist/esm/scripts/promote-7.js +0 -84
- package/dist/esm/scripts/promote-7.js.map +0 -1
- package/dist/esm/scripts/retryJob-9.js.map +0 -1
- /package/dist/esm/scripts/{addDelayedJob-7.d.ts → addDelayedJob-6.d.ts} +0 -0
- /package/dist/esm/scripts/{addPrioritizedJob-8.d.ts → addPrioritizedJob-7.d.ts} +0 -0
- /package/dist/esm/scripts/{addStandardJob-6.d.ts → addStandardJob-7.d.ts} +0 -0
- /package/dist/esm/scripts/{changePriority-5.d.ts → changePriority-6.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToActive-10.d.ts → moveToActive-11.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToDelayed-8.d.ts → moveToDelayed-7.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToFinished-13.d.ts → moveToFinished-14.d.ts} +0 -0
- /package/dist/esm/scripts/{pause-5.d.ts → pause-7.d.ts} +0 -0
- /package/dist/esm/scripts/{promote-7.d.ts → promote-8.d.ts} +0 -0
- /package/dist/esm/scripts/{retryJob-9.d.ts → retryJob-10.d.ts} +0 -0
@@ -22,22 +22,21 @@
|
|
22
22
|
KEYS[11] completed/failed key
|
23
23
|
KEYS[12] jobId key
|
24
24
|
KEYS[13] metrics key
|
25
|
+
KEYS[14] marker key
|
25
26
|
|
26
27
|
ARGV[1] jobId
|
27
28
|
ARGV[2] timestamp
|
28
29
|
ARGV[3] msg property returnvalue / failedReason
|
29
30
|
ARGV[4] return value / failed reason
|
30
31
|
ARGV[5] target (completed/failed)
|
31
|
-
ARGV[6]
|
32
|
-
ARGV[7]
|
33
|
-
ARGV[8]
|
34
|
-
ARGV[9] opts
|
32
|
+
ARGV[6] fetch next?
|
33
|
+
ARGV[7] keys prefix
|
34
|
+
ARGV[8] opts
|
35
35
|
|
36
36
|
opts - token - lock token
|
37
37
|
opts - keepJobs
|
38
38
|
opts - lockDuration - lock duration in milliseconds
|
39
39
|
opts - attempts max attempts
|
40
|
-
opts - attemptsMade
|
41
40
|
opts - maxMetricsSize
|
42
41
|
opts - fpof - fail parent on fail
|
43
42
|
opts - rdof - remove dependency on fail
|
@@ -72,11 +71,10 @@ local rcall = redis.call
|
|
72
71
|
|
73
72
|
local jobIdKey = KEYS[12]
|
74
73
|
if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
75
|
-
local opts = cmsgpack.unpack(ARGV[
|
74
|
+
local opts = cmsgpack.unpack(ARGV[8])
|
76
75
|
|
77
76
|
local token = opts['token']
|
78
77
|
local attempts = opts['attempts']
|
79
|
-
local attemptsMade = opts['attemptsMade']
|
80
78
|
local maxMetricsSize = opts['maxMetricsSize']
|
81
79
|
local maxCount = opts['keepJobs']['count']
|
82
80
|
local maxAge = opts['keepJobs']['age']
|
@@ -120,8 +118,9 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
120
118
|
|
121
119
|
if (numRemovedElements < 1) then return -3 end
|
122
120
|
|
121
|
+
local metaKey = KEYS[9]
|
123
122
|
-- Trim events before emiting them to avoid trimming events emitted in this script
|
124
|
-
trimEvents(
|
123
|
+
trimEvents(metaKey, KEYS[4])
|
125
124
|
|
126
125
|
-- If job has a parent we need to
|
127
126
|
-- 1) remove this job id from parents dependencies
|
@@ -144,17 +143,20 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
144
143
|
else
|
145
144
|
if opts['fpof'] then
|
146
145
|
moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
|
147
|
-
|
146
|
+
parentId, jobIdKey,
|
147
|
+
timestamp)
|
148
148
|
elseif opts['rdof'] then
|
149
149
|
local dependenciesSet = parentKey .. ":dependencies"
|
150
150
|
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
151
151
|
moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
|
152
|
-
|
152
|
+
parentKey, parentId, timestamp)
|
153
153
|
end
|
154
154
|
end
|
155
155
|
end
|
156
156
|
end
|
157
157
|
|
158
|
+
local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
|
159
|
+
|
158
160
|
-- Remove job?
|
159
161
|
if maxCount ~= 0 then
|
160
162
|
local targetSet = KEYS[11]
|
@@ -164,7 +166,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
164
166
|
-- "returnvalue" / "failedReason" and "finishedOn"
|
165
167
|
|
166
168
|
-- Remove old jobs?
|
167
|
-
local prefix = ARGV[
|
169
|
+
local prefix = ARGV[7]
|
168
170
|
|
169
171
|
if maxAge ~= nil then
|
170
172
|
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
@@ -197,13 +199,13 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
197
199
|
|
198
200
|
-- Try to get next job to avoid an extra roundtrip if the queue is not closing,
|
199
201
|
-- and not rate limited.
|
200
|
-
if (ARGV[
|
202
|
+
if (ARGV[6] == "1") then
|
201
203
|
|
202
|
-
local target, paused = getTargetQueueList(
|
204
|
+
local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[8])
|
203
205
|
|
204
206
|
-- Check if there are delayed jobs that can be promoted
|
205
|
-
promoteDelayedJobs(KEYS[7], KEYS[
|
206
|
-
|
207
|
+
promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], KEYS[4], ARGV[7],
|
208
|
+
timestamp, KEYS[10], paused)
|
207
209
|
|
208
210
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
209
211
|
-- Check if we are rate limited first.
|
@@ -217,25 +219,30 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
217
219
|
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
218
220
|
|
219
221
|
if jobId then
|
222
|
+
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
220
223
|
if string.sub(jobId, 1, 2) == "0:" then
|
221
224
|
rcall("LREM", KEYS[2], 1, jobId)
|
222
225
|
|
223
226
|
-- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
|
224
227
|
-- but if ID is 0:0, then there is at least 1 prioritized job to process
|
225
228
|
if jobId == "0:0" then
|
226
|
-
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
|
227
|
-
|
228
|
-
|
229
|
+
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
|
230
|
+
KEYS[10])
|
231
|
+
return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
|
232
|
+
timestamp, maxJobs,
|
233
|
+
expireTime, opts)
|
229
234
|
end
|
230
235
|
else
|
231
|
-
return prepareJobForProcessing(KEYS, ARGV[
|
232
|
-
|
236
|
+
return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
|
237
|
+
timestamp, maxJobs, expireTime,
|
238
|
+
opts)
|
233
239
|
end
|
234
240
|
else
|
235
241
|
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
236
242
|
if jobId then
|
237
|
-
return prepareJobForProcessing(KEYS, ARGV[
|
238
|
-
|
243
|
+
return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
|
244
|
+
timestamp, maxJobs, expireTime,
|
245
|
+
opts)
|
239
246
|
end
|
240
247
|
end
|
241
248
|
|
@@ -11,7 +11,7 @@
|
|
11
11
|
ARGV[2] child key
|
12
12
|
ARGV[3] timestamp
|
13
13
|
ARGV[4] the id of the job
|
14
|
-
|
14
|
+
|
15
15
|
Output:
|
16
16
|
0 - OK
|
17
17
|
1 - There are not pending dependencies.
|
@@ -21,7 +21,8 @@
|
|
21
21
|
]]
|
22
22
|
local rcall = redis.call
|
23
23
|
|
24
|
-
local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp,
|
24
|
+
local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp,
|
25
|
+
lockKey, jobKey, token)
|
25
26
|
if token ~= "0" then
|
26
27
|
if rcall("GET", lockKey) == token then
|
27
28
|
rcall("DEL", lockKey)
|
@@ -46,13 +47,15 @@ end
|
|
46
47
|
if rcall("EXISTS", KEYS[4]) == 1 then
|
47
48
|
if ARGV[2] ~= "" then
|
48
49
|
if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
|
49
|
-
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1],
|
50
|
+
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], KEYS[4],
|
51
|
+
ARGV[1])
|
50
52
|
end
|
51
53
|
|
52
54
|
return 1
|
53
55
|
else
|
54
56
|
if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
|
55
|
-
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1],
|
57
|
+
return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], KEYS[4],
|
58
|
+
ARGV[1])
|
56
59
|
end
|
57
60
|
|
58
61
|
return 1
|
@@ -0,0 +1,41 @@
|
|
1
|
+
--[[
|
2
|
+
Pauses or resumes a queue globably.
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] 'wait' or 'paused''
|
6
|
+
KEYS[2] 'paused' or 'wait'
|
7
|
+
KEYS[3] 'meta'
|
8
|
+
KEYS[4] 'prioritized'
|
9
|
+
KEYS[5] events stream key
|
10
|
+
KEYS[6] 'delayed'
|
11
|
+
KEYS|7] 'marker'
|
12
|
+
|
13
|
+
ARGV[1] 'paused' or 'resumed'
|
14
|
+
|
15
|
+
Event:
|
16
|
+
publish paused or resumed event.
|
17
|
+
]]
|
18
|
+
local rcall = redis.call
|
19
|
+
|
20
|
+
-- Includes
|
21
|
+
--- @include "includes/addDelayMarkerIfNeeded"
|
22
|
+
|
23
|
+
local markerKey = KEYS[7]
|
24
|
+
local hasJobs = rcall("EXISTS", KEYS[1]) == 1
|
25
|
+
if hasJobs then rcall("RENAME", KEYS[1], KEYS[2]) end
|
26
|
+
|
27
|
+
if ARGV[1] == "paused" then
|
28
|
+
rcall("HSET", KEYS[3], "paused", 1)
|
29
|
+
rcall("DEL", markerKey)
|
30
|
+
else
|
31
|
+
rcall("HDEL", KEYS[3], "paused")
|
32
|
+
|
33
|
+
if hasJobs or rcall("ZCARD", KEYS[4]) > 0 then
|
34
|
+
-- Add marker if there are waiting or priority jobs
|
35
|
+
rcall("ZADD", markerKey, 0, "0")
|
36
|
+
else
|
37
|
+
addDelayMarkerIfNeeded(markerKey, KEYS[6])
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
rcall("XADD", KEYS[5], "*", "event", ARGV[1]);
|
@@ -0,0 +1,60 @@
|
|
1
|
+
--[[
|
2
|
+
Promotes a job that is currently "delayed" to the "waiting" state
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] 'delayed'
|
6
|
+
KEYS[2] 'wait'
|
7
|
+
KEYS[3] 'paused'
|
8
|
+
KEYS[4] 'meta'
|
9
|
+
KEYS[5] 'prioritized'
|
10
|
+
KEYS[6] 'pc' priority counter
|
11
|
+
KEYS[7] 'event stream'
|
12
|
+
KEYS[8] 'marker'
|
13
|
+
|
14
|
+
ARGV[1] queue.toKey('')
|
15
|
+
ARGV[2] jobId
|
16
|
+
|
17
|
+
Output:
|
18
|
+
0 - OK
|
19
|
+
-3 - Job not in delayed zset.
|
20
|
+
|
21
|
+
Events:
|
22
|
+
'waiting'
|
23
|
+
]]
|
24
|
+
local rcall = redis.call
|
25
|
+
local jobId = ARGV[2]
|
26
|
+
|
27
|
+
-- Includes
|
28
|
+
--- @include "includes/addJobWithPriority"
|
29
|
+
--- @include "includes/getTargetQueueList"
|
30
|
+
|
31
|
+
if rcall("ZREM", KEYS[1], jobId) == 1 then
|
32
|
+
local jobKey = ARGV[1] .. jobId
|
33
|
+
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
|
34
|
+
local metaKey = KEYS[4]
|
35
|
+
|
36
|
+
-- Remove delayed "marker" from the wait list if there is any.
|
37
|
+
-- Since we are adding a job we do not need the marker anymore.
|
38
|
+
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
39
|
+
local target, paused = getTargetQueueList(metaKey, KEYS[2], KEYS[3])
|
40
|
+
local marker = rcall("LINDEX", target, 0)
|
41
|
+
if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
|
42
|
+
|
43
|
+
if priority == 0 then
|
44
|
+
-- LIFO or FIFO
|
45
|
+
rcall("LPUSH", target, jobId)
|
46
|
+
if not paused then rcall("ZADD", KEYS[8], 0, "0") end
|
47
|
+
else
|
48
|
+
addJobWithPriority(KEYS[8], KEYS[5], priority, jobId, KEYS[6], paused)
|
49
|
+
end
|
50
|
+
|
51
|
+
-- Emit waiting event (wait..ing@token)
|
52
|
+
rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev",
|
53
|
+
"delayed");
|
54
|
+
|
55
|
+
rcall("HSET", jobKey, "delay", 0)
|
56
|
+
|
57
|
+
return 0
|
58
|
+
else
|
59
|
+
return -3
|
60
|
+
end
|
@@ -2,15 +2,16 @@
|
|
2
2
|
Retries a failed job by moving it back to the wait queue.
|
3
3
|
|
4
4
|
Input:
|
5
|
-
KEYS[1]
|
6
|
-
KEYS[2]
|
7
|
-
KEYS[3]
|
8
|
-
KEYS[4]
|
9
|
-
KEYS[5]
|
10
|
-
KEYS[6]
|
11
|
-
KEYS[7]
|
12
|
-
KEYS[8]
|
13
|
-
KEYS[9]
|
5
|
+
KEYS[1] 'active',
|
6
|
+
KEYS[2] 'wait'
|
7
|
+
KEYS[3] 'paused'
|
8
|
+
KEYS[4] job key
|
9
|
+
KEYS[5] 'meta'
|
10
|
+
KEYS[6] events stream
|
11
|
+
KEYS[7] delayed key
|
12
|
+
KEYS[8] prioritized key
|
13
|
+
KEYS[9] 'pc' priority counter
|
14
|
+
KEYS[10] 'marker'
|
14
15
|
|
15
16
|
ARGV[1] key prefix
|
16
17
|
ARGV[2] timestamp
|
@@ -34,9 +35,11 @@ local rcall = redis.call
|
|
34
35
|
--- @include "includes/promoteDelayedJobs"
|
35
36
|
|
36
37
|
local target, paused = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
|
38
|
+
local markerKey = KEYS[10]
|
39
|
+
|
37
40
|
-- Check if there are delayed jobs that we can move to wait.
|
38
41
|
-- test example: when there are delayed jobs between retries
|
39
|
-
promoteDelayedJobs(KEYS[7],
|
42
|
+
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], paused)
|
40
43
|
|
41
44
|
if rcall("EXISTS", KEYS[4]) == 1 then
|
42
45
|
|
@@ -57,9 +60,11 @@ if rcall("EXISTS", KEYS[4]) == 1 then
|
|
57
60
|
if priority == 0 then
|
58
61
|
rcall(ARGV[3], target, ARGV[4])
|
59
62
|
else
|
60
|
-
addJobWithPriority(
|
63
|
+
addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], paused)
|
61
64
|
end
|
62
65
|
|
66
|
+
rcall("HINCRBY", KEYS[4], "atm", 1)
|
67
|
+
|
63
68
|
local maxEvents = rcall("HGET", KEYS[5], "opts.maxLenEvents") or 10000
|
64
69
|
|
65
70
|
-- Emit waiting event
|
@@ -7,6 +7,7 @@ export interface JobJson {
|
|
7
7
|
opts: RedisJobOptions;
|
8
8
|
progress: number | object;
|
9
9
|
attemptsMade: number;
|
10
|
+
attemptsStarted: number;
|
10
11
|
finishedOn?: number;
|
11
12
|
processedOn?: number;
|
12
13
|
timestamp: number;
|
@@ -24,7 +25,7 @@ export interface JobJsonRaw {
|
|
24
25
|
delay: string;
|
25
26
|
opts: string;
|
26
27
|
progress: string;
|
27
|
-
attemptsMade
|
28
|
+
attemptsMade?: string;
|
28
29
|
finishedOn?: string;
|
29
30
|
processedOn?: string;
|
30
31
|
timestamp: string;
|
@@ -34,4 +35,6 @@ export interface JobJsonRaw {
|
|
34
35
|
parentKey?: string;
|
35
36
|
parent?: string;
|
36
37
|
rjk?: string;
|
38
|
+
atm?: string;
|
39
|
+
ats?: string;
|
37
40
|
}
|
@@ -2,6 +2,9 @@ import { JobsOptions, JobJsonSandbox } from '../types';
|
|
2
2
|
import { JobJson } from './job-json';
|
3
3
|
import { ParentKeys } from './parent';
|
4
4
|
export type BulkJobOptions = Omit<JobsOptions, 'repeat'>;
|
5
|
+
export interface MoveToDelayedOpts {
|
6
|
+
skipAttempt?: boolean;
|
7
|
+
}
|
5
8
|
export interface MoveToWaitingChildrenOpts {
|
6
9
|
child?: {
|
7
10
|
id: string;
|
@@ -6,13 +6,12 @@ const content = `--[[
|
|
6
6
|
- adds to delayed zset.
|
7
7
|
- Emits a global event 'delayed' if the job is delayed.
|
8
8
|
Input:
|
9
|
-
KEYS[1] '
|
10
|
-
KEYS[2] '
|
11
|
-
KEYS[3] '
|
12
|
-
KEYS[4] '
|
13
|
-
KEYS[5] '
|
14
|
-
KEYS[6]
|
15
|
-
KEYS[7] events stream key
|
9
|
+
KEYS[1] 'marker',
|
10
|
+
KEYS[2] 'meta'
|
11
|
+
KEYS[3] 'id'
|
12
|
+
KEYS[4] 'delayed'
|
13
|
+
KEYS[5] 'completed'
|
14
|
+
KEYS[6] events stream key
|
16
15
|
ARGV[1] msgpacked arguments array
|
17
16
|
[1] key prefix,
|
18
17
|
[2] custom id (use custom instead of one generated automatically)
|
@@ -29,13 +28,11 @@ const content = `--[[
|
|
29
28
|
jobId - OK
|
30
29
|
-5 - Missing parent key
|
31
30
|
]]
|
32
|
-
local
|
33
|
-
local
|
34
|
-
local
|
35
|
-
local
|
36
|
-
local
|
37
|
-
local completedKey = KEYS[6]
|
38
|
-
local eventsKey = KEYS[7]
|
31
|
+
local metaKey = KEYS[2]
|
32
|
+
local idKey = KEYS[3]
|
33
|
+
local delayedKey = KEYS[4]
|
34
|
+
local completedKey = KEYS[5]
|
35
|
+
local eventsKey = KEYS[6]
|
39
36
|
local jobId
|
40
37
|
local jobIdKey
|
41
38
|
local rcall = redis.call
|
@@ -74,7 +71,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
74
71
|
end
|
75
72
|
--[[
|
76
73
|
Add delay marker if needed.
|
77
|
-
]]
|
74
|
+
]]
|
78
75
|
-- Includes
|
79
76
|
--[[
|
80
77
|
Function to return the next delayed job timestamp.
|
@@ -89,36 +86,20 @@ local function getNextDelayedTimestamp(delayedKey)
|
|
89
86
|
return nextTimestamp
|
90
87
|
end
|
91
88
|
end
|
92
|
-
local function addDelayMarkerIfNeeded(
|
93
|
-
local waitLen = rcall("LLEN", targetKey)
|
94
|
-
if waitLen <= 1 then
|
89
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
95
90
|
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
96
91
|
if nextTimestamp ~= nil then
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
local marker = rcall("LINDEX", targetKey, 0)
|
101
|
-
local oldTimestamp = tonumber(marker:sub(3))
|
102
|
-
if oldTimestamp and oldTimestamp > nextTimestamp then
|
103
|
-
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
104
|
-
end
|
105
|
-
else
|
106
|
-
-- if there is no marker, then we need to add one
|
107
|
-
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
108
|
-
end
|
92
|
+
-- Replace the score of the marker with the newest known
|
93
|
+
-- next timestamp.
|
94
|
+
rcall("ZADD", markerKey, nextTimestamp, "0")
|
109
95
|
end
|
110
|
-
end
|
111
96
|
end
|
112
97
|
--[[
|
113
98
|
Function to check for the meta.paused key to decide if we are paused or not
|
114
99
|
(since an empty list and !EXISTS are not really the same).
|
115
100
|
]]
|
116
|
-
local function
|
117
|
-
|
118
|
-
return waitKey, false
|
119
|
-
else
|
120
|
-
return pausedKey, true
|
121
|
-
end
|
101
|
+
local function isQueuePaused(queueMetaKey)
|
102
|
+
return rcall("HEXISTS", queueMetaKey, "paused") == 1
|
122
103
|
end
|
123
104
|
--[[
|
124
105
|
Validate and move or add dependencies to parent.
|
@@ -131,56 +112,64 @@ end
|
|
131
112
|
--[[
|
132
113
|
Function to add job considering priority.
|
133
114
|
]]
|
134
|
-
|
135
|
-
--[[
|
136
|
-
Function priority marker to wait if needed
|
137
|
-
in order to wake up our workers and to respect priority
|
138
|
-
order as much as possible
|
139
|
-
]]
|
140
|
-
local function addPriorityMarkerIfNeeded(waitKey)
|
141
|
-
local waitLen = rcall("LLEN", waitKey)
|
142
|
-
if waitLen == 0 then
|
143
|
-
rcall("LPUSH", waitKey, "0:0")
|
144
|
-
end
|
145
|
-
end
|
146
|
-
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
115
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
147
116
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
148
117
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
149
118
|
rcall("ZADD", prioritizedKey, score, jobId)
|
150
|
-
if not
|
151
|
-
|
119
|
+
if not isPaused then
|
120
|
+
rcall("ZADD", markerKey, 0, "0")
|
152
121
|
end
|
153
122
|
end
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
local delay = tonumber(jobAttributes[2]) or 0
|
164
|
-
if delay > 0 then
|
165
|
-
local delayedTimestamp = tonumber(timestamp) + delay
|
166
|
-
local score = delayedTimestamp * 0x1000
|
167
|
-
local parentDelayedKey = parentQueueKey .. ":delayed"
|
168
|
-
rcall("ZADD", parentDelayedKey, score, parentId)
|
169
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
|
170
|
-
"delay", delayedTimestamp)
|
171
|
-
addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
|
172
|
-
else
|
173
|
-
if priority == 0 then
|
174
|
-
rcall("RPUSH", parentTarget, parentId)
|
175
|
-
else
|
176
|
-
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
|
177
|
-
parentId, parentQueueKey .. ":pc")
|
178
|
-
end
|
179
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
|
180
|
-
"prev", "waiting-children")
|
181
|
-
end
|
123
|
+
--[[
|
124
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
125
|
+
(since an empty list and !EXISTS are not really the same).
|
126
|
+
]]
|
127
|
+
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
128
|
+
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
129
|
+
return waitKey, false
|
130
|
+
else
|
131
|
+
return pausedKey, true
|
182
132
|
end
|
183
133
|
end
|
134
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
135
|
+
parentKey, parentId, timestamp)
|
136
|
+
local isParentActive = rcall("ZSCORE",
|
137
|
+
parentQueueKey .. ":waiting-children", parentId)
|
138
|
+
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
139
|
+
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
140
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
141
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
142
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
143
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
144
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
145
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
146
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
147
|
+
if delay > 0 then
|
148
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
149
|
+
local score = delayedTimestamp * 0x1000
|
150
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
151
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
152
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
153
|
+
"jobId", parentId, "delay", delayedTimestamp)
|
154
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
155
|
+
else
|
156
|
+
if priority == 0 then
|
157
|
+
local parentTarget, _paused =
|
158
|
+
getTargetQueueList(parentMetaKey, parentWaitKey,
|
159
|
+
parentPausedKey)
|
160
|
+
rcall("RPUSH", parentTarget, parentId)
|
161
|
+
rcall("ZADD", parentMarkerKey, 0, "0")
|
162
|
+
else
|
163
|
+
local isPaused = isQueuePaused(parentMetaKey)
|
164
|
+
addJobWithPriority(parentMarkerKey,
|
165
|
+
parentQueueKey .. ":prioritized", priority,
|
166
|
+
parentId, parentQueueKey .. ":pc", isPaused)
|
167
|
+
end
|
168
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
169
|
+
"jobId", parentId, "prev", "waiting-children")
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
184
173
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
185
174
|
parentId, jobIdKey, returnvalue, timestamp )
|
186
175
|
local processedSet = parentKey .. ":processed"
|
@@ -252,13 +241,13 @@ local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
|
|
252
241
|
rcall("ZADD", delayedKey, score, jobId)
|
253
242
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
254
243
|
"jobId", jobId, "delay", delayedTimestamp)
|
255
|
-
--
|
256
|
-
|
257
|
-
|
258
|
-
|
244
|
+
-- mark that a delayed job is available
|
245
|
+
local isPaused = isQueuePaused(metaKey)
|
246
|
+
if not isPaused then
|
247
|
+
local markerKey = KEYS[1]
|
248
|
+
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
249
|
+
end
|
259
250
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
260
|
-
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
261
|
-
-- fail in this case.
|
262
251
|
if parentDependenciesKey ~= nil then
|
263
252
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
264
253
|
end
|
@@ -267,6 +256,6 @@ return jobId .. "" -- convert to string
|
|
267
256
|
export const addDelayedJob = {
|
268
257
|
name: 'addDelayedJob',
|
269
258
|
content,
|
270
|
-
keys:
|
259
|
+
keys: 6,
|
271
260
|
};
|
272
|
-
//# sourceMappingURL=addDelayedJob-
|
261
|
+
//# sourceMappingURL=addDelayedJob-6.js.map
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"
|
1
|
+
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Pf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|