bullmq 5.46.0 → 5.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/job.js +11 -9
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/scripts.js +4 -3
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/includes/moveParentToFailedIfNeeded.lua +3 -0
- package/dist/cjs/commands/includes/moveParentToWait.lua +49 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +6 -51
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +26 -30
- package/dist/cjs/commands/moveToFinished-14.lua +38 -30
- package/dist/cjs/commands/moveToWaitingChildren-8.lua +13 -8
- package/dist/cjs/scripts/addDelayedJob-6.js +24 -20
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
- package/dist/cjs/scripts/addParentJob-4.js +24 -20
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js +24 -20
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
- package/dist/cjs/scripts/addStandardJob-8.js +24 -20
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
- package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
- package/dist/cjs/scripts/drain-5.js +5 -5
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +56 -54
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +67 -55
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-8.js +43 -33
- package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +5 -5
- package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
- package/dist/cjs/scripts/removeJob-3.js +5 -5
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/job.js +11 -9
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-events.d.ts +1 -1
- package/dist/esm/classes/scripts.js +4 -3
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/includes/moveParentToFailedIfNeeded.lua +3 -0
- package/dist/esm/commands/includes/moveParentToWait.lua +49 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +6 -51
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +26 -30
- package/dist/esm/commands/moveToFinished-14.lua +38 -30
- package/dist/esm/commands/moveToWaitingChildren-8.lua +13 -8
- package/dist/esm/scripts/addDelayedJob-6.js +24 -20
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addJobScheduler-11.js +5 -5
- package/dist/esm/scripts/addParentJob-4.js +24 -20
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js +24 -20
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
- package/dist/esm/scripts/addStandardJob-8.js +24 -20
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
- package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
- package/dist/esm/scripts/drain-5.js +5 -5
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +56 -54
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +67 -55
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-8.js +43 -33
- package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +5 -5
- package/dist/esm/scripts/removeChildDependency-1.js +5 -5
- package/dist/esm/scripts/removeJob-3.js +5 -5
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/job-options.d.ts +12 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
@@ -105,7 +105,7 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
105
105
|
end
|
106
106
|
return waitKey, false
|
107
107
|
end
|
108
|
-
local function
|
108
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
109
109
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
110
110
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
111
111
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -133,10 +133,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
133
133
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
134
134
|
end
|
135
135
|
else
|
136
|
-
|
136
|
+
_moveParentToWait(parentPrefix, parentId)
|
137
137
|
end
|
138
138
|
else
|
139
|
-
|
139
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
140
140
|
end
|
141
141
|
end
|
142
142
|
end
|
@@ -164,10 +164,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
164
164
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
165
165
|
end
|
166
166
|
else
|
167
|
-
|
167
|
+
_moveParentToWait(parentPrefix, parentId)
|
168
168
|
end
|
169
169
|
else
|
170
|
-
|
170
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
171
171
|
end
|
172
172
|
end
|
173
173
|
end
|
@@ -76,9 +76,11 @@ end
|
|
76
76
|
]]
|
77
77
|
-- Includes
|
78
78
|
--[[
|
79
|
-
Validate and move parent to
|
79
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
80
|
+
]]
|
81
|
+
--[[
|
82
|
+
Validate and move parent to a wait status (wait, prioritized or delayed)
|
80
83
|
]]
|
81
|
-
-- Includes
|
82
84
|
--[[
|
83
85
|
Add delay marker if needed.
|
84
86
|
]]
|
@@ -136,12 +138,9 @@ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
|
136
138
|
end
|
137
139
|
return false
|
138
140
|
end
|
139
|
-
local function
|
140
|
-
|
141
|
-
|
142
|
-
parentQueueKey .. ":waiting-children", parentId)
|
143
|
-
if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
|
144
|
-
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
141
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
142
|
+
local isParentWaitingChildren = rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
143
|
+
if isParentWaitingChildren > 0 then
|
145
144
|
local parentWaitKey = parentQueueKey .. ":wait"
|
146
145
|
local parentPausedKey = parentQueueKey .. ":paused"
|
147
146
|
local parentActiveKey = parentQueueKey .. ":active"
|
@@ -150,32 +149,37 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
150
149
|
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
151
150
|
local priority = tonumber(jobAttributes[1]) or 0
|
152
151
|
local delay = tonumber(jobAttributes[2]) or 0
|
152
|
+
-- ignore dependencies if any left
|
153
|
+
rcall("HSET", parentKey, "igdp", 1)
|
153
154
|
if delay > 0 then
|
154
155
|
local delayedTimestamp = tonumber(timestamp) + delay
|
155
156
|
local score = delayedTimestamp * 0x1000
|
156
157
|
local parentDelayedKey = parentQueueKey .. ":delayed"
|
157
158
|
rcall("ZADD", parentDelayedKey, score, parentId)
|
158
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
159
|
-
|
159
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
160
|
+
delayedTimestamp)
|
160
161
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
161
162
|
else
|
162
163
|
if priority == 0 then
|
163
|
-
local parentTarget, isParentPausedOrMaxed =
|
164
|
-
|
165
|
-
|
166
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
167
|
-
parentId)
|
164
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
165
|
+
parentWaitKey, parentPausedKey)
|
166
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
168
167
|
else
|
169
168
|
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
170
|
-
addJobWithPriority(parentMarkerKey,
|
171
|
-
|
172
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
169
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
170
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
173
171
|
end
|
174
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
175
|
-
|
172
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
173
|
+
"waiting-children")
|
176
174
|
end
|
177
175
|
end
|
178
176
|
end
|
177
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
178
|
+
local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
179
|
+
if hasPendingDependencies then
|
180
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
181
|
+
end
|
182
|
+
end
|
179
183
|
--[[
|
180
184
|
Function to remove deduplication key if needed.
|
181
185
|
]]
|
@@ -229,7 +233,7 @@ end
|
|
229
233
|
local getJobKeyPrefix = function (jobKey, jobId)
|
230
234
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
231
235
|
end
|
232
|
-
local function
|
236
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
233
237
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
234
238
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
235
239
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -257,10 +261,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
257
261
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
258
262
|
end
|
259
263
|
else
|
260
|
-
|
264
|
+
_moveParentToWait(parentPrefix, parentId)
|
261
265
|
end
|
262
266
|
else
|
263
|
-
|
267
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
264
268
|
end
|
265
269
|
end
|
266
270
|
end
|
@@ -288,10 +292,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
288
292
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
289
293
|
end
|
290
294
|
else
|
291
|
-
|
295
|
+
_moveParentToWait(parentPrefix, parentId)
|
292
296
|
end
|
293
297
|
else
|
294
|
-
|
298
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
295
299
|
end
|
296
300
|
end
|
297
301
|
end
|
@@ -399,6 +403,8 @@ local function moveParentToFailedIfNeeded(parentQueueKey, parentKey, parentId, j
|
|
399
403
|
parentKey,
|
400
404
|
timestamp
|
401
405
|
)
|
406
|
+
elseif parentData['cpof'] then
|
407
|
+
moveParentToWait(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
402
408
|
elseif parentData['idof'] or parentData['rdof'] then
|
403
409
|
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
404
410
|
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
|
@@ -460,7 +466,9 @@ local maxStalledJobCount = tonumber(ARGV[1])
|
|
460
466
|
local queueKeyPrefix = ARGV[2]
|
461
467
|
local timestamp = ARGV[3]
|
462
468
|
local maxCheckTime = ARGV[4]
|
463
|
-
if rcall("EXISTS", stalledCheckKey) == 1 then
|
469
|
+
if rcall("EXISTS", stalledCheckKey) == 1 then
|
470
|
+
return {{}, {}}
|
471
|
+
end
|
464
472
|
rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
|
465
473
|
-- Trim events before emiting them to avoid trimming events emitted in this script
|
466
474
|
trimEvents(metaKey, eventStreamKey)
|
@@ -484,8 +492,7 @@ if (#stalling > 0) then
|
|
484
492
|
local removed = rcall("LREM", activeKey, 1, jobId)
|
485
493
|
if (removed > 0) then
|
486
494
|
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
487
|
-
local stalledCount =
|
488
|
-
rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
495
|
+
local stalledCount = rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
489
496
|
if (stalledCount > maxStalledJobCount) then
|
490
497
|
local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
|
491
498
|
local rawOpts = jobAttributes[1]
|
@@ -493,38 +500,35 @@ if (#stalling > 0) then
|
|
493
500
|
local opts = cjson.decode(rawOpts)
|
494
501
|
rcall("ZADD", failedKey, timestamp, jobId)
|
495
502
|
removeDeduplicationKeyIfNeeded(queueKeyPrefix, jobAttributes[3])
|
496
|
-
local failedReason =
|
497
|
-
|
498
|
-
rcall("
|
499
|
-
|
500
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
501
|
-
"failed", "jobId", jobId, 'prev', 'active',
|
502
|
-
'failedReason', failedReason)
|
503
|
+
local failedReason = "job stalled more than allowable limit"
|
504
|
+
rcall("HMSET", jobKey, "failedReason", failedReason, "finishedOn", timestamp)
|
505
|
+
rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId", jobId, 'prev', 'active',
|
506
|
+
'failedReason', failedReason)
|
503
507
|
if rawParentData then
|
504
|
-
if opts['fpof'] then
|
508
|
+
if opts['fpof'] or opts['cpof'] then
|
505
509
|
local parentData = cjson.decode(rawParentData)
|
506
510
|
-- TODO: need to remove this job from dependencies set in next breaking change
|
507
511
|
-- no for now as it would imply a breaking change
|
508
512
|
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
509
513
|
local unsuccesssfulSet = parentKey .. ":unsuccessful"
|
510
514
|
rcall("ZADD", unsuccesssfulSet, timestamp, jobKey)
|
511
|
-
|
512
|
-
parentData['queueKey'],
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
timestamp
|
517
|
-
|
515
|
+
if opts['fpof'] then
|
516
|
+
moveParentToFailedIfNeeded(parentData['queueKey'],
|
517
|
+
parentData['queueKey'] .. ':' .. parentData['id'], parentData['id'], jobKey,
|
518
|
+
timestamp)
|
519
|
+
elseif opts['cpof'] then
|
520
|
+
moveParentToWait(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
521
|
+
end
|
518
522
|
elseif opts['idof'] or opts['rdof'] then
|
519
523
|
local parentData = cjson.decode(rawParentData)
|
520
524
|
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
521
525
|
local dependenciesSet = parentKey .. ":dependencies"
|
522
526
|
if rcall("SREM", dependenciesSet, jobKey) == 1 then
|
523
|
-
moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
|
524
|
-
|
527
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet, parentKey,
|
528
|
+
parentData['id'], timestamp)
|
525
529
|
if opts['idof'] then
|
526
|
-
|
527
|
-
|
530
|
+
local failedSet = parentKey .. ":failed"
|
531
|
+
rcall("HSET", failedSet, jobKey, failedReason)
|
528
532
|
end
|
529
533
|
end
|
530
534
|
end
|
@@ -532,15 +536,12 @@ if (#stalling > 0) then
|
|
532
536
|
removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
|
533
537
|
table.insert(failed, jobId)
|
534
538
|
else
|
535
|
-
local target, isPausedOrMaxed =
|
536
|
-
getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
|
539
|
+
local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
|
537
540
|
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
|
538
541
|
addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
|
539
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
540
|
-
"waiting", "jobId", jobId, 'prev', 'active')
|
542
|
+
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, 'prev', 'active')
|
541
543
|
-- Emit the stalled event
|
542
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
543
|
-
"stalled", "jobId", jobId)
|
544
|
+
rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId", jobId)
|
544
545
|
table.insert(stalled, jobId)
|
545
546
|
end
|
546
547
|
end
|
@@ -555,7 +556,8 @@ if (#active > 0) then
|
|
555
556
|
rcall('SADD', stalledKey, unpack(active, from, to))
|
556
557
|
end
|
557
558
|
end
|
558
|
-
return {failed, stalled}
|
559
|
+
return {failed, stalled}
|
560
|
+
`;
|
559
561
|
export const moveStalledJobsToWait = {
|
560
562
|
name: 'moveStalledJobsToWait',
|
561
563
|
content,
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+iBf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -34,6 +34,7 @@ const content = `--[[
|
|
34
34
|
opts - attempts max attempts
|
35
35
|
opts - maxMetricsSize
|
36
36
|
opts - fpof - fail parent on fail
|
37
|
+
opts - cpof - continue parent on fail
|
37
38
|
opts - idof - ignore dependency on fail
|
38
39
|
opts - rdof - remove dependency on fail
|
39
40
|
opts - name - worker name
|
@@ -167,9 +168,11 @@ end
|
|
167
168
|
]]
|
168
169
|
-- Includes
|
169
170
|
--[[
|
170
|
-
Validate and move parent to
|
171
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
172
|
+
]]
|
173
|
+
--[[
|
174
|
+
Validate and move parent to a wait status (wait, prioritized or delayed)
|
171
175
|
]]
|
172
|
-
-- Includes
|
173
176
|
--[[
|
174
177
|
Add delay marker if needed.
|
175
178
|
]]
|
@@ -231,12 +234,9 @@ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
|
231
234
|
end
|
232
235
|
return false
|
233
236
|
end
|
234
|
-
local function
|
235
|
-
|
236
|
-
|
237
|
-
parentQueueKey .. ":waiting-children", parentId)
|
238
|
-
if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
|
239
|
-
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
237
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
238
|
+
local isParentWaitingChildren = rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
239
|
+
if isParentWaitingChildren > 0 then
|
240
240
|
local parentWaitKey = parentQueueKey .. ":wait"
|
241
241
|
local parentPausedKey = parentQueueKey .. ":paused"
|
242
242
|
local parentActiveKey = parentQueueKey .. ":active"
|
@@ -245,32 +245,37 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
245
245
|
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
246
246
|
local priority = tonumber(jobAttributes[1]) or 0
|
247
247
|
local delay = tonumber(jobAttributes[2]) or 0
|
248
|
+
-- ignore dependencies if any left
|
249
|
+
rcall("HSET", parentKey, "igdp", 1)
|
248
250
|
if delay > 0 then
|
249
251
|
local delayedTimestamp = tonumber(timestamp) + delay
|
250
252
|
local score = delayedTimestamp * 0x1000
|
251
253
|
local parentDelayedKey = parentQueueKey .. ":delayed"
|
252
254
|
rcall("ZADD", parentDelayedKey, score, parentId)
|
253
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
254
|
-
|
255
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
256
|
+
delayedTimestamp)
|
255
257
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
256
258
|
else
|
257
259
|
if priority == 0 then
|
258
|
-
local parentTarget, isParentPausedOrMaxed =
|
259
|
-
|
260
|
-
|
261
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
262
|
-
parentId)
|
260
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
261
|
+
parentWaitKey, parentPausedKey)
|
262
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
263
263
|
else
|
264
264
|
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
265
|
-
addJobWithPriority(parentMarkerKey,
|
266
|
-
|
267
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
265
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
266
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
268
267
|
end
|
269
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
270
|
-
|
268
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
269
|
+
"waiting-children")
|
271
270
|
end
|
272
271
|
end
|
273
272
|
end
|
273
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
274
|
+
local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
275
|
+
if hasPendingDependencies then
|
276
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
277
|
+
end
|
278
|
+
end
|
274
279
|
--[[
|
275
280
|
Function to remove deduplication key if needed.
|
276
281
|
]]
|
@@ -324,7 +329,7 @@ end
|
|
324
329
|
local getJobKeyPrefix = function (jobKey, jobId)
|
325
330
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
326
331
|
end
|
327
|
-
local function
|
332
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
328
333
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
329
334
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
330
335
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -352,10 +357,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
352
357
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
353
358
|
end
|
354
359
|
else
|
355
|
-
|
360
|
+
_moveParentToWait(parentPrefix, parentId)
|
356
361
|
end
|
357
362
|
else
|
358
|
-
|
363
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
359
364
|
end
|
360
365
|
end
|
361
366
|
end
|
@@ -383,10 +388,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
383
388
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
384
389
|
end
|
385
390
|
else
|
386
|
-
|
391
|
+
_moveParentToWait(parentPrefix, parentId)
|
387
392
|
end
|
388
393
|
else
|
389
|
-
|
394
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
390
395
|
end
|
391
396
|
end
|
392
397
|
end
|
@@ -494,6 +499,8 @@ local function moveParentToFailedIfNeeded(parentQueueKey, parentKey, parentId, j
|
|
494
499
|
parentKey,
|
495
500
|
timestamp
|
496
501
|
)
|
502
|
+
elseif parentData['cpof'] then
|
503
|
+
moveParentToWait(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
497
504
|
elseif parentData['idof'] or parentData['rdof'] then
|
498
505
|
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
499
506
|
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
|
@@ -655,7 +662,9 @@ local jobIdKey = KEYS[12]
|
|
655
662
|
if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
656
663
|
-- Make sure it does not have pending dependencies
|
657
664
|
-- It must happen before removing lock
|
658
|
-
if ARGV[5] == "completed" and
|
665
|
+
if ARGV[5] == "completed" and
|
666
|
+
not rcall("HGET", jobIdKey, "igdp") and -- check if we should ignore this check
|
667
|
+
rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then
|
659
668
|
return -4
|
660
669
|
end
|
661
670
|
local opts = cmsgpack.unpack(ARGV[8])
|
@@ -682,7 +691,9 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
682
691
|
local timestamp = ARGV[2]
|
683
692
|
-- Remove from active list (if not active we shall return error)
|
684
693
|
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
685
|
-
if (numRemovedElements < 1) then
|
694
|
+
if (numRemovedElements < 1) then
|
695
|
+
return -3
|
696
|
+
end
|
686
697
|
local eventStreamKey = KEYS[4]
|
687
698
|
local metaKey = KEYS[9]
|
688
699
|
-- Trim events before emiting them to avoid trimming events emitted in this script
|
@@ -702,22 +713,24 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
702
713
|
if ARGV[5] == "completed" then
|
703
714
|
local dependenciesSet = parentKey .. ":dependencies"
|
704
715
|
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
705
|
-
updateParentDepsIfNeeded(parentKey, parentQueueKey,
|
706
|
-
|
707
|
-
ARGV[4], timestamp)
|
716
|
+
updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet, parentId, jobIdKey, ARGV[4],
|
717
|
+
timestamp)
|
708
718
|
end
|
709
719
|
else
|
710
|
-
if opts['fpof'] then
|
720
|
+
if opts['fpof'] or opts['cpof'] then
|
711
721
|
local unsuccesssfulSet = parentKey .. ":unsuccessful"
|
712
722
|
rcall("ZADD", unsuccesssfulSet, timestamp, jobIdKey)
|
713
|
-
|
714
|
-
|
715
|
-
|
723
|
+
if opts['fpof'] then
|
724
|
+
moveParentToFailedIfNeeded(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
725
|
+
elseif opts['cpof'] then
|
726
|
+
local failedSet = parentKey .. ":failed"
|
727
|
+
rcall("HSET", failedSet, jobIdKey, ARGV[4])
|
728
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
729
|
+
end
|
716
730
|
elseif opts['idof'] or opts['rdof'] then
|
717
731
|
local dependenciesSet = parentKey .. ":dependencies"
|
718
732
|
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
719
|
-
moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
|
720
|
-
parentKey, parentId, timestamp)
|
733
|
+
moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet, parentKey, parentId, timestamp)
|
721
734
|
if opts['idof'] then
|
722
735
|
local failedSet = parentKey .. ":failed"
|
723
736
|
rcall("HSET", failedSet, jobIdKey, ARGV[4])
|
@@ -750,12 +763,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
750
763
|
removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3])
|
751
764
|
end
|
752
765
|
end
|
753
|
-
rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
|
754
|
-
ARGV[4], "prev", "active")
|
766
|
+
rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3], ARGV[4], "prev", "active")
|
755
767
|
if ARGV[5] == "failed" then
|
756
768
|
if tonumber(attemptsMade) >= tonumber(attempts) then
|
757
|
-
rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId",
|
758
|
-
|
769
|
+
rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId", jobId, "attemptsMade",
|
770
|
+
attemptsMade)
|
759
771
|
end
|
760
772
|
end
|
761
773
|
-- Collect metrics
|
@@ -768,14 +780,18 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
768
780
|
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
|
769
781
|
local markerKey = KEYS[14]
|
770
782
|
-- Check if there are delayed jobs that can be promoted
|
771
|
-
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix,
|
772
|
-
|
783
|
+
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10],
|
784
|
+
isPausedOrMaxed)
|
773
785
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
774
786
|
-- Check if we are rate limited first.
|
775
787
|
local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
|
776
|
-
if expireTime > 0 then
|
788
|
+
if expireTime > 0 then
|
789
|
+
return {0, 0, expireTime, 0}
|
790
|
+
end
|
777
791
|
-- paused or maxed queue
|
778
|
-
if isPausedOrMaxed then
|
792
|
+
if isPausedOrMaxed then
|
793
|
+
return {0, 0, 0, 0}
|
794
|
+
end
|
779
795
|
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
780
796
|
if jobId then
|
781
797
|
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
@@ -784,23 +800,19 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
784
800
|
-- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
|
785
801
|
-- but if ID is 0:0, then there is at least 1 prioritized job to process
|
786
802
|
if jobId == "0:0" then
|
787
|
-
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
|
788
|
-
|
789
|
-
|
790
|
-
timestamp, maxJobs, markerKey,
|
791
|
-
opts)
|
803
|
+
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
804
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
805
|
+
markerKey, opts)
|
792
806
|
end
|
793
807
|
else
|
794
|
-
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
|
795
|
-
|
796
|
-
opts)
|
808
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
|
809
|
+
opts)
|
797
810
|
end
|
798
811
|
else
|
799
812
|
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
800
813
|
if jobId then
|
801
|
-
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
|
802
|
-
|
803
|
-
opts)
|
814
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
|
815
|
+
opts)
|
804
816
|
end
|
805
817
|
end
|
806
818
|
-- Return the timestamp for the next delayed job if any.
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAu0Bf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|