bullmq 5.46.1 → 5.47.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/flow-producer.js +1 -1
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job.js +11 -9
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/scripts.js +4 -3
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/includes/moveParentToFailedIfNeeded.lua +3 -0
- package/dist/cjs/commands/includes/moveParentToWait.lua +49 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +6 -51
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +26 -30
- package/dist/cjs/commands/moveToFinished-14.lua +38 -30
- package/dist/cjs/commands/moveToWaitingChildren-8.lua +13 -8
- package/dist/cjs/scripts/addDelayedJob-6.js +24 -20
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
- package/dist/cjs/scripts/addParentJob-4.js +24 -20
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js +24 -20
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
- package/dist/cjs/scripts/addStandardJob-8.js +24 -20
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
- package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
- package/dist/cjs/scripts/drain-5.js +5 -5
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +56 -54
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +67 -55
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-8.js +43 -33
- package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +5 -5
- package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
- package/dist/cjs/scripts/removeJob-3.js +5 -5
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/flow-producer.js +1 -1
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job.js +11 -9
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/scripts.js +4 -3
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/includes/moveParentToFailedIfNeeded.lua +3 -0
- package/dist/esm/commands/includes/moveParentToWait.lua +49 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +6 -51
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +26 -30
- package/dist/esm/commands/moveToFinished-14.lua +38 -30
- package/dist/esm/commands/moveToWaitingChildren-8.lua +13 -8
- package/dist/esm/scripts/addDelayedJob-6.js +24 -20
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addJobScheduler-11.js +5 -5
- package/dist/esm/scripts/addParentJob-4.js +24 -20
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js +24 -20
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
- package/dist/esm/scripts/addStandardJob-8.js +24 -20
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
- package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
- package/dist/esm/scripts/drain-5.js +5 -5
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +56 -54
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +67 -55
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-8.js +43 -33
- package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +5 -5
- package/dist/esm/scripts/removeChildDependency-1.js +5 -5
- package/dist/esm/scripts/removeJob-3.js +5 -5
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/job-options.d.ts +12 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
@@ -1,55 +1,10 @@
|
|
1
1
|
--[[
|
2
|
-
Validate and move parent to
|
2
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
3
3
|
]]
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
--- @include "isQueuePausedOrMaxed"
|
10
|
-
--- @include "getTargetQueueList"
|
11
|
-
|
12
|
-
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
13
|
-
parentKey, parentId, timestamp)
|
14
|
-
local isParentActive = rcall("ZSCORE",
|
15
|
-
parentQueueKey .. ":waiting-children", parentId)
|
16
|
-
if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
|
17
|
-
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
18
|
-
local parentWaitKey = parentQueueKey .. ":wait"
|
19
|
-
local parentPausedKey = parentQueueKey .. ":paused"
|
20
|
-
local parentActiveKey = parentQueueKey .. ":active"
|
21
|
-
local parentMetaKey = parentQueueKey .. ":meta"
|
22
|
-
|
23
|
-
local parentMarkerKey = parentQueueKey .. ":marker"
|
24
|
-
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
25
|
-
local priority = tonumber(jobAttributes[1]) or 0
|
26
|
-
local delay = tonumber(jobAttributes[2]) or 0
|
27
|
-
|
28
|
-
if delay > 0 then
|
29
|
-
local delayedTimestamp = tonumber(timestamp) + delay
|
30
|
-
local score = delayedTimestamp * 0x1000
|
31
|
-
local parentDelayedKey = parentQueueKey .. ":delayed"
|
32
|
-
rcall("ZADD", parentDelayedKey, score, parentId)
|
33
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
34
|
-
"jobId", parentId, "delay", delayedTimestamp)
|
35
|
-
|
36
|
-
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
37
|
-
else
|
38
|
-
if priority == 0 then
|
39
|
-
local parentTarget, isParentPausedOrMaxed =
|
40
|
-
getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
|
41
|
-
parentPausedKey)
|
42
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
43
|
-
parentId)
|
44
|
-
else
|
45
|
-
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
46
|
-
addJobWithPriority(parentMarkerKey,
|
47
|
-
parentQueueKey .. ":prioritized", priority,
|
48
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
49
|
-
end
|
50
|
-
|
51
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
52
|
-
"jobId", parentId, "prev", "waiting-children")
|
53
|
-
end
|
4
|
+
--- @include "moveParentToWait"
|
5
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
6
|
+
local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
7
|
+
if hasPendingDependencies then
|
8
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
54
9
|
end
|
55
10
|
end
|
@@ -10,7 +10,7 @@
|
|
10
10
|
--- @include "getTargetQueueList"
|
11
11
|
--- @include "removeJobKeys"
|
12
12
|
|
13
|
-
local function
|
13
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
14
14
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
15
15
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
16
16
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -42,10 +42,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
42
42
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
43
43
|
end
|
44
44
|
else
|
45
|
-
|
45
|
+
_moveParentToWait(parentPrefix, parentId)
|
46
46
|
end
|
47
47
|
else
|
48
|
-
|
48
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
49
49
|
end
|
50
50
|
end
|
51
51
|
end
|
@@ -75,10 +75,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
75
75
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
76
76
|
end
|
77
77
|
else
|
78
|
-
|
78
|
+
_moveParentToWait(parentPrefix, parentId)
|
79
79
|
end
|
80
80
|
else
|
81
|
-
|
81
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
82
82
|
end
|
83
83
|
end
|
84
84
|
end
|
@@ -20,7 +20,6 @@
|
|
20
20
|
Events:
|
21
21
|
'stalled' with stalled job id.
|
22
22
|
]]
|
23
|
-
|
24
23
|
local rcall = redis.call
|
25
24
|
|
26
25
|
-- Includes
|
@@ -29,6 +28,7 @@ local rcall = redis.call
|
|
29
28
|
--- @include "includes/getTargetQueueList"
|
30
29
|
--- @include "includes/moveParentToFailedIfNeeded"
|
31
30
|
--- @include "includes/moveParentToWaitIfNeeded"
|
31
|
+
--- @include "includes/moveParentToWait"
|
32
32
|
--- @include "includes/removeDeduplicationKeyIfNeeded"
|
33
33
|
--- @include "includes/removeJobsOnFail"
|
34
34
|
--- @include "includes/trimEvents"
|
@@ -47,7 +47,9 @@ local queueKeyPrefix = ARGV[2]
|
|
47
47
|
local timestamp = ARGV[3]
|
48
48
|
local maxCheckTime = ARGV[4]
|
49
49
|
|
50
|
-
if rcall("EXISTS", stalledCheckKey) == 1 then
|
50
|
+
if rcall("EXISTS", stalledCheckKey) == 1 then
|
51
|
+
return {{}, {}}
|
52
|
+
end
|
51
53
|
|
52
54
|
rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
|
53
55
|
|
@@ -77,8 +79,7 @@ if (#stalling > 0) then
|
|
77
79
|
|
78
80
|
if (removed > 0) then
|
79
81
|
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
80
|
-
local stalledCount =
|
81
|
-
rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
82
|
+
local stalledCount = rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
82
83
|
if (stalledCount > maxStalledJobCount) then
|
83
84
|
local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
|
84
85
|
local rawOpts = jobAttributes[1]
|
@@ -87,39 +88,37 @@ if (#stalling > 0) then
|
|
87
88
|
rcall("ZADD", failedKey, timestamp, jobId)
|
88
89
|
removeDeduplicationKeyIfNeeded(queueKeyPrefix, jobAttributes[3])
|
89
90
|
|
90
|
-
local failedReason =
|
91
|
-
|
92
|
-
rcall("
|
93
|
-
|
94
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
95
|
-
"failed", "jobId", jobId, 'prev', 'active',
|
96
|
-
'failedReason', failedReason)
|
91
|
+
local failedReason = "job stalled more than allowable limit"
|
92
|
+
rcall("HMSET", jobKey, "failedReason", failedReason, "finishedOn", timestamp)
|
93
|
+
rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId", jobId, 'prev', 'active',
|
94
|
+
'failedReason', failedReason)
|
97
95
|
|
98
96
|
if rawParentData then
|
99
|
-
if opts['fpof'] then
|
97
|
+
if opts['fpof'] or opts['cpof'] then
|
100
98
|
local parentData = cjson.decode(rawParentData)
|
101
99
|
-- TODO: need to remove this job from dependencies set in next breaking change
|
102
100
|
-- no for now as it would imply a breaking change
|
103
101
|
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
104
102
|
local unsuccesssfulSet = parentKey .. ":unsuccessful"
|
105
103
|
rcall("ZADD", unsuccesssfulSet, timestamp, jobKey)
|
106
|
-
|
107
|
-
|
108
|
-
parentData['queueKey']
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
104
|
+
|
105
|
+
if opts['fpof'] then
|
106
|
+
moveParentToFailedIfNeeded(parentData['queueKey'],
|
107
|
+
parentData['queueKey'] .. ':' .. parentData['id'], parentData['id'], jobKey,
|
108
|
+
timestamp)
|
109
|
+
elseif opts['cpof'] then
|
110
|
+
moveParentToWait(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
111
|
+
end
|
113
112
|
elseif opts['idof'] or opts['rdof'] then
|
114
113
|
local parentData = cjson.decode(rawParentData)
|
115
114
|
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
116
115
|
local dependenciesSet = parentKey .. ":dependencies"
|
117
116
|
if rcall("SREM", dependenciesSet, jobKey) == 1 then
|
118
|
-
moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
|
119
|
-
|
117
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet, parentKey,
|
118
|
+
parentData['id'], timestamp)
|
120
119
|
if opts['idof'] then
|
121
|
-
|
122
|
-
|
120
|
+
local failedSet = parentKey .. ":failed"
|
121
|
+
rcall("HSET", failedSet, jobKey, failedReason)
|
123
122
|
end
|
124
123
|
end
|
125
124
|
end
|
@@ -129,18 +128,15 @@ if (#stalling > 0) then
|
|
129
128
|
|
130
129
|
table.insert(failed, jobId)
|
131
130
|
else
|
132
|
-
local target, isPausedOrMaxed =
|
133
|
-
getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
|
131
|
+
local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
|
134
132
|
|
135
133
|
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
|
136
134
|
addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
|
137
135
|
|
138
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
139
|
-
"waiting", "jobId", jobId, 'prev', 'active')
|
136
|
+
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, 'prev', 'active')
|
140
137
|
|
141
138
|
-- Emit the stalled event
|
142
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
143
|
-
"stalled", "jobId", jobId)
|
139
|
+
rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId", jobId)
|
144
140
|
table.insert(stalled, jobId)
|
145
141
|
end
|
146
142
|
end
|
@@ -158,4 +154,4 @@ if (#active > 0) then
|
|
158
154
|
end
|
159
155
|
end
|
160
156
|
|
161
|
-
return {failed, stalled}
|
157
|
+
return {failed, stalled}
|
@@ -40,6 +40,7 @@
|
|
40
40
|
opts - attempts max attempts
|
41
41
|
opts - maxMetricsSize
|
42
42
|
opts - fpof - fail parent on fail
|
43
|
+
opts - cpof - continue parent on fail
|
43
44
|
opts - idof - ignore dependency on fail
|
44
45
|
opts - rdof - remove dependency on fail
|
45
46
|
opts - name - worker name
|
@@ -64,6 +65,7 @@ local rcall = redis.call
|
|
64
65
|
--- @include "includes/getTargetQueueList"
|
65
66
|
--- @include "includes/moveJobFromPriorityToActive"
|
66
67
|
--- @include "includes/moveParentToFailedIfNeeded"
|
68
|
+
--- @include "includes/moveParentToWait"
|
67
69
|
--- @include "includes/moveParentToWaitIfNeeded"
|
68
70
|
--- @include "includes/prepareJobForProcessing"
|
69
71
|
--- @include "includes/promoteDelayedJobs"
|
@@ -81,7 +83,9 @@ local jobIdKey = KEYS[12]
|
|
81
83
|
if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
82
84
|
-- Make sure it does not have pending dependencies
|
83
85
|
-- It must happen before removing lock
|
84
|
-
if ARGV[5] == "completed" and
|
86
|
+
if ARGV[5] == "completed" and
|
87
|
+
not rcall("HGET", jobIdKey, "igdp") and -- check if we should ignore this check
|
88
|
+
rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then
|
85
89
|
return -4
|
86
90
|
end
|
87
91
|
|
@@ -117,7 +121,9 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
117
121
|
-- Remove from active list (if not active we shall return error)
|
118
122
|
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
119
123
|
|
120
|
-
if (numRemovedElements < 1) then
|
124
|
+
if (numRemovedElements < 1) then
|
125
|
+
return -3
|
126
|
+
end
|
121
127
|
|
122
128
|
local eventStreamKey = KEYS[4]
|
123
129
|
local metaKey = KEYS[9]
|
@@ -142,22 +148,25 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
142
148
|
if ARGV[5] == "completed" then
|
143
149
|
local dependenciesSet = parentKey .. ":dependencies"
|
144
150
|
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
145
|
-
updateParentDepsIfNeeded(parentKey, parentQueueKey,
|
146
|
-
|
147
|
-
ARGV[4], timestamp)
|
151
|
+
updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet, parentId, jobIdKey, ARGV[4],
|
152
|
+
timestamp)
|
148
153
|
end
|
149
154
|
else
|
150
|
-
if opts['fpof'] then
|
155
|
+
if opts['fpof'] or opts['cpof'] then
|
151
156
|
local unsuccesssfulSet = parentKey .. ":unsuccessful"
|
152
157
|
rcall("ZADD", unsuccesssfulSet, timestamp, jobIdKey)
|
153
|
-
|
154
|
-
|
155
|
-
|
158
|
+
|
159
|
+
if opts['fpof'] then
|
160
|
+
moveParentToFailedIfNeeded(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
161
|
+
elseif opts['cpof'] then
|
162
|
+
local failedSet = parentKey .. ":failed"
|
163
|
+
rcall("HSET", failedSet, jobIdKey, ARGV[4])
|
164
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
165
|
+
end
|
156
166
|
elseif opts['idof'] or opts['rdof'] then
|
157
167
|
local dependenciesSet = parentKey .. ":dependencies"
|
158
168
|
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
159
|
-
moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
|
160
|
-
parentKey, parentId, timestamp)
|
169
|
+
moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet, parentKey, parentId, timestamp)
|
161
170
|
if opts['idof'] then
|
162
171
|
local failedSet = parentKey .. ":failed"
|
163
172
|
rcall("HSET", failedSet, jobIdKey, ARGV[4])
|
@@ -195,13 +204,12 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
195
204
|
end
|
196
205
|
end
|
197
206
|
|
198
|
-
rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
|
199
|
-
ARGV[4], "prev", "active")
|
207
|
+
rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3], ARGV[4], "prev", "active")
|
200
208
|
|
201
209
|
if ARGV[5] == "failed" then
|
202
210
|
if tonumber(attemptsMade) >= tonumber(attempts) then
|
203
|
-
rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId",
|
204
|
-
|
211
|
+
rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId", jobId, "attemptsMade",
|
212
|
+
attemptsMade)
|
205
213
|
end
|
206
214
|
end
|
207
215
|
|
@@ -218,17 +226,21 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
218
226
|
|
219
227
|
local markerKey = KEYS[14]
|
220
228
|
-- Check if there are delayed jobs that can be promoted
|
221
|
-
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix,
|
222
|
-
|
229
|
+
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10],
|
230
|
+
isPausedOrMaxed)
|
223
231
|
|
224
232
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
225
233
|
-- Check if we are rate limited first.
|
226
234
|
local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
|
227
235
|
|
228
|
-
if expireTime > 0 then
|
236
|
+
if expireTime > 0 then
|
237
|
+
return {0, 0, expireTime, 0}
|
238
|
+
end
|
229
239
|
|
230
240
|
-- paused or maxed queue
|
231
|
-
if isPausedOrMaxed then
|
241
|
+
if isPausedOrMaxed then
|
242
|
+
return {0, 0, 0, 0}
|
243
|
+
end
|
232
244
|
|
233
245
|
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
234
246
|
|
@@ -240,23 +252,19 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
240
252
|
-- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
|
241
253
|
-- but if ID is 0:0, then there is at least 1 prioritized job to process
|
242
254
|
if jobId == "0:0" then
|
243
|
-
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
|
244
|
-
|
245
|
-
|
246
|
-
timestamp, maxJobs, markerKey,
|
247
|
-
opts)
|
255
|
+
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
256
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
257
|
+
markerKey, opts)
|
248
258
|
end
|
249
259
|
else
|
250
|
-
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
|
251
|
-
|
252
|
-
opts)
|
260
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
|
261
|
+
opts)
|
253
262
|
end
|
254
263
|
else
|
255
264
|
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
256
265
|
if jobId then
|
257
|
-
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
|
258
|
-
|
259
|
-
opts)
|
266
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
|
267
|
+
opts)
|
260
268
|
end
|
261
269
|
end
|
262
270
|
|
@@ -37,6 +37,7 @@ local jobId = ARGV[4]
|
|
37
37
|
|
38
38
|
--- Includes
|
39
39
|
--- @include "includes/moveParentToFailedIfNeeded"
|
40
|
+
--- @include "includes/moveParentToWait"
|
40
41
|
--- @include "includes/moveParentToWaitIfNeeded"
|
41
42
|
--- @include "includes/removeDeduplicationKeyIfNeeded"
|
42
43
|
--- @include "includes/removeJobsOnFail"
|
@@ -75,18 +76,22 @@ if rcall("EXISTS", jobKey) == 1 then
|
|
75
76
|
local opts = cjson.decode(rawOpts)
|
76
77
|
|
77
78
|
if rawParentData ~= false then
|
78
|
-
if opts['fpof'] then
|
79
|
+
if opts['fpof'] or opts['cpof'] then
|
79
80
|
local parentData = cjson.decode(rawParentData)
|
80
81
|
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
81
82
|
local parentUnsuccesssful = parentKey .. ":unsuccessful"
|
82
83
|
rcall("ZADD", parentUnsuccesssful, timestamp, jobKey)
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
84
|
+
if opts['fpof'] then
|
85
|
+
moveParentToFailedIfNeeded(
|
86
|
+
parentData['queueKey'],
|
87
|
+
parentData['queueKey'] .. ':' .. parentData['id'],
|
88
|
+
parentData['id'],
|
89
|
+
jobKey,
|
90
|
+
timestamp
|
91
|
+
)
|
92
|
+
elseif opts['cpof'] then
|
93
|
+
moveParentToWait(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
94
|
+
end
|
90
95
|
elseif opts['idof'] or opts['rdof'] then
|
91
96
|
local parentData = cjson.decode(rawParentData)
|
92
97
|
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
@@ -162,9 +162,11 @@ end
|
|
162
162
|
]]
|
163
163
|
-- Includes
|
164
164
|
--[[
|
165
|
-
Validate and move parent to
|
165
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
166
|
+
]]
|
167
|
+
--[[
|
168
|
+
Validate and move parent to a wait status (wait, prioritized or delayed)
|
166
169
|
]]
|
167
|
-
-- Includes
|
168
170
|
--[[
|
169
171
|
Function to add job in target list and add marker if needed.
|
170
172
|
]]
|
@@ -234,12 +236,9 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
234
236
|
end
|
235
237
|
return waitKey, false
|
236
238
|
end
|
237
|
-
local function
|
238
|
-
|
239
|
-
|
240
|
-
parentQueueKey .. ":waiting-children", parentId)
|
241
|
-
if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
|
242
|
-
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
239
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
240
|
+
local isParentWaitingChildren = rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
241
|
+
if isParentWaitingChildren > 0 then
|
243
242
|
local parentWaitKey = parentQueueKey .. ":wait"
|
244
243
|
local parentPausedKey = parentQueueKey .. ":paused"
|
245
244
|
local parentActiveKey = parentQueueKey .. ":active"
|
@@ -248,32 +247,37 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
248
247
|
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
249
248
|
local priority = tonumber(jobAttributes[1]) or 0
|
250
249
|
local delay = tonumber(jobAttributes[2]) or 0
|
250
|
+
-- ignore dependencies if any left
|
251
|
+
rcall("HSET", parentKey, "igdp", 1)
|
251
252
|
if delay > 0 then
|
252
253
|
local delayedTimestamp = tonumber(timestamp) + delay
|
253
254
|
local score = delayedTimestamp * 0x1000
|
254
255
|
local parentDelayedKey = parentQueueKey .. ":delayed"
|
255
256
|
rcall("ZADD", parentDelayedKey, score, parentId)
|
256
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
257
|
-
|
257
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
258
|
+
delayedTimestamp)
|
258
259
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
259
260
|
else
|
260
261
|
if priority == 0 then
|
261
|
-
local parentTarget, isParentPausedOrMaxed =
|
262
|
-
|
263
|
-
|
264
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
265
|
-
parentId)
|
262
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
263
|
+
parentWaitKey, parentPausedKey)
|
264
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
266
265
|
else
|
267
266
|
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
268
|
-
addJobWithPriority(parentMarkerKey,
|
269
|
-
|
270
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
267
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
268
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
271
269
|
end
|
272
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
273
|
-
|
270
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
271
|
+
"waiting-children")
|
274
272
|
end
|
275
273
|
end
|
276
274
|
end
|
275
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
276
|
+
local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
277
|
+
if hasPendingDependencies then
|
278
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
279
|
+
end
|
280
|
+
end
|
277
281
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
278
282
|
parentId, jobIdKey, returnvalue, timestamp )
|
279
283
|
local processedSet = parentKey .. ":processed"
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6Xf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -273,7 +273,7 @@ end
|
|
273
273
|
local getJobKeyPrefix = function (jobKey, jobId)
|
274
274
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
275
275
|
end
|
276
|
-
local function
|
276
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
277
277
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
278
278
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
279
279
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -301,10 +301,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
301
301
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
302
302
|
end
|
303
303
|
else
|
304
|
-
|
304
|
+
_moveParentToWait(parentPrefix, parentId)
|
305
305
|
end
|
306
306
|
else
|
307
|
-
|
307
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
308
308
|
end
|
309
309
|
end
|
310
310
|
end
|
@@ -332,10 +332,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
332
332
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
333
333
|
end
|
334
334
|
else
|
335
|
-
|
335
|
+
_moveParentToWait(parentPrefix, parentId)
|
336
336
|
end
|
337
337
|
else
|
338
|
-
|
338
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
339
339
|
end
|
340
340
|
end
|
341
341
|
end
|
@@ -94,9 +94,11 @@ end
|
|
94
94
|
]]
|
95
95
|
-- Includes
|
96
96
|
--[[
|
97
|
-
Validate and move parent to
|
97
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
98
|
+
]]
|
99
|
+
--[[
|
100
|
+
Validate and move parent to a wait status (wait, prioritized or delayed)
|
98
101
|
]]
|
99
|
-
-- Includes
|
100
102
|
--[[
|
101
103
|
Add delay marker if needed.
|
102
104
|
]]
|
@@ -190,12 +192,9 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
190
192
|
end
|
191
193
|
return waitKey, false
|
192
194
|
end
|
193
|
-
local function
|
194
|
-
|
195
|
-
|
196
|
-
parentQueueKey .. ":waiting-children", parentId)
|
197
|
-
if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
|
198
|
-
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
195
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
196
|
+
local isParentWaitingChildren = rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
197
|
+
if isParentWaitingChildren > 0 then
|
199
198
|
local parentWaitKey = parentQueueKey .. ":wait"
|
200
199
|
local parentPausedKey = parentQueueKey .. ":paused"
|
201
200
|
local parentActiveKey = parentQueueKey .. ":active"
|
@@ -204,32 +203,37 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
204
203
|
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
205
204
|
local priority = tonumber(jobAttributes[1]) or 0
|
206
205
|
local delay = tonumber(jobAttributes[2]) or 0
|
206
|
+
-- ignore dependencies if any left
|
207
|
+
rcall("HSET", parentKey, "igdp", 1)
|
207
208
|
if delay > 0 then
|
208
209
|
local delayedTimestamp = tonumber(timestamp) + delay
|
209
210
|
local score = delayedTimestamp * 0x1000
|
210
211
|
local parentDelayedKey = parentQueueKey .. ":delayed"
|
211
212
|
rcall("ZADD", parentDelayedKey, score, parentId)
|
212
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
213
|
-
|
213
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
214
|
+
delayedTimestamp)
|
214
215
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
215
216
|
else
|
216
217
|
if priority == 0 then
|
217
|
-
local parentTarget, isParentPausedOrMaxed =
|
218
|
-
|
219
|
-
|
220
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
221
|
-
parentId)
|
218
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
219
|
+
parentWaitKey, parentPausedKey)
|
220
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
222
221
|
else
|
223
222
|
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
224
|
-
addJobWithPriority(parentMarkerKey,
|
225
|
-
|
226
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
223
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
224
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
227
225
|
end
|
228
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
229
|
-
|
226
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
227
|
+
"waiting-children")
|
230
228
|
end
|
231
229
|
end
|
232
230
|
end
|
231
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
232
|
+
local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
233
|
+
if hasPendingDependencies then
|
234
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
235
|
+
end
|
236
|
+
end
|
233
237
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
234
238
|
parentId, jobIdKey, returnvalue, timestamp )
|
235
239
|
local processedSet = parentKey .. ":processed"
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoVf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|