bullmq 5.41.8 → 5.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/job-scheduler.js +14 -18
- package/dist/cjs/classes/job-scheduler.js.map +1 -1
- package/dist/cjs/classes/job.js +20 -13
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/scripts.js +33 -3
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +4 -8
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addJobScheduler-10.lua +9 -36
- package/dist/cjs/commands/includes/addJobFromScheduler.lua +41 -0
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +11 -1
- package/dist/cjs/commands/includes/removeJobKeys.lua +2 -2
- package/dist/cjs/commands/includes/storeJobScheduler.lua +1 -1
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +5 -0
- package/dist/cjs/commands/moveToFinished-14.lua +2 -0
- package/dist/cjs/commands/moveToWaitingChildren-8.lua +133 -0
- package/dist/cjs/commands/removeJob-3.lua +10 -0
- package/dist/{esm/commands/updateJobScheduler-7.lua → cjs/commands/updateJobScheduler-11.lua} +28 -21
- package/dist/cjs/scripts/addJobScheduler-10.js +80 -70
- package/dist/cjs/scripts/addJobScheduler-10.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +2 -2
- package/dist/cjs/scripts/cleanJobsInSet-3.js +2 -2
- package/dist/cjs/scripts/drain-5.js +2 -2
- package/dist/cjs/scripts/index.js +2 -2
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +17 -3
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +14 -3
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-8.js +549 -0
- package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -0
- package/dist/cjs/scripts/obliterate-2.js +2 -2
- package/dist/cjs/scripts/removeChildDependency-1.js +2 -2
- package/dist/cjs/scripts/removeJob-3.js +10 -2
- package/dist/cjs/scripts/removeJob-3.js.map +1 -1
- package/dist/cjs/scripts/removeJobScheduler-3.js +2 -2
- package/dist/cjs/scripts/removeRepeatable-3.js +2 -2
- package/dist/cjs/scripts/updateJobScheduler-11.js +246 -0
- package/dist/cjs/scripts/updateJobScheduler-11.js.map +1 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/job-scheduler.js +14 -18
- package/dist/esm/classes/job-scheduler.js.map +1 -1
- package/dist/esm/classes/job.js +20 -13
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +1 -1
- package/dist/esm/classes/scripts.js +33 -3
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.js +4 -8
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addJobScheduler-10.lua +9 -36
- package/dist/esm/commands/includes/addJobFromScheduler.lua +41 -0
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +11 -1
- package/dist/esm/commands/includes/removeJobKeys.lua +2 -2
- package/dist/esm/commands/includes/storeJobScheduler.lua +1 -1
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +5 -0
- package/dist/esm/commands/moveToFinished-14.lua +2 -0
- package/dist/esm/commands/moveToWaitingChildren-8.lua +133 -0
- package/dist/esm/commands/removeJob-3.lua +10 -0
- package/dist/{cjs/commands/updateJobScheduler-7.lua → esm/commands/updateJobScheduler-11.lua} +28 -21
- package/dist/esm/scripts/addJobScheduler-10.js +80 -70
- package/dist/esm/scripts/addJobScheduler-10.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +2 -2
- package/dist/esm/scripts/cleanJobsInSet-3.js +2 -2
- package/dist/esm/scripts/drain-5.js +2 -2
- package/dist/esm/scripts/index.d.ts +2 -2
- package/dist/esm/scripts/index.js +2 -2
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +17 -3
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +14 -3
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-8.js +546 -0
- package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -0
- package/dist/esm/scripts/obliterate-2.js +2 -2
- package/dist/esm/scripts/removeChildDependency-1.js +2 -2
- package/dist/esm/scripts/removeJob-3.js +10 -2
- package/dist/esm/scripts/removeJob-3.js.map +1 -1
- package/dist/esm/scripts/removeJobScheduler-3.js +2 -2
- package/dist/esm/scripts/removeRepeatable-3.js +2 -2
- package/dist/esm/scripts/updateJobScheduler-11.js +243 -0
- package/dist/esm/scripts/updateJobScheduler-11.js.map +1 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/moveToWaitingChildren-5.lua +0 -68
- package/dist/cjs/scripts/moveToWaitingChildren-5.js +0 -83
- package/dist/cjs/scripts/moveToWaitingChildren-5.js.map +0 -1
- package/dist/cjs/scripts/updateJobScheduler-7.js +0 -144
- package/dist/cjs/scripts/updateJobScheduler-7.js.map +0 -1
- package/dist/esm/commands/moveToWaitingChildren-5.lua +0 -68
- package/dist/esm/scripts/moveToWaitingChildren-5.js +0 -80
- package/dist/esm/scripts/moveToWaitingChildren-5.js.map +0 -1
- package/dist/esm/scripts/updateJobScheduler-7.js +0 -141
- package/dist/esm/scripts/updateJobScheduler-7.js.map +0 -1
- /package/dist/esm/scripts/{moveToWaitingChildren-5.d.ts → moveToWaitingChildren-8.d.ts} +0 -0
- /package/dist/esm/scripts/{updateJobScheduler-7.d.ts → updateJobScheduler-11.d.ts} +0 -0
@@ -0,0 +1,546 @@
|
|
1
|
+
const content = `--[[
|
2
|
+
Moves job from active to waiting children set.
|
3
|
+
Input:
|
4
|
+
KEYS[1] active key
|
5
|
+
KEYS[2] wait-children key
|
6
|
+
KEYS[3] job key
|
7
|
+
KEYS[4] job dependencies key
|
8
|
+
KEYS[5] job unsuccessful key
|
9
|
+
KEYS[6] stalled key
|
10
|
+
KEYS[7] failed key
|
11
|
+
KEYS[8] events key
|
12
|
+
ARGV[1] token
|
13
|
+
ARGV[2] child key
|
14
|
+
ARGV[3] timestamp
|
15
|
+
ARGV[4] jobId
|
16
|
+
ARGV[5] prefix
|
17
|
+
Output:
|
18
|
+
0 - OK
|
19
|
+
1 - There are not pending dependencies.
|
20
|
+
-1 - Missing job.
|
21
|
+
-2 - Missing lock
|
22
|
+
-3 - Job not in active set
|
23
|
+
]]
|
24
|
+
local rcall = redis.call
|
25
|
+
local activeKey = KEYS[1]
|
26
|
+
local waitingChildrenKey = KEYS[2]
|
27
|
+
local jobKey = KEYS[3]
|
28
|
+
local jobDependenciesKey = KEYS[4]
|
29
|
+
local jobUnsuccessfulKey = KEYS[5]
|
30
|
+
local stalledKey = KEYS[6]
|
31
|
+
local failedKey = KEYS[7]
|
32
|
+
local timestamp = ARGV[3]
|
33
|
+
local jobId = ARGV[4]
|
34
|
+
--- Includes
|
35
|
+
--[[
|
36
|
+
Function to recursively move from waitingChildren to failed.
|
37
|
+
]]
|
38
|
+
-- Includes
|
39
|
+
--[[
|
40
|
+
Validate and move parent to active if needed.
|
41
|
+
]]
|
42
|
+
-- Includes
|
43
|
+
--[[
|
44
|
+
Add delay marker if needed.
|
45
|
+
]]
|
46
|
+
-- Includes
|
47
|
+
--[[
|
48
|
+
Function to return the next delayed job timestamp.
|
49
|
+
]]
|
50
|
+
local function getNextDelayedTimestamp(delayedKey)
|
51
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
52
|
+
if #result then
|
53
|
+
local nextTimestamp = tonumber(result[2])
|
54
|
+
if nextTimestamp ~= nil then
|
55
|
+
return nextTimestamp / 0x1000
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
60
|
+
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
61
|
+
if nextTimestamp ~= nil then
|
62
|
+
-- Replace the score of the marker with the newest known
|
63
|
+
-- next timestamp.
|
64
|
+
rcall("ZADD", markerKey, nextTimestamp, "1")
|
65
|
+
end
|
66
|
+
end
|
67
|
+
--[[
|
68
|
+
Function to add job in target list and add marker if needed.
|
69
|
+
]]
|
70
|
+
-- Includes
|
71
|
+
--[[
|
72
|
+
Add marker if needed when a job is available.
|
73
|
+
]]
|
74
|
+
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
75
|
+
if not isPausedOrMaxed then
|
76
|
+
rcall("ZADD", markerKey, 0, "0")
|
77
|
+
end
|
78
|
+
end
|
79
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
80
|
+
rcall(pushCmd, targetKey, jobId)
|
81
|
+
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
82
|
+
end
|
83
|
+
--[[
|
84
|
+
Function to add job considering priority.
|
85
|
+
]]
|
86
|
+
-- Includes
|
87
|
+
--[[
|
88
|
+
Function to get priority score.
|
89
|
+
]]
|
90
|
+
local function getPriorityScore(priority, priorityCounterKey)
|
91
|
+
local prioCounter = rcall("INCR", priorityCounterKey)
|
92
|
+
return priority * 0x100000000 + prioCounter % 0x100000000
|
93
|
+
end
|
94
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
|
95
|
+
isPausedOrMaxed)
|
96
|
+
local score = getPriorityScore(priority, priorityCounterKey)
|
97
|
+
rcall("ZADD", prioritizedKey, score, jobId)
|
98
|
+
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
99
|
+
end
|
100
|
+
--[[
|
101
|
+
Function to check if queue is paused or maxed
|
102
|
+
(since an empty list and !EXISTS are not really the same).
|
103
|
+
]]
|
104
|
+
local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
105
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
106
|
+
if queueAttributes[1] then
|
107
|
+
return true
|
108
|
+
else
|
109
|
+
if queueAttributes[2] then
|
110
|
+
local activeCount = rcall("LLEN", activeKey)
|
111
|
+
return activeCount >= tonumber(queueAttributes[2])
|
112
|
+
end
|
113
|
+
end
|
114
|
+
return false
|
115
|
+
end
|
116
|
+
--[[
|
117
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
118
|
+
(since an empty list and !EXISTS are not really the same).
|
119
|
+
]]
|
120
|
+
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
121
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
122
|
+
if queueAttributes[1] then
|
123
|
+
return pausedKey, true
|
124
|
+
else
|
125
|
+
if queueAttributes[2] then
|
126
|
+
local activeCount = rcall("LLEN", activeKey)
|
127
|
+
if activeCount >= tonumber(queueAttributes[2]) then
|
128
|
+
return waitKey, true
|
129
|
+
else
|
130
|
+
return waitKey, false
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
return waitKey, false
|
135
|
+
end
|
136
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
137
|
+
parentKey, parentId, timestamp)
|
138
|
+
local isParentActive = rcall("ZSCORE",
|
139
|
+
parentQueueKey .. ":waiting-children", parentId)
|
140
|
+
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
141
|
+
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
142
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
143
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
144
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
145
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
146
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
147
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
148
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
149
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
150
|
+
if delay > 0 then
|
151
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
152
|
+
local score = delayedTimestamp * 0x1000
|
153
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
154
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
155
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
156
|
+
"jobId", parentId, "delay", delayedTimestamp)
|
157
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
158
|
+
else
|
159
|
+
if priority == 0 then
|
160
|
+
local parentTarget, isParentPausedOrMaxed =
|
161
|
+
getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
|
162
|
+
parentPausedKey)
|
163
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
164
|
+
parentId)
|
165
|
+
else
|
166
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
167
|
+
addJobWithPriority(parentMarkerKey,
|
168
|
+
parentQueueKey .. ":prioritized", priority,
|
169
|
+
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
170
|
+
end
|
171
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
172
|
+
"jobId", parentId, "prev", "waiting-children")
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
--[[
|
177
|
+
Function to remove deduplication key if needed.
|
178
|
+
]]
|
179
|
+
local function removeDeduplicationKeyIfNeeded(prefixKey, deduplicationId)
|
180
|
+
if deduplicationId then
|
181
|
+
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
182
|
+
local pttl = rcall("PTTL", deduplicationKey)
|
183
|
+
if pttl == 0 or pttl == -1 then
|
184
|
+
rcall("DEL", deduplicationKey)
|
185
|
+
end
|
186
|
+
end
|
187
|
+
end
|
188
|
+
--[[
|
189
|
+
Functions to remove jobs when removeOnFail option is provided.
|
190
|
+
]]
|
191
|
+
-- Includes
|
192
|
+
--[[
|
193
|
+
Function to remove job.
|
194
|
+
]]
|
195
|
+
-- Includes
|
196
|
+
--[[
|
197
|
+
Function to remove deduplication key.
|
198
|
+
]]
|
199
|
+
local function removeDeduplicationKey(prefixKey, jobKey)
|
200
|
+
local deduplicationId = rcall("HGET", jobKey, "deid")
|
201
|
+
if deduplicationId then
|
202
|
+
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
203
|
+
rcall("DEL", deduplicationKey)
|
204
|
+
end
|
205
|
+
end
|
206
|
+
--[[
|
207
|
+
Function to remove job keys.
|
208
|
+
]]
|
209
|
+
local function removeJobKeys(jobKey)
|
210
|
+
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
211
|
+
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
212
|
+
end
|
213
|
+
--[[
|
214
|
+
Check if this job has a parent. If so we will just remove it from
|
215
|
+
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
216
|
+
which requires code from "moveToFinished"
|
217
|
+
]]
|
218
|
+
-- Includes
|
219
|
+
--[[
|
220
|
+
Functions to destructure job key.
|
221
|
+
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
222
|
+
]]
|
223
|
+
local getJobIdFromKey = function (jobKey)
|
224
|
+
return string.match(jobKey, ".*:(.*)")
|
225
|
+
end
|
226
|
+
local getJobKeyPrefix = function (jobKey, jobId)
|
227
|
+
return string.sub(jobKey, 0, #jobKey - #jobId)
|
228
|
+
end
|
229
|
+
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
230
|
+
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
231
|
+
parentPrefix .. "wait", parentPrefix .. "paused")
|
232
|
+
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
233
|
+
if emitEvent then
|
234
|
+
local parentEventStream = parentPrefix .. "events"
|
235
|
+
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
236
|
+
end
|
237
|
+
end
|
238
|
+
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
|
239
|
+
if parentKey then
|
240
|
+
local parentDependenciesKey = parentKey .. ":dependencies"
|
241
|
+
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
242
|
+
if result > 0 then
|
243
|
+
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
244
|
+
if pendingDependencies == 0 then
|
245
|
+
local parentId = getJobIdFromKey(parentKey)
|
246
|
+
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
247
|
+
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
248
|
+
if numRemovedElements == 1 then
|
249
|
+
if hard then -- remove parent in same queue
|
250
|
+
if parentPrefix == baseKey then
|
251
|
+
removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
|
252
|
+
removeJobKeys(parentKey)
|
253
|
+
if debounceId then
|
254
|
+
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
255
|
+
end
|
256
|
+
else
|
257
|
+
moveParentToWait(parentPrefix, parentId)
|
258
|
+
end
|
259
|
+
else
|
260
|
+
moveParentToWait(parentPrefix, parentId, true)
|
261
|
+
end
|
262
|
+
end
|
263
|
+
end
|
264
|
+
return true
|
265
|
+
end
|
266
|
+
else
|
267
|
+
local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
|
268
|
+
local missedParentKey = parentAttributes[1]
|
269
|
+
if( (type(missedParentKey) == "string") and missedParentKey ~= ""
|
270
|
+
and (rcall("EXISTS", missedParentKey) == 1)) then
|
271
|
+
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
272
|
+
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
273
|
+
if result > 0 then
|
274
|
+
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
275
|
+
if pendingDependencies == 0 then
|
276
|
+
local parentId = getJobIdFromKey(missedParentKey)
|
277
|
+
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
278
|
+
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
279
|
+
if numRemovedElements == 1 then
|
280
|
+
if hard then
|
281
|
+
if parentPrefix == baseKey then
|
282
|
+
removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
|
283
|
+
removeJobKeys(missedParentKey)
|
284
|
+
if parentAttributes[2] then
|
285
|
+
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
286
|
+
end
|
287
|
+
else
|
288
|
+
moveParentToWait(parentPrefix, parentId)
|
289
|
+
end
|
290
|
+
else
|
291
|
+
moveParentToWait(parentPrefix, parentId, true)
|
292
|
+
end
|
293
|
+
end
|
294
|
+
end
|
295
|
+
return true
|
296
|
+
end
|
297
|
+
end
|
298
|
+
end
|
299
|
+
return false
|
300
|
+
end
|
301
|
+
local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
302
|
+
local jobKey = baseKey .. jobId
|
303
|
+
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
304
|
+
if shouldRemoveDeduplicationKey then
|
305
|
+
removeDeduplicationKey(baseKey, jobKey)
|
306
|
+
end
|
307
|
+
removeJobKeys(jobKey)
|
308
|
+
end
|
309
|
+
--[[
|
310
|
+
Functions to remove jobs by max age.
|
311
|
+
]]
|
312
|
+
-- Includes
|
313
|
+
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix,
|
314
|
+
shouldRemoveDebounceKey)
|
315
|
+
local start = timestamp - maxAge * 1000
|
316
|
+
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
|
317
|
+
for i, jobId in ipairs(jobIds) do
|
318
|
+
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
319
|
+
end
|
320
|
+
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
|
321
|
+
end
|
322
|
+
--[[
|
323
|
+
Functions to remove jobs by max count.
|
324
|
+
]]
|
325
|
+
-- Includes
|
326
|
+
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
|
327
|
+
local start = maxCount
|
328
|
+
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
|
329
|
+
for i, jobId in ipairs(jobIds) do
|
330
|
+
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
331
|
+
end
|
332
|
+
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
|
333
|
+
end
|
334
|
+
local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
|
335
|
+
local removeOnFailType = type(opts["removeOnFail"])
|
336
|
+
if removeOnFailType == "number" then
|
337
|
+
removeJobsByMaxCount(opts["removeOnFail"],
|
338
|
+
failedKey, queueKeyPrefix)
|
339
|
+
elseif removeOnFailType == "boolean" then
|
340
|
+
if opts["removeOnFail"] then
|
341
|
+
removeJob(jobId, false, queueKeyPrefix,
|
342
|
+
false --[[remove debounce key]])
|
343
|
+
rcall("ZREM", failedKey, jobId)
|
344
|
+
end
|
345
|
+
elseif removeOnFailType ~= "nil" then
|
346
|
+
local maxAge = opts["removeOnFail"]["age"]
|
347
|
+
local maxCount = opts["removeOnFail"]["count"]
|
348
|
+
if maxAge ~= nil then
|
349
|
+
removeJobsByMaxAge(timestamp, maxAge,
|
350
|
+
failedKey, queueKeyPrefix)
|
351
|
+
end
|
352
|
+
if maxCount ~= nil and maxCount > 0 then
|
353
|
+
removeJobsByMaxCount(maxCount, failedKey,
|
354
|
+
queueKeyPrefix)
|
355
|
+
end
|
356
|
+
end
|
357
|
+
end
|
358
|
+
local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
359
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
360
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
361
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) ~= false then
|
362
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
363
|
+
local parentQueuePrefix = parentQueueKey .. ":"
|
364
|
+
local parentFailedKey = parentQueueKey .. ":failed"
|
365
|
+
rcall("ZADD", parentFailedKey, timestamp, parentId)
|
366
|
+
local failedReason = "child " .. jobIdKey .. " failed"
|
367
|
+
rcall("HSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
|
368
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
|
369
|
+
failedReason, "prev", "waiting-children")
|
370
|
+
local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
|
371
|
+
removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
|
372
|
+
if jobAttributes[1] then
|
373
|
+
local parentData = cjson.decode(jobAttributes[1])
|
374
|
+
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
375
|
+
local grandParentUnsuccesssful = grandParentKey .. ":unsuccessful"
|
376
|
+
rcall("ZADD", grandParentUnsuccesssful, timestamp, parentKey)
|
377
|
+
if parentData['fpof'] then
|
378
|
+
moveParentFromWaitingChildrenToFailed(
|
379
|
+
parentData['queueKey'],
|
380
|
+
parentData['queueKey'] .. ':' .. parentData['id'],
|
381
|
+
parentData['id'],
|
382
|
+
parentKey,
|
383
|
+
timestamp
|
384
|
+
)
|
385
|
+
elseif parentData['idof'] or parentData['rdof'] then
|
386
|
+
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
387
|
+
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
|
388
|
+
if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
|
389
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
|
390
|
+
grandParentKey, parentData['id'], timestamp)
|
391
|
+
if parentData['idof'] then
|
392
|
+
local grandParentFailedSet = grandParentKey .. ":failed"
|
393
|
+
rcall("HSET", grandParentFailedSet, parentKey, failedReason)
|
394
|
+
end
|
395
|
+
end
|
396
|
+
end
|
397
|
+
end
|
398
|
+
local parentRawOpts = jobAttributes[3]
|
399
|
+
local parentOpts = cjson.decode(parentRawOpts)
|
400
|
+
removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
|
401
|
+
elseif rcall("ZSCORE", parentDelayedKey, parentId) ~= false then
|
402
|
+
rcall("ZREM", parentDelayedKey, parentId)
|
403
|
+
local parentQueuePrefix = parentQueueKey .. ":"
|
404
|
+
local parentFailedKey = parentQueueKey .. ":failed"
|
405
|
+
rcall("ZADD", parentFailedKey, timestamp, parentId)
|
406
|
+
local failedReason = "child " .. jobIdKey .. " failed"
|
407
|
+
rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
|
408
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
|
409
|
+
failedReason, "prev", "delayed")
|
410
|
+
local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
|
411
|
+
removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
|
412
|
+
if jobAttributes[1] then
|
413
|
+
local parentData = cjson.decode(jobAttributes[1])
|
414
|
+
if parentData['fpof'] then
|
415
|
+
moveParentFromWaitingChildrenToFailed(
|
416
|
+
parentData['queueKey'],
|
417
|
+
parentData['queueKey'] .. ':' .. parentData['id'],
|
418
|
+
parentData['id'],
|
419
|
+
parentKey,
|
420
|
+
timestamp
|
421
|
+
)
|
422
|
+
elseif parentData['idof'] or parentData['rdof'] then
|
423
|
+
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
424
|
+
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
|
425
|
+
if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
|
426
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
|
427
|
+
grandParentKey, parentData['id'], timestamp)
|
428
|
+
if parentData['idof'] then
|
429
|
+
local grandParentFailedSet = grandParentKey .. ":failed"
|
430
|
+
rcall("HSET", grandParentFailedSet, parentKey, failedReason)
|
431
|
+
end
|
432
|
+
end
|
433
|
+
end
|
434
|
+
end
|
435
|
+
local parentRawOpts = jobAttributes[3]
|
436
|
+
local parentOpts = cjson.decode(parentRawOpts)
|
437
|
+
removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
|
438
|
+
else
|
439
|
+
local grandParentKey = rcall("HGET", parentKey, "parentKey")
|
440
|
+
if grandParentKey then
|
441
|
+
local grandParentUnsuccesssfulSet = grandParentKey .. ":unsuccessful"
|
442
|
+
rcall("ZADD", grandParentUnsuccesssfulSet, timestamp, parentKey)
|
443
|
+
end
|
444
|
+
end
|
445
|
+
end
|
446
|
+
local function removeLock(jobKey, stalledKey, token, jobId)
|
447
|
+
if token ~= "0" then
|
448
|
+
local lockKey = jobKey .. ':lock'
|
449
|
+
local lockToken = rcall("GET", lockKey)
|
450
|
+
if lockToken == token then
|
451
|
+
rcall("DEL", lockKey)
|
452
|
+
rcall("SREM", stalledKey, jobId)
|
453
|
+
else
|
454
|
+
if lockToken then
|
455
|
+
-- Lock exists but token does not match
|
456
|
+
return -6
|
457
|
+
else
|
458
|
+
-- Lock is missing completely
|
459
|
+
return -2
|
460
|
+
end
|
461
|
+
end
|
462
|
+
end
|
463
|
+
return 0
|
464
|
+
end
|
465
|
+
local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId,
|
466
|
+
timestamp)
|
467
|
+
local score = tonumber(timestamp)
|
468
|
+
local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
|
469
|
+
if(numRemovedElements < 1) then
|
470
|
+
return -3
|
471
|
+
end
|
472
|
+
rcall("ZADD", waitingChildrenKey, score, jobId)
|
473
|
+
return 0
|
474
|
+
end
|
475
|
+
if rcall("EXISTS", jobKey) == 1 then
|
476
|
+
if rcall("ZCARD", jobUnsuccessfulKey) ~= 0 then
|
477
|
+
-- TODO: refactor this logic in an include later
|
478
|
+
local jobAttributes = rcall("HMGET", jobKey, "parent", "deid", "opts")
|
479
|
+
removeDeduplicationKeyIfNeeded(ARGV[5], jobAttributes[2])
|
480
|
+
local failedReason = "children are failed"
|
481
|
+
rcall("ZADD", failedKey, timestamp, jobId)
|
482
|
+
rcall("HSET", jobKey, "finishedOn", timestamp)
|
483
|
+
rcall("XADD", KEYS[8], "*", "event", "failed", "jobId", jobId, "failedReason",
|
484
|
+
failedReason, "prev", "active")
|
485
|
+
local rawParentData = jobAttributes[1]
|
486
|
+
local rawOpts = jobAttributes[3]
|
487
|
+
local opts = cjson.decode(rawOpts)
|
488
|
+
if rawParentData ~= false then
|
489
|
+
if opts['fpof'] then
|
490
|
+
local parentData = cjson.decode(rawParentData)
|
491
|
+
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
492
|
+
local parentUnsuccesssful = parentKey .. ":unsuccessful"
|
493
|
+
rcall("ZADD", parentUnsuccesssful, timestamp, jobKey)
|
494
|
+
moveParentFromWaitingChildrenToFailed(
|
495
|
+
parentData['queueKey'],
|
496
|
+
parentData['queueKey'] .. ':' .. parentData['id'],
|
497
|
+
parentData['id'],
|
498
|
+
jobKey,
|
499
|
+
timestamp
|
500
|
+
)
|
501
|
+
elseif opts['idof'] or opts['rdof'] then
|
502
|
+
local parentData = cjson.decode(rawParentData)
|
503
|
+
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
504
|
+
local dependenciesSet = parentKey .. ":dependencies"
|
505
|
+
if rcall("SREM", dependenciesSet, jobKey) == 1 then
|
506
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
|
507
|
+
parentKey, parentData['id'], timestamp)
|
508
|
+
if opts['idof'] then
|
509
|
+
local failedSet = parentKey .. ":failed"
|
510
|
+
rcall("HSET", failedSet, jobKey, failedReason)
|
511
|
+
end
|
512
|
+
end
|
513
|
+
end
|
514
|
+
end
|
515
|
+
removeJobsOnFail(ARGV[5], failedKey, jobId, opts, timestamp)
|
516
|
+
return 0
|
517
|
+
else
|
518
|
+
if ARGV[2] ~= "" then
|
519
|
+
if rcall("SISMEMBER", jobDependenciesKey, ARGV[2]) ~= 0 then
|
520
|
+
local errorCode = removeLock(jobKey, stalledKey, ARGV[1], jobId)
|
521
|
+
if errorCode < 0 then
|
522
|
+
return errorCode
|
523
|
+
end
|
524
|
+
return moveToWaitingChildren(activeKey, waitingChildrenKey, jobId, timestamp)
|
525
|
+
end
|
526
|
+
return 1
|
527
|
+
else
|
528
|
+
if rcall("SCARD", jobDependenciesKey) ~= 0 then
|
529
|
+
local errorCode = removeLock(jobKey, stalledKey, ARGV[1], jobId)
|
530
|
+
if errorCode < 0 then
|
531
|
+
return errorCode
|
532
|
+
end
|
533
|
+
return moveToWaitingChildren(activeKey, waitingChildrenKey, jobId, timestamp)
|
534
|
+
end
|
535
|
+
return 1
|
536
|
+
end
|
537
|
+
end
|
538
|
+
end
|
539
|
+
return -1
|
540
|
+
`;
|
541
|
+
export const moveToWaitingChildren = {
|
542
|
+
name: 'moveToWaitingChildren',
|
543
|
+
content,
|
544
|
+
keys: 8,
|
545
|
+
};
|
546
|
+
//# sourceMappingURL=moveToWaitingChildren-8.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"moveToWaitingChildren-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToWaitingChildren-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2hBf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -38,8 +38,8 @@ end
|
|
38
38
|
Function to remove job keys.
|
39
39
|
]]
|
40
40
|
local function removeJobKeys(jobKey)
|
41
|
-
return rcall("DEL", jobKey, jobKey .. ':logs',
|
42
|
-
jobKey .. ':
|
41
|
+
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
42
|
+
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
43
43
|
end
|
44
44
|
--[[
|
45
45
|
Check if this job has a parent. If so we will just remove it from
|
@@ -71,8 +71,8 @@ end
|
|
71
71
|
Function to remove job keys.
|
72
72
|
]]
|
73
73
|
local function removeJobKeys(jobKey)
|
74
|
-
return rcall("DEL", jobKey, jobKey .. ':logs',
|
75
|
-
jobKey .. ':
|
74
|
+
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
75
|
+
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
76
76
|
end
|
77
77
|
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
78
78
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
@@ -124,8 +124,8 @@ end
|
|
124
124
|
Function to remove job keys.
|
125
125
|
]]
|
126
126
|
local function removeJobKeys(jobKey)
|
127
|
-
return rcall("DEL", jobKey, jobKey .. ':logs',
|
128
|
-
jobKey .. ':
|
127
|
+
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
128
|
+
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
129
129
|
end
|
130
130
|
--[[
|
131
131
|
Check if this job has a parent. If so we will just remove it from
|
@@ -274,6 +274,14 @@ local function removeJob(prefix, jobId, parentKey, removeChildren)
|
|
274
274
|
removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
|
275
275
|
end
|
276
276
|
end
|
277
|
+
local unsuccessful = rcall("ZRANGE", jobKey .. ":unsuccessful", 0, -1)
|
278
|
+
if (#unsuccessful > 0) then
|
279
|
+
for i = 1, #unsuccessful, 1 do
|
280
|
+
local childJobId = getJobIdFromKey(unsuccessful[i])
|
281
|
+
local childJobPrefix = getJobKeyPrefix(unsuccessful[i], childJobId)
|
282
|
+
removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
|
283
|
+
end
|
284
|
+
end
|
277
285
|
end
|
278
286
|
local prev = removeJobFromAnyState(prefix, jobId)
|
279
287
|
removeDeduplicationKey(prefix, jobKey)
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"removeJob-3.js","sourceRoot":"","sources":["../../../src/scripts/removeJob-3.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"removeJob-3.js","sourceRoot":"","sources":["../../../src/scripts/removeJob-3.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkTf,CAAC;AACF,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -18,8 +18,8 @@ local rcall = redis.call
|
|
18
18
|
Function to remove job keys.
|
19
19
|
]]
|
20
20
|
local function removeJobKeys(jobKey)
|
21
|
-
return rcall("DEL", jobKey, jobKey .. ':logs',
|
22
|
-
jobKey .. ':
|
21
|
+
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
22
|
+
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
23
23
|
end
|
24
24
|
local jobSchedulerId = ARGV[1]
|
25
25
|
local prefix = ARGV[2]
|
@@ -21,8 +21,8 @@ local millis = rcall("ZSCORE", KEYS[1], ARGV[2])
|
|
21
21
|
Function to remove job keys.
|
22
22
|
]]
|
23
23
|
local function removeJobKeys(jobKey)
|
24
|
-
return rcall("DEL", jobKey, jobKey .. ':logs',
|
25
|
-
jobKey .. ':
|
24
|
+
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
25
|
+
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
26
26
|
end
|
27
27
|
-- legacy removal TODO: remove in next breaking change
|
28
28
|
if millis then
|