bullmq 5.53.0 → 5.53.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/job.js +1 -1
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/scripts.js +0 -1
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +1 -21
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addJobScheduler-11.lua +1 -1
- package/dist/cjs/commands/includes/storeJobScheduler.lua +1 -0
- package/dist/cjs/commands/{moveStalledJobsToWait-9.lua → moveStalledJobsToWait-8.lua} +23 -40
- package/dist/cjs/scripts/addJobScheduler-11.js +2 -1
- package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/cjs/scripts/index.js +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js +171 -0
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/job.js +1 -1
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +1 -1
- package/dist/esm/classes/scripts.js +0 -1
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +0 -1
- package/dist/esm/classes/worker.js +1 -21
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addJobScheduler-11.lua +1 -1
- package/dist/esm/commands/includes/storeJobScheduler.lua +1 -0
- package/dist/esm/commands/{moveStalledJobsToWait-9.lua → moveStalledJobsToWait-8.lua} +23 -40
- package/dist/esm/scripts/addJobScheduler-11.js +2 -1
- package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +1 -1
- package/dist/esm/scripts/index.js +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-8.js +168 -0
- package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +0 -563
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +0 -1
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +0 -560
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +0 -1
- /package/dist/esm/scripts/{moveStalledJobsToWait-9.d.ts → moveStalledJobsToWait-8.d.ts} +0 -0
@@ -1,563 +0,0 @@
|
|
1
|
-
"use strict";
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
-
exports.moveStalledJobsToWait = void 0;
|
4
|
-
const content = `--[[
|
5
|
-
Move stalled jobs to wait.
|
6
|
-
Input:
|
7
|
-
KEYS[1] 'stalled' (SET)
|
8
|
-
KEYS[2] 'wait', (LIST)
|
9
|
-
KEYS[3] 'active', (LIST)
|
10
|
-
KEYS[4] 'failed', (ZSET)
|
11
|
-
KEYS[5] 'stalled-check', (KEY)
|
12
|
-
KEYS[6] 'meta', (KEY)
|
13
|
-
KEYS[7] 'paused', (LIST)
|
14
|
-
KEYS[8] 'marker'
|
15
|
-
KEYS[9] 'event stream' (STREAM)
|
16
|
-
ARGV[1] Max stalled job count
|
17
|
-
ARGV[2] queue.toKey('')
|
18
|
-
ARGV[3] timestamp
|
19
|
-
ARGV[4] max check time
|
20
|
-
Events:
|
21
|
-
'stalled' with stalled job id.
|
22
|
-
]]
|
23
|
-
local rcall = redis.call
|
24
|
-
-- Includes
|
25
|
-
--[[
|
26
|
-
Function to add job in target list and add marker if needed.
|
27
|
-
]]
|
28
|
-
-- Includes
|
29
|
-
--[[
|
30
|
-
Add marker if needed when a job is available.
|
31
|
-
]]
|
32
|
-
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
33
|
-
if not isPausedOrMaxed then
|
34
|
-
rcall("ZADD", markerKey, 0, "0")
|
35
|
-
end
|
36
|
-
end
|
37
|
-
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
38
|
-
rcall(pushCmd, targetKey, jobId)
|
39
|
-
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
40
|
-
end
|
41
|
-
--[[
|
42
|
-
Function to loop in batches.
|
43
|
-
Just a bit of warning, some commands as ZREM
|
44
|
-
could receive a maximum of 7000 parameters per call.
|
45
|
-
]]
|
46
|
-
local function batches(n, batchSize)
|
47
|
-
local i = 0
|
48
|
-
return function()
|
49
|
-
local from = i * batchSize + 1
|
50
|
-
i = i + 1
|
51
|
-
if (from <= n) then
|
52
|
-
local to = math.min(from + batchSize - 1, n)
|
53
|
-
return from, to
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
--[[
|
58
|
-
Function to check for the meta.paused key to decide if we are paused or not
|
59
|
-
(since an empty list and !EXISTS are not really the same).
|
60
|
-
]]
|
61
|
-
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
62
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
63
|
-
if queueAttributes[1] then
|
64
|
-
return pausedKey, true
|
65
|
-
else
|
66
|
-
if queueAttributes[2] then
|
67
|
-
local activeCount = rcall("LLEN", activeKey)
|
68
|
-
if activeCount >= tonumber(queueAttributes[2]) then
|
69
|
-
return waitKey, true
|
70
|
-
else
|
71
|
-
return waitKey, false
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
return waitKey, false
|
76
|
-
end
|
77
|
-
--[[
|
78
|
-
Function to recursively move from waitingChildren to failed.
|
79
|
-
]]
|
80
|
-
-- Includes
|
81
|
-
--[[
|
82
|
-
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
83
|
-
if no pending dependencies.
|
84
|
-
]]
|
85
|
-
-- Includes
|
86
|
-
--[[
|
87
|
-
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
88
|
-
]]
|
89
|
-
-- Includes
|
90
|
-
--[[
|
91
|
-
Move parent to a wait status (wait, prioritized or delayed)
|
92
|
-
]]
|
93
|
-
-- Includes
|
94
|
-
--[[
|
95
|
-
Add delay marker if needed.
|
96
|
-
]]
|
97
|
-
-- Includes
|
98
|
-
--[[
|
99
|
-
Function to return the next delayed job timestamp.
|
100
|
-
]]
|
101
|
-
local function getNextDelayedTimestamp(delayedKey)
|
102
|
-
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
103
|
-
if #result then
|
104
|
-
local nextTimestamp = tonumber(result[2])
|
105
|
-
if nextTimestamp ~= nil then
|
106
|
-
return nextTimestamp / 0x1000
|
107
|
-
end
|
108
|
-
end
|
109
|
-
end
|
110
|
-
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
111
|
-
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
112
|
-
if nextTimestamp ~= nil then
|
113
|
-
-- Replace the score of the marker with the newest known
|
114
|
-
-- next timestamp.
|
115
|
-
rcall("ZADD", markerKey, nextTimestamp, "1")
|
116
|
-
end
|
117
|
-
end
|
118
|
-
--[[
|
119
|
-
Function to add job considering priority.
|
120
|
-
]]
|
121
|
-
-- Includes
|
122
|
-
--[[
|
123
|
-
Function to get priority score.
|
124
|
-
]]
|
125
|
-
local function getPriorityScore(priority, priorityCounterKey)
|
126
|
-
local prioCounter = rcall("INCR", priorityCounterKey)
|
127
|
-
return priority * 0x100000000 + prioCounter % 0x100000000
|
128
|
-
end
|
129
|
-
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
|
130
|
-
isPausedOrMaxed)
|
131
|
-
local score = getPriorityScore(priority, priorityCounterKey)
|
132
|
-
rcall("ZADD", prioritizedKey, score, jobId)
|
133
|
-
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
134
|
-
end
|
135
|
-
--[[
|
136
|
-
Function to check if queue is paused or maxed
|
137
|
-
(since an empty list and !EXISTS are not really the same).
|
138
|
-
]]
|
139
|
-
local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
140
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
141
|
-
if queueAttributes[1] then
|
142
|
-
return true
|
143
|
-
else
|
144
|
-
if queueAttributes[2] then
|
145
|
-
local activeCount = rcall("LLEN", activeKey)
|
146
|
-
return activeCount >= tonumber(queueAttributes[2])
|
147
|
-
end
|
148
|
-
end
|
149
|
-
return false
|
150
|
-
end
|
151
|
-
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
152
|
-
local parentWaitKey = parentQueueKey .. ":wait"
|
153
|
-
local parentPausedKey = parentQueueKey .. ":paused"
|
154
|
-
local parentActiveKey = parentQueueKey .. ":active"
|
155
|
-
local parentMetaKey = parentQueueKey .. ":meta"
|
156
|
-
local parentMarkerKey = parentQueueKey .. ":marker"
|
157
|
-
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
158
|
-
local priority = tonumber(jobAttributes[1]) or 0
|
159
|
-
local delay = tonumber(jobAttributes[2]) or 0
|
160
|
-
if delay > 0 then
|
161
|
-
local delayedTimestamp = tonumber(timestamp) + delay
|
162
|
-
local score = delayedTimestamp * 0x1000
|
163
|
-
local parentDelayedKey = parentQueueKey .. ":delayed"
|
164
|
-
rcall("ZADD", parentDelayedKey, score, parentId)
|
165
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
166
|
-
delayedTimestamp)
|
167
|
-
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
168
|
-
else
|
169
|
-
if priority == 0 then
|
170
|
-
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
171
|
-
parentWaitKey, parentPausedKey)
|
172
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
173
|
-
else
|
174
|
-
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
175
|
-
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
176
|
-
parentQueueKey .. ":pc", isPausedOrMaxed)
|
177
|
-
end
|
178
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
179
|
-
"waiting-children")
|
180
|
-
end
|
181
|
-
end
|
182
|
-
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
183
|
-
if rcall("EXISTS", parentKey) == 1 then
|
184
|
-
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
185
|
-
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
186
|
-
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
187
|
-
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
188
|
-
end
|
189
|
-
end
|
190
|
-
end
|
191
|
-
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
192
|
-
parentId, timestamp)
|
193
|
-
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
194
|
-
if doNotHavePendingDependencies then
|
195
|
-
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
196
|
-
end
|
197
|
-
end
|
198
|
-
--[[
|
199
|
-
Functions to remove jobs when removeOnFail option is provided.
|
200
|
-
]]
|
201
|
-
-- Includes
|
202
|
-
--[[
|
203
|
-
Function to remove job.
|
204
|
-
]]
|
205
|
-
-- Includes
|
206
|
-
--[[
|
207
|
-
Function to remove deduplication key if needed
|
208
|
-
when a job is being removed.
|
209
|
-
]]
|
210
|
-
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
211
|
-
jobKey, jobId)
|
212
|
-
local deduplicationId = rcall("HGET", jobKey, "deid")
|
213
|
-
if deduplicationId then
|
214
|
-
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
215
|
-
local currentJobId = rcall('GET', deduplicationKey)
|
216
|
-
if currentJobId and currentJobId == jobId then
|
217
|
-
return rcall("DEL", deduplicationKey)
|
218
|
-
end
|
219
|
-
end
|
220
|
-
end
|
221
|
-
--[[
|
222
|
-
Function to remove job keys.
|
223
|
-
]]
|
224
|
-
local function removeJobKeys(jobKey)
|
225
|
-
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
226
|
-
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
227
|
-
end
|
228
|
-
--[[
|
229
|
-
Check if this job has a parent. If so we will just remove it from
|
230
|
-
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
231
|
-
which requires code from "moveToFinished"
|
232
|
-
]]
|
233
|
-
-- Includes
|
234
|
-
--[[
|
235
|
-
Functions to destructure job key.
|
236
|
-
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
237
|
-
]]
|
238
|
-
local getJobIdFromKey = function (jobKey)
|
239
|
-
return string.match(jobKey, ".*:(.*)")
|
240
|
-
end
|
241
|
-
local getJobKeyPrefix = function (jobKey, jobId)
|
242
|
-
return string.sub(jobKey, 0, #jobKey - #jobId)
|
243
|
-
end
|
244
|
-
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
245
|
-
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
246
|
-
parentPrefix .. "wait", parentPrefix .. "paused")
|
247
|
-
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
248
|
-
if emitEvent then
|
249
|
-
local parentEventStream = parentPrefix .. "events"
|
250
|
-
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
251
|
-
end
|
252
|
-
end
|
253
|
-
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
|
254
|
-
if parentKey then
|
255
|
-
local parentDependenciesKey = parentKey .. ":dependencies"
|
256
|
-
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
257
|
-
if result > 0 then
|
258
|
-
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
259
|
-
if pendingDependencies == 0 then
|
260
|
-
local parentId = getJobIdFromKey(parentKey)
|
261
|
-
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
262
|
-
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
263
|
-
if numRemovedElements == 1 then
|
264
|
-
if hard then -- remove parent in same queue
|
265
|
-
if parentPrefix == baseKey then
|
266
|
-
removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
|
267
|
-
removeJobKeys(parentKey)
|
268
|
-
if debounceId then
|
269
|
-
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
270
|
-
end
|
271
|
-
else
|
272
|
-
_moveParentToWait(parentPrefix, parentId)
|
273
|
-
end
|
274
|
-
else
|
275
|
-
_moveParentToWait(parentPrefix, parentId, true)
|
276
|
-
end
|
277
|
-
end
|
278
|
-
end
|
279
|
-
return true
|
280
|
-
end
|
281
|
-
else
|
282
|
-
local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
|
283
|
-
local missedParentKey = parentAttributes[1]
|
284
|
-
if( (type(missedParentKey) == "string") and missedParentKey ~= ""
|
285
|
-
and (rcall("EXISTS", missedParentKey) == 1)) then
|
286
|
-
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
287
|
-
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
288
|
-
if result > 0 then
|
289
|
-
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
290
|
-
if pendingDependencies == 0 then
|
291
|
-
local parentId = getJobIdFromKey(missedParentKey)
|
292
|
-
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
293
|
-
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
294
|
-
if numRemovedElements == 1 then
|
295
|
-
if hard then
|
296
|
-
if parentPrefix == baseKey then
|
297
|
-
removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
|
298
|
-
removeJobKeys(missedParentKey)
|
299
|
-
if parentAttributes[2] then
|
300
|
-
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
301
|
-
end
|
302
|
-
else
|
303
|
-
_moveParentToWait(parentPrefix, parentId)
|
304
|
-
end
|
305
|
-
else
|
306
|
-
_moveParentToWait(parentPrefix, parentId, true)
|
307
|
-
end
|
308
|
-
end
|
309
|
-
end
|
310
|
-
return true
|
311
|
-
end
|
312
|
-
end
|
313
|
-
end
|
314
|
-
return false
|
315
|
-
end
|
316
|
-
local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
317
|
-
local jobKey = baseKey .. jobId
|
318
|
-
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
319
|
-
if shouldRemoveDeduplicationKey then
|
320
|
-
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
|
321
|
-
end
|
322
|
-
removeJobKeys(jobKey)
|
323
|
-
end
|
324
|
-
--[[
|
325
|
-
Functions to remove jobs by max age.
|
326
|
-
]]
|
327
|
-
-- Includes
|
328
|
-
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix,
|
329
|
-
shouldRemoveDebounceKey)
|
330
|
-
local start = timestamp - maxAge * 1000
|
331
|
-
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
|
332
|
-
for i, jobId in ipairs(jobIds) do
|
333
|
-
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
334
|
-
end
|
335
|
-
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
|
336
|
-
end
|
337
|
-
--[[
|
338
|
-
Functions to remove jobs by max count.
|
339
|
-
]]
|
340
|
-
-- Includes
|
341
|
-
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
|
342
|
-
local start = maxCount
|
343
|
-
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
|
344
|
-
for i, jobId in ipairs(jobIds) do
|
345
|
-
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
346
|
-
end
|
347
|
-
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
|
348
|
-
end
|
349
|
-
local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
|
350
|
-
local removeOnFailType = type(opts["removeOnFail"])
|
351
|
-
if removeOnFailType == "number" then
|
352
|
-
removeJobsByMaxCount(opts["removeOnFail"],
|
353
|
-
failedKey, queueKeyPrefix)
|
354
|
-
elseif removeOnFailType == "boolean" then
|
355
|
-
if opts["removeOnFail"] then
|
356
|
-
removeJob(jobId, false, queueKeyPrefix,
|
357
|
-
false --[[remove debounce key]])
|
358
|
-
rcall("ZREM", failedKey, jobId)
|
359
|
-
end
|
360
|
-
elseif removeOnFailType ~= "nil" then
|
361
|
-
local maxAge = opts["removeOnFail"]["age"]
|
362
|
-
local maxCount = opts["removeOnFail"]["count"]
|
363
|
-
if maxAge ~= nil then
|
364
|
-
removeJobsByMaxAge(timestamp, maxAge,
|
365
|
-
failedKey, queueKeyPrefix)
|
366
|
-
end
|
367
|
-
if maxCount ~= nil and maxCount > 0 then
|
368
|
-
removeJobsByMaxCount(maxCount, failedKey,
|
369
|
-
queueKeyPrefix)
|
370
|
-
end
|
371
|
-
end
|
372
|
-
end
|
373
|
-
local moveParentToFailedIfNeeded = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
374
|
-
if rcall("EXISTS", parentKey) == 1 then
|
375
|
-
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
376
|
-
local parentDelayedKey = parentQueueKey .. ":delayed"
|
377
|
-
local parentPrioritizedKey = parentQueueKey .. ":prioritized"
|
378
|
-
local parentWaitingChildrenOrDelayedKey
|
379
|
-
local prevState
|
380
|
-
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
381
|
-
parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
|
382
|
-
prevState = "waiting-children"
|
383
|
-
elseif rcall("ZSCORE", parentDelayedKey, parentId) then
|
384
|
-
parentWaitingChildrenOrDelayedKey = parentDelayedKey
|
385
|
-
prevState = "delayed"
|
386
|
-
rcall("HSET", parentKey, "delay", 0)
|
387
|
-
end
|
388
|
-
if parentWaitingChildrenOrDelayedKey then
|
389
|
-
rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
|
390
|
-
local parentQueuePrefix = parentQueueKey .. ":"
|
391
|
-
local parentFailedKey = parentQueueKey .. ":failed"
|
392
|
-
local deferredFailure = "child " .. jobIdKey .. " failed"
|
393
|
-
rcall("HSET", parentKey, "defa", deferredFailure)
|
394
|
-
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
395
|
-
else
|
396
|
-
if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then
|
397
|
-
local deferredFailure = "child " .. jobIdKey .. " failed"
|
398
|
-
rcall("HSET", parentKey, "defa", deferredFailure)
|
399
|
-
end
|
400
|
-
end
|
401
|
-
end
|
402
|
-
end
|
403
|
-
local moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp)
|
404
|
-
if rawParentData then
|
405
|
-
local parentData = cjson.decode(rawParentData)
|
406
|
-
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
407
|
-
local parentDependenciesChildrenKey = parentKey .. ":dependencies"
|
408
|
-
if parentData['fpof'] then
|
409
|
-
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
410
|
-
local parentUnsuccesssfulChildrenKey = parentKey .. ":unsuccessful"
|
411
|
-
rcall("ZADD", parentUnsuccesssfulChildrenKey, timestamp, childKey)
|
412
|
-
moveParentToFailedIfNeeded(
|
413
|
-
parentData['queueKey'],
|
414
|
-
parentKey,
|
415
|
-
parentData['id'],
|
416
|
-
childKey,
|
417
|
-
timestamp
|
418
|
-
)
|
419
|
-
end
|
420
|
-
elseif parentData['cpof'] then
|
421
|
-
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
422
|
-
local parentFailedChildrenKey = parentKey .. ":failed"
|
423
|
-
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
424
|
-
moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
425
|
-
end
|
426
|
-
elseif parentData['idof'] or parentData['rdof'] then
|
427
|
-
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
428
|
-
moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey,
|
429
|
-
parentKey, parentData['id'], timestamp)
|
430
|
-
if parentData['idof'] then
|
431
|
-
local parentFailedChildrenKey = parentKey .. ":failed"
|
432
|
-
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
433
|
-
end
|
434
|
-
end
|
435
|
-
end
|
436
|
-
end
|
437
|
-
end
|
438
|
-
--[[
|
439
|
-
Function to remove deduplication key if needed
|
440
|
-
when a job is moved to completed or failed states.
|
441
|
-
]]
|
442
|
-
local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
|
443
|
-
deduplicationId, jobId)
|
444
|
-
if deduplicationId then
|
445
|
-
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
446
|
-
local pttl = rcall("PTTL", deduplicationKey)
|
447
|
-
if pttl == 0 then
|
448
|
-
return rcall("DEL", deduplicationKey)
|
449
|
-
end
|
450
|
-
if pttl == -1 then
|
451
|
-
local currentJobId = rcall('GET', deduplicationKey)
|
452
|
-
if currentJobId and currentJobId == jobId then
|
453
|
-
return rcall("DEL", deduplicationKey)
|
454
|
-
end
|
455
|
-
end
|
456
|
-
end
|
457
|
-
end
|
458
|
-
--[[
|
459
|
-
Function to trim events, default 10000.
|
460
|
-
]]
|
461
|
-
-- Includes
|
462
|
-
--[[
|
463
|
-
Function to get max events value or set by default 10000.
|
464
|
-
]]
|
465
|
-
local function getOrSetMaxEvents(metaKey)
|
466
|
-
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
467
|
-
if not maxEvents then
|
468
|
-
maxEvents = 10000
|
469
|
-
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
470
|
-
end
|
471
|
-
return maxEvents
|
472
|
-
end
|
473
|
-
local function trimEvents(metaKey, eventStreamKey)
|
474
|
-
local maxEvents = getOrSetMaxEvents(metaKey)
|
475
|
-
if maxEvents then
|
476
|
-
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
|
477
|
-
else
|
478
|
-
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
|
479
|
-
end
|
480
|
-
end
|
481
|
-
local stalledKey = KEYS[1]
|
482
|
-
local waitKey = KEYS[2]
|
483
|
-
local activeKey = KEYS[3]
|
484
|
-
local failedKey = KEYS[4]
|
485
|
-
local stalledCheckKey = KEYS[5]
|
486
|
-
local metaKey = KEYS[6]
|
487
|
-
local pausedKey = KEYS[7]
|
488
|
-
local markerKey = KEYS[8]
|
489
|
-
local eventStreamKey = KEYS[9]
|
490
|
-
local maxStalledJobCount = tonumber(ARGV[1])
|
491
|
-
local queueKeyPrefix = ARGV[2]
|
492
|
-
local timestamp = ARGV[3]
|
493
|
-
local maxCheckTime = ARGV[4]
|
494
|
-
if rcall("EXISTS", stalledCheckKey) == 1 then
|
495
|
-
return {{}, {}}
|
496
|
-
end
|
497
|
-
rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
|
498
|
-
-- Trim events before emiting them to avoid trimming events emitted in this script
|
499
|
-
trimEvents(metaKey, eventStreamKey)
|
500
|
-
-- Move all stalled jobs to wait
|
501
|
-
local stalling = rcall('SMEMBERS', stalledKey)
|
502
|
-
local stalled = {}
|
503
|
-
local failed = {}
|
504
|
-
if (#stalling > 0) then
|
505
|
-
rcall('DEL', stalledKey)
|
506
|
-
-- Remove from active list
|
507
|
-
for i, jobId in ipairs(stalling) do
|
508
|
-
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
509
|
-
if string.sub(jobId, 1, 2) == "0:" then
|
510
|
-
-- If the jobId is a delay marker ID we just remove it.
|
511
|
-
rcall("LREM", activeKey, 1, jobId)
|
512
|
-
else
|
513
|
-
local jobKey = queueKeyPrefix .. jobId
|
514
|
-
-- Check that the lock is also missing, then we can handle this job as really stalled.
|
515
|
-
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
|
516
|
-
-- Remove from the active queue.
|
517
|
-
local removed = rcall("LREM", activeKey, 1, jobId)
|
518
|
-
if (removed > 0) then
|
519
|
-
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
520
|
-
local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
|
521
|
-
if (stalledCount > maxStalledJobCount) then
|
522
|
-
local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
|
523
|
-
local rawOpts = jobAttributes[1]
|
524
|
-
local rawParentData = jobAttributes[2]
|
525
|
-
local opts = cjson.decode(rawOpts)
|
526
|
-
rcall("ZADD", failedKey, timestamp, jobId)
|
527
|
-
removeDeduplicationKeyIfNeededOnFinalization(queueKeyPrefix, jobAttributes[3], jobId)
|
528
|
-
local failedReason = "job stalled more than allowable limit"
|
529
|
-
rcall("HMSET", jobKey, "failedReason", failedReason, "finishedOn", timestamp)
|
530
|
-
rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId", jobId, 'prev', 'active',
|
531
|
-
'failedReason', failedReason)
|
532
|
-
moveChildFromDependenciesIfNeeded(rawParentData, jobKey, failedReason, timestamp)
|
533
|
-
removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
|
534
|
-
table.insert(failed, jobId)
|
535
|
-
else
|
536
|
-
local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
|
537
|
-
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
|
538
|
-
addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
|
539
|
-
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, 'prev', 'active')
|
540
|
-
-- Emit the stalled event
|
541
|
-
rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId", jobId)
|
542
|
-
table.insert(stalled, jobId)
|
543
|
-
end
|
544
|
-
end
|
545
|
-
end
|
546
|
-
end
|
547
|
-
end
|
548
|
-
end
|
549
|
-
-- Mark potentially stalled jobs
|
550
|
-
local active = rcall('LRANGE', activeKey, 0, -1)
|
551
|
-
if (#active > 0) then
|
552
|
-
for from, to in batches(#active, 7000) do
|
553
|
-
rcall('SADD', stalledKey, unpack(active, from, to))
|
554
|
-
end
|
555
|
-
end
|
556
|
-
return {failed, stalled}
|
557
|
-
`;
|
558
|
-
exports.moveStalledJobsToWait = {
|
559
|
-
name: 'moveStalledJobsToWait',
|
560
|
-
content,
|
561
|
-
keys: 9,
|
562
|
-
};
|
563
|
-
//# sourceMappingURL=moveStalledJobsToWait-9.js.map
|
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyiBf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|