bullmq 5.44.1 → 5.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/async-fifo-queue.js.map +1 -1
- package/dist/cjs/classes/child-processor.js.map +1 -1
- package/dist/cjs/classes/child.js.map +1 -1
- package/dist/cjs/classes/flow-producer.js +15 -4
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job-scheduler.js +1 -1
- package/dist/cjs/classes/job-scheduler.js.map +1 -1
- package/dist/cjs/classes/job.js +137 -54
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-base.js +1 -4
- package/dist/cjs/classes/queue-base.js.map +1 -1
- package/dist/cjs/classes/queue-events-producer.js.map +1 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-getters.js +12 -15
- package/dist/cjs/classes/queue-getters.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/queue.js +12 -13
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/redis-connection.js +1 -1
- package/dist/cjs/classes/redis-connection.js.map +1 -1
- package/dist/cjs/classes/repeat.js +1 -1
- package/dist/cjs/classes/repeat.js.map +1 -1
- package/dist/cjs/classes/scripts.js +18 -7
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +4 -0
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addJobScheduler-11.lua +52 -33
- package/dist/cjs/commands/includes/deduplicateJob.lua +16 -17
- package/dist/cjs/commands/includes/isLocked.lua +1 -0
- package/dist/cjs/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
- package/dist/cjs/commands/includes/moveParentToWait.lua +48 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
- package/dist/cjs/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/cjs/commands/includes/removeJob.lua +2 -2
- package/dist/cjs/commands/includes/removeJobWithChildren.lua +94 -0
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/cjs/commands/includes/storeJob.lua +1 -1
- package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +18 -53
- package/dist/cjs/commands/moveToFinished-14.lua +43 -51
- package/dist/cjs/commands/moveToWaitingChildren-8.lua +5 -32
- package/dist/cjs/commands/removeJob-3.lua +14 -72
- package/dist/cjs/commands/removeUnprocessedChildren-2.lua +31 -0
- package/dist/cjs/commands/updateJobScheduler-12.lua +29 -23
- package/dist/cjs/scripts/addDelayedJob-6.js +73 -55
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +59 -40
- package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +73 -55
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js +73 -55
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +14 -9
- package/dist/cjs/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/cjs/scripts/addStandardJob-8.js +73 -55
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
- package/dist/cjs/scripts/cleanJobsInSet-3.js +14 -9
- package/dist/cjs/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/cjs/scripts/drain-5.js +14 -9
- package/dist/cjs/scripts/drain-5.js.map +1 -1
- package/dist/cjs/scripts/index.js +1 -0
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +157 -155
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +183 -157
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-8.js +145 -137
- package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +14 -9
- package/dist/cjs/scripts/obliterate-2.js.map +1 -1
- package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
- package/dist/cjs/scripts/removeJob-3.js +97 -63
- package/dist/cjs/scripts/removeJob-3.js.map +1 -1
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js +338 -0
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js.map +1 -0
- package/dist/cjs/scripts/updateJobScheduler-12.js +27 -21
- package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/types/index.js +1 -0
- package/dist/cjs/types/index.js.map +1 -1
- package/dist/cjs/types/job-progress.js +3 -0
- package/dist/cjs/types/job-progress.js.map +1 -0
- package/dist/cjs/utils.js +12 -1
- package/dist/cjs/utils.js.map +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/async-fifo-queue.js.map +1 -1
- package/dist/esm/classes/child-processor.js.map +1 -1
- package/dist/esm/classes/child.js.map +1 -1
- package/dist/esm/classes/flow-producer.js +15 -4
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job-scheduler.js +1 -1
- package/dist/esm/classes/job-scheduler.js.map +1 -1
- package/dist/esm/classes/job.d.ts +49 -14
- package/dist/esm/classes/job.js +138 -55
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-base.d.ts +1 -4
- package/dist/esm/classes/queue-base.js +1 -4
- package/dist/esm/classes/queue-base.js.map +1 -1
- package/dist/esm/classes/queue-events-producer.d.ts +2 -2
- package/dist/esm/classes/queue-events-producer.js.map +1 -1
- package/dist/esm/classes/queue-events.d.ts +118 -29
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-getters.d.ts +11 -14
- package/dist/esm/classes/queue-getters.js +12 -15
- package/dist/esm/classes/queue-getters.js.map +1 -1
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +14 -15
- package/dist/esm/classes/queue.js +12 -13
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/redis-connection.js +1 -1
- package/dist/esm/classes/redis-connection.js.map +1 -1
- package/dist/esm/classes/repeat.js +1 -1
- package/dist/esm/classes/repeat.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +4 -3
- package/dist/esm/classes/scripts.js +18 -7
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +2 -1
- package/dist/esm/classes/worker.js +5 -1
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addJobScheduler-11.lua +52 -33
- package/dist/esm/commands/includes/deduplicateJob.lua +16 -17
- package/dist/esm/commands/includes/isLocked.lua +1 -0
- package/dist/esm/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
- package/dist/esm/commands/includes/moveParentToWait.lua +48 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
- package/dist/esm/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/esm/commands/includes/removeJob.lua +2 -2
- package/dist/esm/commands/includes/removeJobWithChildren.lua +94 -0
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/esm/commands/includes/storeJob.lua +1 -1
- package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +18 -53
- package/dist/esm/commands/moveToFinished-14.lua +43 -51
- package/dist/esm/commands/moveToWaitingChildren-8.lua +5 -32
- package/dist/esm/commands/removeJob-3.lua +14 -72
- package/dist/esm/commands/removeUnprocessedChildren-2.lua +31 -0
- package/dist/esm/commands/updateJobScheduler-12.lua +29 -23
- package/dist/esm/interfaces/job-json.d.ts +5 -2
- package/dist/esm/interfaces/minimal-job.d.ts +29 -11
- package/dist/esm/interfaces/queue-options.d.ts +6 -2
- package/dist/esm/interfaces/repeat-options.d.ts +1 -1
- package/dist/esm/interfaces/sandboxed-job.d.ts +2 -2
- package/dist/esm/interfaces/sandboxed-options.d.ts +1 -1
- package/dist/esm/interfaces/telemetry.d.ts +18 -15
- package/dist/esm/interfaces/worker-options.d.ts +10 -10
- package/dist/esm/scripts/addDelayedJob-6.js +73 -55
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addJobScheduler-11.js +59 -40
- package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/esm/scripts/addParentJob-4.js +73 -55
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js +73 -55
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +14 -9
- package/dist/esm/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/esm/scripts/addStandardJob-8.js +73 -55
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
- package/dist/esm/scripts/cleanJobsInSet-3.js +14 -9
- package/dist/esm/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/esm/scripts/drain-5.js +14 -9
- package/dist/esm/scripts/drain-5.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +1 -0
- package/dist/esm/scripts/index.js +1 -0
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +157 -155
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +183 -157
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-8.js +145 -137
- package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +14 -9
- package/dist/esm/scripts/obliterate-2.js.map +1 -1
- package/dist/esm/scripts/removeChildDependency-1.js +5 -5
- package/dist/esm/scripts/removeJob-3.js +97 -63
- package/dist/esm/scripts/removeJob-3.js.map +1 -1
- package/dist/esm/scripts/removeUnprocessedChildren-2.d.ts +5 -0
- package/dist/esm/scripts/removeUnprocessedChildren-2.js +335 -0
- package/dist/esm/scripts/removeUnprocessedChildren-2.js.map +1 -0
- package/dist/esm/scripts/updateJobScheduler-12.js +27 -21
- package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/index.d.ts +1 -0
- package/dist/esm/types/index.js +1 -0
- package/dist/esm/types/index.js.map +1 -1
- package/dist/esm/types/job-options.d.ts +13 -2
- package/dist/esm/types/job-progress.d.ts +1 -0
- package/dist/esm/types/job-progress.js +2 -0
- package/dist/esm/types/job-progress.js.map +1 -0
- package/dist/esm/utils.d.ts +17 -0
- package/dist/esm/utils.js +11 -0
- package/dist/esm/utils.js.map +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +26 -21
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
- package/dist/cjs/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
- package/dist/esm/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
@@ -34,6 +34,7 @@ const content = `--[[
|
|
34
34
|
opts - attempts max attempts
|
35
35
|
opts - maxMetricsSize
|
36
36
|
opts - fpof - fail parent on fail
|
37
|
+
opts - cpof - continue parent on fail
|
37
38
|
opts - idof - ignore dependency on fail
|
38
39
|
opts - rdof - remove dependency on fail
|
39
40
|
opts - name - worker name
|
@@ -167,7 +168,16 @@ end
|
|
167
168
|
]]
|
168
169
|
-- Includes
|
169
170
|
--[[
|
170
|
-
Validate and move parent to
|
171
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
172
|
+
if no pending dependencies.
|
173
|
+
]]
|
174
|
+
-- Includes
|
175
|
+
--[[
|
176
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
177
|
+
]]
|
178
|
+
-- Includes
|
179
|
+
--[[
|
180
|
+
Move parent to a wait status (wait, prioritized or delayed)
|
171
181
|
]]
|
172
182
|
-- Includes
|
173
183
|
--[[
|
@@ -231,58 +241,55 @@ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
|
231
241
|
end
|
232
242
|
return false
|
233
243
|
end
|
234
|
-
local function
|
235
|
-
|
236
|
-
local
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
local
|
247
|
-
local
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
244
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
245
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
246
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
247
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
248
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
249
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
250
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
251
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
252
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
253
|
+
-- ignore dependencies if any left
|
254
|
+
rcall("HSET", parentKey, "igdp", 1)
|
255
|
+
if delay > 0 then
|
256
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
257
|
+
local score = delayedTimestamp * 0x1000
|
258
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
259
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
260
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
261
|
+
delayedTimestamp)
|
262
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
263
|
+
else
|
264
|
+
if priority == 0 then
|
265
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
266
|
+
parentWaitKey, parentPausedKey)
|
267
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
256
268
|
else
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
parentPausedKey)
|
261
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
262
|
-
parentId)
|
263
|
-
else
|
264
|
-
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
265
|
-
addJobWithPriority(parentMarkerKey,
|
266
|
-
parentQueueKey .. ":prioritized", priority,
|
267
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
268
|
-
end
|
269
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
270
|
-
"jobId", parentId, "prev", "waiting-children")
|
269
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
270
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
271
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
271
272
|
end
|
273
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
274
|
+
"waiting-children")
|
272
275
|
end
|
273
276
|
end
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
local pttl = rcall("PTTL", deduplicationKey)
|
281
|
-
if pttl == 0 or pttl == -1 then
|
282
|
-
rcall("DEL", deduplicationKey)
|
277
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
278
|
+
if rcall("EXISTS", parentKey) == 1 then
|
279
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
280
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
281
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
282
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
283
283
|
end
|
284
284
|
end
|
285
285
|
end
|
286
|
+
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
287
|
+
parentId, timestamp)
|
288
|
+
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
289
|
+
if doNotHavePendingDependencies then
|
290
|
+
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
291
|
+
end
|
292
|
+
end
|
286
293
|
--[[
|
287
294
|
Functions to remove jobs when removeOnFail option is provided.
|
288
295
|
]]
|
@@ -292,13 +299,18 @@ end
|
|
292
299
|
]]
|
293
300
|
-- Includes
|
294
301
|
--[[
|
295
|
-
Function to remove deduplication key
|
302
|
+
Function to remove deduplication key if needed
|
303
|
+
when a job is being removed.
|
296
304
|
]]
|
297
|
-
local function
|
305
|
+
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
306
|
+
jobKey, jobId)
|
298
307
|
local deduplicationId = rcall("HGET", jobKey, "deid")
|
299
308
|
if deduplicationId then
|
300
309
|
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
301
|
-
rcall(
|
310
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
311
|
+
if currentJobId and currentJobId == jobId then
|
312
|
+
return rcall("DEL", deduplicationKey)
|
313
|
+
end
|
302
314
|
end
|
303
315
|
end
|
304
316
|
--[[
|
@@ -324,7 +336,7 @@ end
|
|
324
336
|
local getJobKeyPrefix = function (jobKey, jobId)
|
325
337
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
326
338
|
end
|
327
|
-
local function
|
339
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
328
340
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
329
341
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
330
342
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -352,10 +364,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
352
364
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
353
365
|
end
|
354
366
|
else
|
355
|
-
|
367
|
+
_moveParentToWait(parentPrefix, parentId)
|
356
368
|
end
|
357
369
|
else
|
358
|
-
|
370
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
359
371
|
end
|
360
372
|
end
|
361
373
|
end
|
@@ -383,10 +395,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
383
395
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
384
396
|
end
|
385
397
|
else
|
386
|
-
|
398
|
+
_moveParentToWait(parentPrefix, parentId)
|
387
399
|
end
|
388
400
|
else
|
389
|
-
|
401
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
390
402
|
end
|
391
403
|
end
|
392
404
|
end
|
@@ -400,7 +412,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
400
412
|
local jobKey = baseKey .. jobId
|
401
413
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
402
414
|
if shouldRemoveDeduplicationKey then
|
403
|
-
|
415
|
+
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
|
404
416
|
end
|
405
417
|
removeJobKeys(jobKey)
|
406
418
|
end
|
@@ -453,68 +465,71 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
|
|
453
465
|
end
|
454
466
|
end
|
455
467
|
end
|
456
|
-
local
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
prevState
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
|
468
|
+
local moveParentToFailedIfNeeded
|
469
|
+
local moveChildFromDependenciesIfNeeded
|
470
|
+
moveParentToFailedIfNeeded = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
471
|
+
if rcall("EXISTS", parentKey) == 1 then
|
472
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
473
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
474
|
+
local parentPrioritizedKey = parentQueueKey .. ":prioritized"
|
475
|
+
local parentWaitingChildrenOrDelayedKey
|
476
|
+
local prevState
|
477
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
478
|
+
parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
|
479
|
+
prevState = "waiting-children"
|
480
|
+
elseif rcall("ZSCORE", parentDelayedKey, parentId) then
|
481
|
+
parentWaitingChildrenOrDelayedKey = parentDelayedKey
|
482
|
+
prevState = "delayed"
|
483
|
+
rcall("HSET", parentKey, "delay", 0)
|
484
|
+
end
|
485
|
+
if parentWaitingChildrenOrDelayedKey then
|
486
|
+
rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
|
487
|
+
local parentQueuePrefix = parentQueueKey .. ":"
|
488
|
+
local parentFailedKey = parentQueueKey .. ":failed"
|
489
|
+
local deferredFailure = "child " .. jobIdKey .. " failed"
|
490
|
+
rcall("HSET", parentKey, "defa", deferredFailure)
|
491
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
492
|
+
else
|
493
|
+
if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then
|
494
|
+
local deferredFailure = "child " .. jobIdKey .. " failed"
|
495
|
+
rcall("HSET", parentKey, "defa", deferredFailure)
|
496
|
+
end
|
497
|
+
end
|
471
498
|
end
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
local
|
476
|
-
|
477
|
-
local
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
if jobAttributes[1] then
|
484
|
-
local parentData = cjson.decode(jobAttributes[1])
|
485
|
-
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
486
|
-
local grandParentUnsuccesssful = grandParentKey .. ":unsuccessful"
|
487
|
-
rcall("ZADD", grandParentUnsuccesssful, timestamp, parentKey)
|
488
|
-
if parentData['fpof'] then
|
489
|
-
moveParentFromWaitingChildrenToFailed(
|
499
|
+
end
|
500
|
+
moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp)
|
501
|
+
if rawParentData then
|
502
|
+
local parentData = cjson.decode(rawParentData)
|
503
|
+
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
504
|
+
local parentDependenciesChildrenKey = parentKey .. ":dependencies"
|
505
|
+
if parentData['fpof'] then
|
506
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
507
|
+
local parentUnsuccesssfulChildrenKey = parentKey .. ":unsuccessful"
|
508
|
+
rcall("ZADD", parentUnsuccesssfulChildrenKey, timestamp, childKey)
|
509
|
+
moveParentToFailedIfNeeded(
|
490
510
|
parentData['queueKey'],
|
491
|
-
parentData['queueKey'] .. ':' .. parentData['id'],
|
492
|
-
parentData['id'],
|
493
511
|
parentKey,
|
512
|
+
parentData['id'],
|
513
|
+
childKey,
|
494
514
|
timestamp
|
495
515
|
)
|
496
|
-
|
497
|
-
|
498
|
-
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
|
516
|
+
end
|
517
|
+
elseif parentData['cpof'] then
|
518
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
519
|
+
local parentFailedChildrenKey = parentKey .. ":failed"
|
520
|
+
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
521
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
522
|
+
end
|
523
|
+
elseif parentData['idof'] or parentData['rdof'] then
|
524
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
525
|
+
moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey,
|
526
|
+
parentKey, parentData['id'], timestamp)
|
527
|
+
if parentData['idof'] then
|
528
|
+
local parentFailedChildrenKey = parentKey .. ":failed"
|
529
|
+
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
506
530
|
end
|
507
531
|
end
|
508
532
|
end
|
509
|
-
local parentRawOpts = jobAttributes[3]
|
510
|
-
local parentOpts = cjson.decode(parentRawOpts)
|
511
|
-
removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
|
512
|
-
else
|
513
|
-
local grandParentKey = rcall("HGET", parentKey, "parentKey")
|
514
|
-
if grandParentKey then
|
515
|
-
local grandParentUnsuccesssfulSet = grandParentKey .. ":unsuccessful"
|
516
|
-
rcall("ZADD", grandParentUnsuccesssfulSet, timestamp, parentKey)
|
517
|
-
end
|
518
533
|
end
|
519
534
|
end
|
520
535
|
--[[
|
@@ -586,6 +601,26 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
|
|
586
601
|
addBaseMarkerIfNeeded(markerKey, isPaused)
|
587
602
|
end
|
588
603
|
end
|
604
|
+
--[[
|
605
|
+
Function to remove deduplication key if needed
|
606
|
+
when a job is moved to completed or failed states.
|
607
|
+
]]
|
608
|
+
local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
|
609
|
+
deduplicationId, jobId)
|
610
|
+
if deduplicationId then
|
611
|
+
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
612
|
+
local pttl = rcall("PTTL", deduplicationKey)
|
613
|
+
if pttl == 0 then
|
614
|
+
return rcall("DEL", deduplicationKey)
|
615
|
+
end
|
616
|
+
if pttl == -1 then
|
617
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
618
|
+
if currentJobId and currentJobId == jobId then
|
619
|
+
return rcall("DEL", deduplicationKey)
|
620
|
+
end
|
621
|
+
end
|
622
|
+
end
|
623
|
+
end
|
589
624
|
local function removeLock(jobKey, stalledKey, token, jobId)
|
590
625
|
if token ~= "0" then
|
591
626
|
local lockKey = jobKey .. ':lock'
|
@@ -636,7 +671,7 @@ local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDepende
|
|
636
671
|
parentId, jobIdKey, returnvalue, timestamp )
|
637
672
|
local processedSet = parentKey .. ":processed"
|
638
673
|
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
639
|
-
|
674
|
+
moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
640
675
|
end
|
641
676
|
--[[
|
642
677
|
Function to update a bunch of fields in a job.
|
@@ -650,7 +685,14 @@ local function updateJobFields(jobKey, msgpackedFields)
|
|
650
685
|
end
|
651
686
|
end
|
652
687
|
local jobIdKey = KEYS[12]
|
653
|
-
if rcall("EXISTS", jobIdKey) == 1 then --
|
688
|
+
if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
689
|
+
-- Make sure it does not have pending dependencies
|
690
|
+
-- It must happen before removing lock
|
691
|
+
if ARGV[5] == "completed" and
|
692
|
+
not rcall("HGET", jobIdKey, "igdp") and -- check if we should ignore this check
|
693
|
+
rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then
|
694
|
+
return -4
|
695
|
+
end
|
654
696
|
local opts = cmsgpack.unpack(ARGV[8])
|
655
697
|
local token = opts['token']
|
656
698
|
local errorCode = removeLock(jobIdKey, KEYS[5], token, ARGV[1])
|
@@ -662,14 +704,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
662
704
|
local maxMetricsSize = opts['maxMetricsSize']
|
663
705
|
local maxCount = opts['keepJobs']['count']
|
664
706
|
local maxAge = opts['keepJobs']['age']
|
665
|
-
if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
|
666
|
-
return -4
|
667
|
-
end
|
668
707
|
local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid")
|
669
708
|
local parentKey = jobAttributes[1] or ""
|
670
709
|
local parentId = ""
|
671
710
|
local parentQueueKey = ""
|
672
|
-
if jobAttributes[2] then
|
711
|
+
if jobAttributes[2] then -- TODO: need to revisit this logic if it's still needed
|
673
712
|
local jsonDecodedParent = cjson.decode(jobAttributes[2])
|
674
713
|
parentId = jsonDecodedParent['id']
|
675
714
|
parentQueueKey = jsonDecodedParent['queueKey']
|
@@ -678,13 +717,15 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
678
717
|
local timestamp = ARGV[2]
|
679
718
|
-- Remove from active list (if not active we shall return error)
|
680
719
|
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
681
|
-
if (numRemovedElements < 1) then
|
720
|
+
if (numRemovedElements < 1) then
|
721
|
+
return -3
|
722
|
+
end
|
682
723
|
local eventStreamKey = KEYS[4]
|
683
724
|
local metaKey = KEYS[9]
|
684
725
|
-- Trim events before emiting them to avoid trimming events emitted in this script
|
685
726
|
trimEvents(metaKey, eventStreamKey)
|
686
727
|
local prefix = ARGV[7]
|
687
|
-
|
728
|
+
removeDeduplicationKeyIfNeededOnFinalization(prefix, jobAttributes[3], jobId)
|
688
729
|
-- If job has a parent we need to
|
689
730
|
-- 1) remove this job id from parents dependencies
|
690
731
|
-- 2) move the job Id to parent "processed" set
|
@@ -698,28 +739,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
698
739
|
if ARGV[5] == "completed" then
|
699
740
|
local dependenciesSet = parentKey .. ":dependencies"
|
700
741
|
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
701
|
-
updateParentDepsIfNeeded(parentKey, parentQueueKey,
|
702
|
-
|
703
|
-
ARGV[4], timestamp)
|
742
|
+
updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet, parentId, jobIdKey, ARGV[4],
|
743
|
+
timestamp)
|
704
744
|
end
|
705
745
|
else
|
706
|
-
|
707
|
-
local unsuccesssfulSet = parentKey .. ":unsuccessful"
|
708
|
-
rcall("ZADD", unsuccesssfulSet, timestamp, jobIdKey)
|
709
|
-
moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
|
710
|
-
parentId, jobIdKey,
|
711
|
-
timestamp)
|
712
|
-
elseif opts['idof'] or opts['rdof'] then
|
713
|
-
local dependenciesSet = parentKey .. ":dependencies"
|
714
|
-
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
715
|
-
moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
|
716
|
-
parentKey, parentId, timestamp)
|
717
|
-
if opts['idof'] then
|
718
|
-
local failedSet = parentKey .. ":failed"
|
719
|
-
rcall("HSET", failedSet, jobIdKey, ARGV[4])
|
720
|
-
end
|
721
|
-
end
|
722
|
-
end
|
746
|
+
moveChildFromDependenciesIfNeeded(jobAttributes[2], jobIdKey, ARGV[4], timestamp)
|
723
747
|
end
|
724
748
|
end
|
725
749
|
local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
|
@@ -728,8 +752,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
728
752
|
local targetSet = KEYS[11]
|
729
753
|
-- Add to complete/failed set
|
730
754
|
rcall("ZADD", targetSet, timestamp, jobId)
|
731
|
-
rcall("
|
755
|
+
rcall("HSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
|
732
756
|
-- "returnvalue" / "failedReason" and "finishedOn"
|
757
|
+
if ARGV[5] == "failed" then
|
758
|
+
rcall("HDEL", jobIdKey, "defa")
|
759
|
+
end
|
733
760
|
-- Remove old jobs?
|
734
761
|
if maxAge ~= nil then
|
735
762
|
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
@@ -746,12 +773,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
746
773
|
removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3])
|
747
774
|
end
|
748
775
|
end
|
749
|
-
rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
|
750
|
-
ARGV[4])
|
776
|
+
rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3], ARGV[4], "prev", "active")
|
751
777
|
if ARGV[5] == "failed" then
|
752
778
|
if tonumber(attemptsMade) >= tonumber(attempts) then
|
753
|
-
rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId",
|
754
|
-
|
779
|
+
rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId", jobId, "attemptsMade",
|
780
|
+
attemptsMade)
|
755
781
|
end
|
756
782
|
end
|
757
783
|
-- Collect metrics
|
@@ -764,14 +790,18 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
764
790
|
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
|
765
791
|
local markerKey = KEYS[14]
|
766
792
|
-- Check if there are delayed jobs that can be promoted
|
767
|
-
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix,
|
768
|
-
|
793
|
+
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10],
|
794
|
+
isPausedOrMaxed)
|
769
795
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
770
796
|
-- Check if we are rate limited first.
|
771
797
|
local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
|
772
|
-
if expireTime > 0 then
|
798
|
+
if expireTime > 0 then
|
799
|
+
return {0, 0, expireTime, 0}
|
800
|
+
end
|
773
801
|
-- paused or maxed queue
|
774
|
-
if isPausedOrMaxed then
|
802
|
+
if isPausedOrMaxed then
|
803
|
+
return {0, 0, 0, 0}
|
804
|
+
end
|
775
805
|
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
776
806
|
if jobId then
|
777
807
|
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
@@ -780,23 +810,19 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
780
810
|
-- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
|
781
811
|
-- but if ID is 0:0, then there is at least 1 prioritized job to process
|
782
812
|
if jobId == "0:0" then
|
783
|
-
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
|
784
|
-
|
785
|
-
|
786
|
-
timestamp, maxJobs, markerKey,
|
787
|
-
opts)
|
813
|
+
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
814
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
815
|
+
markerKey, opts)
|
788
816
|
end
|
789
817
|
else
|
790
|
-
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
|
791
|
-
|
792
|
-
opts)
|
818
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
|
819
|
+
opts)
|
793
820
|
end
|
794
821
|
else
|
795
822
|
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
796
823
|
if jobId then
|
797
|
-
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
|
798
|
-
|
799
|
-
opts)
|
824
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
|
825
|
+
opts)
|
800
826
|
end
|
801
827
|
end
|
802
828
|
-- Return the timestamp for the next delayed job if any.
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAi1Bf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|