bullmq 5.44.1 → 5.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/async-fifo-queue.js.map +1 -1
- package/dist/cjs/classes/child-processor.js.map +1 -1
- package/dist/cjs/classes/child.js.map +1 -1
- package/dist/cjs/classes/flow-producer.js +15 -4
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job-scheduler.js +1 -1
- package/dist/cjs/classes/job-scheduler.js.map +1 -1
- package/dist/cjs/classes/job.js +137 -54
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-base.js +1 -4
- package/dist/cjs/classes/queue-base.js.map +1 -1
- package/dist/cjs/classes/queue-events-producer.js.map +1 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-getters.js +12 -15
- package/dist/cjs/classes/queue-getters.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/queue.js +12 -13
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/redis-connection.js +1 -1
- package/dist/cjs/classes/redis-connection.js.map +1 -1
- package/dist/cjs/classes/repeat.js +1 -1
- package/dist/cjs/classes/repeat.js.map +1 -1
- package/dist/cjs/classes/scripts.js +18 -7
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +4 -0
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addJobScheduler-11.lua +52 -33
- package/dist/cjs/commands/includes/deduplicateJob.lua +16 -17
- package/dist/cjs/commands/includes/isLocked.lua +1 -0
- package/dist/cjs/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
- package/dist/cjs/commands/includes/moveParentToWait.lua +48 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
- package/dist/cjs/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/cjs/commands/includes/removeJob.lua +2 -2
- package/dist/cjs/commands/includes/removeJobWithChildren.lua +94 -0
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/cjs/commands/includes/storeJob.lua +1 -1
- package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +18 -53
- package/dist/cjs/commands/moveToFinished-14.lua +43 -51
- package/dist/cjs/commands/moveToWaitingChildren-8.lua +5 -32
- package/dist/cjs/commands/removeJob-3.lua +14 -72
- package/dist/cjs/commands/removeUnprocessedChildren-2.lua +31 -0
- package/dist/cjs/commands/updateJobScheduler-12.lua +29 -23
- package/dist/cjs/scripts/addDelayedJob-6.js +73 -55
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +59 -40
- package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +73 -55
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js +73 -55
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +14 -9
- package/dist/cjs/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/cjs/scripts/addStandardJob-8.js +73 -55
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
- package/dist/cjs/scripts/cleanJobsInSet-3.js +14 -9
- package/dist/cjs/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/cjs/scripts/drain-5.js +14 -9
- package/dist/cjs/scripts/drain-5.js.map +1 -1
- package/dist/cjs/scripts/index.js +1 -0
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +157 -155
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +183 -157
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-8.js +145 -137
- package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +14 -9
- package/dist/cjs/scripts/obliterate-2.js.map +1 -1
- package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
- package/dist/cjs/scripts/removeJob-3.js +97 -63
- package/dist/cjs/scripts/removeJob-3.js.map +1 -1
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js +338 -0
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js.map +1 -0
- package/dist/cjs/scripts/updateJobScheduler-12.js +27 -21
- package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/types/index.js +1 -0
- package/dist/cjs/types/index.js.map +1 -1
- package/dist/cjs/types/job-progress.js +3 -0
- package/dist/cjs/types/job-progress.js.map +1 -0
- package/dist/cjs/utils.js +12 -1
- package/dist/cjs/utils.js.map +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/async-fifo-queue.js.map +1 -1
- package/dist/esm/classes/child-processor.js.map +1 -1
- package/dist/esm/classes/child.js.map +1 -1
- package/dist/esm/classes/flow-producer.js +15 -4
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job-scheduler.js +1 -1
- package/dist/esm/classes/job-scheduler.js.map +1 -1
- package/dist/esm/classes/job.d.ts +49 -14
- package/dist/esm/classes/job.js +138 -55
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-base.d.ts +1 -4
- package/dist/esm/classes/queue-base.js +1 -4
- package/dist/esm/classes/queue-base.js.map +1 -1
- package/dist/esm/classes/queue-events-producer.d.ts +2 -2
- package/dist/esm/classes/queue-events-producer.js.map +1 -1
- package/dist/esm/classes/queue-events.d.ts +118 -29
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-getters.d.ts +11 -14
- package/dist/esm/classes/queue-getters.js +12 -15
- package/dist/esm/classes/queue-getters.js.map +1 -1
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +14 -15
- package/dist/esm/classes/queue.js +12 -13
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/redis-connection.js +1 -1
- package/dist/esm/classes/redis-connection.js.map +1 -1
- package/dist/esm/classes/repeat.js +1 -1
- package/dist/esm/classes/repeat.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +4 -3
- package/dist/esm/classes/scripts.js +18 -7
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +2 -1
- package/dist/esm/classes/worker.js +5 -1
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addJobScheduler-11.lua +52 -33
- package/dist/esm/commands/includes/deduplicateJob.lua +16 -17
- package/dist/esm/commands/includes/isLocked.lua +1 -0
- package/dist/esm/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
- package/dist/esm/commands/includes/moveParentToWait.lua +48 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
- package/dist/esm/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/esm/commands/includes/removeJob.lua +2 -2
- package/dist/esm/commands/includes/removeJobWithChildren.lua +94 -0
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/esm/commands/includes/storeJob.lua +1 -1
- package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +18 -53
- package/dist/esm/commands/moveToFinished-14.lua +43 -51
- package/dist/esm/commands/moveToWaitingChildren-8.lua +5 -32
- package/dist/esm/commands/removeJob-3.lua +14 -72
- package/dist/esm/commands/removeUnprocessedChildren-2.lua +31 -0
- package/dist/esm/commands/updateJobScheduler-12.lua +29 -23
- package/dist/esm/interfaces/job-json.d.ts +5 -2
- package/dist/esm/interfaces/minimal-job.d.ts +29 -11
- package/dist/esm/interfaces/queue-options.d.ts +6 -2
- package/dist/esm/interfaces/repeat-options.d.ts +1 -1
- package/dist/esm/interfaces/sandboxed-job.d.ts +2 -2
- package/dist/esm/interfaces/sandboxed-options.d.ts +1 -1
- package/dist/esm/interfaces/telemetry.d.ts +18 -15
- package/dist/esm/interfaces/worker-options.d.ts +10 -10
- package/dist/esm/scripts/addDelayedJob-6.js +73 -55
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addJobScheduler-11.js +59 -40
- package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/esm/scripts/addParentJob-4.js +73 -55
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js +73 -55
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +14 -9
- package/dist/esm/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/esm/scripts/addStandardJob-8.js +73 -55
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
- package/dist/esm/scripts/cleanJobsInSet-3.js +14 -9
- package/dist/esm/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/esm/scripts/drain-5.js +14 -9
- package/dist/esm/scripts/drain-5.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +1 -0
- package/dist/esm/scripts/index.js +1 -0
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +157 -155
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +183 -157
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-8.js +145 -137
- package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +14 -9
- package/dist/esm/scripts/obliterate-2.js.map +1 -1
- package/dist/esm/scripts/removeChildDependency-1.js +5 -5
- package/dist/esm/scripts/removeJob-3.js +97 -63
- package/dist/esm/scripts/removeJob-3.js.map +1 -1
- package/dist/esm/scripts/removeUnprocessedChildren-2.d.ts +5 -0
- package/dist/esm/scripts/removeUnprocessedChildren-2.js +335 -0
- package/dist/esm/scripts/removeUnprocessedChildren-2.js.map +1 -0
- package/dist/esm/scripts/updateJobScheduler-12.js +27 -21
- package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/index.d.ts +1 -0
- package/dist/esm/types/index.js +1 -0
- package/dist/esm/types/index.js.map +1 -1
- package/dist/esm/types/job-options.d.ts +13 -2
- package/dist/esm/types/job-progress.d.ts +1 -0
- package/dist/esm/types/job-progress.js +2 -0
- package/dist/esm/types/job-progress.js.map +1 -0
- package/dist/esm/utils.d.ts +17 -0
- package/dist/esm/utils.js +11 -0
- package/dist/esm/utils.js.map +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +26 -21
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
- package/dist/cjs/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
- package/dist/esm/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
@@ -37,6 +37,7 @@ const content = `--[[
|
|
37
37
|
opts - attempts max attempts
|
38
38
|
opts - maxMetricsSize
|
39
39
|
opts - fpof - fail parent on fail
|
40
|
+
opts - cpof - continue parent on fail
|
40
41
|
opts - idof - ignore dependency on fail
|
41
42
|
opts - rdof - remove dependency on fail
|
42
43
|
opts - name - worker name
|
@@ -170,7 +171,16 @@ end
|
|
170
171
|
]]
|
171
172
|
-- Includes
|
172
173
|
--[[
|
173
|
-
Validate and move parent to
|
174
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
175
|
+
if no pending dependencies.
|
176
|
+
]]
|
177
|
+
-- Includes
|
178
|
+
--[[
|
179
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
180
|
+
]]
|
181
|
+
-- Includes
|
182
|
+
--[[
|
183
|
+
Move parent to a wait status (wait, prioritized or delayed)
|
174
184
|
]]
|
175
185
|
-- Includes
|
176
186
|
--[[
|
@@ -234,58 +244,55 @@ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
|
234
244
|
end
|
235
245
|
return false
|
236
246
|
end
|
237
|
-
local function
|
238
|
-
|
239
|
-
local
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
local
|
250
|
-
local
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
247
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
248
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
249
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
250
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
251
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
252
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
253
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
254
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
255
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
256
|
+
-- ignore dependencies if any left
|
257
|
+
rcall("HSET", parentKey, "igdp", 1)
|
258
|
+
if delay > 0 then
|
259
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
260
|
+
local score = delayedTimestamp * 0x1000
|
261
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
262
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
263
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
264
|
+
delayedTimestamp)
|
265
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
266
|
+
else
|
267
|
+
if priority == 0 then
|
268
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
269
|
+
parentWaitKey, parentPausedKey)
|
270
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
259
271
|
else
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
parentPausedKey)
|
264
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
265
|
-
parentId)
|
266
|
-
else
|
267
|
-
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
268
|
-
addJobWithPriority(parentMarkerKey,
|
269
|
-
parentQueueKey .. ":prioritized", priority,
|
270
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
271
|
-
end
|
272
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
273
|
-
"jobId", parentId, "prev", "waiting-children")
|
272
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
273
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
274
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
274
275
|
end
|
276
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
277
|
+
"waiting-children")
|
275
278
|
end
|
276
279
|
end
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
local pttl = rcall("PTTL", deduplicationKey)
|
284
|
-
if pttl == 0 or pttl == -1 then
|
285
|
-
rcall("DEL", deduplicationKey)
|
280
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
281
|
+
if rcall("EXISTS", parentKey) == 1 then
|
282
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
283
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
284
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
285
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
286
286
|
end
|
287
287
|
end
|
288
288
|
end
|
289
|
+
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
290
|
+
parentId, timestamp)
|
291
|
+
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
292
|
+
if doNotHavePendingDependencies then
|
293
|
+
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
294
|
+
end
|
295
|
+
end
|
289
296
|
--[[
|
290
297
|
Functions to remove jobs when removeOnFail option is provided.
|
291
298
|
]]
|
@@ -295,13 +302,18 @@ end
|
|
295
302
|
]]
|
296
303
|
-- Includes
|
297
304
|
--[[
|
298
|
-
Function to remove deduplication key
|
305
|
+
Function to remove deduplication key if needed
|
306
|
+
when a job is being removed.
|
299
307
|
]]
|
300
|
-
local function
|
308
|
+
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
309
|
+
jobKey, jobId)
|
301
310
|
local deduplicationId = rcall("HGET", jobKey, "deid")
|
302
311
|
if deduplicationId then
|
303
312
|
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
304
|
-
rcall(
|
313
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
314
|
+
if currentJobId and currentJobId == jobId then
|
315
|
+
return rcall("DEL", deduplicationKey)
|
316
|
+
end
|
305
317
|
end
|
306
318
|
end
|
307
319
|
--[[
|
@@ -327,7 +339,7 @@ end
|
|
327
339
|
local getJobKeyPrefix = function (jobKey, jobId)
|
328
340
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
329
341
|
end
|
330
|
-
local function
|
342
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
331
343
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
332
344
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
333
345
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -355,10 +367,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
355
367
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
356
368
|
end
|
357
369
|
else
|
358
|
-
|
370
|
+
_moveParentToWait(parentPrefix, parentId)
|
359
371
|
end
|
360
372
|
else
|
361
|
-
|
373
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
362
374
|
end
|
363
375
|
end
|
364
376
|
end
|
@@ -386,10 +398,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
386
398
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
387
399
|
end
|
388
400
|
else
|
389
|
-
|
401
|
+
_moveParentToWait(parentPrefix, parentId)
|
390
402
|
end
|
391
403
|
else
|
392
|
-
|
404
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
393
405
|
end
|
394
406
|
end
|
395
407
|
end
|
@@ -403,7 +415,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
403
415
|
local jobKey = baseKey .. jobId
|
404
416
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
405
417
|
if shouldRemoveDeduplicationKey then
|
406
|
-
|
418
|
+
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
|
407
419
|
end
|
408
420
|
removeJobKeys(jobKey)
|
409
421
|
end
|
@@ -456,68 +468,71 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
|
|
456
468
|
end
|
457
469
|
end
|
458
470
|
end
|
459
|
-
local
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
prevState
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
471
|
+
local moveParentToFailedIfNeeded
|
472
|
+
local moveChildFromDependenciesIfNeeded
|
473
|
+
moveParentToFailedIfNeeded = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
474
|
+
if rcall("EXISTS", parentKey) == 1 then
|
475
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
476
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
477
|
+
local parentPrioritizedKey = parentQueueKey .. ":prioritized"
|
478
|
+
local parentWaitingChildrenOrDelayedKey
|
479
|
+
local prevState
|
480
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
481
|
+
parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
|
482
|
+
prevState = "waiting-children"
|
483
|
+
elseif rcall("ZSCORE", parentDelayedKey, parentId) then
|
484
|
+
parentWaitingChildrenOrDelayedKey = parentDelayedKey
|
485
|
+
prevState = "delayed"
|
486
|
+
rcall("HSET", parentKey, "delay", 0)
|
487
|
+
end
|
488
|
+
if parentWaitingChildrenOrDelayedKey then
|
489
|
+
rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
|
490
|
+
local parentQueuePrefix = parentQueueKey .. ":"
|
491
|
+
local parentFailedKey = parentQueueKey .. ":failed"
|
492
|
+
local deferredFailure = "child " .. jobIdKey .. " failed"
|
493
|
+
rcall("HSET", parentKey, "defa", deferredFailure)
|
494
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
495
|
+
else
|
496
|
+
if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then
|
497
|
+
local deferredFailure = "child " .. jobIdKey .. " failed"
|
498
|
+
rcall("HSET", parentKey, "defa", deferredFailure)
|
499
|
+
end
|
500
|
+
end
|
474
501
|
end
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
local
|
479
|
-
|
480
|
-
local
|
481
|
-
|
482
|
-
|
483
|
-
|
484
|
-
|
485
|
-
|
486
|
-
if jobAttributes[1] then
|
487
|
-
local parentData = cjson.decode(jobAttributes[1])
|
488
|
-
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
489
|
-
local grandParentUnsuccesssful = grandParentKey .. ":unsuccessful"
|
490
|
-
rcall("ZADD", grandParentUnsuccesssful, timestamp, parentKey)
|
491
|
-
if parentData['fpof'] then
|
492
|
-
moveParentFromWaitingChildrenToFailed(
|
502
|
+
end
|
503
|
+
moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp)
|
504
|
+
if rawParentData then
|
505
|
+
local parentData = cjson.decode(rawParentData)
|
506
|
+
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
507
|
+
local parentDependenciesChildrenKey = parentKey .. ":dependencies"
|
508
|
+
if parentData['fpof'] then
|
509
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
510
|
+
local parentUnsuccesssfulChildrenKey = parentKey .. ":unsuccessful"
|
511
|
+
rcall("ZADD", parentUnsuccesssfulChildrenKey, timestamp, childKey)
|
512
|
+
moveParentToFailedIfNeeded(
|
493
513
|
parentData['queueKey'],
|
494
|
-
parentData['queueKey'] .. ':' .. parentData['id'],
|
495
|
-
parentData['id'],
|
496
514
|
parentKey,
|
515
|
+
parentData['id'],
|
516
|
+
childKey,
|
497
517
|
timestamp
|
498
518
|
)
|
499
|
-
|
500
|
-
|
501
|
-
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
519
|
+
end
|
520
|
+
elseif parentData['cpof'] then
|
521
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
522
|
+
local parentFailedChildrenKey = parentKey .. ":failed"
|
523
|
+
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
524
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
525
|
+
end
|
526
|
+
elseif parentData['idof'] or parentData['rdof'] then
|
527
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
528
|
+
moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey,
|
529
|
+
parentKey, parentData['id'], timestamp)
|
530
|
+
if parentData['idof'] then
|
531
|
+
local parentFailedChildrenKey = parentKey .. ":failed"
|
532
|
+
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
509
533
|
end
|
510
534
|
end
|
511
535
|
end
|
512
|
-
local parentRawOpts = jobAttributes[3]
|
513
|
-
local parentOpts = cjson.decode(parentRawOpts)
|
514
|
-
removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
|
515
|
-
else
|
516
|
-
local grandParentKey = rcall("HGET", parentKey, "parentKey")
|
517
|
-
if grandParentKey then
|
518
|
-
local grandParentUnsuccesssfulSet = grandParentKey .. ":unsuccessful"
|
519
|
-
rcall("ZADD", grandParentUnsuccesssfulSet, timestamp, parentKey)
|
520
|
-
end
|
521
536
|
end
|
522
537
|
end
|
523
538
|
--[[
|
@@ -589,6 +604,26 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
|
|
589
604
|
addBaseMarkerIfNeeded(markerKey, isPaused)
|
590
605
|
end
|
591
606
|
end
|
607
|
+
--[[
|
608
|
+
Function to remove deduplication key if needed
|
609
|
+
when a job is moved to completed or failed states.
|
610
|
+
]]
|
611
|
+
local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
|
612
|
+
deduplicationId, jobId)
|
613
|
+
if deduplicationId then
|
614
|
+
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
615
|
+
local pttl = rcall("PTTL", deduplicationKey)
|
616
|
+
if pttl == 0 then
|
617
|
+
return rcall("DEL", deduplicationKey)
|
618
|
+
end
|
619
|
+
if pttl == -1 then
|
620
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
621
|
+
if currentJobId and currentJobId == jobId then
|
622
|
+
return rcall("DEL", deduplicationKey)
|
623
|
+
end
|
624
|
+
end
|
625
|
+
end
|
626
|
+
end
|
592
627
|
local function removeLock(jobKey, stalledKey, token, jobId)
|
593
628
|
if token ~= "0" then
|
594
629
|
local lockKey = jobKey .. ':lock'
|
@@ -639,7 +674,7 @@ local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDepende
|
|
639
674
|
parentId, jobIdKey, returnvalue, timestamp )
|
640
675
|
local processedSet = parentKey .. ":processed"
|
641
676
|
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
642
|
-
|
677
|
+
moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
643
678
|
end
|
644
679
|
--[[
|
645
680
|
Function to update a bunch of fields in a job.
|
@@ -653,7 +688,14 @@ local function updateJobFields(jobKey, msgpackedFields)
|
|
653
688
|
end
|
654
689
|
end
|
655
690
|
local jobIdKey = KEYS[12]
|
656
|
-
if rcall("EXISTS", jobIdKey) == 1 then --
|
691
|
+
if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
692
|
+
-- Make sure it does not have pending dependencies
|
693
|
+
-- It must happen before removing lock
|
694
|
+
if ARGV[5] == "completed" and
|
695
|
+
not rcall("HGET", jobIdKey, "igdp") and -- check if we should ignore this check
|
696
|
+
rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then
|
697
|
+
return -4
|
698
|
+
end
|
657
699
|
local opts = cmsgpack.unpack(ARGV[8])
|
658
700
|
local token = opts['token']
|
659
701
|
local errorCode = removeLock(jobIdKey, KEYS[5], token, ARGV[1])
|
@@ -665,14 +707,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
665
707
|
local maxMetricsSize = opts['maxMetricsSize']
|
666
708
|
local maxCount = opts['keepJobs']['count']
|
667
709
|
local maxAge = opts['keepJobs']['age']
|
668
|
-
if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
|
669
|
-
return -4
|
670
|
-
end
|
671
710
|
local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid")
|
672
711
|
local parentKey = jobAttributes[1] or ""
|
673
712
|
local parentId = ""
|
674
713
|
local parentQueueKey = ""
|
675
|
-
if jobAttributes[2] then
|
714
|
+
if jobAttributes[2] then -- TODO: need to revisit this logic if it's still needed
|
676
715
|
local jsonDecodedParent = cjson.decode(jobAttributes[2])
|
677
716
|
parentId = jsonDecodedParent['id']
|
678
717
|
parentQueueKey = jsonDecodedParent['queueKey']
|
@@ -681,13 +720,15 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
681
720
|
local timestamp = ARGV[2]
|
682
721
|
-- Remove from active list (if not active we shall return error)
|
683
722
|
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
684
|
-
if (numRemovedElements < 1) then
|
723
|
+
if (numRemovedElements < 1) then
|
724
|
+
return -3
|
725
|
+
end
|
685
726
|
local eventStreamKey = KEYS[4]
|
686
727
|
local metaKey = KEYS[9]
|
687
728
|
-- Trim events before emiting them to avoid trimming events emitted in this script
|
688
729
|
trimEvents(metaKey, eventStreamKey)
|
689
730
|
local prefix = ARGV[7]
|
690
|
-
|
731
|
+
removeDeduplicationKeyIfNeededOnFinalization(prefix, jobAttributes[3], jobId)
|
691
732
|
-- If job has a parent we need to
|
692
733
|
-- 1) remove this job id from parents dependencies
|
693
734
|
-- 2) move the job Id to parent "processed" set
|
@@ -701,28 +742,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
701
742
|
if ARGV[5] == "completed" then
|
702
743
|
local dependenciesSet = parentKey .. ":dependencies"
|
703
744
|
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
704
|
-
updateParentDepsIfNeeded(parentKey, parentQueueKey,
|
705
|
-
|
706
|
-
ARGV[4], timestamp)
|
745
|
+
updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet, parentId, jobIdKey, ARGV[4],
|
746
|
+
timestamp)
|
707
747
|
end
|
708
748
|
else
|
709
|
-
|
710
|
-
local unsuccesssfulSet = parentKey .. ":unsuccessful"
|
711
|
-
rcall("ZADD", unsuccesssfulSet, timestamp, jobIdKey)
|
712
|
-
moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
|
713
|
-
parentId, jobIdKey,
|
714
|
-
timestamp)
|
715
|
-
elseif opts['idof'] or opts['rdof'] then
|
716
|
-
local dependenciesSet = parentKey .. ":dependencies"
|
717
|
-
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
718
|
-
moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
|
719
|
-
parentKey, parentId, timestamp)
|
720
|
-
if opts['idof'] then
|
721
|
-
local failedSet = parentKey .. ":failed"
|
722
|
-
rcall("HSET", failedSet, jobIdKey, ARGV[4])
|
723
|
-
end
|
724
|
-
end
|
725
|
-
end
|
749
|
+
moveChildFromDependenciesIfNeeded(jobAttributes[2], jobIdKey, ARGV[4], timestamp)
|
726
750
|
end
|
727
751
|
end
|
728
752
|
local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
|
@@ -731,8 +755,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
731
755
|
local targetSet = KEYS[11]
|
732
756
|
-- Add to complete/failed set
|
733
757
|
rcall("ZADD", targetSet, timestamp, jobId)
|
734
|
-
rcall("
|
758
|
+
rcall("HSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
|
735
759
|
-- "returnvalue" / "failedReason" and "finishedOn"
|
760
|
+
if ARGV[5] == "failed" then
|
761
|
+
rcall("HDEL", jobIdKey, "defa")
|
762
|
+
end
|
736
763
|
-- Remove old jobs?
|
737
764
|
if maxAge ~= nil then
|
738
765
|
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
@@ -749,12 +776,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
749
776
|
removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3])
|
750
777
|
end
|
751
778
|
end
|
752
|
-
rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
|
753
|
-
ARGV[4])
|
779
|
+
rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3], ARGV[4], "prev", "active")
|
754
780
|
if ARGV[5] == "failed" then
|
755
781
|
if tonumber(attemptsMade) >= tonumber(attempts) then
|
756
|
-
rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId",
|
757
|
-
|
782
|
+
rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId", jobId, "attemptsMade",
|
783
|
+
attemptsMade)
|
758
784
|
end
|
759
785
|
end
|
760
786
|
-- Collect metrics
|
@@ -767,14 +793,18 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
767
793
|
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
|
768
794
|
local markerKey = KEYS[14]
|
769
795
|
-- Check if there are delayed jobs that can be promoted
|
770
|
-
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix,
|
771
|
-
|
796
|
+
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10],
|
797
|
+
isPausedOrMaxed)
|
772
798
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
773
799
|
-- Check if we are rate limited first.
|
774
800
|
local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
|
775
|
-
if expireTime > 0 then
|
801
|
+
if expireTime > 0 then
|
802
|
+
return {0, 0, expireTime, 0}
|
803
|
+
end
|
776
804
|
-- paused or maxed queue
|
777
|
-
if isPausedOrMaxed then
|
805
|
+
if isPausedOrMaxed then
|
806
|
+
return {0, 0, 0, 0}
|
807
|
+
end
|
778
808
|
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
779
809
|
if jobId then
|
780
810
|
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
@@ -783,23 +813,19 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
783
813
|
-- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
|
784
814
|
-- but if ID is 0:0, then there is at least 1 prioritized job to process
|
785
815
|
if jobId == "0:0" then
|
786
|
-
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
|
787
|
-
|
788
|
-
|
789
|
-
timestamp, maxJobs, markerKey,
|
790
|
-
opts)
|
816
|
+
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
817
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
818
|
+
markerKey, opts)
|
791
819
|
end
|
792
820
|
else
|
793
|
-
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
|
794
|
-
|
795
|
-
opts)
|
821
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
|
822
|
+
opts)
|
796
823
|
end
|
797
824
|
else
|
798
825
|
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
799
826
|
if jobId then
|
800
|
-
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
|
801
|
-
|
802
|
-
opts)
|
827
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
|
828
|
+
opts)
|
803
829
|
end
|
804
830
|
end
|
805
831
|
-- Return the timestamp for the next delayed job if any.
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAi1Bf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|