bullmq 5.66.5 → 5.66.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/commands/includes/moveChildFromDependenciesIfNeeded.lua +4 -11
- package/dist/cjs/commands/includes/removeJobsByMaxAge.lua +2 -3
- package/dist/cjs/commands/moveToFinished-14.lua +2 -1
- package/dist/cjs/interfaces/index.js +0 -1
- package/dist/cjs/interfaces/index.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +152 -186
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/types/index.js +1 -0
- package/dist/cjs/types/index.js.map +1 -1
- package/dist/{esm/interfaces → cjs/types}/keep-jobs.js.map +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/scripts.d.ts +2 -2
- package/dist/esm/commands/includes/moveChildFromDependenciesIfNeeded.lua +4 -11
- package/dist/esm/commands/includes/removeJobsByMaxAge.lua +2 -3
- package/dist/esm/commands/moveToFinished-14.lua +2 -1
- package/dist/esm/interfaces/base-job-options.d.ts +1 -1
- package/dist/esm/interfaces/index.d.ts +0 -1
- package/dist/esm/interfaces/index.js +0 -1
- package/dist/esm/interfaces/index.js.map +1 -1
- package/dist/esm/interfaces/worker-options.d.ts +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +152 -186
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/index.d.ts +1 -0
- package/dist/esm/types/index.js +1 -0
- package/dist/esm/types/index.js.map +1 -1
- package/dist/esm/{interfaces → types}/keep-jobs.d.ts +12 -3
- package/dist/{cjs/interfaces → esm/types}/keep-jobs.js.map +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +6 -6
- package/dist/cjs/commands/includes/removeJobsOnFail.lua +0 -36
- package/dist/esm/commands/includes/removeJobsOnFail.lua +0 -36
- /package/dist/cjs/{interfaces → types}/keep-jobs.js +0 -0
- /package/dist/esm/{interfaces → types}/keep-jobs.js +0 -0
|
@@ -6,28 +6,21 @@
|
|
|
6
6
|
--- @include "moveParentToWaitIfNoPendingDependencies"
|
|
7
7
|
--- @include "moveParentToWaitIfNeeded"
|
|
8
8
|
--- @include "moveParentToWait"
|
|
9
|
-
--- @include "removeJobsOnFail"
|
|
10
9
|
|
|
11
|
-
local
|
|
10
|
+
local handleChildFailureAndMoveParentToWait = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
|
12
11
|
if rcall("EXISTS", parentKey) == 1 then
|
|
13
12
|
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
|
14
13
|
local parentDelayedKey = parentQueueKey .. ":delayed"
|
|
15
|
-
local parentPrioritizedKey = parentQueueKey .. ":prioritized"
|
|
16
14
|
local parentWaitingChildrenOrDelayedKey
|
|
17
|
-
local prevState
|
|
18
15
|
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
|
19
16
|
parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
|
|
20
|
-
prevState = "waiting-children"
|
|
21
17
|
elseif rcall("ZSCORE", parentDelayedKey, parentId) then
|
|
22
18
|
parentWaitingChildrenOrDelayedKey = parentDelayedKey
|
|
23
|
-
prevState = "delayed"
|
|
24
19
|
rcall("HSET", parentKey, "delay", 0)
|
|
25
20
|
end
|
|
26
21
|
|
|
27
22
|
if parentWaitingChildrenOrDelayedKey then
|
|
28
23
|
rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
|
|
29
|
-
local parentQueuePrefix = parentQueueKey .. ":"
|
|
30
|
-
local parentFailedKey = parentQueueKey .. ":failed"
|
|
31
24
|
local deferredFailure = "child " .. jobIdKey .. " failed"
|
|
32
25
|
rcall("HSET", parentKey, "defa", deferredFailure)
|
|
33
26
|
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
|
@@ -47,9 +40,9 @@ local moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, fai
|
|
|
47
40
|
local parentDependenciesChildrenKey = parentKey .. ":dependencies"
|
|
48
41
|
if parentData['fpof'] then
|
|
49
42
|
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
|
50
|
-
local
|
|
51
|
-
rcall("ZADD",
|
|
52
|
-
|
|
43
|
+
local parentUnsuccessfulChildrenKey = parentKey .. ":unsuccessful"
|
|
44
|
+
rcall("ZADD", parentUnsuccessfulChildrenKey, timestamp, childKey)
|
|
45
|
+
handleChildFailureAndMoveParentToWait(
|
|
53
46
|
parentData['queueKey'],
|
|
54
47
|
parentKey,
|
|
55
48
|
parentData['id'],
|
|
@@ -5,10 +5,9 @@
|
|
|
5
5
|
-- Includes
|
|
6
6
|
--- @include "removeJob"
|
|
7
7
|
|
|
8
|
-
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix,
|
|
9
|
-
shouldRemoveDebounceKey)
|
|
8
|
+
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit)
|
|
10
9
|
local start = timestamp - maxAge * 1000
|
|
11
|
-
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
|
|
10
|
+
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf", "LIMIT", 0, maxLimit)
|
|
12
11
|
for i, jobId in ipairs(jobIds) do
|
|
13
12
|
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
|
14
13
|
end
|
|
@@ -107,6 +107,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
|
107
107
|
local maxMetricsSize = opts['maxMetricsSize']
|
|
108
108
|
local maxCount = opts['keepJobs']['count']
|
|
109
109
|
local maxAge = opts['keepJobs']['age']
|
|
110
|
+
local maxLimit = opts['keepJobs']['limit'] or 1000
|
|
110
111
|
|
|
111
112
|
local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid")
|
|
112
113
|
local parentKey = jobAttributes[1] or ""
|
|
@@ -175,7 +176,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
|
175
176
|
|
|
176
177
|
-- Remove old jobs?
|
|
177
178
|
if maxAge ~= nil then
|
|
178
|
-
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
|
179
|
+
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit)
|
|
179
180
|
end
|
|
180
181
|
|
|
181
182
|
if maxCount ~= nil and maxCount > 0 then
|
|
@@ -10,7 +10,6 @@ tslib_1.__exportStar(require("./flow-job"), exports);
|
|
|
10
10
|
tslib_1.__exportStar(require("./ioredis-events"), exports);
|
|
11
11
|
tslib_1.__exportStar(require("./job-json"), exports);
|
|
12
12
|
tslib_1.__exportStar(require("./job-scheduler-json"), exports);
|
|
13
|
-
tslib_1.__exportStar(require("./keep-jobs"), exports);
|
|
14
13
|
tslib_1.__exportStar(require("./lock-manager-worker-context"), exports);
|
|
15
14
|
tslib_1.__exportStar(require("./metrics-options"), exports);
|
|
16
15
|
tslib_1.__exportStar(require("./metrics"), exports);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/interfaces/index.ts"],"names":[],"mappings":";;;AAAA,6DAAmC;AACnC,4DAAkC;AAClC,6DAAmC;AACnC,0DAAgC;AAChC,uDAA6B;AAC7B,qDAA2B;AAC3B,2DAAiC;AACjC,qDAA2B;AAC3B,+DAAqC;AACrC,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/interfaces/index.ts"],"names":[],"mappings":";;;AAAA,6DAAmC;AACnC,4DAAkC;AAClC,6DAAmC;AACnC,0DAAgC;AAChC,uDAA6B;AAC7B,qDAA2B;AAC3B,2DAAiC;AACjC,qDAA2B;AAC3B,+DAAqC;AACrC,wEAA8C;AAC9C,4DAAkC;AAClC,oDAA0B;AAC1B,wDAA8B;AAC9B,0DAAgC;AAChC,2DAAiC;AACjC,mDAAyB;AACzB,2DAAiC;AACjC,uDAA6B;AAC7B,0DAAgC;AAChC,iEAAuC;AACvC,0DAAgC;AAChC,0DAAgC;AAChC,2DAAiC;AACjC,+DAAqC;AACrC,2DAAiC;AACjC,0DAAgC;AAChC,iEAAuC;AACvC,oEAA0C;AAC1C,0DAAgC;AAChC,8DAAoC;AACpC,2DAAiC;AACjC,sDAA4B;AAC5B,qDAA2B"}
|
|
@@ -292,200 +292,19 @@ local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDep
|
|
|
292
292
|
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
|
293
293
|
end
|
|
294
294
|
end
|
|
295
|
-
|
|
296
|
-
Functions to remove jobs when removeOnFail option is provided.
|
|
297
|
-
]]
|
|
298
|
-
-- Includes
|
|
299
|
-
--[[
|
|
300
|
-
Function to remove job.
|
|
301
|
-
]]
|
|
302
|
-
-- Includes
|
|
303
|
-
--[[
|
|
304
|
-
Function to remove deduplication key if needed
|
|
305
|
-
when a job is being removed.
|
|
306
|
-
]]
|
|
307
|
-
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
|
308
|
-
jobId, deduplicationId)
|
|
309
|
-
if deduplicationId then
|
|
310
|
-
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
|
311
|
-
local currentJobId = rcall('GET', deduplicationKey)
|
|
312
|
-
if currentJobId and currentJobId == jobId then
|
|
313
|
-
return rcall("DEL", deduplicationKey)
|
|
314
|
-
end
|
|
315
|
-
end
|
|
316
|
-
end
|
|
317
|
-
--[[
|
|
318
|
-
Function to remove job keys.
|
|
319
|
-
]]
|
|
320
|
-
local function removeJobKeys(jobKey)
|
|
321
|
-
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
|
322
|
-
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
|
323
|
-
end
|
|
324
|
-
--[[
|
|
325
|
-
Check if this job has a parent. If so we will just remove it from
|
|
326
|
-
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
|
327
|
-
which requires code from "moveToFinished"
|
|
328
|
-
]]
|
|
329
|
-
-- Includes
|
|
330
|
-
--[[
|
|
331
|
-
Functions to destructure job key.
|
|
332
|
-
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
|
333
|
-
]]
|
|
334
|
-
local getJobIdFromKey = function (jobKey)
|
|
335
|
-
return string.match(jobKey, ".*:(.*)")
|
|
336
|
-
end
|
|
337
|
-
local getJobKeyPrefix = function (jobKey, jobId)
|
|
338
|
-
return string.sub(jobKey, 0, #jobKey - #jobId)
|
|
339
|
-
end
|
|
340
|
-
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
|
341
|
-
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
|
342
|
-
parentPrefix .. "wait", parentPrefix .. "paused")
|
|
343
|
-
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
|
344
|
-
if emitEvent then
|
|
345
|
-
local parentEventStream = parentPrefix .. "events"
|
|
346
|
-
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
|
347
|
-
end
|
|
348
|
-
end
|
|
349
|
-
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
|
|
350
|
-
if parentKey then
|
|
351
|
-
local parentDependenciesKey = parentKey .. ":dependencies"
|
|
352
|
-
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
353
|
-
if result > 0 then
|
|
354
|
-
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
|
355
|
-
if pendingDependencies == 0 then
|
|
356
|
-
local parentId = getJobIdFromKey(parentKey)
|
|
357
|
-
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
|
358
|
-
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
|
359
|
-
if numRemovedElements == 1 then
|
|
360
|
-
if hard then -- remove parent in same queue
|
|
361
|
-
if parentPrefix == baseKey then
|
|
362
|
-
removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
|
|
363
|
-
removeJobKeys(parentKey)
|
|
364
|
-
if debounceId then
|
|
365
|
-
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
|
366
|
-
end
|
|
367
|
-
else
|
|
368
|
-
_moveParentToWait(parentPrefix, parentId)
|
|
369
|
-
end
|
|
370
|
-
else
|
|
371
|
-
_moveParentToWait(parentPrefix, parentId, true)
|
|
372
|
-
end
|
|
373
|
-
end
|
|
374
|
-
end
|
|
375
|
-
return true
|
|
376
|
-
end
|
|
377
|
-
else
|
|
378
|
-
local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
|
|
379
|
-
local missedParentKey = parentAttributes[1]
|
|
380
|
-
if( (type(missedParentKey) == "string") and missedParentKey ~= ""
|
|
381
|
-
and (rcall("EXISTS", missedParentKey) == 1)) then
|
|
382
|
-
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
|
383
|
-
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
384
|
-
if result > 0 then
|
|
385
|
-
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
|
386
|
-
if pendingDependencies == 0 then
|
|
387
|
-
local parentId = getJobIdFromKey(missedParentKey)
|
|
388
|
-
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
|
389
|
-
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
|
390
|
-
if numRemovedElements == 1 then
|
|
391
|
-
if hard then
|
|
392
|
-
if parentPrefix == baseKey then
|
|
393
|
-
removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
|
|
394
|
-
removeJobKeys(missedParentKey)
|
|
395
|
-
if parentAttributes[2] then
|
|
396
|
-
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
|
397
|
-
end
|
|
398
|
-
else
|
|
399
|
-
_moveParentToWait(parentPrefix, parentId)
|
|
400
|
-
end
|
|
401
|
-
else
|
|
402
|
-
_moveParentToWait(parentPrefix, parentId, true)
|
|
403
|
-
end
|
|
404
|
-
end
|
|
405
|
-
end
|
|
406
|
-
return true
|
|
407
|
-
end
|
|
408
|
-
end
|
|
409
|
-
end
|
|
410
|
-
return false
|
|
411
|
-
end
|
|
412
|
-
local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
413
|
-
local jobKey = baseKey .. jobId
|
|
414
|
-
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
|
415
|
-
if shouldRemoveDeduplicationKey then
|
|
416
|
-
local deduplicationId = rcall("HGET", jobKey, "deid")
|
|
417
|
-
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobId, deduplicationId)
|
|
418
|
-
end
|
|
419
|
-
removeJobKeys(jobKey)
|
|
420
|
-
end
|
|
421
|
-
--[[
|
|
422
|
-
Functions to remove jobs by max age.
|
|
423
|
-
]]
|
|
424
|
-
-- Includes
|
|
425
|
-
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix,
|
|
426
|
-
shouldRemoveDebounceKey)
|
|
427
|
-
local start = timestamp - maxAge * 1000
|
|
428
|
-
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
|
|
429
|
-
for i, jobId in ipairs(jobIds) do
|
|
430
|
-
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
|
431
|
-
end
|
|
432
|
-
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
|
|
433
|
-
end
|
|
434
|
-
--[[
|
|
435
|
-
Functions to remove jobs by max count.
|
|
436
|
-
]]
|
|
437
|
-
-- Includes
|
|
438
|
-
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
|
|
439
|
-
local start = maxCount
|
|
440
|
-
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
|
|
441
|
-
for i, jobId in ipairs(jobIds) do
|
|
442
|
-
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
|
443
|
-
end
|
|
444
|
-
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
|
|
445
|
-
end
|
|
446
|
-
local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
|
|
447
|
-
local removeOnFailType = type(opts["removeOnFail"])
|
|
448
|
-
if removeOnFailType == "number" then
|
|
449
|
-
removeJobsByMaxCount(opts["removeOnFail"],
|
|
450
|
-
failedKey, queueKeyPrefix)
|
|
451
|
-
elseif removeOnFailType == "boolean" then
|
|
452
|
-
if opts["removeOnFail"] then
|
|
453
|
-
removeJob(jobId, false, queueKeyPrefix,
|
|
454
|
-
false --[[remove debounce key]])
|
|
455
|
-
rcall("ZREM", failedKey, jobId)
|
|
456
|
-
end
|
|
457
|
-
elseif removeOnFailType ~= "nil" then
|
|
458
|
-
local maxAge = opts["removeOnFail"]["age"]
|
|
459
|
-
local maxCount = opts["removeOnFail"]["count"]
|
|
460
|
-
if maxAge ~= nil then
|
|
461
|
-
removeJobsByMaxAge(timestamp, maxAge,
|
|
462
|
-
failedKey, queueKeyPrefix)
|
|
463
|
-
end
|
|
464
|
-
if maxCount ~= nil and maxCount > 0 then
|
|
465
|
-
removeJobsByMaxCount(maxCount, failedKey,
|
|
466
|
-
queueKeyPrefix)
|
|
467
|
-
end
|
|
468
|
-
end
|
|
469
|
-
end
|
|
470
|
-
local moveParentToFailedIfNeeded = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
|
295
|
+
local handleChildFailureAndMoveParentToWait = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
|
471
296
|
if rcall("EXISTS", parentKey) == 1 then
|
|
472
297
|
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
|
473
298
|
local parentDelayedKey = parentQueueKey .. ":delayed"
|
|
474
|
-
local parentPrioritizedKey = parentQueueKey .. ":prioritized"
|
|
475
299
|
local parentWaitingChildrenOrDelayedKey
|
|
476
|
-
local prevState
|
|
477
300
|
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
|
478
301
|
parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
|
|
479
|
-
prevState = "waiting-children"
|
|
480
302
|
elseif rcall("ZSCORE", parentDelayedKey, parentId) then
|
|
481
303
|
parentWaitingChildrenOrDelayedKey = parentDelayedKey
|
|
482
|
-
prevState = "delayed"
|
|
483
304
|
rcall("HSET", parentKey, "delay", 0)
|
|
484
305
|
end
|
|
485
306
|
if parentWaitingChildrenOrDelayedKey then
|
|
486
307
|
rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
|
|
487
|
-
local parentQueuePrefix = parentQueueKey .. ":"
|
|
488
|
-
local parentFailedKey = parentQueueKey .. ":failed"
|
|
489
308
|
local deferredFailure = "child " .. jobIdKey .. " failed"
|
|
490
309
|
rcall("HSET", parentKey, "defa", deferredFailure)
|
|
491
310
|
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
|
@@ -504,9 +323,9 @@ local moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, fai
|
|
|
504
323
|
local parentDependenciesChildrenKey = parentKey .. ":dependencies"
|
|
505
324
|
if parentData['fpof'] then
|
|
506
325
|
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
|
507
|
-
local
|
|
508
|
-
rcall("ZADD",
|
|
509
|
-
|
|
326
|
+
local parentUnsuccessfulChildrenKey = parentKey .. ":unsuccessful"
|
|
327
|
+
rcall("ZADD", parentUnsuccessfulChildrenKey, timestamp, childKey)
|
|
328
|
+
handleChildFailureAndMoveParentToWait(
|
|
510
329
|
parentData['queueKey'],
|
|
511
330
|
parentKey,
|
|
512
331
|
parentData['id'],
|
|
@@ -622,6 +441,152 @@ local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
|
|
|
622
441
|
end
|
|
623
442
|
end
|
|
624
443
|
end
|
|
444
|
+
--[[
|
|
445
|
+
Function to remove job keys.
|
|
446
|
+
]]
|
|
447
|
+
local function removeJobKeys(jobKey)
|
|
448
|
+
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
|
449
|
+
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
|
450
|
+
end
|
|
451
|
+
--[[
|
|
452
|
+
Functions to remove jobs by max age.
|
|
453
|
+
]]
|
|
454
|
+
-- Includes
|
|
455
|
+
--[[
|
|
456
|
+
Function to remove job.
|
|
457
|
+
]]
|
|
458
|
+
-- Includes
|
|
459
|
+
--[[
|
|
460
|
+
Function to remove deduplication key if needed
|
|
461
|
+
when a job is being removed.
|
|
462
|
+
]]
|
|
463
|
+
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
|
464
|
+
jobId, deduplicationId)
|
|
465
|
+
if deduplicationId then
|
|
466
|
+
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
|
467
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
|
468
|
+
if currentJobId and currentJobId == jobId then
|
|
469
|
+
return rcall("DEL", deduplicationKey)
|
|
470
|
+
end
|
|
471
|
+
end
|
|
472
|
+
end
|
|
473
|
+
--[[
|
|
474
|
+
Check if this job has a parent. If so we will just remove it from
|
|
475
|
+
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
|
476
|
+
which requires code from "moveToFinished"
|
|
477
|
+
]]
|
|
478
|
+
-- Includes
|
|
479
|
+
--[[
|
|
480
|
+
Functions to destructure job key.
|
|
481
|
+
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
|
482
|
+
]]
|
|
483
|
+
local getJobIdFromKey = function (jobKey)
|
|
484
|
+
return string.match(jobKey, ".*:(.*)")
|
|
485
|
+
end
|
|
486
|
+
local getJobKeyPrefix = function (jobKey, jobId)
|
|
487
|
+
return string.sub(jobKey, 0, #jobKey - #jobId)
|
|
488
|
+
end
|
|
489
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
|
490
|
+
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
|
491
|
+
parentPrefix .. "wait", parentPrefix .. "paused")
|
|
492
|
+
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
|
493
|
+
if emitEvent then
|
|
494
|
+
local parentEventStream = parentPrefix .. "events"
|
|
495
|
+
rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
|
496
|
+
end
|
|
497
|
+
end
|
|
498
|
+
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
|
|
499
|
+
if parentKey then
|
|
500
|
+
local parentDependenciesKey = parentKey .. ":dependencies"
|
|
501
|
+
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
502
|
+
if result > 0 then
|
|
503
|
+
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
|
504
|
+
if pendingDependencies == 0 then
|
|
505
|
+
local parentId = getJobIdFromKey(parentKey)
|
|
506
|
+
local parentPrefix = getJobKeyPrefix(parentKey, parentId)
|
|
507
|
+
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
|
508
|
+
if numRemovedElements == 1 then
|
|
509
|
+
if hard then -- remove parent in same queue
|
|
510
|
+
if parentPrefix == baseKey then
|
|
511
|
+
removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
|
|
512
|
+
removeJobKeys(parentKey)
|
|
513
|
+
if debounceId then
|
|
514
|
+
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
|
515
|
+
end
|
|
516
|
+
else
|
|
517
|
+
_moveParentToWait(parentPrefix, parentId)
|
|
518
|
+
end
|
|
519
|
+
else
|
|
520
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
|
521
|
+
end
|
|
522
|
+
end
|
|
523
|
+
end
|
|
524
|
+
return true
|
|
525
|
+
end
|
|
526
|
+
else
|
|
527
|
+
local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
|
|
528
|
+
local missedParentKey = parentAttributes[1]
|
|
529
|
+
if( (type(missedParentKey) == "string") and missedParentKey ~= ""
|
|
530
|
+
and (rcall("EXISTS", missedParentKey) == 1)) then
|
|
531
|
+
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
|
532
|
+
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
533
|
+
if result > 0 then
|
|
534
|
+
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
|
535
|
+
if pendingDependencies == 0 then
|
|
536
|
+
local parentId = getJobIdFromKey(missedParentKey)
|
|
537
|
+
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
|
538
|
+
local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
|
539
|
+
if numRemovedElements == 1 then
|
|
540
|
+
if hard then
|
|
541
|
+
if parentPrefix == baseKey then
|
|
542
|
+
removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
|
|
543
|
+
removeJobKeys(missedParentKey)
|
|
544
|
+
if parentAttributes[2] then
|
|
545
|
+
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
|
546
|
+
end
|
|
547
|
+
else
|
|
548
|
+
_moveParentToWait(parentPrefix, parentId)
|
|
549
|
+
end
|
|
550
|
+
else
|
|
551
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
|
552
|
+
end
|
|
553
|
+
end
|
|
554
|
+
end
|
|
555
|
+
return true
|
|
556
|
+
end
|
|
557
|
+
end
|
|
558
|
+
end
|
|
559
|
+
return false
|
|
560
|
+
end
|
|
561
|
+
local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
562
|
+
local jobKey = baseKey .. jobId
|
|
563
|
+
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
|
564
|
+
if shouldRemoveDeduplicationKey then
|
|
565
|
+
local deduplicationId = rcall("HGET", jobKey, "deid")
|
|
566
|
+
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobId, deduplicationId)
|
|
567
|
+
end
|
|
568
|
+
removeJobKeys(jobKey)
|
|
569
|
+
end
|
|
570
|
+
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit)
|
|
571
|
+
local start = timestamp - maxAge * 1000
|
|
572
|
+
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf", "LIMIT", 0, maxLimit)
|
|
573
|
+
for i, jobId in ipairs(jobIds) do
|
|
574
|
+
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
|
575
|
+
end
|
|
576
|
+
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
|
|
577
|
+
end
|
|
578
|
+
--[[
|
|
579
|
+
Functions to remove jobs by max count.
|
|
580
|
+
]]
|
|
581
|
+
-- Includes
|
|
582
|
+
local function removeJobsByMaxCount(maxCount, targetSet, prefix)
|
|
583
|
+
local start = maxCount
|
|
584
|
+
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
|
|
585
|
+
for i, jobId in ipairs(jobIds) do
|
|
586
|
+
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
|
587
|
+
end
|
|
588
|
+
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
|
|
589
|
+
end
|
|
625
590
|
local function removeLock(jobKey, stalledKey, token, jobId)
|
|
626
591
|
if token ~= "0" then
|
|
627
592
|
local lockKey = jobKey .. ':lock'
|
|
@@ -708,6 +673,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
|
708
673
|
local maxMetricsSize = opts['maxMetricsSize']
|
|
709
674
|
local maxCount = opts['keepJobs']['count']
|
|
710
675
|
local maxAge = opts['keepJobs']['age']
|
|
676
|
+
local maxLimit = opts['keepJobs']['limit'] or 1000
|
|
711
677
|
local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid")
|
|
712
678
|
local parentKey = jobAttributes[1] or ""
|
|
713
679
|
local parentId = ""
|
|
@@ -763,7 +729,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
|
763
729
|
end
|
|
764
730
|
-- Remove old jobs?
|
|
765
731
|
if maxAge ~= nil then
|
|
766
|
-
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
|
732
|
+
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix, maxLimit)
|
|
767
733
|
end
|
|
768
734
|
if maxCount ~= nil and maxCount > 0 then
|
|
769
735
|
removeJobsByMaxCount(maxCount, targetSet, prefix)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
|
1
|
+
{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkzBf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|