bullmq 5.58.9 → 5.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/child-pool.js +8 -1
- package/dist/cjs/classes/child-pool.js.map +1 -1
- package/dist/cjs/classes/queue.js +9 -0
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/scripts.js +4 -1
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +42 -30
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/changePriority-7.lua +1 -1
- package/dist/cjs/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
- package/dist/cjs/commands/includes/deduplicateJob.lua +53 -20
- package/dist/cjs/commands/includes/getTargetQueueList.lua +5 -5
- package/dist/cjs/commands/includes/prepareJobForProcessing.lua +2 -4
- package/dist/cjs/commands/moveJobFromActiveToWait-9.lua +1 -1
- package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -1
- package/dist/cjs/commands/moveToActive-11.lua +7 -4
- package/dist/cjs/commands/moveToDelayed-8.lua +2 -1
- package/dist/cjs/commands/moveToFinished-14.lua +10 -7
- package/dist/cjs/scripts/addDelayedJob-6.js +52 -23
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
- package/dist/cjs/scripts/addParentJob-5.js +52 -23
- package/dist/cjs/scripts/addParentJob-5.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-9.js +52 -23
- package/dist/cjs/scripts/addPrioritizedJob-9.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
- package/dist/cjs/scripts/addStandardJob-9.js +52 -23
- package/dist/cjs/scripts/addStandardJob-9.js.map +1 -1
- package/dist/cjs/scripts/changePriority-7.js +5 -5
- package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
- package/dist/cjs/scripts/drain-5.js +5 -5
- package/dist/cjs/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
- package/dist/cjs/scripts/getRateLimitTtl-2.js.map +1 -0
- package/dist/cjs/scripts/index.js +1 -1
- package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +6 -6
- package/dist/cjs/scripts/moveJobsToWait-8.js +5 -5
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js +9 -9
- package/dist/cjs/scripts/moveToActive-11.js +13 -12
- package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
- package/dist/cjs/scripts/moveToDelayed-8.js +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +16 -15
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +5 -5
- package/dist/cjs/scripts/promote-9.js +5 -5
- package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
- package/dist/cjs/scripts/removeJob-2.js +5 -5
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js +5 -5
- package/dist/cjs/scripts/reprocessJob-8.js +5 -5
- package/dist/cjs/scripts/retryJob-11.js +5 -5
- package/dist/cjs/scripts/updateJobScheduler-12.js +5 -5
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/child-pool.js +8 -1
- package/dist/esm/classes/child-pool.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +6 -0
- package/dist/esm/classes/queue.js +9 -0
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/scripts.js +4 -1
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +1 -0
- package/dist/esm/classes/worker.js +42 -30
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/changePriority-7.lua +1 -1
- package/dist/esm/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
- package/dist/esm/commands/includes/deduplicateJob.lua +53 -20
- package/dist/esm/commands/includes/getTargetQueueList.lua +5 -5
- package/dist/esm/commands/includes/prepareJobForProcessing.lua +2 -4
- package/dist/esm/commands/moveJobFromActiveToWait-9.lua +1 -1
- package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -1
- package/dist/esm/commands/moveToActive-11.lua +7 -4
- package/dist/esm/commands/moveToDelayed-8.lua +2 -1
- package/dist/esm/commands/moveToFinished-14.lua +10 -7
- package/dist/esm/interfaces/base-job-options.d.ts +1 -1
- package/dist/esm/interfaces/worker-options.d.ts +8 -0
- package/dist/esm/scripts/addDelayedJob-6.js +52 -23
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addJobScheduler-11.js +5 -5
- package/dist/esm/scripts/addParentJob-5.js +52 -23
- package/dist/esm/scripts/addParentJob-5.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-9.js +52 -23
- package/dist/esm/scripts/addPrioritizedJob-9.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
- package/dist/esm/scripts/addStandardJob-9.js +52 -23
- package/dist/esm/scripts/addStandardJob-9.js.map +1 -1
- package/dist/esm/scripts/changePriority-7.js +5 -5
- package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
- package/dist/esm/scripts/drain-5.js +5 -5
- package/dist/esm/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
- package/dist/esm/scripts/getRateLimitTtl-2.js.map +1 -0
- package/dist/esm/scripts/index.d.ts +1 -1
- package/dist/esm/scripts/index.js +1 -1
- package/dist/esm/scripts/moveJobFromActiveToWait-9.js +6 -6
- package/dist/esm/scripts/moveJobsToWait-8.js +5 -5
- package/dist/esm/scripts/moveStalledJobsToWait-8.js +9 -9
- package/dist/esm/scripts/moveToActive-11.js +13 -12
- package/dist/esm/scripts/moveToActive-11.js.map +1 -1
- package/dist/esm/scripts/moveToDelayed-8.js +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +16 -15
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +5 -5
- package/dist/esm/scripts/promote-9.js +5 -5
- package/dist/esm/scripts/removeChildDependency-1.js +5 -5
- package/dist/esm/scripts/removeJob-2.js +5 -5
- package/dist/esm/scripts/removeUnprocessedChildren-2.js +5 -5
- package/dist/esm/scripts/reprocessJob-8.js +5 -5
- package/dist/esm/scripts/retryJob-11.js +5 -5
- package/dist/esm/scripts/updateJobScheduler-12.js +5 -5
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
- package/dist/cjs/scripts/getRateLimitTtl-1.js.map +0 -1
- package/dist/esm/scripts/getRateLimitTtl-1.js.map +0 -1
- /package/dist/esm/scripts/{getRateLimitTtl-1.d.ts → getRateLimitTtl-2.d.ts} +0 -0
|
@@ -50,7 +50,7 @@ if rcall("EXISTS", jobKey) == 1 then
|
|
|
50
50
|
local prioritizedKey = KEYS[4]
|
|
51
51
|
local priorityCounterKey = KEYS[6]
|
|
52
52
|
local markerKey = KEYS[7]
|
|
53
|
-
|
|
53
|
+
|
|
54
54
|
-- Re-add with the new priority
|
|
55
55
|
if rcall("ZREM", prioritizedKey, jobId) > 0 then
|
|
56
56
|
reAddJobWithNewPriority( prioritizedKey, markerKey, target,
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
Input:
|
|
5
5
|
KEYS[1] 'limiter'
|
|
6
|
+
KEYS[2] 'meta'
|
|
6
7
|
|
|
7
8
|
ARGV[1] maxJobs
|
|
8
9
|
]]
|
|
@@ -16,5 +17,10 @@ local rateLimiterKey = KEYS[1]
|
|
|
16
17
|
if ARGV[1] ~= "0" then
|
|
17
18
|
return getRateLimitTTL(tonumber(ARGV[1]), rateLimiterKey)
|
|
18
19
|
else
|
|
20
|
+
local rateLimitMax = rcall("HGET", KEYS[2], "max")
|
|
21
|
+
if rateLimitMax then
|
|
22
|
+
return getRateLimitTTL(tonumber(rateLimitMax), rateLimiterKey)
|
|
23
|
+
end
|
|
24
|
+
|
|
19
25
|
return rcall("PTTL", rateLimiterKey)
|
|
20
26
|
end
|
|
@@ -3,39 +3,71 @@
|
|
|
3
3
|
]]
|
|
4
4
|
-- Includes
|
|
5
5
|
--- @include "removeJobKeys"
|
|
6
|
+
|
|
7
|
+
local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents, currentDeduplicatedJobId,
|
|
8
|
+
jobId, deduplicationId, prefix)
|
|
9
|
+
if rcall("ZREM", delayedKey, currentDeduplicatedJobId) > 0 then
|
|
10
|
+
removeJobKeys(prefix .. currentDeduplicatedJobId)
|
|
11
|
+
rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDeduplicatedJobId,
|
|
12
|
+
"prev", "delayed")
|
|
13
|
+
|
|
14
|
+
-- TODO remove debounced event in next breaking change
|
|
15
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
|
|
16
|
+
jobId, "debounceId", deduplicationId)
|
|
17
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
|
18
|
+
jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDeduplicatedJobId)
|
|
19
|
+
|
|
20
|
+
return true
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
return false
|
|
24
|
+
end
|
|
25
|
+
|
|
6
26
|
local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
7
27
|
prefix)
|
|
8
28
|
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
|
9
29
|
if deduplicationId then
|
|
10
30
|
local ttl = deduplicationOpts['ttl']
|
|
11
|
-
if deduplicationOpts['replace']
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
if
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
31
|
+
if deduplicationOpts['replace'] then
|
|
32
|
+
if ttl and ttl > 0 then
|
|
33
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
|
34
|
+
if currentDebounceJobId then
|
|
35
|
+
local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
36
|
+
currentDebounceJobId, jobId, deduplicationId, prefix)
|
|
37
|
+
if isRemoved then
|
|
38
|
+
if deduplicationOpts['extend'] then
|
|
39
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
|
40
|
+
else
|
|
41
|
+
rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
|
|
42
|
+
end
|
|
43
|
+
return
|
|
21
44
|
else
|
|
22
|
-
|
|
45
|
+
return currentDebounceJobId
|
|
23
46
|
end
|
|
24
|
-
|
|
25
|
-
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
|
26
|
-
jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
|
|
27
|
-
return
|
|
28
47
|
else
|
|
29
|
-
|
|
48
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
|
49
|
+
return
|
|
30
50
|
end
|
|
31
51
|
else
|
|
32
|
-
rcall('
|
|
33
|
-
|
|
52
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
|
53
|
+
if currentDebounceJobId then
|
|
54
|
+
local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
55
|
+
currentDebounceJobId, jobId, deduplicationId, prefix)
|
|
56
|
+
|
|
57
|
+
if isRemoved then
|
|
58
|
+
rcall('SET', deduplicationKey, jobId)
|
|
59
|
+
return
|
|
60
|
+
else
|
|
61
|
+
return currentDebounceJobId
|
|
62
|
+
end
|
|
63
|
+
else
|
|
64
|
+
rcall('SET', deduplicationKey, jobId)
|
|
65
|
+
return
|
|
66
|
+
end
|
|
34
67
|
end
|
|
35
68
|
else
|
|
36
|
-
local ttl = deduplicationOpts['ttl']
|
|
37
69
|
local deduplicationKeyExists
|
|
38
|
-
if ttl then
|
|
70
|
+
if ttl and ttl > 0 then
|
|
39
71
|
if deduplicationOpts['extend'] then
|
|
40
72
|
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
|
41
73
|
if currentDebounceJobId then
|
|
@@ -58,6 +90,7 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
58
90
|
|
|
59
91
|
if deduplicationKeyExists then
|
|
60
92
|
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
|
93
|
+
-- TODO remove debounced event in next breaking change
|
|
61
94
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
|
|
62
95
|
currentDebounceJobId, "debounceId", deduplicationId)
|
|
63
96
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
|
@@ -4,19 +4,19 @@
|
|
|
4
4
|
]]
|
|
5
5
|
|
|
6
6
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
7
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
7
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
8
8
|
|
|
9
9
|
if queueAttributes[1] then
|
|
10
|
-
return pausedKey, true
|
|
10
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
11
11
|
else
|
|
12
12
|
if queueAttributes[2] then
|
|
13
13
|
local activeCount = rcall("LLEN", activeKey)
|
|
14
14
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
15
|
-
return waitKey, true
|
|
15
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
16
16
|
else
|
|
17
|
-
return waitKey, false
|
|
17
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
18
18
|
end
|
|
19
19
|
end
|
|
20
20
|
end
|
|
21
|
-
return waitKey, false
|
|
21
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
22
22
|
end
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
--- @include "addBaseMarkerIfNeeded"
|
|
11
11
|
|
|
12
12
|
local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
|
|
13
|
-
jobId, processedOn, maxJobs, markerKey, opts)
|
|
13
|
+
jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
|
|
14
14
|
local jobKey = keyPrefix .. jobId
|
|
15
15
|
|
|
16
16
|
-- Check if we need to perform rate limiting.
|
|
@@ -18,16 +18,14 @@ local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey
|
|
|
18
18
|
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
|
19
19
|
|
|
20
20
|
if jobCounter == 1 then
|
|
21
|
-
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
|
|
22
21
|
local integerDuration = math.floor(math.abs(limiterDuration))
|
|
23
22
|
rcall("PEXPIRE", rateLimiterKey, integerDuration)
|
|
24
23
|
end
|
|
25
24
|
end
|
|
26
25
|
|
|
27
|
-
local lockKey = jobKey .. ':lock'
|
|
28
|
-
|
|
29
26
|
-- get a lock
|
|
30
27
|
if opts['token'] ~= "0" then
|
|
28
|
+
local lockKey = jobKey .. ':lock'
|
|
31
29
|
rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
|
|
32
30
|
end
|
|
33
31
|
|
|
@@ -51,14 +51,15 @@ local opts = cmsgpack.unpack(ARGV[3])
|
|
|
51
51
|
--- @include "includes/prepareJobForProcessing"
|
|
52
52
|
--- @include "includes/promoteDelayedJobs"
|
|
53
53
|
|
|
54
|
-
local target, isPausedOrMaxed = getTargetQueueList(KEYS[9],
|
|
54
|
+
local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(KEYS[9],
|
|
55
|
+
activeKey, waitKey, KEYS[8])
|
|
55
56
|
|
|
56
57
|
-- Check if there are delayed jobs that we can move to wait.
|
|
57
58
|
local markerKey = KEYS[11]
|
|
58
59
|
promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
|
|
59
60
|
ARGV[2], KEYS[10], isPausedOrMaxed)
|
|
60
61
|
|
|
61
|
-
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
|
62
|
+
local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
|
|
62
63
|
local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
|
|
63
64
|
|
|
64
65
|
-- Check if we are rate limited first.
|
|
@@ -67,6 +68,8 @@ if expireTime > 0 then return {0, 0, expireTime, 0} end
|
|
|
67
68
|
-- paused or maxed queue
|
|
68
69
|
if isPausedOrMaxed then return {0, 0, 0, 0} end
|
|
69
70
|
|
|
71
|
+
local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
|
|
72
|
+
|
|
70
73
|
-- no job ID, try non-blocking move from wait to active
|
|
71
74
|
local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
|
72
75
|
|
|
@@ -78,12 +81,12 @@ end
|
|
|
78
81
|
|
|
79
82
|
if jobId then
|
|
80
83
|
return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
|
|
81
|
-
maxJobs, markerKey, opts)
|
|
84
|
+
maxJobs, limiterDuration, markerKey, opts)
|
|
82
85
|
else
|
|
83
86
|
jobId = moveJobFromPrioritizedToActive(KEYS[3], activeKey, KEYS[10])
|
|
84
87
|
if jobId then
|
|
85
88
|
return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
|
|
86
|
-
maxJobs, markerKey, opts)
|
|
89
|
+
maxJobs, limiterDuration, markerKey, opts)
|
|
87
90
|
end
|
|
88
91
|
end
|
|
89
92
|
|
|
@@ -50,11 +50,12 @@ if rcall("EXISTS", jobKey) == 1 then
|
|
|
50
50
|
local delayedKey = KEYS[4]
|
|
51
51
|
local jobId = ARGV[3]
|
|
52
52
|
local delay = tonumber(ARGV[5])
|
|
53
|
-
local score, delayedTimestamp = getDelayedScore(delayedKey, ARGV[2], delay)
|
|
54
53
|
|
|
55
54
|
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
|
56
55
|
if numRemovedElements < 1 then return -3 end
|
|
57
56
|
|
|
57
|
+
local score, delayedTimestamp = getDelayedScore(delayedKey, ARGV[2], delay)
|
|
58
|
+
|
|
58
59
|
if ARGV[6] == "0" then
|
|
59
60
|
rcall("HINCRBY", jobKey, "atm", 1)
|
|
60
61
|
end
|
|
@@ -209,14 +209,15 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
|
209
209
|
-- and not rate limited.
|
|
210
210
|
if (ARGV[6] == "1") then
|
|
211
211
|
|
|
212
|
-
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2],
|
|
212
|
+
local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(metaKey, KEYS[2],
|
|
213
|
+
KEYS[1], KEYS[8])
|
|
213
214
|
|
|
214
215
|
local markerKey = KEYS[14]
|
|
215
216
|
-- Check if there are delayed jobs that can be promoted
|
|
216
217
|
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10],
|
|
217
218
|
isPausedOrMaxed)
|
|
218
219
|
|
|
219
|
-
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
|
220
|
+
local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
|
|
220
221
|
-- Check if we are rate limited first.
|
|
221
222
|
local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
|
|
222
223
|
|
|
@@ -229,6 +230,8 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
|
229
230
|
return {0, 0, 0, 0}
|
|
230
231
|
end
|
|
231
232
|
|
|
233
|
+
local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
|
|
234
|
+
|
|
232
235
|
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
|
233
236
|
|
|
234
237
|
if jobId then
|
|
@@ -241,17 +244,17 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
|
241
244
|
if jobId == "0:0" then
|
|
242
245
|
jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
|
|
243
246
|
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
|
244
|
-
markerKey, opts)
|
|
247
|
+
limiterDuration, markerKey, opts)
|
|
245
248
|
end
|
|
246
249
|
else
|
|
247
|
-
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
|
248
|
-
opts)
|
|
250
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
|
251
|
+
limiterDuration, markerKey, opts)
|
|
249
252
|
end
|
|
250
253
|
else
|
|
251
254
|
jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
|
|
252
255
|
if jobId then
|
|
253
|
-
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
|
254
|
-
opts)
|
|
256
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
|
257
|
+
limiterDuration, markerKey, opts)
|
|
255
258
|
end
|
|
256
259
|
end
|
|
257
260
|
|
|
@@ -123,37 +123,65 @@ local function removeJobKeys(jobKey)
|
|
|
123
123
|
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
|
124
124
|
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
|
125
125
|
end
|
|
126
|
+
local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents, currentDeduplicatedJobId,
|
|
127
|
+
jobId, deduplicationId, prefix)
|
|
128
|
+
if rcall("ZREM", delayedKey, currentDeduplicatedJobId) > 0 then
|
|
129
|
+
removeJobKeys(prefix .. currentDeduplicatedJobId)
|
|
130
|
+
rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDeduplicatedJobId,
|
|
131
|
+
"prev", "delayed")
|
|
132
|
+
-- TODO remove debounced event in next breaking change
|
|
133
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
|
|
134
|
+
jobId, "debounceId", deduplicationId)
|
|
135
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
|
136
|
+
jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDeduplicatedJobId)
|
|
137
|
+
return true
|
|
138
|
+
end
|
|
139
|
+
return false
|
|
140
|
+
end
|
|
126
141
|
local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
127
142
|
prefix)
|
|
128
143
|
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
|
129
144
|
if deduplicationId then
|
|
130
145
|
local ttl = deduplicationOpts['ttl']
|
|
131
|
-
if deduplicationOpts['replace']
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
if
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
146
|
+
if deduplicationOpts['replace'] then
|
|
147
|
+
if ttl and ttl > 0 then
|
|
148
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
|
149
|
+
if currentDebounceJobId then
|
|
150
|
+
local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
151
|
+
currentDebounceJobId, jobId, deduplicationId, prefix)
|
|
152
|
+
if isRemoved then
|
|
153
|
+
if deduplicationOpts['extend'] then
|
|
154
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
|
155
|
+
else
|
|
156
|
+
rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
|
|
157
|
+
end
|
|
158
|
+
return
|
|
140
159
|
else
|
|
141
|
-
|
|
160
|
+
return currentDebounceJobId
|
|
142
161
|
end
|
|
143
|
-
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
|
144
|
-
jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
|
|
145
|
-
return
|
|
146
162
|
else
|
|
147
|
-
|
|
163
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
|
164
|
+
return
|
|
148
165
|
end
|
|
149
166
|
else
|
|
150
|
-
rcall('
|
|
151
|
-
|
|
167
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
|
168
|
+
if currentDebounceJobId then
|
|
169
|
+
local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
170
|
+
currentDebounceJobId, jobId, deduplicationId, prefix)
|
|
171
|
+
if isRemoved then
|
|
172
|
+
rcall('SET', deduplicationKey, jobId)
|
|
173
|
+
return
|
|
174
|
+
else
|
|
175
|
+
return currentDebounceJobId
|
|
176
|
+
end
|
|
177
|
+
else
|
|
178
|
+
rcall('SET', deduplicationKey, jobId)
|
|
179
|
+
return
|
|
180
|
+
end
|
|
152
181
|
end
|
|
153
182
|
else
|
|
154
|
-
local ttl = deduplicationOpts['ttl']
|
|
155
183
|
local deduplicationKeyExists
|
|
156
|
-
if ttl then
|
|
184
|
+
if ttl and ttl > 0 then
|
|
157
185
|
if deduplicationOpts['extend'] then
|
|
158
186
|
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
|
159
187
|
if currentDebounceJobId then
|
|
@@ -175,6 +203,7 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
175
203
|
end
|
|
176
204
|
if deduplicationKeyExists then
|
|
177
205
|
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
|
206
|
+
-- TODO remove debounced event in next breaking change
|
|
178
207
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
|
|
179
208
|
currentDebounceJobId, "debounceId", deduplicationId)
|
|
180
209
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
|
@@ -277,20 +306,20 @@ end
|
|
|
277
306
|
(since an empty list and !EXISTS are not really the same).
|
|
278
307
|
]]
|
|
279
308
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
280
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
309
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
281
310
|
if queueAttributes[1] then
|
|
282
|
-
return pausedKey, true
|
|
311
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
283
312
|
else
|
|
284
313
|
if queueAttributes[2] then
|
|
285
314
|
local activeCount = rcall("LLEN", activeKey)
|
|
286
315
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
287
|
-
return waitKey, true
|
|
316
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
288
317
|
else
|
|
289
|
-
return waitKey, false
|
|
318
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
290
319
|
end
|
|
291
320
|
end
|
|
292
321
|
end
|
|
293
|
-
return waitKey, false
|
|
322
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
294
323
|
end
|
|
295
324
|
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
|
296
325
|
local parentWaitKey = parentQueueKey .. ":wait"
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
|
1
|
+
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAudf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
|
@@ -182,20 +182,20 @@ end
|
|
|
182
182
|
(since an empty list and !EXISTS are not really the same).
|
|
183
183
|
]]
|
|
184
184
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
185
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
185
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
186
186
|
if queueAttributes[1] then
|
|
187
|
-
return pausedKey, true
|
|
187
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
188
188
|
else
|
|
189
189
|
if queueAttributes[2] then
|
|
190
190
|
local activeCount = rcall("LLEN", activeKey)
|
|
191
191
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
192
|
-
return waitKey, true
|
|
192
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
193
193
|
else
|
|
194
|
-
return waitKey, false
|
|
194
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
195
195
|
end
|
|
196
196
|
end
|
|
197
197
|
end
|
|
198
|
-
return waitKey, false
|
|
198
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
199
199
|
end
|
|
200
200
|
--[[
|
|
201
201
|
Function to add job in target list and add marker if needed.
|
|
@@ -56,37 +56,65 @@ local function removeJobKeys(jobKey)
|
|
|
56
56
|
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
|
57
57
|
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
|
58
58
|
end
|
|
59
|
+
local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents, currentDeduplicatedJobId,
|
|
60
|
+
jobId, deduplicationId, prefix)
|
|
61
|
+
if rcall("ZREM", delayedKey, currentDeduplicatedJobId) > 0 then
|
|
62
|
+
removeJobKeys(prefix .. currentDeduplicatedJobId)
|
|
63
|
+
rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDeduplicatedJobId,
|
|
64
|
+
"prev", "delayed")
|
|
65
|
+
-- TODO remove debounced event in next breaking change
|
|
66
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
|
|
67
|
+
jobId, "debounceId", deduplicationId)
|
|
68
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
|
69
|
+
jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDeduplicatedJobId)
|
|
70
|
+
return true
|
|
71
|
+
end
|
|
72
|
+
return false
|
|
73
|
+
end
|
|
59
74
|
local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
60
75
|
prefix)
|
|
61
76
|
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
|
62
77
|
if deduplicationId then
|
|
63
78
|
local ttl = deduplicationOpts['ttl']
|
|
64
|
-
if deduplicationOpts['replace']
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
if
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
79
|
+
if deduplicationOpts['replace'] then
|
|
80
|
+
if ttl and ttl > 0 then
|
|
81
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
|
82
|
+
if currentDebounceJobId then
|
|
83
|
+
local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
84
|
+
currentDebounceJobId, jobId, deduplicationId, prefix)
|
|
85
|
+
if isRemoved then
|
|
86
|
+
if deduplicationOpts['extend'] then
|
|
87
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
|
88
|
+
else
|
|
89
|
+
rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
|
|
90
|
+
end
|
|
91
|
+
return
|
|
73
92
|
else
|
|
74
|
-
|
|
93
|
+
return currentDebounceJobId
|
|
75
94
|
end
|
|
76
|
-
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
|
77
|
-
jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
|
|
78
|
-
return
|
|
79
95
|
else
|
|
80
|
-
|
|
96
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
|
97
|
+
return
|
|
81
98
|
end
|
|
82
99
|
else
|
|
83
|
-
rcall('
|
|
84
|
-
|
|
100
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
|
101
|
+
if currentDebounceJobId then
|
|
102
|
+
local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
|
|
103
|
+
currentDebounceJobId, jobId, deduplicationId, prefix)
|
|
104
|
+
if isRemoved then
|
|
105
|
+
rcall('SET', deduplicationKey, jobId)
|
|
106
|
+
return
|
|
107
|
+
else
|
|
108
|
+
return currentDebounceJobId
|
|
109
|
+
end
|
|
110
|
+
else
|
|
111
|
+
rcall('SET', deduplicationKey, jobId)
|
|
112
|
+
return
|
|
113
|
+
end
|
|
85
114
|
end
|
|
86
115
|
else
|
|
87
|
-
local ttl = deduplicationOpts['ttl']
|
|
88
116
|
local deduplicationKeyExists
|
|
89
|
-
if ttl then
|
|
117
|
+
if ttl and ttl > 0 then
|
|
90
118
|
if deduplicationOpts['extend'] then
|
|
91
119
|
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
|
92
120
|
if currentDebounceJobId then
|
|
@@ -108,6 +136,7 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
|
|
|
108
136
|
end
|
|
109
137
|
if deduplicationKeyExists then
|
|
110
138
|
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
|
139
|
+
-- TODO remove debounced event in next breaking change
|
|
111
140
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
|
|
112
141
|
currentDebounceJobId, "debounceId", deduplicationId)
|
|
113
142
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
|
@@ -234,20 +263,20 @@ end
|
|
|
234
263
|
(since an empty list and !EXISTS are not really the same).
|
|
235
264
|
]]
|
|
236
265
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
237
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
266
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
238
267
|
if queueAttributes[1] then
|
|
239
|
-
return pausedKey, true
|
|
268
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
240
269
|
else
|
|
241
270
|
if queueAttributes[2] then
|
|
242
271
|
local activeCount = rcall("LLEN", activeKey)
|
|
243
272
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
244
|
-
return waitKey, true
|
|
273
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
245
274
|
else
|
|
246
|
-
return waitKey, false
|
|
275
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
247
276
|
end
|
|
248
277
|
end
|
|
249
278
|
end
|
|
250
|
-
return waitKey, false
|
|
279
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
251
280
|
end
|
|
252
281
|
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
|
253
282
|
local parentWaitKey = parentQueueKey .. ":wait"
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"addParentJob-5.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-5.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
|
1
|
+
{"version":3,"file":"addParentJob-5.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-5.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+af,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|