bullmq 5.10.4 → 5.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/job.js +6 -0
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js +2 -1
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/queue.js +9 -0
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/scripts.js +1 -0
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/commands/addDelayedJob-6.lua +11 -1
- package/dist/cjs/commands/addParentJob-4.lua +10 -1
- package/dist/cjs/commands/addPrioritizedJob-8.lua +11 -2
- package/dist/cjs/commands/addStandardJob-8.lua +11 -2
- package/dist/cjs/commands/includes/cleanList.lua +1 -1
- package/dist/cjs/commands/includes/cleanSet.lua +2 -2
- package/dist/cjs/commands/includes/debounceJob.lua +23 -0
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +6 -3
- package/dist/cjs/commands/includes/removeDebounceKey.lua +12 -0
- package/dist/cjs/commands/includes/removeDebounceKeyIfNeeded.lua +14 -0
- package/dist/cjs/commands/includes/removeJob.lua +5 -1
- package/dist/cjs/commands/includes/removeJobs.lua +1 -1
- package/dist/cjs/commands/includes/removeJobsByMaxAge.lua +3 -2
- package/dist/cjs/commands/includes/removeJobsByMaxCount.lua +1 -1
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +11 -4
- package/dist/cjs/commands/includes/storeJob.lua +7 -1
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +6 -2
- package/dist/cjs/commands/moveToFinished-14.lua +14 -11
- package/dist/cjs/commands/removeChildDependency-1.lua +1 -1
- package/dist/cjs/commands/removeJob-2.lua +3 -1
- package/dist/cjs/interfaces/debounce-options.js +3 -0
- package/dist/cjs/interfaces/debounce-options.js.map +1 -0
- package/dist/cjs/interfaces/index.js +1 -0
- package/dist/cjs/interfaces/index.js.map +1 -1
- package/dist/cjs/scripts/addDelayedJob-6.js +34 -1
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +34 -1
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js +34 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/cjs/scripts/addStandardJob-8.js +34 -1
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
- package/dist/cjs/scripts/cleanJobsInSet-2.js +28 -8
- package/dist/cjs/scripts/cleanJobsInSet-2.js.map +1 -1
- package/dist/cjs/scripts/drain-4.js +26 -6
- package/dist/cjs/scripts/drain-4.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +49 -13
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +56 -21
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +26 -6
- package/dist/cjs/scripts/obliterate-2.js.map +1 -1
- package/dist/cjs/scripts/removeChildDependency-1.js +12 -5
- package/dist/cjs/scripts/removeChildDependency-1.js.map +1 -1
- package/dist/cjs/scripts/removeJob-2.js +23 -5
- package/dist/cjs/scripts/removeJob-2.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/esm/classes/job.d.ts +4 -0
- package/dist/esm/classes/job.js +6 -0
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-events.d.ts +9 -0
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-keys.js +2 -1
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +6 -0
- package/dist/esm/classes/queue.js +9 -0
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/scripts.js +1 -0
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/commands/addDelayedJob-6.lua +11 -1
- package/dist/esm/commands/addParentJob-4.lua +10 -1
- package/dist/esm/commands/addPrioritizedJob-8.lua +11 -2
- package/dist/esm/commands/addStandardJob-8.lua +11 -2
- package/dist/esm/commands/includes/cleanList.lua +1 -1
- package/dist/esm/commands/includes/cleanSet.lua +2 -2
- package/dist/esm/commands/includes/debounceJob.lua +23 -0
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +6 -3
- package/dist/esm/commands/includes/removeDebounceKey.lua +12 -0
- package/dist/esm/commands/includes/removeDebounceKeyIfNeeded.lua +14 -0
- package/dist/esm/commands/includes/removeJob.lua +5 -1
- package/dist/esm/commands/includes/removeJobs.lua +1 -1
- package/dist/esm/commands/includes/removeJobsByMaxAge.lua +3 -2
- package/dist/esm/commands/includes/removeJobsByMaxCount.lua +1 -1
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +11 -4
- package/dist/esm/commands/includes/storeJob.lua +7 -1
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +6 -2
- package/dist/esm/commands/moveToFinished-14.lua +14 -11
- package/dist/esm/commands/removeChildDependency-1.lua +1 -1
- package/dist/esm/commands/removeJob-2.lua +3 -1
- package/dist/esm/interfaces/debounce-options.d.ts +13 -0
- package/dist/esm/interfaces/debounce-options.js +2 -0
- package/dist/esm/interfaces/debounce-options.js.map +1 -0
- package/dist/esm/interfaces/index.d.ts +1 -0
- package/dist/esm/interfaces/index.js +1 -0
- package/dist/esm/interfaces/index.js.map +1 -1
- package/dist/esm/interfaces/job-json.d.ts +2 -0
- package/dist/esm/scripts/addDelayedJob-6.js +34 -1
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addParentJob-4.js +34 -1
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js +34 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/esm/scripts/addStandardJob-8.js +34 -1
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
- package/dist/esm/scripts/cleanJobsInSet-2.js +28 -8
- package/dist/esm/scripts/cleanJobsInSet-2.js.map +1 -1
- package/dist/esm/scripts/drain-4.js +26 -6
- package/dist/esm/scripts/drain-4.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +49 -13
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +56 -21
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +26 -6
- package/dist/esm/scripts/obliterate-2.js.map +1 -1
- package/dist/esm/scripts/removeChildDependency-1.js +12 -5
- package/dist/esm/scripts/removeChildDependency-1.js.map +1 -1
- package/dist/esm/scripts/removeJob-2.js +23 -5
- package/dist/esm/scripts/removeJob-2.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/job-options.d.ts +9 -1
- package/package.json +1 -1
@@ -0,0 +1,12 @@
|
|
1
|
+
--[[
|
2
|
+
Function to remove debounce key.
|
3
|
+
]]
|
4
|
+
|
5
|
+
local function removeDebounceKey(prefixKey, jobKey)
|
6
|
+
local debounceId = rcall("HGET", jobKey, "deid")
|
7
|
+
if debounceId then
|
8
|
+
local debounceKey = prefixKey .. "de:" .. debounceId
|
9
|
+
rcall("DEL", debounceKey)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
@@ -0,0 +1,14 @@
|
|
1
|
+
--[[
|
2
|
+
Function to remove debounce key if needed.
|
3
|
+
]]
|
4
|
+
|
5
|
+
local function removeDebounceKeyIfNeeded(prefixKey, debounceId)
|
6
|
+
if debounceId then
|
7
|
+
local debounceKey = prefixKey .. "de:" .. debounceId
|
8
|
+
local pttl = rcall("PTTL", debounceKey)
|
9
|
+
|
10
|
+
if pttl == 0 or pttl == -1 then
|
11
|
+
rcall("DEL", debounceKey)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -3,11 +3,15 @@
|
|
3
3
|
]]
|
4
4
|
|
5
5
|
-- Includes
|
6
|
+
--- @include "removeDebounceKey"
|
6
7
|
--- @include "removeJobKeys"
|
7
8
|
--- @include "removeParentDependencyKey"
|
8
9
|
|
9
|
-
local function removeJob(jobId, hard, baseKey)
|
10
|
+
local function removeJob(jobId, hard, baseKey, shouldRemoveDebounceKey)
|
10
11
|
local jobKey = baseKey .. jobId
|
11
12
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
13
|
+
if shouldRemoveDebounceKey then
|
14
|
+
removeDebounceKey(baseKey, jobKey)
|
15
|
+
end
|
12
16
|
removeJobKeys(jobKey)
|
13
17
|
end
|
@@ -5,11 +5,12 @@
|
|
5
5
|
-- Includes
|
6
6
|
--- @include "removeJob"
|
7
7
|
|
8
|
-
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix
|
8
|
+
local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix,
|
9
|
+
shouldRemoveDebounceKey)
|
9
10
|
local start = timestamp - maxAge * 1000
|
10
11
|
local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
|
11
12
|
for i, jobId in ipairs(jobIds) do
|
12
|
-
removeJob(jobId, false, prefix)
|
13
|
+
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
13
14
|
end
|
14
15
|
rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
|
15
16
|
end
|
@@ -9,7 +9,7 @@ local function removeJobsByMaxCount(maxCount, targetSet, prefix)
|
|
9
9
|
local start = maxCount
|
10
10
|
local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
|
11
11
|
for i, jobId in ipairs(jobIds) do
|
12
|
-
removeJob(jobId, false, prefix)
|
12
|
+
removeJob(jobId, false, prefix, false --[[remove debounce key]])
|
13
13
|
end
|
14
14
|
rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
|
15
15
|
end
|
@@ -21,7 +21,7 @@ local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
24
|
-
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
24
|
+
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
|
25
25
|
if parentKey then
|
26
26
|
local parentDependenciesKey = parentKey .. ":dependencies"
|
27
27
|
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
@@ -36,8 +36,11 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
|
36
36
|
if numRemovedElements == 1 then
|
37
37
|
if hard then -- remove parent in same queue
|
38
38
|
if parentPrefix == baseKey then
|
39
|
-
removeParentDependencyKey(parentKey, hard, nil, baseKey)
|
39
|
+
removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
|
40
40
|
removeJobKeys(parentKey)
|
41
|
+
if debounceId then
|
42
|
+
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
43
|
+
end
|
41
44
|
else
|
42
45
|
moveParentToWait(parentPrefix, parentId)
|
43
46
|
end
|
@@ -49,7 +52,8 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
|
49
52
|
return true
|
50
53
|
end
|
51
54
|
else
|
52
|
-
local
|
55
|
+
local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
|
56
|
+
local missedParentKey = parentAttributes[1]
|
53
57
|
if( (type(missedParentKey) == "string") and missedParentKey ~= ""
|
54
58
|
and (rcall("EXISTS", missedParentKey) == 1)) then
|
55
59
|
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
@@ -65,8 +69,11 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
|
65
69
|
if numRemovedElements == 1 then
|
66
70
|
if hard then
|
67
71
|
if parentPrefix == baseKey then
|
68
|
-
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
|
72
|
+
removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
|
69
73
|
removeJobKeys(missedParentKey)
|
74
|
+
if parentAttributes[2] then
|
75
|
+
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
76
|
+
end
|
70
77
|
else
|
71
78
|
moveParentToWait(parentPrefix, parentId)
|
72
79
|
end
|
@@ -6,7 +6,8 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
6
6
|
local jsonOpts = cjson.encode(opts)
|
7
7
|
local delay = opts['delay'] or 0
|
8
8
|
local priority = opts['priority'] or 0
|
9
|
-
|
9
|
+
local debounceId = opts['de'] and opts['de']['id']
|
10
|
+
|
10
11
|
local optionalValues = {}
|
11
12
|
if parentKey ~= nil then
|
12
13
|
table.insert(optionalValues, "parentKey")
|
@@ -20,6 +21,11 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
20
21
|
table.insert(optionalValues, repeatJobKey)
|
21
22
|
end
|
22
23
|
|
24
|
+
if debounceId then
|
25
|
+
table.insert(optionalValues, "deid")
|
26
|
+
table.insert(optionalValues, debounceId)
|
27
|
+
end
|
28
|
+
|
23
29
|
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
24
30
|
"timestamp", timestamp, "delay", delay, "priority", priority,
|
25
31
|
unpack(optionalValues))
|
@@ -29,6 +29,7 @@ local rcall = redis.call
|
|
29
29
|
--- @include "includes/getTargetQueueList"
|
30
30
|
--- @include "includes/moveParentFromWaitingChildrenToFailed"
|
31
31
|
--- @include "includes/moveParentToWaitIfNeeded"
|
32
|
+
--- @include "includes/removeDebounceKeyIfNeeded"
|
32
33
|
--- @include "includes/removeJob"
|
33
34
|
--- @include "includes/removeJobsByMaxAge"
|
34
35
|
--- @include "includes/removeJobsByMaxCount"
|
@@ -83,12 +84,14 @@ if (#stalling > 0) then
|
|
83
84
|
local stalledCount =
|
84
85
|
rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
85
86
|
if (stalledCount > MAX_STALLED_JOB_COUNT) then
|
86
|
-
local jobAttributes = rcall("HMGET", jobKey, "opts", "parent")
|
87
|
+
local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
|
87
88
|
local rawOpts = jobAttributes[1]
|
88
89
|
local rawParentData = jobAttributes[2]
|
89
90
|
local opts = cjson.decode(rawOpts)
|
90
91
|
local removeOnFailType = type(opts["removeOnFail"])
|
91
92
|
rcall("ZADD", failedKey, timestamp, jobId)
|
93
|
+
removeDebounceKeyIfNeeded(queueKeyPrefix, jobAttributes[3])
|
94
|
+
|
92
95
|
local failedReason =
|
93
96
|
"job stalled more than allowable limit"
|
94
97
|
rcall("HMSET", jobKey, "failedReason", failedReason,
|
@@ -122,7 +125,8 @@ if (#stalling > 0) then
|
|
122
125
|
failedKey, queueKeyPrefix)
|
123
126
|
elseif removeOnFailType == "boolean" then
|
124
127
|
if opts["removeOnFail"] then
|
125
|
-
removeJob(jobId, false, queueKeyPrefix
|
128
|
+
removeJob(jobId, false, queueKeyPrefix,
|
129
|
+
false --[[remove debounce key]])
|
126
130
|
rcall("ZREM", failedKey, jobId)
|
127
131
|
end
|
128
132
|
elseif removeOnFailType ~= "nil" then
|
@@ -65,6 +65,7 @@ local rcall = redis.call
|
|
65
65
|
--- @include "includes/moveParentToWaitIfNeeded"
|
66
66
|
--- @include "includes/prepareJobForProcessing"
|
67
67
|
--- @include "includes/promoteDelayedJobs"
|
68
|
+
--- @include "includes/removeDebounceKeyIfNeeded"
|
68
69
|
--- @include "includes/removeJobKeys"
|
69
70
|
--- @include "includes/removeJobsByMaxAge"
|
70
71
|
--- @include "includes/removeJobsByMaxCount"
|
@@ -93,12 +94,12 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
93
94
|
return -4
|
94
95
|
end
|
95
96
|
|
96
|
-
local
|
97
|
-
local parentKey =
|
97
|
+
local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid")
|
98
|
+
local parentKey = jobAttributes[1] or ""
|
98
99
|
local parentId = ""
|
99
100
|
local parentQueueKey = ""
|
100
|
-
if
|
101
|
-
local jsonDecodedParent = cjson.decode(
|
101
|
+
if jobAttributes[2] ~= false then
|
102
|
+
local jsonDecodedParent = cjson.decode(jobAttributes[2])
|
102
103
|
parentId = jsonDecodedParent['id']
|
103
104
|
parentQueueKey = jsonDecodedParent['queueKey']
|
104
105
|
end
|
@@ -116,6 +117,10 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
116
117
|
-- Trim events before emiting them to avoid trimming events emitted in this script
|
117
118
|
trimEvents(metaKey, eventStreamKey)
|
118
119
|
|
120
|
+
local prefix = ARGV[7]
|
121
|
+
|
122
|
+
removeDebounceKeyIfNeeded(prefix, jobAttributes[3])
|
123
|
+
|
119
124
|
-- If job has a parent we need to
|
120
125
|
-- 1) remove this job id from parents dependencies
|
121
126
|
-- 2) move the job Id to parent "processed" set
|
@@ -164,8 +169,6 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
164
169
|
-- "returnvalue" / "failedReason" and "finishedOn"
|
165
170
|
|
166
171
|
-- Remove old jobs?
|
167
|
-
local prefix = ARGV[7]
|
168
|
-
|
169
172
|
if maxAge ~= nil then
|
170
173
|
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
171
174
|
end
|
@@ -179,7 +182,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
179
182
|
-- TODO: when a child is removed when finished, result or failure in parent
|
180
183
|
-- must not be deleted, those value references should be deleted when the parent
|
181
184
|
-- is deleted
|
182
|
-
removeParentDependencyKey(jobIdKey, false, parentKey)
|
185
|
+
removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3])
|
183
186
|
end
|
184
187
|
end
|
185
188
|
|
@@ -205,7 +208,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
205
208
|
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
|
206
209
|
|
207
210
|
-- Check if there are delayed jobs that can be promoted
|
208
|
-
promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], eventStreamKey,
|
211
|
+
promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], eventStreamKey, prefix,
|
209
212
|
timestamp, KEYS[10], isPausedOrMaxed)
|
210
213
|
|
211
214
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
@@ -229,19 +232,19 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
229
232
|
if jobId == "0:0" then
|
230
233
|
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
|
231
234
|
KEYS[10])
|
232
|
-
return prepareJobForProcessing(
|
235
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
|
233
236
|
timestamp, maxJobs,
|
234
237
|
opts)
|
235
238
|
end
|
236
239
|
else
|
237
|
-
return prepareJobForProcessing(
|
240
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
|
238
241
|
timestamp, maxJobs,
|
239
242
|
opts)
|
240
243
|
end
|
241
244
|
else
|
242
245
|
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
243
246
|
if jobId then
|
244
|
-
return prepareJobForProcessing(
|
247
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
|
245
248
|
timestamp, maxJobs,
|
246
249
|
opts)
|
247
250
|
end
|
@@ -25,7 +25,7 @@ if rcall("EXISTS", jobKey) ~= 1 then return -1 end
|
|
25
25
|
|
26
26
|
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
27
27
|
|
28
|
-
if removeParentDependencyKey(jobKey, false, parentKey, KEYS[1]) then
|
28
|
+
if removeParentDependencyKey(jobKey, false, parentKey, KEYS[1], nil) then
|
29
29
|
rcall("HDEL", jobKey, "parentKey", "parent")
|
30
30
|
|
31
31
|
return 0
|
@@ -19,6 +19,7 @@ local rcall = redis.call
|
|
19
19
|
--- @include "includes/destructureJobKey"
|
20
20
|
--- @include "includes/getOrSetMaxEvents"
|
21
21
|
--- @include "includes/isLocked"
|
22
|
+
--- @include "includes/removeDebounceKey"
|
22
23
|
--- @include "includes/removeJobFromAnyState"
|
23
24
|
--- @include "includes/removeJobKeys"
|
24
25
|
--- @include "includes/removeParentDependencyKey"
|
@@ -26,7 +27,7 @@ local rcall = redis.call
|
|
26
27
|
local function removeJob( prefix, jobId, parentKey, removeChildren)
|
27
28
|
local jobKey = prefix .. jobId;
|
28
29
|
|
29
|
-
removeParentDependencyKey(jobKey, false, parentKey)
|
30
|
+
removeParentDependencyKey(jobKey, false, parentKey, nil)
|
30
31
|
|
31
32
|
if removeChildren == "1" then
|
32
33
|
-- Check if this job has children
|
@@ -66,6 +67,7 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
|
|
66
67
|
|
67
68
|
local prev = removeJobFromAnyState(prefix, jobId)
|
68
69
|
|
70
|
+
removeDebounceKey(prefix, jobKey)
|
69
71
|
if removeJobKeys(jobKey) > 0 then
|
70
72
|
local maxEvents = getOrSetMaxEvents(KEYS[2])
|
71
73
|
rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"debounce-options.js","sourceRoot":"","sources":["../../../src/interfaces/debounce-options.ts"],"names":[],"mappings":""}
|
@@ -3,6 +3,7 @@ export * from './backoff-options';
|
|
3
3
|
export * from './base-job-options';
|
4
4
|
export * from './child-message';
|
5
5
|
export * from './connection';
|
6
|
+
export * from './debounce-options';
|
6
7
|
export * from './flow-job';
|
7
8
|
export * from './ioredis-events';
|
8
9
|
export * from './job-json';
|
@@ -3,6 +3,7 @@ export * from './backoff-options';
|
|
3
3
|
export * from './base-job-options';
|
4
4
|
export * from './child-message';
|
5
5
|
export * from './connection';
|
6
|
+
export * from './debounce-options';
|
6
7
|
export * from './flow-job';
|
7
8
|
export * from './ioredis-events';
|
8
9
|
export * from './job-json';
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC"}
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC"}
|
@@ -17,6 +17,7 @@ export interface JobJson {
|
|
17
17
|
parent?: ParentKeys;
|
18
18
|
parentKey?: string;
|
19
19
|
repeatJobKey?: string;
|
20
|
+
debounceId?: string;
|
20
21
|
processedBy?: string;
|
21
22
|
}
|
22
23
|
export interface JobJsonRaw {
|
@@ -35,6 +36,7 @@ export interface JobJsonRaw {
|
|
35
36
|
returnvalue: string;
|
36
37
|
parentKey?: string;
|
37
38
|
parent?: string;
|
39
|
+
deid?: string;
|
38
40
|
rjk?: string;
|
39
41
|
atm?: string;
|
40
42
|
ats?: string;
|
@@ -22,6 +22,7 @@ const content = `--[[
|
|
22
22
|
[7] parent dependencies key.
|
23
23
|
[8] parent? {id, queueKey}
|
24
24
|
[9] repeat job key
|
25
|
+
[10] debounce key
|
25
26
|
ARGV[2] Json stringified job data
|
26
27
|
ARGV[3] msgpacked options
|
27
28
|
Output:
|
@@ -39,8 +40,9 @@ local rcall = redis.call
|
|
39
40
|
local args = cmsgpack.unpack(ARGV[1])
|
40
41
|
local data = ARGV[2]
|
41
42
|
local parentKey = args[5]
|
42
|
-
local repeatJobKey = args[9]
|
43
43
|
local parent = args[8]
|
44
|
+
local repeatJobKey = args[9]
|
45
|
+
local debounceKey = args[10]
|
44
46
|
local parentData
|
45
47
|
-- Includes
|
46
48
|
--[[
|
@@ -67,6 +69,27 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
67
69
|
rcall("ZADD", markerKey, nextTimestamp, "1")
|
68
70
|
end
|
69
71
|
end
|
72
|
+
--[[
|
73
|
+
Function to debounce a job.
|
74
|
+
]]
|
75
|
+
local function debounceJob(prefixKey, debounceOpts, jobId, debounceKey, eventsKey, maxEvents)
|
76
|
+
local debounceId = debounceOpts and debounceOpts['id']
|
77
|
+
if debounceId then
|
78
|
+
local ttl = debounceOpts['ttl']
|
79
|
+
local debounceKeyExists
|
80
|
+
if ttl then
|
81
|
+
debounceKeyExists = not rcall('SET', debounceKey, jobId, 'PX', ttl, 'NX')
|
82
|
+
else
|
83
|
+
debounceKeyExists = not rcall('SET', debounceKey, jobId, 'NX')
|
84
|
+
end
|
85
|
+
if debounceKeyExists then
|
86
|
+
local currentDebounceJobId = rcall('GET', debounceKey)
|
87
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
88
|
+
"debounced", "jobId", currentDebounceJobId, "debounceId", debounceId)
|
89
|
+
return currentDebounceJobId
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
70
93
|
--[[
|
71
94
|
Bake in the job id first 12 bits into the timestamp
|
72
95
|
to guarantee correct execution order of delayed jobs
|
@@ -272,6 +295,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
272
295
|
local jsonOpts = cjson.encode(opts)
|
273
296
|
local delay = opts['delay'] or 0
|
274
297
|
local priority = opts['priority'] or 0
|
298
|
+
local debounceId = opts['de'] and opts['de']['id']
|
275
299
|
local optionalValues = {}
|
276
300
|
if parentKey ~= nil then
|
277
301
|
table.insert(optionalValues, "parentKey")
|
@@ -283,6 +307,10 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
283
307
|
table.insert(optionalValues, "rjk")
|
284
308
|
table.insert(optionalValues, repeatJobKey)
|
285
309
|
end
|
310
|
+
if debounceId then
|
311
|
+
table.insert(optionalValues, "deid")
|
312
|
+
table.insert(optionalValues, debounceId)
|
313
|
+
end
|
286
314
|
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
287
315
|
"timestamp", timestamp, "delay", delay, "priority", priority,
|
288
316
|
unpack(optionalValues))
|
@@ -310,6 +338,11 @@ else
|
|
310
338
|
maxEvents, timestamp)
|
311
339
|
end
|
312
340
|
end
|
341
|
+
local debouncedJobId = debounceJob(args[1], opts['de'],
|
342
|
+
jobId, debounceKey, eventsKey, maxEvents)
|
343
|
+
if debouncedJobId then
|
344
|
+
return debouncedJobId
|
345
|
+
end
|
313
346
|
-- Store the job.
|
314
347
|
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
315
348
|
opts, timestamp, parentKey, parentData,
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyWf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -18,6 +18,7 @@ const content = `--[[
|
|
18
18
|
[7] parent dependencies key.
|
19
19
|
[8] parent? {id, queueKey}
|
20
20
|
[9] repeat job key
|
21
|
+
[10] debounce key
|
21
22
|
ARGV[2] Json stringified job data
|
22
23
|
ARGV[3] msgpacked options
|
23
24
|
Output:
|
@@ -35,10 +36,32 @@ local args = cmsgpack.unpack(ARGV[1])
|
|
35
36
|
local data = ARGV[2]
|
36
37
|
local opts = cmsgpack.unpack(ARGV[3])
|
37
38
|
local parentKey = args[5]
|
38
|
-
local repeatJobKey = args[9]
|
39
39
|
local parent = args[8]
|
40
|
+
local repeatJobKey = args[9]
|
41
|
+
local debounceKey = args[10]
|
40
42
|
local parentData
|
41
43
|
-- Includes
|
44
|
+
--[[
|
45
|
+
Function to debounce a job.
|
46
|
+
]]
|
47
|
+
local function debounceJob(prefixKey, debounceOpts, jobId, debounceKey, eventsKey, maxEvents)
|
48
|
+
local debounceId = debounceOpts and debounceOpts['id']
|
49
|
+
if debounceId then
|
50
|
+
local ttl = debounceOpts['ttl']
|
51
|
+
local debounceKeyExists
|
52
|
+
if ttl then
|
53
|
+
debounceKeyExists = not rcall('SET', debounceKey, jobId, 'PX', ttl, 'NX')
|
54
|
+
else
|
55
|
+
debounceKeyExists = not rcall('SET', debounceKey, jobId, 'NX')
|
56
|
+
end
|
57
|
+
if debounceKeyExists then
|
58
|
+
local currentDebounceJobId = rcall('GET', debounceKey)
|
59
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
60
|
+
"debounced", "jobId", currentDebounceJobId, "debounceId", debounceId)
|
61
|
+
return currentDebounceJobId
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
42
65
|
--[[
|
43
66
|
Function to get max events value or set by default 10000.
|
44
67
|
]]
|
@@ -244,6 +267,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
244
267
|
local jsonOpts = cjson.encode(opts)
|
245
268
|
local delay = opts['delay'] or 0
|
246
269
|
local priority = opts['priority'] or 0
|
270
|
+
local debounceId = opts['de'] and opts['de']['id']
|
247
271
|
local optionalValues = {}
|
248
272
|
if parentKey ~= nil then
|
249
273
|
table.insert(optionalValues, "parentKey")
|
@@ -255,6 +279,10 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
255
279
|
table.insert(optionalValues, "rjk")
|
256
280
|
table.insert(optionalValues, repeatJobKey)
|
257
281
|
end
|
282
|
+
if debounceId then
|
283
|
+
table.insert(optionalValues, "deid")
|
284
|
+
table.insert(optionalValues, debounceId)
|
285
|
+
end
|
258
286
|
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
259
287
|
"timestamp", timestamp, "delay", delay, "priority", priority,
|
260
288
|
unpack(optionalValues))
|
@@ -281,6 +309,11 @@ else
|
|
281
309
|
maxEvents, timestamp)
|
282
310
|
end
|
283
311
|
end
|
312
|
+
local debouncedJobId = debounceJob(args[1], opts['de'],
|
313
|
+
jobId, debounceKey, eventsKey, maxEvents)
|
314
|
+
if debouncedJobId then
|
315
|
+
return debouncedJobId
|
316
|
+
end
|
284
317
|
-- Store the job.
|
285
318
|
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
286
319
|
parentKey, parentData, repeatJobKey)
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwUf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -22,6 +22,7 @@ const content = `--[[
|
|
22
22
|
[7] parent dependencies key.
|
23
23
|
[8] parent? {id, queueKey}
|
24
24
|
[9] repeat job key
|
25
|
+
[10] debounce key
|
25
26
|
ARGV[2] Json stringified job data
|
26
27
|
ARGV[3] msgpacked options
|
27
28
|
Output:
|
@@ -42,8 +43,9 @@ local args = cmsgpack.unpack(ARGV[1])
|
|
42
43
|
local data = ARGV[2]
|
43
44
|
local opts = cmsgpack.unpack(ARGV[3])
|
44
45
|
local parentKey = args[5]
|
45
|
-
local repeatJobKey = args[9]
|
46
46
|
local parent = args[8]
|
47
|
+
local repeatJobKey = args[9]
|
48
|
+
local debounceKey = args[10]
|
47
49
|
local parentData
|
48
50
|
-- Includes
|
49
51
|
--[[
|
@@ -65,6 +67,27 @@ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, pr
|
|
65
67
|
rcall("ZADD", prioritizedKey, score, jobId)
|
66
68
|
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
67
69
|
end
|
70
|
+
--[[
|
71
|
+
Function to debounce a job.
|
72
|
+
]]
|
73
|
+
local function debounceJob(prefixKey, debounceOpts, jobId, debounceKey, eventsKey, maxEvents)
|
74
|
+
local debounceId = debounceOpts and debounceOpts['id']
|
75
|
+
if debounceId then
|
76
|
+
local ttl = debounceOpts['ttl']
|
77
|
+
local debounceKeyExists
|
78
|
+
if ttl then
|
79
|
+
debounceKeyExists = not rcall('SET', debounceKey, jobId, 'PX', ttl, 'NX')
|
80
|
+
else
|
81
|
+
debounceKeyExists = not rcall('SET', debounceKey, jobId, 'NX')
|
82
|
+
end
|
83
|
+
if debounceKeyExists then
|
84
|
+
local currentDebounceJobId = rcall('GET', debounceKey)
|
85
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
86
|
+
"debounced", "jobId", currentDebounceJobId, "debounceId", debounceId)
|
87
|
+
return currentDebounceJobId
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
68
91
|
--[[
|
69
92
|
Function to store a job
|
70
93
|
]]
|
@@ -73,6 +96,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
73
96
|
local jsonOpts = cjson.encode(opts)
|
74
97
|
local delay = opts['delay'] or 0
|
75
98
|
local priority = opts['priority'] or 0
|
99
|
+
local debounceId = opts['de'] and opts['de']['id']
|
76
100
|
local optionalValues = {}
|
77
101
|
if parentKey ~= nil then
|
78
102
|
table.insert(optionalValues, "parentKey")
|
@@ -84,6 +108,10 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
84
108
|
table.insert(optionalValues, "rjk")
|
85
109
|
table.insert(optionalValues, repeatJobKey)
|
86
110
|
end
|
111
|
+
if debounceId then
|
112
|
+
table.insert(optionalValues, "deid")
|
113
|
+
table.insert(optionalValues, debounceId)
|
114
|
+
end
|
87
115
|
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
88
116
|
"timestamp", timestamp, "delay", delay, "priority", priority,
|
89
117
|
unpack(optionalValues))
|
@@ -288,6 +316,11 @@ else
|
|
288
316
|
maxEvents, timestamp)
|
289
317
|
end
|
290
318
|
end
|
319
|
+
local debouncedJobId = debounceJob(args[1], opts['de'],
|
320
|
+
jobId, debounceKey, eventsKey, maxEvents)
|
321
|
+
if debouncedJobId then
|
322
|
+
return debouncedJobId
|
323
|
+
end
|
291
324
|
-- Store the job.
|
292
325
|
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
293
326
|
opts, timestamp, parentKey, parentData,
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addPrioritizedJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addPrioritizedJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkVf,CAAC;AACF,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,IAAI,EAAE,mBAAmB;IACzB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|