bullmq 5.59.0 → 5.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/queue.js +16 -0
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/scripts.js +4 -1
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +5 -8
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/changePriority-7.lua +1 -1
- package/dist/cjs/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
- package/dist/cjs/commands/includes/getTargetQueueList.lua +5 -5
- package/dist/cjs/commands/includes/prepareJobForProcessing.lua +2 -4
- package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -1
- package/dist/cjs/commands/moveToActive-11.lua +7 -4
- package/dist/cjs/commands/moveToFinished-14.lua +10 -7
- package/dist/cjs/scripts/addDelayedJob-6.js +5 -5
- package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
- package/dist/cjs/scripts/addParentJob-5.js +5 -5
- package/dist/cjs/scripts/addPrioritizedJob-9.js +5 -5
- package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
- package/dist/cjs/scripts/addStandardJob-9.js +5 -5
- package/dist/cjs/scripts/changePriority-7.js +5 -5
- package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
- package/dist/cjs/scripts/drain-5.js +5 -5
- package/dist/cjs/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
- package/dist/cjs/scripts/getRateLimitTtl-2.js.map +1 -0
- package/dist/cjs/scripts/index.js +1 -1
- package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +5 -5
- package/dist/cjs/scripts/moveJobsToWait-8.js +5 -5
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js +9 -9
- package/dist/cjs/scripts/moveToActive-11.js +13 -12
- package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +16 -15
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +5 -5
- package/dist/cjs/scripts/promote-9.js +5 -5
- package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
- package/dist/cjs/scripts/removeJob-2.js +5 -5
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js +5 -5
- package/dist/cjs/scripts/reprocessJob-8.js +5 -5
- package/dist/cjs/scripts/retryJob-11.js +5 -5
- package/dist/cjs/scripts/updateJobScheduler-12.js +5 -5
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/queue.d.ts +10 -0
- package/dist/esm/classes/queue.js +16 -0
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/scripts.js +4 -1
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.js +5 -8
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/changePriority-7.lua +1 -1
- package/dist/esm/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
- package/dist/esm/commands/includes/getTargetQueueList.lua +5 -5
- package/dist/esm/commands/includes/prepareJobForProcessing.lua +2 -4
- package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -1
- package/dist/esm/commands/moveToActive-11.lua +7 -4
- package/dist/esm/commands/moveToFinished-14.lua +10 -7
- package/dist/esm/scripts/addDelayedJob-6.js +5 -5
- package/dist/esm/scripts/addJobScheduler-11.js +5 -5
- package/dist/esm/scripts/addParentJob-5.js +5 -5
- package/dist/esm/scripts/addPrioritizedJob-9.js +5 -5
- package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
- package/dist/esm/scripts/addStandardJob-9.js +5 -5
- package/dist/esm/scripts/changePriority-7.js +5 -5
- package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
- package/dist/esm/scripts/drain-5.js +5 -5
- package/dist/esm/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
- package/dist/esm/scripts/getRateLimitTtl-2.js.map +1 -0
- package/dist/esm/scripts/index.d.ts +1 -1
- package/dist/esm/scripts/index.js +1 -1
- package/dist/esm/scripts/moveJobFromActiveToWait-9.js +5 -5
- package/dist/esm/scripts/moveJobsToWait-8.js +5 -5
- package/dist/esm/scripts/moveStalledJobsToWait-8.js +9 -9
- package/dist/esm/scripts/moveToActive-11.js +13 -12
- package/dist/esm/scripts/moveToActive-11.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +16 -15
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +5 -5
- package/dist/esm/scripts/promote-9.js +5 -5
- package/dist/esm/scripts/removeChildDependency-1.js +5 -5
- package/dist/esm/scripts/removeJob-2.js +5 -5
- package/dist/esm/scripts/removeUnprocessedChildren-2.js +5 -5
- package/dist/esm/scripts/reprocessJob-8.js +5 -5
- package/dist/esm/scripts/retryJob-11.js +5 -5
- package/dist/esm/scripts/updateJobScheduler-12.js +5 -5
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
- package/dist/cjs/scripts/getRateLimitTtl-1.js.map +0 -1
- package/dist/esm/scripts/getRateLimitTtl-1.js.map +0 -1
- /package/dist/esm/scripts/{getRateLimitTtl-1.d.ts → getRateLimitTtl-2.d.ts} +0 -0
|
@@ -51,14 +51,15 @@ local opts = cmsgpack.unpack(ARGV[3])
|
|
|
51
51
|
--- @include "includes/prepareJobForProcessing"
|
|
52
52
|
--- @include "includes/promoteDelayedJobs"
|
|
53
53
|
|
|
54
|
-
local target, isPausedOrMaxed = getTargetQueueList(KEYS[9],
|
|
54
|
+
local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(KEYS[9],
|
|
55
|
+
activeKey, waitKey, KEYS[8])
|
|
55
56
|
|
|
56
57
|
-- Check if there are delayed jobs that we can move to wait.
|
|
57
58
|
local markerKey = KEYS[11]
|
|
58
59
|
promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
|
|
59
60
|
ARGV[2], KEYS[10], isPausedOrMaxed)
|
|
60
61
|
|
|
61
|
-
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
|
62
|
+
local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
|
|
62
63
|
local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
|
|
63
64
|
|
|
64
65
|
-- Check if we are rate limited first.
|
|
@@ -67,6 +68,8 @@ if expireTime > 0 then return {0, 0, expireTime, 0} end
|
|
|
67
68
|
-- paused or maxed queue
|
|
68
69
|
if isPausedOrMaxed then return {0, 0, 0, 0} end
|
|
69
70
|
|
|
71
|
+
local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
|
|
72
|
+
|
|
70
73
|
-- no job ID, try non-blocking move from wait to active
|
|
71
74
|
local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
|
72
75
|
|
|
@@ -78,12 +81,12 @@ end
|
|
|
78
81
|
|
|
79
82
|
if jobId then
|
|
80
83
|
return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
|
|
81
|
-
maxJobs, markerKey, opts)
|
|
84
|
+
maxJobs, limiterDuration, markerKey, opts)
|
|
82
85
|
else
|
|
83
86
|
jobId = moveJobFromPrioritizedToActive(KEYS[3], activeKey, KEYS[10])
|
|
84
87
|
if jobId then
|
|
85
88
|
return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
|
|
86
|
-
maxJobs, markerKey, opts)
|
|
89
|
+
maxJobs, limiterDuration, markerKey, opts)
|
|
87
90
|
end
|
|
88
91
|
end
|
|
89
92
|
|
|
@@ -209,14 +209,15 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
|
209
209
|
-- and not rate limited.
|
|
210
210
|
if (ARGV[6] == "1") then
|
|
211
211
|
|
|
212
|
-
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2],
|
|
212
|
+
local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(metaKey, KEYS[2],
|
|
213
|
+
KEYS[1], KEYS[8])
|
|
213
214
|
|
|
214
215
|
local markerKey = KEYS[14]
|
|
215
216
|
-- Check if there are delayed jobs that can be promoted
|
|
216
217
|
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10],
|
|
217
218
|
isPausedOrMaxed)
|
|
218
219
|
|
|
219
|
-
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
|
220
|
+
local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
|
|
220
221
|
-- Check if we are rate limited first.
|
|
221
222
|
local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
|
|
222
223
|
|
|
@@ -229,6 +230,8 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
|
229
230
|
return {0, 0, 0, 0}
|
|
230
231
|
end
|
|
231
232
|
|
|
233
|
+
local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
|
|
234
|
+
|
|
232
235
|
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
|
233
236
|
|
|
234
237
|
if jobId then
|
|
@@ -241,17 +244,17 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
|
241
244
|
if jobId == "0:0" then
|
|
242
245
|
jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
|
|
243
246
|
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
|
244
|
-
markerKey, opts)
|
|
247
|
+
limiterDuration, markerKey, opts)
|
|
245
248
|
end
|
|
246
249
|
else
|
|
247
|
-
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
|
248
|
-
opts)
|
|
250
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
|
251
|
+
limiterDuration, markerKey, opts)
|
|
249
252
|
end
|
|
250
253
|
else
|
|
251
254
|
jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
|
|
252
255
|
if jobId then
|
|
253
|
-
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
|
254
|
-
opts)
|
|
256
|
+
return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
|
|
257
|
+
limiterDuration, markerKey, opts)
|
|
255
258
|
end
|
|
256
259
|
end
|
|
257
260
|
|
|
@@ -306,20 +306,20 @@ end
|
|
|
306
306
|
(since an empty list and !EXISTS are not really the same).
|
|
307
307
|
]]
|
|
308
308
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
309
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
309
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
310
310
|
if queueAttributes[1] then
|
|
311
|
-
return pausedKey, true
|
|
311
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
312
312
|
else
|
|
313
313
|
if queueAttributes[2] then
|
|
314
314
|
local activeCount = rcall("LLEN", activeKey)
|
|
315
315
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
316
|
-
return waitKey, true
|
|
316
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
317
317
|
else
|
|
318
|
-
return waitKey, false
|
|
318
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
319
319
|
end
|
|
320
320
|
end
|
|
321
321
|
end
|
|
322
|
-
return waitKey, false
|
|
322
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
323
323
|
end
|
|
324
324
|
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
|
325
325
|
local parentWaitKey = parentQueueKey .. ":wait"
|
|
@@ -182,20 +182,20 @@ end
|
|
|
182
182
|
(since an empty list and !EXISTS are not really the same).
|
|
183
183
|
]]
|
|
184
184
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
185
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
185
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
186
186
|
if queueAttributes[1] then
|
|
187
|
-
return pausedKey, true
|
|
187
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
188
188
|
else
|
|
189
189
|
if queueAttributes[2] then
|
|
190
190
|
local activeCount = rcall("LLEN", activeKey)
|
|
191
191
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
192
|
-
return waitKey, true
|
|
192
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
193
193
|
else
|
|
194
|
-
return waitKey, false
|
|
194
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
195
195
|
end
|
|
196
196
|
end
|
|
197
197
|
end
|
|
198
|
-
return waitKey, false
|
|
198
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
199
199
|
end
|
|
200
200
|
--[[
|
|
201
201
|
Function to add job in target list and add marker if needed.
|
|
@@ -263,20 +263,20 @@ end
|
|
|
263
263
|
(since an empty list and !EXISTS are not really the same).
|
|
264
264
|
]]
|
|
265
265
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
266
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
266
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
267
267
|
if queueAttributes[1] then
|
|
268
|
-
return pausedKey, true
|
|
268
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
269
269
|
else
|
|
270
270
|
if queueAttributes[2] then
|
|
271
271
|
local activeCount = rcall("LLEN", activeKey)
|
|
272
272
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
273
|
-
return waitKey, true
|
|
273
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
274
274
|
else
|
|
275
|
-
return waitKey, false
|
|
275
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
276
276
|
end
|
|
277
277
|
end
|
|
278
278
|
end
|
|
279
|
-
return waitKey, false
|
|
279
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
280
280
|
end
|
|
281
281
|
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
|
282
282
|
local parentWaitKey = parentQueueKey .. ":wait"
|
|
@@ -300,20 +300,20 @@ end
|
|
|
300
300
|
(since an empty list and !EXISTS are not really the same).
|
|
301
301
|
]]
|
|
302
302
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
303
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
303
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
304
304
|
if queueAttributes[1] then
|
|
305
|
-
return pausedKey, true
|
|
305
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
306
306
|
else
|
|
307
307
|
if queueAttributes[2] then
|
|
308
308
|
local activeCount = rcall("LLEN", activeKey)
|
|
309
309
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
310
|
-
return waitKey, true
|
|
310
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
311
311
|
else
|
|
312
|
-
return waitKey, false
|
|
312
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
313
313
|
end
|
|
314
314
|
end
|
|
315
315
|
end
|
|
316
|
-
return waitKey, false
|
|
316
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
317
317
|
end
|
|
318
318
|
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
|
319
319
|
local parentWaitKey = parentQueueKey .. ":wait"
|
|
@@ -90,20 +90,20 @@ end
|
|
|
90
90
|
(since an empty list and !EXISTS are not really the same).
|
|
91
91
|
]]
|
|
92
92
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
93
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
93
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
94
94
|
if queueAttributes[1] then
|
|
95
|
-
return pausedKey, true
|
|
95
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
96
96
|
else
|
|
97
97
|
if queueAttributes[2] then
|
|
98
98
|
local activeCount = rcall("LLEN", activeKey)
|
|
99
99
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
100
|
-
return waitKey, true
|
|
100
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
101
101
|
else
|
|
102
|
-
return waitKey, false
|
|
102
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
103
103
|
end
|
|
104
104
|
end
|
|
105
105
|
end
|
|
106
|
-
return waitKey, false
|
|
106
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
107
107
|
end
|
|
108
108
|
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
|
109
109
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
|
@@ -188,20 +188,20 @@ end
|
|
|
188
188
|
(since an empty list and !EXISTS are not really the same).
|
|
189
189
|
]]
|
|
190
190
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
191
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
191
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
192
192
|
if queueAttributes[1] then
|
|
193
|
-
return pausedKey, true
|
|
193
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
194
194
|
else
|
|
195
195
|
if queueAttributes[2] then
|
|
196
196
|
local activeCount = rcall("LLEN", activeKey)
|
|
197
197
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
198
|
-
return waitKey, true
|
|
198
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
199
199
|
else
|
|
200
|
-
return waitKey, false
|
|
200
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
201
201
|
end
|
|
202
202
|
end
|
|
203
203
|
end
|
|
204
|
-
return waitKey, false
|
|
204
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
205
205
|
end
|
|
206
206
|
--[[
|
|
207
207
|
Function to handle the case when job is duplicated.
|
|
@@ -62,20 +62,20 @@ end
|
|
|
62
62
|
(since an empty list and !EXISTS are not really the same).
|
|
63
63
|
]]
|
|
64
64
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
65
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
65
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
66
66
|
if queueAttributes[1] then
|
|
67
|
-
return pausedKey, true
|
|
67
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
68
68
|
else
|
|
69
69
|
if queueAttributes[2] then
|
|
70
70
|
local activeCount = rcall("LLEN", activeKey)
|
|
71
71
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
72
|
-
return waitKey, true
|
|
72
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
73
73
|
else
|
|
74
|
-
return waitKey, false
|
|
74
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
75
75
|
end
|
|
76
76
|
end
|
|
77
77
|
end
|
|
78
|
-
return waitKey, false
|
|
78
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
79
79
|
end
|
|
80
80
|
--[[
|
|
81
81
|
Function to push back job considering priority in front of same prioritized jobs.
|
|
@@ -126,20 +126,20 @@ end
|
|
|
126
126
|
(since an empty list and !EXISTS are not really the same).
|
|
127
127
|
]]
|
|
128
128
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
129
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
129
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
130
130
|
if queueAttributes[1] then
|
|
131
|
-
return pausedKey, true
|
|
131
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
132
132
|
else
|
|
133
133
|
if queueAttributes[2] then
|
|
134
134
|
local activeCount = rcall("LLEN", activeKey)
|
|
135
135
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
136
|
-
return waitKey, true
|
|
136
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
137
137
|
else
|
|
138
|
-
return waitKey, false
|
|
138
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
139
139
|
end
|
|
140
140
|
end
|
|
141
141
|
end
|
|
142
|
-
return waitKey, false
|
|
142
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
143
143
|
end
|
|
144
144
|
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
|
145
145
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
|
@@ -98,20 +98,20 @@ end
|
|
|
98
98
|
(since an empty list and !EXISTS are not really the same).
|
|
99
99
|
]]
|
|
100
100
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
101
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
101
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
102
102
|
if queueAttributes[1] then
|
|
103
|
-
return pausedKey, true
|
|
103
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
104
104
|
else
|
|
105
105
|
if queueAttributes[2] then
|
|
106
106
|
local activeCount = rcall("LLEN", activeKey)
|
|
107
107
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
108
|
-
return waitKey, true
|
|
108
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
109
109
|
else
|
|
110
|
-
return waitKey, false
|
|
110
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
111
111
|
end
|
|
112
112
|
end
|
|
113
113
|
end
|
|
114
|
-
return waitKey, false
|
|
114
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
115
115
|
end
|
|
116
116
|
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
|
117
117
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
|
@@ -5,6 +5,7 @@ const content = `--[[
|
|
|
5
5
|
Get rate limit ttl
|
|
6
6
|
Input:
|
|
7
7
|
KEYS[1] 'limiter'
|
|
8
|
+
KEYS[2] 'meta'
|
|
8
9
|
ARGV[1] maxJobs
|
|
9
10
|
]]
|
|
10
11
|
local rcall = redis.call
|
|
@@ -28,12 +29,16 @@ local rateLimiterKey = KEYS[1]
|
|
|
28
29
|
if ARGV[1] ~= "0" then
|
|
29
30
|
return getRateLimitTTL(tonumber(ARGV[1]), rateLimiterKey)
|
|
30
31
|
else
|
|
32
|
+
local rateLimitMax = rcall("HGET", KEYS[2], "max")
|
|
33
|
+
if rateLimitMax then
|
|
34
|
+
return getRateLimitTTL(tonumber(rateLimitMax), rateLimiterKey)
|
|
35
|
+
end
|
|
31
36
|
return rcall("PTTL", rateLimiterKey)
|
|
32
37
|
end
|
|
33
38
|
`;
|
|
34
39
|
exports.getRateLimitTtl = {
|
|
35
40
|
name: 'getRateLimitTtl',
|
|
36
41
|
content,
|
|
37
|
-
keys:
|
|
42
|
+
keys: 2,
|
|
38
43
|
};
|
|
39
|
-
//# sourceMappingURL=getRateLimitTtl-
|
|
44
|
+
//# sourceMappingURL=getRateLimitTtl-2.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getRateLimitTtl-2.js","sourceRoot":"","sources":["../../../src/scripts/getRateLimitTtl-2.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkCf,CAAC;AACW,QAAA,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
|
@@ -20,7 +20,7 @@ tslib_1.__exportStar(require("./getDependencyCounts-4"), exports);
|
|
|
20
20
|
tslib_1.__exportStar(require("./getJobScheduler-1"), exports);
|
|
21
21
|
tslib_1.__exportStar(require("./getMetrics-2"), exports);
|
|
22
22
|
tslib_1.__exportStar(require("./getRanges-1"), exports);
|
|
23
|
-
tslib_1.__exportStar(require("./getRateLimitTtl-
|
|
23
|
+
tslib_1.__exportStar(require("./getRateLimitTtl-2"), exports);
|
|
24
24
|
tslib_1.__exportStar(require("./getState-8"), exports);
|
|
25
25
|
tslib_1.__exportStar(require("./getStateV2-8"), exports);
|
|
26
26
|
tslib_1.__exportStar(require("./isFinished-3"), exports);
|
|
@@ -60,20 +60,20 @@ end
|
|
|
60
60
|
(since an empty list and !EXISTS are not really the same).
|
|
61
61
|
]]
|
|
62
62
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
63
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
63
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
64
64
|
if queueAttributes[1] then
|
|
65
|
-
return pausedKey, true
|
|
65
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
66
66
|
else
|
|
67
67
|
if queueAttributes[2] then
|
|
68
68
|
local activeCount = rcall("LLEN", activeKey)
|
|
69
69
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
70
|
-
return waitKey, true
|
|
70
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
71
71
|
else
|
|
72
|
-
return waitKey, false
|
|
72
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
73
73
|
end
|
|
74
74
|
end
|
|
75
75
|
end
|
|
76
|
-
return waitKey, false
|
|
76
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
77
77
|
end
|
|
78
78
|
local function removeLock(jobKey, stalledKey, token, jobId)
|
|
79
79
|
if token ~= "0" then
|
|
@@ -64,20 +64,20 @@ end
|
|
|
64
64
|
(since an empty list and !EXISTS are not really the same).
|
|
65
65
|
]]
|
|
66
66
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
67
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
67
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
68
68
|
if queueAttributes[1] then
|
|
69
|
-
return pausedKey, true
|
|
69
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
70
70
|
else
|
|
71
71
|
if queueAttributes[2] then
|
|
72
72
|
local activeCount = rcall("LLEN", activeKey)
|
|
73
73
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
74
|
-
return waitKey, true
|
|
74
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
75
75
|
else
|
|
76
|
-
return waitKey, false
|
|
76
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
77
77
|
end
|
|
78
78
|
end
|
|
79
79
|
end
|
|
80
|
-
return waitKey, false
|
|
80
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
81
81
|
end
|
|
82
82
|
local metaKey = KEYS[6]
|
|
83
83
|
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[7], KEYS[4], KEYS[5])
|
|
@@ -53,30 +53,30 @@ local function batches(n, batchSize)
|
|
|
53
53
|
end
|
|
54
54
|
end
|
|
55
55
|
end
|
|
56
|
+
--[[
|
|
57
|
+
Function to move job to wait to be picked up by a waiting worker.
|
|
58
|
+
]]
|
|
59
|
+
-- Includes
|
|
56
60
|
--[[
|
|
57
61
|
Function to check for the meta.paused key to decide if we are paused or not
|
|
58
62
|
(since an empty list and !EXISTS are not really the same).
|
|
59
63
|
]]
|
|
60
64
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
61
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
65
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
62
66
|
if queueAttributes[1] then
|
|
63
|
-
return pausedKey, true
|
|
67
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
64
68
|
else
|
|
65
69
|
if queueAttributes[2] then
|
|
66
70
|
local activeCount = rcall("LLEN", activeKey)
|
|
67
71
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
68
|
-
return waitKey, true
|
|
72
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
69
73
|
else
|
|
70
|
-
return waitKey, false
|
|
74
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
71
75
|
end
|
|
72
76
|
end
|
|
73
77
|
end
|
|
74
|
-
return waitKey, false
|
|
78
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
75
79
|
end
|
|
76
|
-
--[[
|
|
77
|
-
Function to move job to wait to be picked up by a waiting worker.
|
|
78
|
-
]]
|
|
79
|
-
-- Includes
|
|
80
80
|
local function moveJobToWait(metaKey, activeKey, waitKey, pausedKey, markerKey, eventStreamKey,
|
|
81
81
|
jobId, pushCmd)
|
|
82
82
|
local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
|
|
@@ -71,20 +71,20 @@ end
|
|
|
71
71
|
(since an empty list and !EXISTS are not really the same).
|
|
72
72
|
]]
|
|
73
73
|
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
74
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
|
74
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
|
|
75
75
|
if queueAttributes[1] then
|
|
76
|
-
return pausedKey, true
|
|
76
|
+
return pausedKey, true, queueAttributes[3], queueAttributes[4]
|
|
77
77
|
else
|
|
78
78
|
if queueAttributes[2] then
|
|
79
79
|
local activeCount = rcall("LLEN", activeKey)
|
|
80
80
|
if activeCount >= tonumber(queueAttributes[2]) then
|
|
81
|
-
return waitKey, true
|
|
81
|
+
return waitKey, true, queueAttributes[3], queueAttributes[4]
|
|
82
82
|
else
|
|
83
|
-
return waitKey, false
|
|
83
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
84
84
|
end
|
|
85
85
|
end
|
|
86
86
|
end
|
|
87
|
-
return waitKey, false
|
|
87
|
+
return waitKey, false, queueAttributes[3], queueAttributes[4]
|
|
88
88
|
end
|
|
89
89
|
--[[
|
|
90
90
|
Function to move job from prioritized state to active.
|
|
@@ -115,20 +115,19 @@ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
|
115
115
|
end
|
|
116
116
|
end
|
|
117
117
|
local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
|
|
118
|
-
jobId, processedOn, maxJobs, markerKey, opts)
|
|
118
|
+
jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
|
|
119
119
|
local jobKey = keyPrefix .. jobId
|
|
120
120
|
-- Check if we need to perform rate limiting.
|
|
121
121
|
if maxJobs then
|
|
122
122
|
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
|
123
123
|
if jobCounter == 1 then
|
|
124
|
-
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
|
|
125
124
|
local integerDuration = math.floor(math.abs(limiterDuration))
|
|
126
125
|
rcall("PEXPIRE", rateLimiterKey, integerDuration)
|
|
127
126
|
end
|
|
128
127
|
end
|
|
129
|
-
local lockKey = jobKey .. ':lock'
|
|
130
128
|
-- get a lock
|
|
131
129
|
if opts['token'] ~= "0" then
|
|
130
|
+
local lockKey = jobKey .. ':lock'
|
|
132
131
|
rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
|
|
133
132
|
end
|
|
134
133
|
local optionalValues = {}
|
|
@@ -202,17 +201,19 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
|
|
|
202
201
|
addBaseMarkerIfNeeded(markerKey, isPaused)
|
|
203
202
|
end
|
|
204
203
|
end
|
|
205
|
-
local target, isPausedOrMaxed = getTargetQueueList(KEYS[9],
|
|
204
|
+
local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(KEYS[9],
|
|
205
|
+
activeKey, waitKey, KEYS[8])
|
|
206
206
|
-- Check if there are delayed jobs that we can move to wait.
|
|
207
207
|
local markerKey = KEYS[11]
|
|
208
208
|
promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
|
|
209
209
|
ARGV[2], KEYS[10], isPausedOrMaxed)
|
|
210
|
-
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
|
210
|
+
local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
|
|
211
211
|
local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
|
|
212
212
|
-- Check if we are rate limited first.
|
|
213
213
|
if expireTime > 0 then return {0, 0, expireTime, 0} end
|
|
214
214
|
-- paused or maxed queue
|
|
215
215
|
if isPausedOrMaxed then return {0, 0, 0, 0} end
|
|
216
|
+
local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
|
|
216
217
|
-- no job ID, try non-blocking move from wait to active
|
|
217
218
|
local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
|
218
219
|
-- Markers in waitlist DEPRECATED in v5: Will be completely removed in v6.
|
|
@@ -222,12 +223,12 @@ if jobId and string.sub(jobId, 1, 2) == "0:" then
|
|
|
222
223
|
end
|
|
223
224
|
if jobId then
|
|
224
225
|
return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
|
|
225
|
-
maxJobs, markerKey, opts)
|
|
226
|
+
maxJobs, limiterDuration, markerKey, opts)
|
|
226
227
|
else
|
|
227
228
|
jobId = moveJobFromPrioritizedToActive(KEYS[3], activeKey, KEYS[10])
|
|
228
229
|
if jobId then
|
|
229
230
|
return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
|
|
230
|
-
maxJobs, markerKey, opts)
|
|
231
|
+
maxJobs, limiterDuration, markerKey, opts)
|
|
231
232
|
end
|
|
232
233
|
end
|
|
233
234
|
-- Return the timestamp for the next delayed job if any.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
|
1
|
+
{"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Of,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|