bullmq 5.59.0 → 5.61.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/dist/cjs/classes/queue.js +16 -0
  2. package/dist/cjs/classes/queue.js.map +1 -1
  3. package/dist/cjs/classes/scripts.js +4 -1
  4. package/dist/cjs/classes/scripts.js.map +1 -1
  5. package/dist/cjs/classes/worker.js +5 -8
  6. package/dist/cjs/classes/worker.js.map +1 -1
  7. package/dist/cjs/commands/changePriority-7.lua +1 -1
  8. package/dist/cjs/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
  9. package/dist/cjs/commands/includes/getTargetQueueList.lua +5 -5
  10. package/dist/cjs/commands/includes/prepareJobForProcessing.lua +2 -4
  11. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -1
  12. package/dist/cjs/commands/moveToActive-11.lua +7 -4
  13. package/dist/cjs/commands/moveToFinished-14.lua +10 -7
  14. package/dist/cjs/scripts/addDelayedJob-6.js +5 -5
  15. package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
  16. package/dist/cjs/scripts/addParentJob-5.js +5 -5
  17. package/dist/cjs/scripts/addPrioritizedJob-9.js +5 -5
  18. package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
  19. package/dist/cjs/scripts/addStandardJob-9.js +5 -5
  20. package/dist/cjs/scripts/changePriority-7.js +5 -5
  21. package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
  22. package/dist/cjs/scripts/drain-5.js +5 -5
  23. package/dist/cjs/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
  24. package/dist/cjs/scripts/getRateLimitTtl-2.js.map +1 -0
  25. package/dist/cjs/scripts/index.js +1 -1
  26. package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +5 -5
  27. package/dist/cjs/scripts/moveJobsToWait-8.js +5 -5
  28. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +9 -9
  29. package/dist/cjs/scripts/moveToActive-11.js +13 -12
  30. package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
  31. package/dist/cjs/scripts/moveToFinished-14.js +16 -15
  32. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  33. package/dist/cjs/scripts/obliterate-2.js +5 -5
  34. package/dist/cjs/scripts/promote-9.js +5 -5
  35. package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
  36. package/dist/cjs/scripts/removeJob-2.js +5 -5
  37. package/dist/cjs/scripts/removeUnprocessedChildren-2.js +5 -5
  38. package/dist/cjs/scripts/reprocessJob-8.js +5 -5
  39. package/dist/cjs/scripts/retryJob-11.js +5 -5
  40. package/dist/cjs/scripts/updateJobScheduler-12.js +5 -5
  41. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  42. package/dist/cjs/version.js +1 -1
  43. package/dist/esm/classes/queue.d.ts +10 -0
  44. package/dist/esm/classes/queue.js +16 -0
  45. package/dist/esm/classes/queue.js.map +1 -1
  46. package/dist/esm/classes/scripts.js +4 -1
  47. package/dist/esm/classes/scripts.js.map +1 -1
  48. package/dist/esm/classes/worker.js +5 -8
  49. package/dist/esm/classes/worker.js.map +1 -1
  50. package/dist/esm/commands/changePriority-7.lua +1 -1
  51. package/dist/esm/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
  52. package/dist/esm/commands/includes/getTargetQueueList.lua +5 -5
  53. package/dist/esm/commands/includes/prepareJobForProcessing.lua +2 -4
  54. package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -1
  55. package/dist/esm/commands/moveToActive-11.lua +7 -4
  56. package/dist/esm/commands/moveToFinished-14.lua +10 -7
  57. package/dist/esm/scripts/addDelayedJob-6.js +5 -5
  58. package/dist/esm/scripts/addJobScheduler-11.js +5 -5
  59. package/dist/esm/scripts/addParentJob-5.js +5 -5
  60. package/dist/esm/scripts/addPrioritizedJob-9.js +5 -5
  61. package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
  62. package/dist/esm/scripts/addStandardJob-9.js +5 -5
  63. package/dist/esm/scripts/changePriority-7.js +5 -5
  64. package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
  65. package/dist/esm/scripts/drain-5.js +5 -5
  66. package/dist/esm/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
  67. package/dist/esm/scripts/getRateLimitTtl-2.js.map +1 -0
  68. package/dist/esm/scripts/index.d.ts +1 -1
  69. package/dist/esm/scripts/index.js +1 -1
  70. package/dist/esm/scripts/moveJobFromActiveToWait-9.js +5 -5
  71. package/dist/esm/scripts/moveJobsToWait-8.js +5 -5
  72. package/dist/esm/scripts/moveStalledJobsToWait-8.js +9 -9
  73. package/dist/esm/scripts/moveToActive-11.js +13 -12
  74. package/dist/esm/scripts/moveToActive-11.js.map +1 -1
  75. package/dist/esm/scripts/moveToFinished-14.js +16 -15
  76. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  77. package/dist/esm/scripts/obliterate-2.js +5 -5
  78. package/dist/esm/scripts/promote-9.js +5 -5
  79. package/dist/esm/scripts/removeChildDependency-1.js +5 -5
  80. package/dist/esm/scripts/removeJob-2.js +5 -5
  81. package/dist/esm/scripts/removeUnprocessedChildren-2.js +5 -5
  82. package/dist/esm/scripts/reprocessJob-8.js +5 -5
  83. package/dist/esm/scripts/retryJob-11.js +5 -5
  84. package/dist/esm/scripts/updateJobScheduler-12.js +5 -5
  85. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  86. package/dist/esm/version.d.ts +1 -1
  87. package/dist/esm/version.js +1 -1
  88. package/package.json +1 -1
  89. package/dist/cjs/scripts/getRateLimitTtl-1.js.map +0 -1
  90. package/dist/esm/scripts/getRateLimitTtl-1.js.map +0 -1
  91. /package/dist/esm/scripts/{getRateLimitTtl-1.d.ts → getRateLimitTtl-2.d.ts} +0 -0
@@ -51,14 +51,15 @@ local opts = cmsgpack.unpack(ARGV[3])
51
51
  --- @include "includes/prepareJobForProcessing"
52
52
  --- @include "includes/promoteDelayedJobs"
53
53
 
54
- local target, isPausedOrMaxed = getTargetQueueList(KEYS[9], activeKey, waitKey, KEYS[8])
54
+ local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(KEYS[9],
55
+ activeKey, waitKey, KEYS[8])
55
56
 
56
57
  -- Check if there are delayed jobs that we can move to wait.
57
58
  local markerKey = KEYS[11]
58
59
  promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
59
60
  ARGV[2], KEYS[10], isPausedOrMaxed)
60
61
 
61
- local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
62
+ local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
62
63
  local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
63
64
 
64
65
  -- Check if we are rate limited first.
@@ -67,6 +68,8 @@ if expireTime > 0 then return {0, 0, expireTime, 0} end
67
68
  -- paused or maxed queue
68
69
  if isPausedOrMaxed then return {0, 0, 0, 0} end
69
70
 
71
+ local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
72
+
70
73
  -- no job ID, try non-blocking move from wait to active
71
74
  local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
72
75
 
@@ -78,12 +81,12 @@ end
78
81
 
79
82
  if jobId then
80
83
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
81
- maxJobs, markerKey, opts)
84
+ maxJobs, limiterDuration, markerKey, opts)
82
85
  else
83
86
  jobId = moveJobFromPrioritizedToActive(KEYS[3], activeKey, KEYS[10])
84
87
  if jobId then
85
88
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
86
- maxJobs, markerKey, opts)
89
+ maxJobs, limiterDuration, markerKey, opts)
87
90
  end
88
91
  end
89
92
 
@@ -209,14 +209,15 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
209
209
  -- and not rate limited.
210
210
  if (ARGV[6] == "1") then
211
211
 
212
- local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
212
+ local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(metaKey, KEYS[2],
213
+ KEYS[1], KEYS[8])
213
214
 
214
215
  local markerKey = KEYS[14]
215
216
  -- Check if there are delayed jobs that can be promoted
216
217
  promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10],
217
218
  isPausedOrMaxed)
218
219
 
219
- local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
220
+ local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
220
221
  -- Check if we are rate limited first.
221
222
  local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
222
223
 
@@ -229,6 +230,8 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
229
230
  return {0, 0, 0, 0}
230
231
  end
231
232
 
233
+ local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
234
+
232
235
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
233
236
 
234
237
  if jobId then
@@ -241,17 +244,17 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
241
244
  if jobId == "0:0" then
242
245
  jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
243
246
  return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
244
- markerKey, opts)
247
+ limiterDuration, markerKey, opts)
245
248
  end
246
249
  else
247
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
248
- opts)
250
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
251
+ limiterDuration, markerKey, opts)
249
252
  end
250
253
  else
251
254
  jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
252
255
  if jobId then
253
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
254
- opts)
256
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
257
+ limiterDuration, markerKey, opts)
255
258
  end
256
259
  end
257
260
 
@@ -306,20 +306,20 @@ end
306
306
  (since an empty list and !EXISTS are not really the same).
307
307
  ]]
308
308
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
309
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
309
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
310
310
  if queueAttributes[1] then
311
- return pausedKey, true
311
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
312
312
  else
313
313
  if queueAttributes[2] then
314
314
  local activeCount = rcall("LLEN", activeKey)
315
315
  if activeCount >= tonumber(queueAttributes[2]) then
316
- return waitKey, true
316
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
317
317
  else
318
- return waitKey, false
318
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
319
319
  end
320
320
  end
321
321
  end
322
- return waitKey, false
322
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
323
323
  end
324
324
  local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
325
325
  local parentWaitKey = parentQueueKey .. ":wait"
@@ -182,20 +182,20 @@ end
182
182
  (since an empty list and !EXISTS are not really the same).
183
183
  ]]
184
184
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
185
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
185
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
186
186
  if queueAttributes[1] then
187
- return pausedKey, true
187
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
188
188
  else
189
189
  if queueAttributes[2] then
190
190
  local activeCount = rcall("LLEN", activeKey)
191
191
  if activeCount >= tonumber(queueAttributes[2]) then
192
- return waitKey, true
192
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
193
193
  else
194
- return waitKey, false
194
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
195
195
  end
196
196
  end
197
197
  end
198
- return waitKey, false
198
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
199
199
  end
200
200
  --[[
201
201
  Function to add job in target list and add marker if needed.
@@ -263,20 +263,20 @@ end
263
263
  (since an empty list and !EXISTS are not really the same).
264
264
  ]]
265
265
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
266
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
266
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
267
267
  if queueAttributes[1] then
268
- return pausedKey, true
268
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
269
269
  else
270
270
  if queueAttributes[2] then
271
271
  local activeCount = rcall("LLEN", activeKey)
272
272
  if activeCount >= tonumber(queueAttributes[2]) then
273
- return waitKey, true
273
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
274
274
  else
275
- return waitKey, false
275
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
276
276
  end
277
277
  end
278
278
  end
279
- return waitKey, false
279
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
280
280
  end
281
281
  local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
282
282
  local parentWaitKey = parentQueueKey .. ":wait"
@@ -300,20 +300,20 @@ end
300
300
  (since an empty list and !EXISTS are not really the same).
301
301
  ]]
302
302
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
303
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
303
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
304
304
  if queueAttributes[1] then
305
- return pausedKey, true
305
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
306
306
  else
307
307
  if queueAttributes[2] then
308
308
  local activeCount = rcall("LLEN", activeKey)
309
309
  if activeCount >= tonumber(queueAttributes[2]) then
310
- return waitKey, true
310
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
311
311
  else
312
- return waitKey, false
312
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
313
313
  end
314
314
  end
315
315
  end
316
- return waitKey, false
316
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
317
317
  end
318
318
  local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
319
319
  local parentWaitKey = parentQueueKey .. ":wait"
@@ -90,20 +90,20 @@ end
90
90
  (since an empty list and !EXISTS are not really the same).
91
91
  ]]
92
92
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
93
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
93
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
94
94
  if queueAttributes[1] then
95
- return pausedKey, true
95
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
96
96
  else
97
97
  if queueAttributes[2] then
98
98
  local activeCount = rcall("LLEN", activeKey)
99
99
  if activeCount >= tonumber(queueAttributes[2]) then
100
- return waitKey, true
100
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
101
101
  else
102
- return waitKey, false
102
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
103
103
  end
104
104
  end
105
105
  end
106
- return waitKey, false
106
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
107
107
  end
108
108
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
109
109
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -188,20 +188,20 @@ end
188
188
  (since an empty list and !EXISTS are not really the same).
189
189
  ]]
190
190
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
191
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
191
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
192
192
  if queueAttributes[1] then
193
- return pausedKey, true
193
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
194
194
  else
195
195
  if queueAttributes[2] then
196
196
  local activeCount = rcall("LLEN", activeKey)
197
197
  if activeCount >= tonumber(queueAttributes[2]) then
198
- return waitKey, true
198
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
199
199
  else
200
- return waitKey, false
200
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
201
201
  end
202
202
  end
203
203
  end
204
- return waitKey, false
204
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
205
205
  end
206
206
  --[[
207
207
  Function to handle the case when job is duplicated.
@@ -62,20 +62,20 @@ end
62
62
  (since an empty list and !EXISTS are not really the same).
63
63
  ]]
64
64
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
65
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
65
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
66
66
  if queueAttributes[1] then
67
- return pausedKey, true
67
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
68
68
  else
69
69
  if queueAttributes[2] then
70
70
  local activeCount = rcall("LLEN", activeKey)
71
71
  if activeCount >= tonumber(queueAttributes[2]) then
72
- return waitKey, true
72
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
73
73
  else
74
- return waitKey, false
74
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
75
75
  end
76
76
  end
77
77
  end
78
- return waitKey, false
78
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
79
79
  end
80
80
  --[[
81
81
  Function to push back job considering priority in front of same prioritized jobs.
@@ -126,20 +126,20 @@ end
126
126
  (since an empty list and !EXISTS are not really the same).
127
127
  ]]
128
128
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
129
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
129
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
130
130
  if queueAttributes[1] then
131
- return pausedKey, true
131
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
132
132
  else
133
133
  if queueAttributes[2] then
134
134
  local activeCount = rcall("LLEN", activeKey)
135
135
  if activeCount >= tonumber(queueAttributes[2]) then
136
- return waitKey, true
136
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
137
137
  else
138
- return waitKey, false
138
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
139
139
  end
140
140
  end
141
141
  end
142
- return waitKey, false
142
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
143
143
  end
144
144
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
145
145
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -98,20 +98,20 @@ end
98
98
  (since an empty list and !EXISTS are not really the same).
99
99
  ]]
100
100
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
101
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
101
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
102
102
  if queueAttributes[1] then
103
- return pausedKey, true
103
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
104
104
  else
105
105
  if queueAttributes[2] then
106
106
  local activeCount = rcall("LLEN", activeKey)
107
107
  if activeCount >= tonumber(queueAttributes[2]) then
108
- return waitKey, true
108
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
109
109
  else
110
- return waitKey, false
110
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
111
111
  end
112
112
  end
113
113
  end
114
- return waitKey, false
114
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
115
115
  end
116
116
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
117
117
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -5,6 +5,7 @@ const content = `--[[
5
5
  Get rate limit ttl
6
6
  Input:
7
7
  KEYS[1] 'limiter'
8
+ KEYS[2] 'meta'
8
9
  ARGV[1] maxJobs
9
10
  ]]
10
11
  local rcall = redis.call
@@ -28,12 +29,16 @@ local rateLimiterKey = KEYS[1]
28
29
  if ARGV[1] ~= "0" then
29
30
  return getRateLimitTTL(tonumber(ARGV[1]), rateLimiterKey)
30
31
  else
32
+ local rateLimitMax = rcall("HGET", KEYS[2], "max")
33
+ if rateLimitMax then
34
+ return getRateLimitTTL(tonumber(rateLimitMax), rateLimiterKey)
35
+ end
31
36
  return rcall("PTTL", rateLimiterKey)
32
37
  end
33
38
  `;
34
39
  exports.getRateLimitTtl = {
35
40
  name: 'getRateLimitTtl',
36
41
  content,
37
- keys: 1,
42
+ keys: 2,
38
43
  };
39
- //# sourceMappingURL=getRateLimitTtl-1.js.map
44
+ //# sourceMappingURL=getRateLimitTtl-2.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"getRateLimitTtl-2.js","sourceRoot":"","sources":["../../../src/scripts/getRateLimitTtl-2.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkCf,CAAC;AACW,QAAA,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -20,7 +20,7 @@ tslib_1.__exportStar(require("./getDependencyCounts-4"), exports);
20
20
  tslib_1.__exportStar(require("./getJobScheduler-1"), exports);
21
21
  tslib_1.__exportStar(require("./getMetrics-2"), exports);
22
22
  tslib_1.__exportStar(require("./getRanges-1"), exports);
23
- tslib_1.__exportStar(require("./getRateLimitTtl-1"), exports);
23
+ tslib_1.__exportStar(require("./getRateLimitTtl-2"), exports);
24
24
  tslib_1.__exportStar(require("./getState-8"), exports);
25
25
  tslib_1.__exportStar(require("./getStateV2-8"), exports);
26
26
  tslib_1.__exportStar(require("./isFinished-3"), exports);
@@ -60,20 +60,20 @@ end
60
60
  (since an empty list and !EXISTS are not really the same).
61
61
  ]]
62
62
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
63
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
63
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
64
64
  if queueAttributes[1] then
65
- return pausedKey, true
65
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
66
66
  else
67
67
  if queueAttributes[2] then
68
68
  local activeCount = rcall("LLEN", activeKey)
69
69
  if activeCount >= tonumber(queueAttributes[2]) then
70
- return waitKey, true
70
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
71
71
  else
72
- return waitKey, false
72
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
73
73
  end
74
74
  end
75
75
  end
76
- return waitKey, false
76
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
77
77
  end
78
78
  local function removeLock(jobKey, stalledKey, token, jobId)
79
79
  if token ~= "0" then
@@ -64,20 +64,20 @@ end
64
64
  (since an empty list and !EXISTS are not really the same).
65
65
  ]]
66
66
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
67
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
67
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
68
68
  if queueAttributes[1] then
69
- return pausedKey, true
69
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
70
70
  else
71
71
  if queueAttributes[2] then
72
72
  local activeCount = rcall("LLEN", activeKey)
73
73
  if activeCount >= tonumber(queueAttributes[2]) then
74
- return waitKey, true
74
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
75
75
  else
76
- return waitKey, false
76
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
77
77
  end
78
78
  end
79
79
  end
80
- return waitKey, false
80
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
81
81
  end
82
82
  local metaKey = KEYS[6]
83
83
  local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[7], KEYS[4], KEYS[5])
@@ -53,30 +53,30 @@ local function batches(n, batchSize)
53
53
  end
54
54
  end
55
55
  end
56
+ --[[
57
+ Function to move job to wait to be picked up by a waiting worker.
58
+ ]]
59
+ -- Includes
56
60
  --[[
57
61
  Function to check for the meta.paused key to decide if we are paused or not
58
62
  (since an empty list and !EXISTS are not really the same).
59
63
  ]]
60
64
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
61
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
65
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
62
66
  if queueAttributes[1] then
63
- return pausedKey, true
67
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
64
68
  else
65
69
  if queueAttributes[2] then
66
70
  local activeCount = rcall("LLEN", activeKey)
67
71
  if activeCount >= tonumber(queueAttributes[2]) then
68
- return waitKey, true
72
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
69
73
  else
70
- return waitKey, false
74
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
71
75
  end
72
76
  end
73
77
  end
74
- return waitKey, false
78
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
75
79
  end
76
- --[[
77
- Function to move job to wait to be picked up by a waiting worker.
78
- ]]
79
- -- Includes
80
80
  local function moveJobToWait(metaKey, activeKey, waitKey, pausedKey, markerKey, eventStreamKey,
81
81
  jobId, pushCmd)
82
82
  local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
@@ -71,20 +71,20 @@ end
71
71
  (since an empty list and !EXISTS are not really the same).
72
72
  ]]
73
73
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
74
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
74
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
75
75
  if queueAttributes[1] then
76
- return pausedKey, true
76
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
77
77
  else
78
78
  if queueAttributes[2] then
79
79
  local activeCount = rcall("LLEN", activeKey)
80
80
  if activeCount >= tonumber(queueAttributes[2]) then
81
- return waitKey, true
81
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
82
82
  else
83
- return waitKey, false
83
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
84
84
  end
85
85
  end
86
86
  end
87
- return waitKey, false
87
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
88
88
  end
89
89
  --[[
90
90
  Function to move job from prioritized state to active.
@@ -115,20 +115,19 @@ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
115
115
  end
116
116
  end
117
117
  local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
118
- jobId, processedOn, maxJobs, markerKey, opts)
118
+ jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
119
119
  local jobKey = keyPrefix .. jobId
120
120
  -- Check if we need to perform rate limiting.
121
121
  if maxJobs then
122
122
  local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
123
123
  if jobCounter == 1 then
124
- local limiterDuration = opts['limiter'] and opts['limiter']['duration']
125
124
  local integerDuration = math.floor(math.abs(limiterDuration))
126
125
  rcall("PEXPIRE", rateLimiterKey, integerDuration)
127
126
  end
128
127
  end
129
- local lockKey = jobKey .. ':lock'
130
128
  -- get a lock
131
129
  if opts['token'] ~= "0" then
130
+ local lockKey = jobKey .. ':lock'
132
131
  rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
133
132
  end
134
133
  local optionalValues = {}
@@ -202,17 +201,19 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
202
201
  addBaseMarkerIfNeeded(markerKey, isPaused)
203
202
  end
204
203
  end
205
- local target, isPausedOrMaxed = getTargetQueueList(KEYS[9], activeKey, waitKey, KEYS[8])
204
+ local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(KEYS[9],
205
+ activeKey, waitKey, KEYS[8])
206
206
  -- Check if there are delayed jobs that we can move to wait.
207
207
  local markerKey = KEYS[11]
208
208
  promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
209
209
  ARGV[2], KEYS[10], isPausedOrMaxed)
210
- local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
210
+ local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
211
211
  local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
212
212
  -- Check if we are rate limited first.
213
213
  if expireTime > 0 then return {0, 0, expireTime, 0} end
214
214
  -- paused or maxed queue
215
215
  if isPausedOrMaxed then return {0, 0, 0, 0} end
216
+ local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
216
217
  -- no job ID, try non-blocking move from wait to active
217
218
  local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
218
219
  -- Markers in waitlist DEPRECATED in v5: Will be completely removed in v6.
@@ -222,12 +223,12 @@ if jobId and string.sub(jobId, 1, 2) == "0:" then
222
223
  end
223
224
  if jobId then
224
225
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
225
- maxJobs, markerKey, opts)
226
+ maxJobs, limiterDuration, markerKey, opts)
226
227
  else
227
228
  jobId = moveJobFromPrioritizedToActive(KEYS[3], activeKey, KEYS[10])
228
229
  if jobId then
229
230
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
230
- maxJobs, markerKey, opts)
231
+ maxJobs, limiterDuration, markerKey, opts)
231
232
  end
232
233
  end
233
234
  -- Return the timestamp for the next delayed job if any.
@@ -1 +1 @@
1
- {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyOf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Of,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}