bullmq 5.59.0 → 5.61.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/dist/cjs/classes/queue.js +16 -0
  2. package/dist/cjs/classes/queue.js.map +1 -1
  3. package/dist/cjs/classes/scripts.js +4 -1
  4. package/dist/cjs/classes/scripts.js.map +1 -1
  5. package/dist/cjs/classes/worker.js +5 -8
  6. package/dist/cjs/classes/worker.js.map +1 -1
  7. package/dist/cjs/commands/changePriority-7.lua +1 -1
  8. package/dist/cjs/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
  9. package/dist/cjs/commands/includes/getTargetQueueList.lua +5 -5
  10. package/dist/cjs/commands/includes/prepareJobForProcessing.lua +2 -4
  11. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -1
  12. package/dist/cjs/commands/moveToActive-11.lua +7 -4
  13. package/dist/cjs/commands/moveToFinished-14.lua +10 -7
  14. package/dist/cjs/scripts/addDelayedJob-6.js +5 -5
  15. package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
  16. package/dist/cjs/scripts/addParentJob-5.js +5 -5
  17. package/dist/cjs/scripts/addPrioritizedJob-9.js +5 -5
  18. package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
  19. package/dist/cjs/scripts/addStandardJob-9.js +5 -5
  20. package/dist/cjs/scripts/changePriority-7.js +5 -5
  21. package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
  22. package/dist/cjs/scripts/drain-5.js +5 -5
  23. package/dist/cjs/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
  24. package/dist/cjs/scripts/getRateLimitTtl-2.js.map +1 -0
  25. package/dist/cjs/scripts/index.js +1 -1
  26. package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +5 -5
  27. package/dist/cjs/scripts/moveJobsToWait-8.js +5 -5
  28. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +9 -9
  29. package/dist/cjs/scripts/moveToActive-11.js +13 -12
  30. package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
  31. package/dist/cjs/scripts/moveToFinished-14.js +16 -15
  32. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  33. package/dist/cjs/scripts/obliterate-2.js +5 -5
  34. package/dist/cjs/scripts/promote-9.js +5 -5
  35. package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
  36. package/dist/cjs/scripts/removeJob-2.js +5 -5
  37. package/dist/cjs/scripts/removeUnprocessedChildren-2.js +5 -5
  38. package/dist/cjs/scripts/reprocessJob-8.js +5 -5
  39. package/dist/cjs/scripts/retryJob-11.js +5 -5
  40. package/dist/cjs/scripts/updateJobScheduler-12.js +5 -5
  41. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  42. package/dist/cjs/version.js +1 -1
  43. package/dist/esm/classes/queue.d.ts +10 -0
  44. package/dist/esm/classes/queue.js +16 -0
  45. package/dist/esm/classes/queue.js.map +1 -1
  46. package/dist/esm/classes/scripts.js +4 -1
  47. package/dist/esm/classes/scripts.js.map +1 -1
  48. package/dist/esm/classes/worker.js +5 -8
  49. package/dist/esm/classes/worker.js.map +1 -1
  50. package/dist/esm/commands/changePriority-7.lua +1 -1
  51. package/dist/esm/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
  52. package/dist/esm/commands/includes/getTargetQueueList.lua +5 -5
  53. package/dist/esm/commands/includes/prepareJobForProcessing.lua +2 -4
  54. package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -1
  55. package/dist/esm/commands/moveToActive-11.lua +7 -4
  56. package/dist/esm/commands/moveToFinished-14.lua +10 -7
  57. package/dist/esm/scripts/addDelayedJob-6.js +5 -5
  58. package/dist/esm/scripts/addJobScheduler-11.js +5 -5
  59. package/dist/esm/scripts/addParentJob-5.js +5 -5
  60. package/dist/esm/scripts/addPrioritizedJob-9.js +5 -5
  61. package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
  62. package/dist/esm/scripts/addStandardJob-9.js +5 -5
  63. package/dist/esm/scripts/changePriority-7.js +5 -5
  64. package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
  65. package/dist/esm/scripts/drain-5.js +5 -5
  66. package/dist/esm/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
  67. package/dist/esm/scripts/getRateLimitTtl-2.js.map +1 -0
  68. package/dist/esm/scripts/index.d.ts +1 -1
  69. package/dist/esm/scripts/index.js +1 -1
  70. package/dist/esm/scripts/moveJobFromActiveToWait-9.js +5 -5
  71. package/dist/esm/scripts/moveJobsToWait-8.js +5 -5
  72. package/dist/esm/scripts/moveStalledJobsToWait-8.js +9 -9
  73. package/dist/esm/scripts/moveToActive-11.js +13 -12
  74. package/dist/esm/scripts/moveToActive-11.js.map +1 -1
  75. package/dist/esm/scripts/moveToFinished-14.js +16 -15
  76. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  77. package/dist/esm/scripts/obliterate-2.js +5 -5
  78. package/dist/esm/scripts/promote-9.js +5 -5
  79. package/dist/esm/scripts/removeChildDependency-1.js +5 -5
  80. package/dist/esm/scripts/removeJob-2.js +5 -5
  81. package/dist/esm/scripts/removeUnprocessedChildren-2.js +5 -5
  82. package/dist/esm/scripts/reprocessJob-8.js +5 -5
  83. package/dist/esm/scripts/retryJob-11.js +5 -5
  84. package/dist/esm/scripts/updateJobScheduler-12.js +5 -5
  85. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  86. package/dist/esm/version.d.ts +1 -1
  87. package/dist/esm/version.js +1 -1
  88. package/package.json +1 -1
  89. package/dist/cjs/scripts/getRateLimitTtl-1.js.map +0 -1
  90. package/dist/esm/scripts/getRateLimitTtl-1.js.map +0 -1
  91. /package/dist/esm/scripts/{getRateLimitTtl-1.d.ts → getRateLimitTtl-2.d.ts} +0 -0
@@ -209,14 +209,15 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
209
209
  -- and not rate limited.
210
210
  if (ARGV[6] == "1") then
211
211
 
212
- local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
212
+ local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(metaKey, KEYS[2],
213
+ KEYS[1], KEYS[8])
213
214
 
214
215
  local markerKey = KEYS[14]
215
216
  -- Check if there are delayed jobs that can be promoted
216
217
  promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10],
217
218
  isPausedOrMaxed)
218
219
 
219
- local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
220
+ local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
220
221
  -- Check if we are rate limited first.
221
222
  local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
222
223
 
@@ -229,6 +230,8 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
229
230
  return {0, 0, 0, 0}
230
231
  end
231
232
 
233
+ local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
234
+
232
235
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
233
236
 
234
237
  if jobId then
@@ -241,17 +244,17 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
241
244
  if jobId == "0:0" then
242
245
  jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
243
246
  return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
244
- markerKey, opts)
247
+ limiterDuration, markerKey, opts)
245
248
  end
246
249
  else
247
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
248
- opts)
250
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
251
+ limiterDuration, markerKey, opts)
249
252
  end
250
253
  else
251
254
  jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
252
255
  if jobId then
253
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
254
- opts)
256
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
257
+ limiterDuration, markerKey, opts)
255
258
  end
256
259
  end
257
260
 
@@ -303,20 +303,20 @@ end
303
303
  (since an empty list and !EXISTS are not really the same).
304
304
  ]]
305
305
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
306
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
306
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
307
307
  if queueAttributes[1] then
308
- return pausedKey, true
308
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
309
309
  else
310
310
  if queueAttributes[2] then
311
311
  local activeCount = rcall("LLEN", activeKey)
312
312
  if activeCount >= tonumber(queueAttributes[2]) then
313
- return waitKey, true
313
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
314
314
  else
315
- return waitKey, false
315
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
316
316
  end
317
317
  end
318
318
  end
319
- return waitKey, false
319
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
320
320
  end
321
321
  local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
322
322
  local parentWaitKey = parentQueueKey .. ":wait"
@@ -179,20 +179,20 @@ end
179
179
  (since an empty list and !EXISTS are not really the same).
180
180
  ]]
181
181
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
182
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
182
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
183
183
  if queueAttributes[1] then
184
- return pausedKey, true
184
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
185
185
  else
186
186
  if queueAttributes[2] then
187
187
  local activeCount = rcall("LLEN", activeKey)
188
188
  if activeCount >= tonumber(queueAttributes[2]) then
189
- return waitKey, true
189
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
190
190
  else
191
- return waitKey, false
191
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
192
192
  end
193
193
  end
194
194
  end
195
- return waitKey, false
195
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
196
196
  end
197
197
  --[[
198
198
  Function to add job in target list and add marker if needed.
@@ -260,20 +260,20 @@ end
260
260
  (since an empty list and !EXISTS are not really the same).
261
261
  ]]
262
262
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
263
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
263
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
264
264
  if queueAttributes[1] then
265
- return pausedKey, true
265
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
266
266
  else
267
267
  if queueAttributes[2] then
268
268
  local activeCount = rcall("LLEN", activeKey)
269
269
  if activeCount >= tonumber(queueAttributes[2]) then
270
- return waitKey, true
270
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
271
271
  else
272
- return waitKey, false
272
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
273
273
  end
274
274
  end
275
275
  end
276
- return waitKey, false
276
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
277
277
  end
278
278
  local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
279
279
  local parentWaitKey = parentQueueKey .. ":wait"
@@ -297,20 +297,20 @@ end
297
297
  (since an empty list and !EXISTS are not really the same).
298
298
  ]]
299
299
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
300
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
300
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
301
301
  if queueAttributes[1] then
302
- return pausedKey, true
302
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
303
303
  else
304
304
  if queueAttributes[2] then
305
305
  local activeCount = rcall("LLEN", activeKey)
306
306
  if activeCount >= tonumber(queueAttributes[2]) then
307
- return waitKey, true
307
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
308
308
  else
309
- return waitKey, false
309
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
310
310
  end
311
311
  end
312
312
  end
313
- return waitKey, false
313
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
314
314
  end
315
315
  local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
316
316
  local parentWaitKey = parentQueueKey .. ":wait"
@@ -87,20 +87,20 @@ end
87
87
  (since an empty list and !EXISTS are not really the same).
88
88
  ]]
89
89
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
90
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
90
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
91
91
  if queueAttributes[1] then
92
- return pausedKey, true
92
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
93
93
  else
94
94
  if queueAttributes[2] then
95
95
  local activeCount = rcall("LLEN", activeKey)
96
96
  if activeCount >= tonumber(queueAttributes[2]) then
97
- return waitKey, true
97
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
98
98
  else
99
- return waitKey, false
99
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
100
100
  end
101
101
  end
102
102
  end
103
- return waitKey, false
103
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
104
104
  end
105
105
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
106
106
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -185,20 +185,20 @@ end
185
185
  (since an empty list and !EXISTS are not really the same).
186
186
  ]]
187
187
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
188
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
188
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
189
189
  if queueAttributes[1] then
190
- return pausedKey, true
190
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
191
191
  else
192
192
  if queueAttributes[2] then
193
193
  local activeCount = rcall("LLEN", activeKey)
194
194
  if activeCount >= tonumber(queueAttributes[2]) then
195
- return waitKey, true
195
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
196
196
  else
197
- return waitKey, false
197
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
198
198
  end
199
199
  end
200
200
  end
201
- return waitKey, false
201
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
202
202
  end
203
203
  --[[
204
204
  Function to handle the case when job is duplicated.
@@ -59,20 +59,20 @@ end
59
59
  (since an empty list and !EXISTS are not really the same).
60
60
  ]]
61
61
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
62
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
62
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
63
63
  if queueAttributes[1] then
64
- return pausedKey, true
64
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
65
65
  else
66
66
  if queueAttributes[2] then
67
67
  local activeCount = rcall("LLEN", activeKey)
68
68
  if activeCount >= tonumber(queueAttributes[2]) then
69
- return waitKey, true
69
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
70
70
  else
71
- return waitKey, false
71
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
72
72
  end
73
73
  end
74
74
  end
75
- return waitKey, false
75
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
76
76
  end
77
77
  --[[
78
78
  Function to push back job considering priority in front of same prioritized jobs.
@@ -123,20 +123,20 @@ end
123
123
  (since an empty list and !EXISTS are not really the same).
124
124
  ]]
125
125
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
126
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
126
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
127
127
  if queueAttributes[1] then
128
- return pausedKey, true
128
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
129
129
  else
130
130
  if queueAttributes[2] then
131
131
  local activeCount = rcall("LLEN", activeKey)
132
132
  if activeCount >= tonumber(queueAttributes[2]) then
133
- return waitKey, true
133
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
134
134
  else
135
- return waitKey, false
135
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
136
136
  end
137
137
  end
138
138
  end
139
- return waitKey, false
139
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
140
140
  end
141
141
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
142
142
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -95,20 +95,20 @@ end
95
95
  (since an empty list and !EXISTS are not really the same).
96
96
  ]]
97
97
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
98
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
98
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
99
99
  if queueAttributes[1] then
100
- return pausedKey, true
100
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
101
101
  else
102
102
  if queueAttributes[2] then
103
103
  local activeCount = rcall("LLEN", activeKey)
104
104
  if activeCount >= tonumber(queueAttributes[2]) then
105
- return waitKey, true
105
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
106
106
  else
107
- return waitKey, false
107
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
108
108
  end
109
109
  end
110
110
  end
111
- return waitKey, false
111
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
112
112
  end
113
113
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
114
114
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -2,6 +2,7 @@ const content = `--[[
2
2
  Get rate limit ttl
3
3
  Input:
4
4
  KEYS[1] 'limiter'
5
+ KEYS[2] 'meta'
5
6
  ARGV[1] maxJobs
6
7
  ]]
7
8
  local rcall = redis.call
@@ -25,12 +26,16 @@ local rateLimiterKey = KEYS[1]
25
26
  if ARGV[1] ~= "0" then
26
27
  return getRateLimitTTL(tonumber(ARGV[1]), rateLimiterKey)
27
28
  else
29
+ local rateLimitMax = rcall("HGET", KEYS[2], "max")
30
+ if rateLimitMax then
31
+ return getRateLimitTTL(tonumber(rateLimitMax), rateLimiterKey)
32
+ end
28
33
  return rcall("PTTL", rateLimiterKey)
29
34
  end
30
35
  `;
31
36
  export const getRateLimitTtl = {
32
37
  name: 'getRateLimitTtl',
33
38
  content,
34
- keys: 1,
39
+ keys: 2,
35
40
  };
36
- //# sourceMappingURL=getRateLimitTtl-1.js.map
41
+ //# sourceMappingURL=getRateLimitTtl-2.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"getRateLimitTtl-2.js","sourceRoot":"","sources":["../../../src/scripts/getRateLimitTtl-2.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkCf,CAAC;AACF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -17,7 +17,7 @@ export * from './getDependencyCounts-4';
17
17
  export * from './getJobScheduler-1';
18
18
  export * from './getMetrics-2';
19
19
  export * from './getRanges-1';
20
- export * from './getRateLimitTtl-1';
20
+ export * from './getRateLimitTtl-2';
21
21
  export * from './getState-8';
22
22
  export * from './getStateV2-8';
23
23
  export * from './isFinished-3';
@@ -17,7 +17,7 @@ export * from './getDependencyCounts-4';
17
17
  export * from './getJobScheduler-1';
18
18
  export * from './getMetrics-2';
19
19
  export * from './getRanges-1';
20
- export * from './getRateLimitTtl-1';
20
+ export * from './getRateLimitTtl-2';
21
21
  export * from './getState-8';
22
22
  export * from './getStateV2-8';
23
23
  export * from './isFinished-3';
@@ -57,20 +57,20 @@ end
57
57
  (since an empty list and !EXISTS are not really the same).
58
58
  ]]
59
59
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
60
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
60
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
61
61
  if queueAttributes[1] then
62
- return pausedKey, true
62
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
63
63
  else
64
64
  if queueAttributes[2] then
65
65
  local activeCount = rcall("LLEN", activeKey)
66
66
  if activeCount >= tonumber(queueAttributes[2]) then
67
- return waitKey, true
67
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
68
68
  else
69
- return waitKey, false
69
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
70
70
  end
71
71
  end
72
72
  end
73
- return waitKey, false
73
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
74
74
  end
75
75
  local function removeLock(jobKey, stalledKey, token, jobId)
76
76
  if token ~= "0" then
@@ -61,20 +61,20 @@ end
61
61
  (since an empty list and !EXISTS are not really the same).
62
62
  ]]
63
63
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
64
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
64
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
65
65
  if queueAttributes[1] then
66
- return pausedKey, true
66
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
67
67
  else
68
68
  if queueAttributes[2] then
69
69
  local activeCount = rcall("LLEN", activeKey)
70
70
  if activeCount >= tonumber(queueAttributes[2]) then
71
- return waitKey, true
71
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
72
72
  else
73
- return waitKey, false
73
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
74
74
  end
75
75
  end
76
76
  end
77
- return waitKey, false
77
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
78
78
  end
79
79
  local metaKey = KEYS[6]
80
80
  local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[7], KEYS[4], KEYS[5])
@@ -50,30 +50,30 @@ local function batches(n, batchSize)
50
50
  end
51
51
  end
52
52
  end
53
+ --[[
54
+ Function to move job to wait to be picked up by a waiting worker.
55
+ ]]
56
+ -- Includes
53
57
  --[[
54
58
  Function to check for the meta.paused key to decide if we are paused or not
55
59
  (since an empty list and !EXISTS are not really the same).
56
60
  ]]
57
61
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
58
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
62
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
59
63
  if queueAttributes[1] then
60
- return pausedKey, true
64
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
61
65
  else
62
66
  if queueAttributes[2] then
63
67
  local activeCount = rcall("LLEN", activeKey)
64
68
  if activeCount >= tonumber(queueAttributes[2]) then
65
- return waitKey, true
69
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
66
70
  else
67
- return waitKey, false
71
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
68
72
  end
69
73
  end
70
74
  end
71
- return waitKey, false
75
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
72
76
  end
73
- --[[
74
- Function to move job to wait to be picked up by a waiting worker.
75
- ]]
76
- -- Includes
77
77
  local function moveJobToWait(metaKey, activeKey, waitKey, pausedKey, markerKey, eventStreamKey,
78
78
  jobId, pushCmd)
79
79
  local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
@@ -68,20 +68,20 @@ end
68
68
  (since an empty list and !EXISTS are not really the same).
69
69
  ]]
70
70
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
71
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
71
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
72
72
  if queueAttributes[1] then
73
- return pausedKey, true
73
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
74
74
  else
75
75
  if queueAttributes[2] then
76
76
  local activeCount = rcall("LLEN", activeKey)
77
77
  if activeCount >= tonumber(queueAttributes[2]) then
78
- return waitKey, true
78
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
79
79
  else
80
- return waitKey, false
80
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
81
81
  end
82
82
  end
83
83
  end
84
- return waitKey, false
84
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
85
85
  end
86
86
  --[[
87
87
  Function to move job from prioritized state to active.
@@ -112,20 +112,19 @@ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
112
112
  end
113
113
  end
114
114
  local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
115
- jobId, processedOn, maxJobs, markerKey, opts)
115
+ jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
116
116
  local jobKey = keyPrefix .. jobId
117
117
  -- Check if we need to perform rate limiting.
118
118
  if maxJobs then
119
119
  local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
120
120
  if jobCounter == 1 then
121
- local limiterDuration = opts['limiter'] and opts['limiter']['duration']
122
121
  local integerDuration = math.floor(math.abs(limiterDuration))
123
122
  rcall("PEXPIRE", rateLimiterKey, integerDuration)
124
123
  end
125
124
  end
126
- local lockKey = jobKey .. ':lock'
127
125
  -- get a lock
128
126
  if opts['token'] ~= "0" then
127
+ local lockKey = jobKey .. ':lock'
129
128
  rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
130
129
  end
131
130
  local optionalValues = {}
@@ -199,17 +198,19 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
199
198
  addBaseMarkerIfNeeded(markerKey, isPaused)
200
199
  end
201
200
  end
202
- local target, isPausedOrMaxed = getTargetQueueList(KEYS[9], activeKey, waitKey, KEYS[8])
201
+ local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(KEYS[9],
202
+ activeKey, waitKey, KEYS[8])
203
203
  -- Check if there are delayed jobs that we can move to wait.
204
204
  local markerKey = KEYS[11]
205
205
  promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
206
206
  ARGV[2], KEYS[10], isPausedOrMaxed)
207
- local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
207
+ local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
208
208
  local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
209
209
  -- Check if we are rate limited first.
210
210
  if expireTime > 0 then return {0, 0, expireTime, 0} end
211
211
  -- paused or maxed queue
212
212
  if isPausedOrMaxed then return {0, 0, 0, 0} end
213
+ local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
213
214
  -- no job ID, try non-blocking move from wait to active
214
215
  local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
215
216
  -- Markers in waitlist DEPRECATED in v5: Will be completely removed in v6.
@@ -219,12 +220,12 @@ if jobId and string.sub(jobId, 1, 2) == "0:" then
219
220
  end
220
221
  if jobId then
221
222
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
222
- maxJobs, markerKey, opts)
223
+ maxJobs, limiterDuration, markerKey, opts)
223
224
  else
224
225
  jobId = moveJobFromPrioritizedToActive(KEYS[3], activeKey, KEYS[10])
225
226
  if jobId then
226
227
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
227
- maxJobs, markerKey, opts)
228
+ maxJobs, limiterDuration, markerKey, opts)
228
229
  end
229
230
  end
230
231
  -- Return the timestamp for the next delayed job if any.
@@ -1 +1 @@
1
- {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyOf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Of,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -137,20 +137,20 @@ end
137
137
  (since an empty list and !EXISTS are not really the same).
138
138
  ]]
139
139
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
140
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
140
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
141
141
  if queueAttributes[1] then
142
- return pausedKey, true
142
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
143
143
  else
144
144
  if queueAttributes[2] then
145
145
  local activeCount = rcall("LLEN", activeKey)
146
146
  if activeCount >= tonumber(queueAttributes[2]) then
147
- return waitKey, true
147
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
148
148
  else
149
- return waitKey, false
149
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
150
150
  end
151
151
  end
152
152
  end
153
- return waitKey, false
153
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
154
154
  end
155
155
  --[[
156
156
  Function to move job from prioritized state to active.
@@ -538,20 +538,19 @@ end
538
538
  ]]
539
539
  -- Includes
540
540
  local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
541
- jobId, processedOn, maxJobs, markerKey, opts)
541
+ jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
542
542
  local jobKey = keyPrefix .. jobId
543
543
  -- Check if we need to perform rate limiting.
544
544
  if maxJobs then
545
545
  local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
546
546
  if jobCounter == 1 then
547
- local limiterDuration = opts['limiter'] and opts['limiter']['duration']
548
547
  local integerDuration = math.floor(math.abs(limiterDuration))
549
548
  rcall("PEXPIRE", rateLimiterKey, integerDuration)
550
549
  end
551
550
  end
552
- local lockKey = jobKey .. ':lock'
553
551
  -- get a lock
554
552
  if opts['token'] ~= "0" then
553
+ local lockKey = jobKey .. ':lock'
555
554
  rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
556
555
  end
557
556
  local optionalValues = {}
@@ -789,12 +788,13 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
789
788
  -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
790
789
  -- and not rate limited.
791
790
  if (ARGV[6] == "1") then
792
- local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
791
+ local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(metaKey, KEYS[2],
792
+ KEYS[1], KEYS[8])
793
793
  local markerKey = KEYS[14]
794
794
  -- Check if there are delayed jobs that can be promoted
795
795
  promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10],
796
796
  isPausedOrMaxed)
797
- local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
797
+ local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
798
798
  -- Check if we are rate limited first.
799
799
  local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
800
800
  if expireTime > 0 then
@@ -804,6 +804,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
804
804
  if isPausedOrMaxed then
805
805
  return {0, 0, 0, 0}
806
806
  end
807
+ local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
807
808
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
808
809
  if jobId then
809
810
  -- Markers in waitlist DEPRECATED in v5: Remove in v6.
@@ -814,17 +815,17 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
814
815
  if jobId == "0:0" then
815
816
  jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
816
817
  return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
817
- markerKey, opts)
818
+ limiterDuration, markerKey, opts)
818
819
  end
819
820
  else
820
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
821
- opts)
821
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
822
+ limiterDuration, markerKey, opts)
822
823
  end
823
824
  else
824
825
  jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
825
826
  if jobId then
826
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
827
- opts)
827
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
828
+ limiterDuration, markerKey, opts)
828
829
  end
829
830
  end
830
831
  -- Return the timestamp for the next delayed job if any.
@@ -1 +1 @@
1
- {"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAm1Bf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAo1Bf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}