bullmq 5.59.0 → 5.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/dist/cjs/classes/queue.js +9 -0
  2. package/dist/cjs/classes/queue.js.map +1 -1
  3. package/dist/cjs/classes/scripts.js +4 -1
  4. package/dist/cjs/classes/scripts.js.map +1 -1
  5. package/dist/cjs/commands/changePriority-7.lua +1 -1
  6. package/dist/cjs/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
  7. package/dist/cjs/commands/includes/getTargetQueueList.lua +5 -5
  8. package/dist/cjs/commands/includes/prepareJobForProcessing.lua +2 -4
  9. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -1
  10. package/dist/cjs/commands/moveToActive-11.lua +7 -4
  11. package/dist/cjs/commands/moveToFinished-14.lua +10 -7
  12. package/dist/cjs/scripts/addDelayedJob-6.js +5 -5
  13. package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
  14. package/dist/cjs/scripts/addParentJob-5.js +5 -5
  15. package/dist/cjs/scripts/addPrioritizedJob-9.js +5 -5
  16. package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
  17. package/dist/cjs/scripts/addStandardJob-9.js +5 -5
  18. package/dist/cjs/scripts/changePriority-7.js +5 -5
  19. package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
  20. package/dist/cjs/scripts/drain-5.js +5 -5
  21. package/dist/cjs/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
  22. package/dist/cjs/scripts/getRateLimitTtl-2.js.map +1 -0
  23. package/dist/cjs/scripts/index.js +1 -1
  24. package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +5 -5
  25. package/dist/cjs/scripts/moveJobsToWait-8.js +5 -5
  26. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +9 -9
  27. package/dist/cjs/scripts/moveToActive-11.js +13 -12
  28. package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
  29. package/dist/cjs/scripts/moveToFinished-14.js +16 -15
  30. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  31. package/dist/cjs/scripts/obliterate-2.js +5 -5
  32. package/dist/cjs/scripts/promote-9.js +5 -5
  33. package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
  34. package/dist/cjs/scripts/removeJob-2.js +5 -5
  35. package/dist/cjs/scripts/removeUnprocessedChildren-2.js +5 -5
  36. package/dist/cjs/scripts/reprocessJob-8.js +5 -5
  37. package/dist/cjs/scripts/retryJob-11.js +5 -5
  38. package/dist/cjs/scripts/updateJobScheduler-12.js +5 -5
  39. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  40. package/dist/cjs/version.js +1 -1
  41. package/dist/esm/classes/queue.d.ts +6 -0
  42. package/dist/esm/classes/queue.js +9 -0
  43. package/dist/esm/classes/queue.js.map +1 -1
  44. package/dist/esm/classes/scripts.js +4 -1
  45. package/dist/esm/classes/scripts.js.map +1 -1
  46. package/dist/esm/commands/changePriority-7.lua +1 -1
  47. package/dist/esm/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
  48. package/dist/esm/commands/includes/getTargetQueueList.lua +5 -5
  49. package/dist/esm/commands/includes/prepareJobForProcessing.lua +2 -4
  50. package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -1
  51. package/dist/esm/commands/moveToActive-11.lua +7 -4
  52. package/dist/esm/commands/moveToFinished-14.lua +10 -7
  53. package/dist/esm/scripts/addDelayedJob-6.js +5 -5
  54. package/dist/esm/scripts/addJobScheduler-11.js +5 -5
  55. package/dist/esm/scripts/addParentJob-5.js +5 -5
  56. package/dist/esm/scripts/addPrioritizedJob-9.js +5 -5
  57. package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
  58. package/dist/esm/scripts/addStandardJob-9.js +5 -5
  59. package/dist/esm/scripts/changePriority-7.js +5 -5
  60. package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
  61. package/dist/esm/scripts/drain-5.js +5 -5
  62. package/dist/esm/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
  63. package/dist/esm/scripts/getRateLimitTtl-2.js.map +1 -0
  64. package/dist/esm/scripts/index.d.ts +1 -1
  65. package/dist/esm/scripts/index.js +1 -1
  66. package/dist/esm/scripts/moveJobFromActiveToWait-9.js +5 -5
  67. package/dist/esm/scripts/moveJobsToWait-8.js +5 -5
  68. package/dist/esm/scripts/moveStalledJobsToWait-8.js +9 -9
  69. package/dist/esm/scripts/moveToActive-11.js +13 -12
  70. package/dist/esm/scripts/moveToActive-11.js.map +1 -1
  71. package/dist/esm/scripts/moveToFinished-14.js +16 -15
  72. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  73. package/dist/esm/scripts/obliterate-2.js +5 -5
  74. package/dist/esm/scripts/promote-9.js +5 -5
  75. package/dist/esm/scripts/removeChildDependency-1.js +5 -5
  76. package/dist/esm/scripts/removeJob-2.js +5 -5
  77. package/dist/esm/scripts/removeUnprocessedChildren-2.js +5 -5
  78. package/dist/esm/scripts/reprocessJob-8.js +5 -5
  79. package/dist/esm/scripts/retryJob-11.js +5 -5
  80. package/dist/esm/scripts/updateJobScheduler-12.js +5 -5
  81. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  82. package/dist/esm/version.d.ts +1 -1
  83. package/dist/esm/version.js +1 -1
  84. package/package.json +1 -1
  85. package/dist/cjs/scripts/getRateLimitTtl-1.js.map +0 -1
  86. package/dist/esm/scripts/getRateLimitTtl-1.js.map +0 -1
  87. /package/dist/esm/scripts/{getRateLimitTtl-1.d.ts → getRateLimitTtl-2.d.ts} +0 -0
@@ -59,20 +59,20 @@ end
59
59
  (since an empty list and !EXISTS are not really the same).
60
60
  ]]
61
61
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
62
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
62
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
63
63
  if queueAttributes[1] then
64
- return pausedKey, true
64
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
65
65
  else
66
66
  if queueAttributes[2] then
67
67
  local activeCount = rcall("LLEN", activeKey)
68
68
  if activeCount >= tonumber(queueAttributes[2]) then
69
- return waitKey, true
69
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
70
70
  else
71
- return waitKey, false
71
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
72
72
  end
73
73
  end
74
74
  end
75
- return waitKey, false
75
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
76
76
  end
77
77
  --[[
78
78
  Function to push back job considering priority in front of same prioritized jobs.
@@ -123,20 +123,20 @@ end
123
123
  (since an empty list and !EXISTS are not really the same).
124
124
  ]]
125
125
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
126
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
126
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
127
127
  if queueAttributes[1] then
128
- return pausedKey, true
128
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
129
129
  else
130
130
  if queueAttributes[2] then
131
131
  local activeCount = rcall("LLEN", activeKey)
132
132
  if activeCount >= tonumber(queueAttributes[2]) then
133
- return waitKey, true
133
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
134
134
  else
135
- return waitKey, false
135
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
136
136
  end
137
137
  end
138
138
  end
139
- return waitKey, false
139
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
140
140
  end
141
141
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
142
142
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -95,20 +95,20 @@ end
95
95
  (since an empty list and !EXISTS are not really the same).
96
96
  ]]
97
97
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
98
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
98
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
99
99
  if queueAttributes[1] then
100
- return pausedKey, true
100
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
101
101
  else
102
102
  if queueAttributes[2] then
103
103
  local activeCount = rcall("LLEN", activeKey)
104
104
  if activeCount >= tonumber(queueAttributes[2]) then
105
- return waitKey, true
105
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
106
106
  else
107
- return waitKey, false
107
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
108
108
  end
109
109
  end
110
110
  end
111
- return waitKey, false
111
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
112
112
  end
113
113
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
114
114
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -2,6 +2,7 @@ const content = `--[[
2
2
  Get rate limit ttl
3
3
  Input:
4
4
  KEYS[1] 'limiter'
5
+ KEYS[2] 'meta'
5
6
  ARGV[1] maxJobs
6
7
  ]]
7
8
  local rcall = redis.call
@@ -25,12 +26,16 @@ local rateLimiterKey = KEYS[1]
25
26
  if ARGV[1] ~= "0" then
26
27
  return getRateLimitTTL(tonumber(ARGV[1]), rateLimiterKey)
27
28
  else
29
+ local rateLimitMax = rcall("HGET", KEYS[2], "max")
30
+ if rateLimitMax then
31
+ return getRateLimitTTL(tonumber(rateLimitMax), rateLimiterKey)
32
+ end
28
33
  return rcall("PTTL", rateLimiterKey)
29
34
  end
30
35
  `;
31
36
  export const getRateLimitTtl = {
32
37
  name: 'getRateLimitTtl',
33
38
  content,
34
- keys: 1,
39
+ keys: 2,
35
40
  };
36
- //# sourceMappingURL=getRateLimitTtl-1.js.map
41
+ //# sourceMappingURL=getRateLimitTtl-2.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"getRateLimitTtl-2.js","sourceRoot":"","sources":["../../../src/scripts/getRateLimitTtl-2.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkCf,CAAC;AACF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -17,7 +17,7 @@ export * from './getDependencyCounts-4';
17
17
  export * from './getJobScheduler-1';
18
18
  export * from './getMetrics-2';
19
19
  export * from './getRanges-1';
20
- export * from './getRateLimitTtl-1';
20
+ export * from './getRateLimitTtl-2';
21
21
  export * from './getState-8';
22
22
  export * from './getStateV2-8';
23
23
  export * from './isFinished-3';
@@ -17,7 +17,7 @@ export * from './getDependencyCounts-4';
17
17
  export * from './getJobScheduler-1';
18
18
  export * from './getMetrics-2';
19
19
  export * from './getRanges-1';
20
- export * from './getRateLimitTtl-1';
20
+ export * from './getRateLimitTtl-2';
21
21
  export * from './getState-8';
22
22
  export * from './getStateV2-8';
23
23
  export * from './isFinished-3';
@@ -57,20 +57,20 @@ end
57
57
  (since an empty list and !EXISTS are not really the same).
58
58
  ]]
59
59
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
60
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
60
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
61
61
  if queueAttributes[1] then
62
- return pausedKey, true
62
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
63
63
  else
64
64
  if queueAttributes[2] then
65
65
  local activeCount = rcall("LLEN", activeKey)
66
66
  if activeCount >= tonumber(queueAttributes[2]) then
67
- return waitKey, true
67
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
68
68
  else
69
- return waitKey, false
69
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
70
70
  end
71
71
  end
72
72
  end
73
- return waitKey, false
73
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
74
74
  end
75
75
  local function removeLock(jobKey, stalledKey, token, jobId)
76
76
  if token ~= "0" then
@@ -61,20 +61,20 @@ end
61
61
  (since an empty list and !EXISTS are not really the same).
62
62
  ]]
63
63
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
64
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
64
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
65
65
  if queueAttributes[1] then
66
- return pausedKey, true
66
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
67
67
  else
68
68
  if queueAttributes[2] then
69
69
  local activeCount = rcall("LLEN", activeKey)
70
70
  if activeCount >= tonumber(queueAttributes[2]) then
71
- return waitKey, true
71
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
72
72
  else
73
- return waitKey, false
73
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
74
74
  end
75
75
  end
76
76
  end
77
- return waitKey, false
77
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
78
78
  end
79
79
  local metaKey = KEYS[6]
80
80
  local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[7], KEYS[4], KEYS[5])
@@ -50,30 +50,30 @@ local function batches(n, batchSize)
50
50
  end
51
51
  end
52
52
  end
53
+ --[[
54
+ Function to move job to wait to be picked up by a waiting worker.
55
+ ]]
56
+ -- Includes
53
57
  --[[
54
58
  Function to check for the meta.paused key to decide if we are paused or not
55
59
  (since an empty list and !EXISTS are not really the same).
56
60
  ]]
57
61
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
58
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
62
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
59
63
  if queueAttributes[1] then
60
- return pausedKey, true
64
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
61
65
  else
62
66
  if queueAttributes[2] then
63
67
  local activeCount = rcall("LLEN", activeKey)
64
68
  if activeCount >= tonumber(queueAttributes[2]) then
65
- return waitKey, true
69
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
66
70
  else
67
- return waitKey, false
71
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
68
72
  end
69
73
  end
70
74
  end
71
- return waitKey, false
75
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
72
76
  end
73
- --[[
74
- Function to move job to wait to be picked up by a waiting worker.
75
- ]]
76
- -- Includes
77
77
  local function moveJobToWait(metaKey, activeKey, waitKey, pausedKey, markerKey, eventStreamKey,
78
78
  jobId, pushCmd)
79
79
  local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
@@ -68,20 +68,20 @@ end
68
68
  (since an empty list and !EXISTS are not really the same).
69
69
  ]]
70
70
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
71
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
71
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
72
72
  if queueAttributes[1] then
73
- return pausedKey, true
73
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
74
74
  else
75
75
  if queueAttributes[2] then
76
76
  local activeCount = rcall("LLEN", activeKey)
77
77
  if activeCount >= tonumber(queueAttributes[2]) then
78
- return waitKey, true
78
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
79
79
  else
80
- return waitKey, false
80
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
81
81
  end
82
82
  end
83
83
  end
84
- return waitKey, false
84
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
85
85
  end
86
86
  --[[
87
87
  Function to move job from prioritized state to active.
@@ -112,20 +112,19 @@ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
112
112
  end
113
113
  end
114
114
  local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
115
- jobId, processedOn, maxJobs, markerKey, opts)
115
+ jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
116
116
  local jobKey = keyPrefix .. jobId
117
117
  -- Check if we need to perform rate limiting.
118
118
  if maxJobs then
119
119
  local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
120
120
  if jobCounter == 1 then
121
- local limiterDuration = opts['limiter'] and opts['limiter']['duration']
122
121
  local integerDuration = math.floor(math.abs(limiterDuration))
123
122
  rcall("PEXPIRE", rateLimiterKey, integerDuration)
124
123
  end
125
124
  end
126
- local lockKey = jobKey .. ':lock'
127
125
  -- get a lock
128
126
  if opts['token'] ~= "0" then
127
+ local lockKey = jobKey .. ':lock'
129
128
  rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
130
129
  end
131
130
  local optionalValues = {}
@@ -199,17 +198,19 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
199
198
  addBaseMarkerIfNeeded(markerKey, isPaused)
200
199
  end
201
200
  end
202
- local target, isPausedOrMaxed = getTargetQueueList(KEYS[9], activeKey, waitKey, KEYS[8])
201
+ local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(KEYS[9],
202
+ activeKey, waitKey, KEYS[8])
203
203
  -- Check if there are delayed jobs that we can move to wait.
204
204
  local markerKey = KEYS[11]
205
205
  promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
206
206
  ARGV[2], KEYS[10], isPausedOrMaxed)
207
- local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
207
+ local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
208
208
  local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
209
209
  -- Check if we are rate limited first.
210
210
  if expireTime > 0 then return {0, 0, expireTime, 0} end
211
211
  -- paused or maxed queue
212
212
  if isPausedOrMaxed then return {0, 0, 0, 0} end
213
+ local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
213
214
  -- no job ID, try non-blocking move from wait to active
214
215
  local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
215
216
  -- Markers in waitlist DEPRECATED in v5: Will be completely removed in v6.
@@ -219,12 +220,12 @@ if jobId and string.sub(jobId, 1, 2) == "0:" then
219
220
  end
220
221
  if jobId then
221
222
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
222
- maxJobs, markerKey, opts)
223
+ maxJobs, limiterDuration, markerKey, opts)
223
224
  else
224
225
  jobId = moveJobFromPrioritizedToActive(KEYS[3], activeKey, KEYS[10])
225
226
  if jobId then
226
227
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
227
- maxJobs, markerKey, opts)
228
+ maxJobs, limiterDuration, markerKey, opts)
228
229
  end
229
230
  end
230
231
  -- Return the timestamp for the next delayed job if any.
@@ -1 +1 @@
1
- {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyOf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Of,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -137,20 +137,20 @@ end
137
137
  (since an empty list and !EXISTS are not really the same).
138
138
  ]]
139
139
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
140
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
140
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
141
141
  if queueAttributes[1] then
142
- return pausedKey, true
142
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
143
143
  else
144
144
  if queueAttributes[2] then
145
145
  local activeCount = rcall("LLEN", activeKey)
146
146
  if activeCount >= tonumber(queueAttributes[2]) then
147
- return waitKey, true
147
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
148
148
  else
149
- return waitKey, false
149
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
150
150
  end
151
151
  end
152
152
  end
153
- return waitKey, false
153
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
154
154
  end
155
155
  --[[
156
156
  Function to move job from prioritized state to active.
@@ -538,20 +538,19 @@ end
538
538
  ]]
539
539
  -- Includes
540
540
  local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
541
- jobId, processedOn, maxJobs, markerKey, opts)
541
+ jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
542
542
  local jobKey = keyPrefix .. jobId
543
543
  -- Check if we need to perform rate limiting.
544
544
  if maxJobs then
545
545
  local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
546
546
  if jobCounter == 1 then
547
- local limiterDuration = opts['limiter'] and opts['limiter']['duration']
548
547
  local integerDuration = math.floor(math.abs(limiterDuration))
549
548
  rcall("PEXPIRE", rateLimiterKey, integerDuration)
550
549
  end
551
550
  end
552
- local lockKey = jobKey .. ':lock'
553
551
  -- get a lock
554
552
  if opts['token'] ~= "0" then
553
+ local lockKey = jobKey .. ':lock'
555
554
  rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
556
555
  end
557
556
  local optionalValues = {}
@@ -789,12 +788,13 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
789
788
  -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
790
789
  -- and not rate limited.
791
790
  if (ARGV[6] == "1") then
792
- local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
791
+ local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(metaKey, KEYS[2],
792
+ KEYS[1], KEYS[8])
793
793
  local markerKey = KEYS[14]
794
794
  -- Check if there are delayed jobs that can be promoted
795
795
  promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10],
796
796
  isPausedOrMaxed)
797
- local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
797
+ local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
798
798
  -- Check if we are rate limited first.
799
799
  local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
800
800
  if expireTime > 0 then
@@ -804,6 +804,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
804
804
  if isPausedOrMaxed then
805
805
  return {0, 0, 0, 0}
806
806
  end
807
+ local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
807
808
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
808
809
  if jobId then
809
810
  -- Markers in waitlist DEPRECATED in v5: Remove in v6.
@@ -814,17 +815,17 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
814
815
  if jobId == "0:0" then
815
816
  jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
816
817
  return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
817
- markerKey, opts)
818
+ limiterDuration, markerKey, opts)
818
819
  end
819
820
  else
820
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
821
- opts)
821
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
822
+ limiterDuration, markerKey, opts)
822
823
  end
823
824
  else
824
825
  jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
825
826
  if jobId then
826
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
827
- opts)
827
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
828
+ limiterDuration, markerKey, opts)
828
829
  end
829
830
  end
830
831
  -- Return the timestamp for the next delayed job if any.
@@ -1 +1 @@
1
- {"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAm1Bf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAo1Bf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -83,20 +83,20 @@ end
83
83
  (since an empty list and !EXISTS are not really the same).
84
84
  ]]
85
85
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
86
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
86
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
87
87
  if queueAttributes[1] then
88
- return pausedKey, true
88
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
89
89
  else
90
90
  if queueAttributes[2] then
91
91
  local activeCount = rcall("LLEN", activeKey)
92
92
  if activeCount >= tonumber(queueAttributes[2]) then
93
- return waitKey, true
93
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
94
94
  else
95
- return waitKey, false
95
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
96
96
  end
97
97
  end
98
98
  end
99
- return waitKey, false
99
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
100
100
  end
101
101
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
102
102
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -59,20 +59,20 @@ end
59
59
  (since an empty list and !EXISTS are not really the same).
60
60
  ]]
61
61
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
62
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
62
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
63
63
  if queueAttributes[1] then
64
- return pausedKey, true
64
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
65
65
  else
66
66
  if queueAttributes[2] then
67
67
  local activeCount = rcall("LLEN", activeKey)
68
68
  if activeCount >= tonumber(queueAttributes[2]) then
69
- return waitKey, true
69
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
70
70
  else
71
- return waitKey, false
71
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
72
72
  end
73
73
  end
74
74
  end
75
- return waitKey, false
75
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
76
76
  end
77
77
  if rcall("ZREM", KEYS[1], jobId) == 1 then
78
78
  local jobKey = ARGV[1] .. jobId
@@ -52,20 +52,20 @@ end
52
52
  (since an empty list and !EXISTS are not really the same).
53
53
  ]]
54
54
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
55
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
55
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
56
56
  if queueAttributes[1] then
57
- return pausedKey, true
57
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
58
58
  else
59
59
  if queueAttributes[2] then
60
60
  local activeCount = rcall("LLEN", activeKey)
61
61
  if activeCount >= tonumber(queueAttributes[2]) then
62
- return waitKey, true
62
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
63
63
  else
64
- return waitKey, false
64
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
65
65
  end
66
66
  end
67
67
  end
68
- return waitKey, false
68
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
69
69
  end
70
70
  --[[
71
71
  Function to remove job keys.
@@ -167,20 +167,20 @@ end
167
167
  (since an empty list and !EXISTS are not really the same).
168
168
  ]]
169
169
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
170
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
170
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
171
171
  if queueAttributes[1] then
172
- return pausedKey, true
172
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
173
173
  else
174
174
  if queueAttributes[2] then
175
175
  local activeCount = rcall("LLEN", activeKey)
176
176
  if activeCount >= tonumber(queueAttributes[2]) then
177
- return waitKey, true
177
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
178
178
  else
179
- return waitKey, false
179
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
180
180
  end
181
181
  end
182
182
  end
183
- return waitKey, false
183
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
184
184
  end
185
185
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
186
186
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -135,20 +135,20 @@ end
135
135
  (since an empty list and !EXISTS are not really the same).
136
136
  ]]
137
137
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
138
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
138
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
139
139
  if queueAttributes[1] then
140
- return pausedKey, true
140
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
141
141
  else
142
142
  if queueAttributes[2] then
143
143
  local activeCount = rcall("LLEN", activeKey)
144
144
  if activeCount >= tonumber(queueAttributes[2]) then
145
- return waitKey, true
145
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
146
146
  else
147
- return waitKey, false
147
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
148
148
  end
149
149
  end
150
150
  end
151
- return waitKey, false
151
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
152
152
  end
153
153
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
154
154
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -52,20 +52,20 @@ end
52
52
  (since an empty list and !EXISTS are not really the same).
53
53
  ]]
54
54
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
55
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
55
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
56
56
  if queueAttributes[1] then
57
- return pausedKey, true
57
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
58
58
  else
59
59
  if queueAttributes[2] then
60
60
  local activeCount = rcall("LLEN", activeKey)
61
61
  if activeCount >= tonumber(queueAttributes[2]) then
62
- return waitKey, true
62
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
63
63
  else
64
- return waitKey, false
64
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
65
65
  end
66
66
  end
67
67
  end
68
- return waitKey, false
68
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
69
69
  end
70
70
  local jobKey = KEYS[1]
71
71
  if rcall("EXISTS", jobKey) == 1 then
@@ -77,20 +77,20 @@ end
77
77
  (since an empty list and !EXISTS are not really the same).
78
78
  ]]
79
79
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
80
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
80
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
81
81
  if queueAttributes[1] then
82
- return pausedKey, true
82
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
83
83
  else
84
84
  if queueAttributes[2] then
85
85
  local activeCount = rcall("LLEN", activeKey)
86
86
  if activeCount >= tonumber(queueAttributes[2]) then
87
- return waitKey, true
87
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
88
88
  else
89
- return waitKey, false
89
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
90
90
  end
91
91
  end
92
92
  end
93
- return waitKey, false
93
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
94
94
  end
95
95
  --[[
96
96
  Function to check if queue is paused or maxed
@@ -171,20 +171,20 @@ end
171
171
  (since an empty list and !EXISTS are not really the same).
172
172
  ]]
173
173
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
174
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
174
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
175
175
  if queueAttributes[1] then
176
- return pausedKey, true
176
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
177
177
  else
178
178
  if queueAttributes[2] then
179
179
  local activeCount = rcall("LLEN", activeKey)
180
180
  if activeCount >= tonumber(queueAttributes[2]) then
181
- return waitKey, true
181
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
182
182
  else
183
- return waitKey, false
183
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
184
184
  end
185
185
  end
186
186
  end
187
- return waitKey, false
187
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
188
188
  end
189
189
  --[[
190
190
  Function to add job in target list and add marker if needed.