bullmq 5.58.9 → 5.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/dist/cjs/classes/child-pool.js +8 -1
  2. package/dist/cjs/classes/child-pool.js.map +1 -1
  3. package/dist/cjs/classes/queue.js +9 -0
  4. package/dist/cjs/classes/queue.js.map +1 -1
  5. package/dist/cjs/classes/scripts.js +4 -1
  6. package/dist/cjs/classes/scripts.js.map +1 -1
  7. package/dist/cjs/classes/worker.js +42 -30
  8. package/dist/cjs/classes/worker.js.map +1 -1
  9. package/dist/cjs/commands/changePriority-7.lua +1 -1
  10. package/dist/cjs/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
  11. package/dist/cjs/commands/includes/deduplicateJob.lua +53 -20
  12. package/dist/cjs/commands/includes/getTargetQueueList.lua +5 -5
  13. package/dist/cjs/commands/includes/prepareJobForProcessing.lua +2 -4
  14. package/dist/cjs/commands/moveJobFromActiveToWait-9.lua +1 -1
  15. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -1
  16. package/dist/cjs/commands/moveToActive-11.lua +7 -4
  17. package/dist/cjs/commands/moveToDelayed-8.lua +2 -1
  18. package/dist/cjs/commands/moveToFinished-14.lua +10 -7
  19. package/dist/cjs/scripts/addDelayedJob-6.js +52 -23
  20. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  21. package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
  22. package/dist/cjs/scripts/addParentJob-5.js +52 -23
  23. package/dist/cjs/scripts/addParentJob-5.js.map +1 -1
  24. package/dist/cjs/scripts/addPrioritizedJob-9.js +52 -23
  25. package/dist/cjs/scripts/addPrioritizedJob-9.js.map +1 -1
  26. package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
  27. package/dist/cjs/scripts/addStandardJob-9.js +52 -23
  28. package/dist/cjs/scripts/addStandardJob-9.js.map +1 -1
  29. package/dist/cjs/scripts/changePriority-7.js +5 -5
  30. package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
  31. package/dist/cjs/scripts/drain-5.js +5 -5
  32. package/dist/cjs/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
  33. package/dist/cjs/scripts/getRateLimitTtl-2.js.map +1 -0
  34. package/dist/cjs/scripts/index.js +1 -1
  35. package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +6 -6
  36. package/dist/cjs/scripts/moveJobsToWait-8.js +5 -5
  37. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +9 -9
  38. package/dist/cjs/scripts/moveToActive-11.js +13 -12
  39. package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
  40. package/dist/cjs/scripts/moveToDelayed-8.js +1 -1
  41. package/dist/cjs/scripts/moveToFinished-14.js +16 -15
  42. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  43. package/dist/cjs/scripts/obliterate-2.js +5 -5
  44. package/dist/cjs/scripts/promote-9.js +5 -5
  45. package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
  46. package/dist/cjs/scripts/removeJob-2.js +5 -5
  47. package/dist/cjs/scripts/removeUnprocessedChildren-2.js +5 -5
  48. package/dist/cjs/scripts/reprocessJob-8.js +5 -5
  49. package/dist/cjs/scripts/retryJob-11.js +5 -5
  50. package/dist/cjs/scripts/updateJobScheduler-12.js +5 -5
  51. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  52. package/dist/cjs/version.js +1 -1
  53. package/dist/esm/classes/child-pool.js +8 -1
  54. package/dist/esm/classes/child-pool.js.map +1 -1
  55. package/dist/esm/classes/queue.d.ts +6 -0
  56. package/dist/esm/classes/queue.js +9 -0
  57. package/dist/esm/classes/queue.js.map +1 -1
  58. package/dist/esm/classes/scripts.js +4 -1
  59. package/dist/esm/classes/scripts.js.map +1 -1
  60. package/dist/esm/classes/worker.d.ts +1 -0
  61. package/dist/esm/classes/worker.js +42 -30
  62. package/dist/esm/classes/worker.js.map +1 -1
  63. package/dist/esm/commands/changePriority-7.lua +1 -1
  64. package/dist/esm/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
  65. package/dist/esm/commands/includes/deduplicateJob.lua +53 -20
  66. package/dist/esm/commands/includes/getTargetQueueList.lua +5 -5
  67. package/dist/esm/commands/includes/prepareJobForProcessing.lua +2 -4
  68. package/dist/esm/commands/moveJobFromActiveToWait-9.lua +1 -1
  69. package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -1
  70. package/dist/esm/commands/moveToActive-11.lua +7 -4
  71. package/dist/esm/commands/moveToDelayed-8.lua +2 -1
  72. package/dist/esm/commands/moveToFinished-14.lua +10 -7
  73. package/dist/esm/interfaces/base-job-options.d.ts +1 -1
  74. package/dist/esm/interfaces/worker-options.d.ts +8 -0
  75. package/dist/esm/scripts/addDelayedJob-6.js +52 -23
  76. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  77. package/dist/esm/scripts/addJobScheduler-11.js +5 -5
  78. package/dist/esm/scripts/addParentJob-5.js +52 -23
  79. package/dist/esm/scripts/addParentJob-5.js.map +1 -1
  80. package/dist/esm/scripts/addPrioritizedJob-9.js +52 -23
  81. package/dist/esm/scripts/addPrioritizedJob-9.js.map +1 -1
  82. package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
  83. package/dist/esm/scripts/addStandardJob-9.js +52 -23
  84. package/dist/esm/scripts/addStandardJob-9.js.map +1 -1
  85. package/dist/esm/scripts/changePriority-7.js +5 -5
  86. package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
  87. package/dist/esm/scripts/drain-5.js +5 -5
  88. package/dist/esm/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
  89. package/dist/esm/scripts/getRateLimitTtl-2.js.map +1 -0
  90. package/dist/esm/scripts/index.d.ts +1 -1
  91. package/dist/esm/scripts/index.js +1 -1
  92. package/dist/esm/scripts/moveJobFromActiveToWait-9.js +6 -6
  93. package/dist/esm/scripts/moveJobsToWait-8.js +5 -5
  94. package/dist/esm/scripts/moveStalledJobsToWait-8.js +9 -9
  95. package/dist/esm/scripts/moveToActive-11.js +13 -12
  96. package/dist/esm/scripts/moveToActive-11.js.map +1 -1
  97. package/dist/esm/scripts/moveToDelayed-8.js +1 -1
  98. package/dist/esm/scripts/moveToFinished-14.js +16 -15
  99. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  100. package/dist/esm/scripts/obliterate-2.js +5 -5
  101. package/dist/esm/scripts/promote-9.js +5 -5
  102. package/dist/esm/scripts/removeChildDependency-1.js +5 -5
  103. package/dist/esm/scripts/removeJob-2.js +5 -5
  104. package/dist/esm/scripts/removeUnprocessedChildren-2.js +5 -5
  105. package/dist/esm/scripts/reprocessJob-8.js +5 -5
  106. package/dist/esm/scripts/retryJob-11.js +5 -5
  107. package/dist/esm/scripts/updateJobScheduler-12.js +5 -5
  108. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  109. package/dist/esm/version.d.ts +1 -1
  110. package/dist/esm/version.js +1 -1
  111. package/package.json +1 -1
  112. package/dist/cjs/scripts/getRateLimitTtl-1.js.map +0 -1
  113. package/dist/esm/scripts/getRateLimitTtl-1.js.map +0 -1
  114. /package/dist/esm/scripts/{getRateLimitTtl-1.d.ts → getRateLimitTtl-2.d.ts} +0 -0
@@ -85,37 +85,65 @@ local function removeJobKeys(jobKey)
85
85
  return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
86
86
  jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
87
87
  end
88
+ local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents, currentDeduplicatedJobId,
89
+ jobId, deduplicationId, prefix)
90
+ if rcall("ZREM", delayedKey, currentDeduplicatedJobId) > 0 then
91
+ removeJobKeys(prefix .. currentDeduplicatedJobId)
92
+ rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDeduplicatedJobId,
93
+ "prev", "delayed")
94
+ -- TODO remove debounced event in next breaking change
95
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
96
+ jobId, "debounceId", deduplicationId)
97
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
98
+ jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDeduplicatedJobId)
99
+ return true
100
+ end
101
+ return false
102
+ end
88
103
  local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
89
104
  prefix)
90
105
  local deduplicationId = deduplicationOpts and deduplicationOpts['id']
91
106
  if deduplicationId then
92
107
  local ttl = deduplicationOpts['ttl']
93
- if deduplicationOpts['replace'] and ttl and ttl > 0 then
94
- local currentDebounceJobId = rcall('GET', deduplicationKey)
95
- if currentDebounceJobId then
96
- if rcall("ZREM", delayedKey, currentDebounceJobId) > 0 then
97
- removeJobKeys(prefix .. currentDebounceJobId)
98
- rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDebounceJobId,
99
- "prev", "delayed")
100
- if deduplicationOpts['extend'] then
101
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
108
+ if deduplicationOpts['replace'] then
109
+ if ttl and ttl > 0 then
110
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
111
+ if currentDebounceJobId then
112
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
113
+ currentDebounceJobId, jobId, deduplicationId, prefix)
114
+ if isRemoved then
115
+ if deduplicationOpts['extend'] then
116
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
117
+ else
118
+ rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
119
+ end
120
+ return
102
121
  else
103
- rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
122
+ return currentDebounceJobId
104
123
  end
105
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
106
- jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
107
- return
108
124
  else
109
- return currentDebounceJobId
125
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
126
+ return
110
127
  end
111
128
  else
112
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
113
- return
129
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
130
+ if currentDebounceJobId then
131
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
132
+ currentDebounceJobId, jobId, deduplicationId, prefix)
133
+ if isRemoved then
134
+ rcall('SET', deduplicationKey, jobId)
135
+ return
136
+ else
137
+ return currentDebounceJobId
138
+ end
139
+ else
140
+ rcall('SET', deduplicationKey, jobId)
141
+ return
142
+ end
114
143
  end
115
144
  else
116
- local ttl = deduplicationOpts['ttl']
117
145
  local deduplicationKeyExists
118
- if ttl then
146
+ if ttl and ttl > 0 then
119
147
  if deduplicationOpts['extend'] then
120
148
  local currentDebounceJobId = rcall('GET', deduplicationKey)
121
149
  if currentDebounceJobId then
@@ -137,6 +165,7 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
137
165
  end
138
166
  if deduplicationKeyExists then
139
167
  local currentDebounceJobId = rcall('GET', deduplicationKey)
168
+ -- TODO remove debounced event in next breaking change
140
169
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
141
170
  currentDebounceJobId, "debounceId", deduplicationId)
142
171
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
@@ -268,20 +297,20 @@ end
268
297
  (since an empty list and !EXISTS are not really the same).
269
298
  ]]
270
299
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
271
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
300
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
272
301
  if queueAttributes[1] then
273
- return pausedKey, true
302
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
274
303
  else
275
304
  if queueAttributes[2] then
276
305
  local activeCount = rcall("LLEN", activeKey)
277
306
  if activeCount >= tonumber(queueAttributes[2]) then
278
- return waitKey, true
307
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
279
308
  else
280
- return waitKey, false
309
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
281
310
  end
282
311
  end
283
312
  end
284
- return waitKey, false
313
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
285
314
  end
286
315
  local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
287
316
  local parentWaitKey = parentQueueKey .. ":wait"
@@ -1 +1 @@
1
- {"version":3,"file":"addPrioritizedJob-9.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4Zf,CAAC;AACF,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,IAAI,EAAE,mBAAmB;IACzB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addPrioritizedJob-9.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAybf,CAAC;AACF,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,IAAI,EAAE,mBAAmB;IACzB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -87,20 +87,20 @@ end
87
87
  (since an empty list and !EXISTS are not really the same).
88
88
  ]]
89
89
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
90
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
90
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
91
91
  if queueAttributes[1] then
92
- return pausedKey, true
92
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
93
93
  else
94
94
  if queueAttributes[2] then
95
95
  local activeCount = rcall("LLEN", activeKey)
96
96
  if activeCount >= tonumber(queueAttributes[2]) then
97
- return waitKey, true
97
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
98
98
  else
99
- return waitKey, false
99
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
100
100
  end
101
101
  end
102
102
  end
103
- return waitKey, false
103
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
104
104
  end
105
105
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
106
106
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -79,37 +79,65 @@ local function removeJobKeys(jobKey)
79
79
  return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
80
80
  jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
81
81
  end
82
+ local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents, currentDeduplicatedJobId,
83
+ jobId, deduplicationId, prefix)
84
+ if rcall("ZREM", delayedKey, currentDeduplicatedJobId) > 0 then
85
+ removeJobKeys(prefix .. currentDeduplicatedJobId)
86
+ rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDeduplicatedJobId,
87
+ "prev", "delayed")
88
+ -- TODO remove debounced event in next breaking change
89
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
90
+ jobId, "debounceId", deduplicationId)
91
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
92
+ jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDeduplicatedJobId)
93
+ return true
94
+ end
95
+ return false
96
+ end
82
97
  local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
83
98
  prefix)
84
99
  local deduplicationId = deduplicationOpts and deduplicationOpts['id']
85
100
  if deduplicationId then
86
101
  local ttl = deduplicationOpts['ttl']
87
- if deduplicationOpts['replace'] and ttl and ttl > 0 then
88
- local currentDebounceJobId = rcall('GET', deduplicationKey)
89
- if currentDebounceJobId then
90
- if rcall("ZREM", delayedKey, currentDebounceJobId) > 0 then
91
- removeJobKeys(prefix .. currentDebounceJobId)
92
- rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDebounceJobId,
93
- "prev", "delayed")
94
- if deduplicationOpts['extend'] then
95
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
102
+ if deduplicationOpts['replace'] then
103
+ if ttl and ttl > 0 then
104
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
105
+ if currentDebounceJobId then
106
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
107
+ currentDebounceJobId, jobId, deduplicationId, prefix)
108
+ if isRemoved then
109
+ if deduplicationOpts['extend'] then
110
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
111
+ else
112
+ rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
113
+ end
114
+ return
96
115
  else
97
- rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
116
+ return currentDebounceJobId
98
117
  end
99
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
100
- jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
101
- return
102
118
  else
103
- return currentDebounceJobId
119
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
120
+ return
104
121
  end
105
122
  else
106
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
107
- return
123
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
124
+ if currentDebounceJobId then
125
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
126
+ currentDebounceJobId, jobId, deduplicationId, prefix)
127
+ if isRemoved then
128
+ rcall('SET', deduplicationKey, jobId)
129
+ return
130
+ else
131
+ return currentDebounceJobId
132
+ end
133
+ else
134
+ rcall('SET', deduplicationKey, jobId)
135
+ return
136
+ end
108
137
  end
109
138
  else
110
- local ttl = deduplicationOpts['ttl']
111
139
  local deduplicationKeyExists
112
- if ttl then
140
+ if ttl and ttl > 0 then
113
141
  if deduplicationOpts['extend'] then
114
142
  local currentDebounceJobId = rcall('GET', deduplicationKey)
115
143
  if currentDebounceJobId then
@@ -131,6 +159,7 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
131
159
  end
132
160
  if deduplicationKeyExists then
133
161
  local currentDebounceJobId = rcall('GET', deduplicationKey)
162
+ -- TODO remove debounced event in next breaking change
134
163
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
135
164
  currentDebounceJobId, "debounceId", deduplicationId)
136
165
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
@@ -156,20 +185,20 @@ end
156
185
  (since an empty list and !EXISTS are not really the same).
157
186
  ]]
158
187
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
159
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
188
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
160
189
  if queueAttributes[1] then
161
- return pausedKey, true
190
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
162
191
  else
163
192
  if queueAttributes[2] then
164
193
  local activeCount = rcall("LLEN", activeKey)
165
194
  if activeCount >= tonumber(queueAttributes[2]) then
166
- return waitKey, true
195
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
167
196
  else
168
- return waitKey, false
197
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
169
198
  end
170
199
  end
171
200
  end
172
- return waitKey, false
201
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
173
202
  end
174
203
  --[[
175
204
  Function to handle the case when job is duplicated.
@@ -1 +1 @@
1
- {"version":3,"file":"addStandardJob-9.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgaf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addStandardJob-9.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6bf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -59,20 +59,20 @@ end
59
59
  (since an empty list and !EXISTS are not really the same).
60
60
  ]]
61
61
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
62
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
62
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
63
63
  if queueAttributes[1] then
64
- return pausedKey, true
64
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
65
65
  else
66
66
  if queueAttributes[2] then
67
67
  local activeCount = rcall("LLEN", activeKey)
68
68
  if activeCount >= tonumber(queueAttributes[2]) then
69
- return waitKey, true
69
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
70
70
  else
71
- return waitKey, false
71
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
72
72
  end
73
73
  end
74
74
  end
75
- return waitKey, false
75
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
76
76
  end
77
77
  --[[
78
78
  Function to push back job considering priority in front of same prioritized jobs.
@@ -123,20 +123,20 @@ end
123
123
  (since an empty list and !EXISTS are not really the same).
124
124
  ]]
125
125
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
126
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
126
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
127
127
  if queueAttributes[1] then
128
- return pausedKey, true
128
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
129
129
  else
130
130
  if queueAttributes[2] then
131
131
  local activeCount = rcall("LLEN", activeKey)
132
132
  if activeCount >= tonumber(queueAttributes[2]) then
133
- return waitKey, true
133
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
134
134
  else
135
- return waitKey, false
135
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
136
136
  end
137
137
  end
138
138
  end
139
- return waitKey, false
139
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
140
140
  end
141
141
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
142
142
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -95,20 +95,20 @@ end
95
95
  (since an empty list and !EXISTS are not really the same).
96
96
  ]]
97
97
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
98
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
98
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
99
99
  if queueAttributes[1] then
100
- return pausedKey, true
100
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
101
101
  else
102
102
  if queueAttributes[2] then
103
103
  local activeCount = rcall("LLEN", activeKey)
104
104
  if activeCount >= tonumber(queueAttributes[2]) then
105
- return waitKey, true
105
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
106
106
  else
107
- return waitKey, false
107
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
108
108
  end
109
109
  end
110
110
  end
111
- return waitKey, false
111
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
112
112
  end
113
113
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
114
114
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -2,6 +2,7 @@ const content = `--[[
2
2
  Get rate limit ttl
3
3
  Input:
4
4
  KEYS[1] 'limiter'
5
+ KEYS[2] 'meta'
5
6
  ARGV[1] maxJobs
6
7
  ]]
7
8
  local rcall = redis.call
@@ -25,12 +26,16 @@ local rateLimiterKey = KEYS[1]
25
26
  if ARGV[1] ~= "0" then
26
27
  return getRateLimitTTL(tonumber(ARGV[1]), rateLimiterKey)
27
28
  else
29
+ local rateLimitMax = rcall("HGET", KEYS[2], "max")
30
+ if rateLimitMax then
31
+ return getRateLimitTTL(tonumber(rateLimitMax), rateLimiterKey)
32
+ end
28
33
  return rcall("PTTL", rateLimiterKey)
29
34
  end
30
35
  `;
31
36
  export const getRateLimitTtl = {
32
37
  name: 'getRateLimitTtl',
33
38
  content,
34
- keys: 1,
39
+ keys: 2,
35
40
  };
36
- //# sourceMappingURL=getRateLimitTtl-1.js.map
41
+ //# sourceMappingURL=getRateLimitTtl-2.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"getRateLimitTtl-2.js","sourceRoot":"","sources":["../../../src/scripts/getRateLimitTtl-2.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkCf,CAAC;AACF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -17,7 +17,7 @@ export * from './getDependencyCounts-4';
17
17
  export * from './getJobScheduler-1';
18
18
  export * from './getMetrics-2';
19
19
  export * from './getRanges-1';
20
- export * from './getRateLimitTtl-1';
20
+ export * from './getRateLimitTtl-2';
21
21
  export * from './getState-8';
22
22
  export * from './getStateV2-8';
23
23
  export * from './isFinished-3';
@@ -17,7 +17,7 @@ export * from './getDependencyCounts-4';
17
17
  export * from './getJobScheduler-1';
18
18
  export * from './getMetrics-2';
19
19
  export * from './getRanges-1';
20
- export * from './getRateLimitTtl-1';
20
+ export * from './getRateLimitTtl-2';
21
21
  export * from './getState-8';
22
22
  export * from './getStateV2-8';
23
23
  export * from './isFinished-3';
@@ -9,7 +9,7 @@ const content = `--[[
9
9
  KEYS[6] limiter key
10
10
  KEYS[7] prioritized key
11
11
  KEYS[8] marker key
12
- KEYS[9] event key
12
+ KEYS[9] event key
13
13
  ARGV[1] job id
14
14
  ARGV[2] lock token
15
15
  ARGV[3] job id key
@@ -57,20 +57,20 @@ end
57
57
  (since an empty list and !EXISTS are not really the same).
58
58
  ]]
59
59
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
60
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
60
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
61
61
  if queueAttributes[1] then
62
- return pausedKey, true
62
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
63
63
  else
64
64
  if queueAttributes[2] then
65
65
  local activeCount = rcall("LLEN", activeKey)
66
66
  if activeCount >= tonumber(queueAttributes[2]) then
67
- return waitKey, true
67
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
68
68
  else
69
- return waitKey, false
69
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
70
70
  end
71
71
  end
72
72
  end
73
- return waitKey, false
73
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
74
74
  end
75
75
  local function removeLock(jobKey, stalledKey, token, jobId)
76
76
  if token ~= "0" then
@@ -61,20 +61,20 @@ end
61
61
  (since an empty list and !EXISTS are not really the same).
62
62
  ]]
63
63
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
64
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
64
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
65
65
  if queueAttributes[1] then
66
- return pausedKey, true
66
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
67
67
  else
68
68
  if queueAttributes[2] then
69
69
  local activeCount = rcall("LLEN", activeKey)
70
70
  if activeCount >= tonumber(queueAttributes[2]) then
71
- return waitKey, true
71
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
72
72
  else
73
- return waitKey, false
73
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
74
74
  end
75
75
  end
76
76
  end
77
- return waitKey, false
77
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
78
78
  end
79
79
  local metaKey = KEYS[6]
80
80
  local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[7], KEYS[4], KEYS[5])
@@ -50,30 +50,30 @@ local function batches(n, batchSize)
50
50
  end
51
51
  end
52
52
  end
53
+ --[[
54
+ Function to move job to wait to be picked up by a waiting worker.
55
+ ]]
56
+ -- Includes
53
57
  --[[
54
58
  Function to check for the meta.paused key to decide if we are paused or not
55
59
  (since an empty list and !EXISTS are not really the same).
56
60
  ]]
57
61
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
58
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
62
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
59
63
  if queueAttributes[1] then
60
- return pausedKey, true
64
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
61
65
  else
62
66
  if queueAttributes[2] then
63
67
  local activeCount = rcall("LLEN", activeKey)
64
68
  if activeCount >= tonumber(queueAttributes[2]) then
65
- return waitKey, true
69
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
66
70
  else
67
- return waitKey, false
71
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
68
72
  end
69
73
  end
70
74
  end
71
- return waitKey, false
75
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
72
76
  end
73
- --[[
74
- Function to move job to wait to be picked up by a waiting worker.
75
- ]]
76
- -- Includes
77
77
  local function moveJobToWait(metaKey, activeKey, waitKey, pausedKey, markerKey, eventStreamKey,
78
78
  jobId, pushCmd)
79
79
  local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
@@ -68,20 +68,20 @@ end
68
68
  (since an empty list and !EXISTS are not really the same).
69
69
  ]]
70
70
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
71
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
71
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
72
72
  if queueAttributes[1] then
73
- return pausedKey, true
73
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
74
74
  else
75
75
  if queueAttributes[2] then
76
76
  local activeCount = rcall("LLEN", activeKey)
77
77
  if activeCount >= tonumber(queueAttributes[2]) then
78
- return waitKey, true
78
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
79
79
  else
80
- return waitKey, false
80
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
81
81
  end
82
82
  end
83
83
  end
84
- return waitKey, false
84
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
85
85
  end
86
86
  --[[
87
87
  Function to move job from prioritized state to active.
@@ -112,20 +112,19 @@ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
112
112
  end
113
113
  end
114
114
  local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
115
- jobId, processedOn, maxJobs, markerKey, opts)
115
+ jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
116
116
  local jobKey = keyPrefix .. jobId
117
117
  -- Check if we need to perform rate limiting.
118
118
  if maxJobs then
119
119
  local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
120
120
  if jobCounter == 1 then
121
- local limiterDuration = opts['limiter'] and opts['limiter']['duration']
122
121
  local integerDuration = math.floor(math.abs(limiterDuration))
123
122
  rcall("PEXPIRE", rateLimiterKey, integerDuration)
124
123
  end
125
124
  end
126
- local lockKey = jobKey .. ':lock'
127
125
  -- get a lock
128
126
  if opts['token'] ~= "0" then
127
+ local lockKey = jobKey .. ':lock'
129
128
  rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
130
129
  end
131
130
  local optionalValues = {}
@@ -199,17 +198,19 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
199
198
  addBaseMarkerIfNeeded(markerKey, isPaused)
200
199
  end
201
200
  end
202
- local target, isPausedOrMaxed = getTargetQueueList(KEYS[9], activeKey, waitKey, KEYS[8])
201
+ local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(KEYS[9],
202
+ activeKey, waitKey, KEYS[8])
203
203
  -- Check if there are delayed jobs that we can move to wait.
204
204
  local markerKey = KEYS[11]
205
205
  promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
206
206
  ARGV[2], KEYS[10], isPausedOrMaxed)
207
- local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
207
+ local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
208
208
  local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
209
209
  -- Check if we are rate limited first.
210
210
  if expireTime > 0 then return {0, 0, expireTime, 0} end
211
211
  -- paused or maxed queue
212
212
  if isPausedOrMaxed then return {0, 0, 0, 0} end
213
+ local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
213
214
  -- no job ID, try non-blocking move from wait to active
214
215
  local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
215
216
  -- Markers in waitlist DEPRECATED in v5: Will be completely removed in v6.
@@ -219,12 +220,12 @@ if jobId and string.sub(jobId, 1, 2) == "0:" then
219
220
  end
220
221
  if jobId then
221
222
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
222
- maxJobs, markerKey, opts)
223
+ maxJobs, limiterDuration, markerKey, opts)
223
224
  else
224
225
  jobId = moveJobFromPrioritizedToActive(KEYS[3], activeKey, KEYS[10])
225
226
  if jobId then
226
227
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
227
- maxJobs, markerKey, opts)
228
+ maxJobs, limiterDuration, markerKey, opts)
228
229
  end
229
230
  end
230
231
  -- Return the timestamp for the next delayed job if any.
@@ -1 +1 @@
1
- {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyOf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Of,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -126,9 +126,9 @@ if rcall("EXISTS", jobKey) == 1 then
126
126
  local delayedKey = KEYS[4]
127
127
  local jobId = ARGV[3]
128
128
  local delay = tonumber(ARGV[5])
129
- local score, delayedTimestamp = getDelayedScore(delayedKey, ARGV[2], delay)
130
129
  local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
131
130
  if numRemovedElements < 1 then return -3 end
131
+ local score, delayedTimestamp = getDelayedScore(delayedKey, ARGV[2], delay)
132
132
  if ARGV[6] == "0" then
133
133
  rcall("HINCRBY", jobKey, "atm", 1)
134
134
  end