bullmq 5.58.9 → 5.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/dist/cjs/classes/child-pool.js +8 -1
  2. package/dist/cjs/classes/child-pool.js.map +1 -1
  3. package/dist/cjs/classes/queue.js +9 -0
  4. package/dist/cjs/classes/queue.js.map +1 -1
  5. package/dist/cjs/classes/scripts.js +4 -1
  6. package/dist/cjs/classes/scripts.js.map +1 -1
  7. package/dist/cjs/classes/worker.js +42 -30
  8. package/dist/cjs/classes/worker.js.map +1 -1
  9. package/dist/cjs/commands/changePriority-7.lua +1 -1
  10. package/dist/cjs/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
  11. package/dist/cjs/commands/includes/deduplicateJob.lua +53 -20
  12. package/dist/cjs/commands/includes/getTargetQueueList.lua +5 -5
  13. package/dist/cjs/commands/includes/prepareJobForProcessing.lua +2 -4
  14. package/dist/cjs/commands/moveJobFromActiveToWait-9.lua +1 -1
  15. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -1
  16. package/dist/cjs/commands/moveToActive-11.lua +7 -4
  17. package/dist/cjs/commands/moveToDelayed-8.lua +2 -1
  18. package/dist/cjs/commands/moveToFinished-14.lua +10 -7
  19. package/dist/cjs/scripts/addDelayedJob-6.js +52 -23
  20. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  21. package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
  22. package/dist/cjs/scripts/addParentJob-5.js +52 -23
  23. package/dist/cjs/scripts/addParentJob-5.js.map +1 -1
  24. package/dist/cjs/scripts/addPrioritizedJob-9.js +52 -23
  25. package/dist/cjs/scripts/addPrioritizedJob-9.js.map +1 -1
  26. package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
  27. package/dist/cjs/scripts/addStandardJob-9.js +52 -23
  28. package/dist/cjs/scripts/addStandardJob-9.js.map +1 -1
  29. package/dist/cjs/scripts/changePriority-7.js +5 -5
  30. package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
  31. package/dist/cjs/scripts/drain-5.js +5 -5
  32. package/dist/cjs/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
  33. package/dist/cjs/scripts/getRateLimitTtl-2.js.map +1 -0
  34. package/dist/cjs/scripts/index.js +1 -1
  35. package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +6 -6
  36. package/dist/cjs/scripts/moveJobsToWait-8.js +5 -5
  37. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +9 -9
  38. package/dist/cjs/scripts/moveToActive-11.js +13 -12
  39. package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
  40. package/dist/cjs/scripts/moveToDelayed-8.js +1 -1
  41. package/dist/cjs/scripts/moveToFinished-14.js +16 -15
  42. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  43. package/dist/cjs/scripts/obliterate-2.js +5 -5
  44. package/dist/cjs/scripts/promote-9.js +5 -5
  45. package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
  46. package/dist/cjs/scripts/removeJob-2.js +5 -5
  47. package/dist/cjs/scripts/removeUnprocessedChildren-2.js +5 -5
  48. package/dist/cjs/scripts/reprocessJob-8.js +5 -5
  49. package/dist/cjs/scripts/retryJob-11.js +5 -5
  50. package/dist/cjs/scripts/updateJobScheduler-12.js +5 -5
  51. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  52. package/dist/cjs/version.js +1 -1
  53. package/dist/esm/classes/child-pool.js +8 -1
  54. package/dist/esm/classes/child-pool.js.map +1 -1
  55. package/dist/esm/classes/queue.d.ts +6 -0
  56. package/dist/esm/classes/queue.js +9 -0
  57. package/dist/esm/classes/queue.js.map +1 -1
  58. package/dist/esm/classes/scripts.js +4 -1
  59. package/dist/esm/classes/scripts.js.map +1 -1
  60. package/dist/esm/classes/worker.d.ts +1 -0
  61. package/dist/esm/classes/worker.js +42 -30
  62. package/dist/esm/classes/worker.js.map +1 -1
  63. package/dist/esm/commands/changePriority-7.lua +1 -1
  64. package/dist/esm/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
  65. package/dist/esm/commands/includes/deduplicateJob.lua +53 -20
  66. package/dist/esm/commands/includes/getTargetQueueList.lua +5 -5
  67. package/dist/esm/commands/includes/prepareJobForProcessing.lua +2 -4
  68. package/dist/esm/commands/moveJobFromActiveToWait-9.lua +1 -1
  69. package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -1
  70. package/dist/esm/commands/moveToActive-11.lua +7 -4
  71. package/dist/esm/commands/moveToDelayed-8.lua +2 -1
  72. package/dist/esm/commands/moveToFinished-14.lua +10 -7
  73. package/dist/esm/interfaces/base-job-options.d.ts +1 -1
  74. package/dist/esm/interfaces/worker-options.d.ts +8 -0
  75. package/dist/esm/scripts/addDelayedJob-6.js +52 -23
  76. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  77. package/dist/esm/scripts/addJobScheduler-11.js +5 -5
  78. package/dist/esm/scripts/addParentJob-5.js +52 -23
  79. package/dist/esm/scripts/addParentJob-5.js.map +1 -1
  80. package/dist/esm/scripts/addPrioritizedJob-9.js +52 -23
  81. package/dist/esm/scripts/addPrioritizedJob-9.js.map +1 -1
  82. package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
  83. package/dist/esm/scripts/addStandardJob-9.js +52 -23
  84. package/dist/esm/scripts/addStandardJob-9.js.map +1 -1
  85. package/dist/esm/scripts/changePriority-7.js +5 -5
  86. package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
  87. package/dist/esm/scripts/drain-5.js +5 -5
  88. package/dist/esm/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
  89. package/dist/esm/scripts/getRateLimitTtl-2.js.map +1 -0
  90. package/dist/esm/scripts/index.d.ts +1 -1
  91. package/dist/esm/scripts/index.js +1 -1
  92. package/dist/esm/scripts/moveJobFromActiveToWait-9.js +6 -6
  93. package/dist/esm/scripts/moveJobsToWait-8.js +5 -5
  94. package/dist/esm/scripts/moveStalledJobsToWait-8.js +9 -9
  95. package/dist/esm/scripts/moveToActive-11.js +13 -12
  96. package/dist/esm/scripts/moveToActive-11.js.map +1 -1
  97. package/dist/esm/scripts/moveToDelayed-8.js +1 -1
  98. package/dist/esm/scripts/moveToFinished-14.js +16 -15
  99. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  100. package/dist/esm/scripts/obliterate-2.js +5 -5
  101. package/dist/esm/scripts/promote-9.js +5 -5
  102. package/dist/esm/scripts/removeChildDependency-1.js +5 -5
  103. package/dist/esm/scripts/removeJob-2.js +5 -5
  104. package/dist/esm/scripts/removeUnprocessedChildren-2.js +5 -5
  105. package/dist/esm/scripts/reprocessJob-8.js +5 -5
  106. package/dist/esm/scripts/retryJob-11.js +5 -5
  107. package/dist/esm/scripts/updateJobScheduler-12.js +5 -5
  108. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  109. package/dist/esm/version.d.ts +1 -1
  110. package/dist/esm/version.js +1 -1
  111. package/package.json +1 -1
  112. package/dist/cjs/scripts/getRateLimitTtl-1.js.map +0 -1
  113. package/dist/esm/scripts/getRateLimitTtl-1.js.map +0 -1
  114. /package/dist/esm/scripts/{getRateLimitTtl-1.d.ts → getRateLimitTtl-2.d.ts} +0 -0
@@ -88,37 +88,65 @@ local function removeJobKeys(jobKey)
88
88
  return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
89
89
  jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
90
90
  end
91
+ local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents, currentDeduplicatedJobId,
92
+ jobId, deduplicationId, prefix)
93
+ if rcall("ZREM", delayedKey, currentDeduplicatedJobId) > 0 then
94
+ removeJobKeys(prefix .. currentDeduplicatedJobId)
95
+ rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDeduplicatedJobId,
96
+ "prev", "delayed")
97
+ -- TODO remove debounced event in next breaking change
98
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
99
+ jobId, "debounceId", deduplicationId)
100
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
101
+ jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDeduplicatedJobId)
102
+ return true
103
+ end
104
+ return false
105
+ end
91
106
  local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
92
107
  prefix)
93
108
  local deduplicationId = deduplicationOpts and deduplicationOpts['id']
94
109
  if deduplicationId then
95
110
  local ttl = deduplicationOpts['ttl']
96
- if deduplicationOpts['replace'] and ttl and ttl > 0 then
97
- local currentDebounceJobId = rcall('GET', deduplicationKey)
98
- if currentDebounceJobId then
99
- if rcall("ZREM", delayedKey, currentDebounceJobId) > 0 then
100
- removeJobKeys(prefix .. currentDebounceJobId)
101
- rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDebounceJobId,
102
- "prev", "delayed")
103
- if deduplicationOpts['extend'] then
104
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
111
+ if deduplicationOpts['replace'] then
112
+ if ttl and ttl > 0 then
113
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
114
+ if currentDebounceJobId then
115
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
116
+ currentDebounceJobId, jobId, deduplicationId, prefix)
117
+ if isRemoved then
118
+ if deduplicationOpts['extend'] then
119
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
120
+ else
121
+ rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
122
+ end
123
+ return
105
124
  else
106
- rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
125
+ return currentDebounceJobId
107
126
  end
108
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
109
- jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
110
- return
111
127
  else
112
- return currentDebounceJobId
128
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
129
+ return
113
130
  end
114
131
  else
115
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
116
- return
132
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
133
+ if currentDebounceJobId then
134
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
135
+ currentDebounceJobId, jobId, deduplicationId, prefix)
136
+ if isRemoved then
137
+ rcall('SET', deduplicationKey, jobId)
138
+ return
139
+ else
140
+ return currentDebounceJobId
141
+ end
142
+ else
143
+ rcall('SET', deduplicationKey, jobId)
144
+ return
145
+ end
117
146
  end
118
147
  else
119
- local ttl = deduplicationOpts['ttl']
120
148
  local deduplicationKeyExists
121
- if ttl then
149
+ if ttl and ttl > 0 then
122
150
  if deduplicationOpts['extend'] then
123
151
  local currentDebounceJobId = rcall('GET', deduplicationKey)
124
152
  if currentDebounceJobId then
@@ -140,6 +168,7 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
140
168
  end
141
169
  if deduplicationKeyExists then
142
170
  local currentDebounceJobId = rcall('GET', deduplicationKey)
171
+ -- TODO remove debounced event in next breaking change
143
172
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
144
173
  currentDebounceJobId, "debounceId", deduplicationId)
145
174
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
@@ -271,20 +300,20 @@ end
271
300
  (since an empty list and !EXISTS are not really the same).
272
301
  ]]
273
302
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
274
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
303
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
275
304
  if queueAttributes[1] then
276
- return pausedKey, true
305
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
277
306
  else
278
307
  if queueAttributes[2] then
279
308
  local activeCount = rcall("LLEN", activeKey)
280
309
  if activeCount >= tonumber(queueAttributes[2]) then
281
- return waitKey, true
310
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
282
311
  else
283
- return waitKey, false
312
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
284
313
  end
285
314
  end
286
315
  end
287
- return waitKey, false
316
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
288
317
  end
289
318
  local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
290
319
  local parentWaitKey = parentQueueKey .. ":wait"
@@ -1 +1 @@
1
- {"version":3,"file":"addPrioritizedJob-9.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4Zf,CAAC;AACW,QAAA,iBAAiB,GAAG;IAC/B,IAAI,EAAE,mBAAmB;IACzB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addPrioritizedJob-9.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAybf,CAAC;AACW,QAAA,iBAAiB,GAAG;IAC/B,IAAI,EAAE,mBAAmB;IACzB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -90,20 +90,20 @@ end
90
90
  (since an empty list and !EXISTS are not really the same).
91
91
  ]]
92
92
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
93
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
93
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
94
94
  if queueAttributes[1] then
95
- return pausedKey, true
95
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
96
96
  else
97
97
  if queueAttributes[2] then
98
98
  local activeCount = rcall("LLEN", activeKey)
99
99
  if activeCount >= tonumber(queueAttributes[2]) then
100
- return waitKey, true
100
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
101
101
  else
102
- return waitKey, false
102
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
103
103
  end
104
104
  end
105
105
  end
106
- return waitKey, false
106
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
107
107
  end
108
108
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
109
109
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -82,37 +82,65 @@ local function removeJobKeys(jobKey)
82
82
  return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
83
83
  jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
84
84
  end
85
+ local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents, currentDeduplicatedJobId,
86
+ jobId, deduplicationId, prefix)
87
+ if rcall("ZREM", delayedKey, currentDeduplicatedJobId) > 0 then
88
+ removeJobKeys(prefix .. currentDeduplicatedJobId)
89
+ rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDeduplicatedJobId,
90
+ "prev", "delayed")
91
+ -- TODO remove debounced event in next breaking change
92
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
93
+ jobId, "debounceId", deduplicationId)
94
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
95
+ jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDeduplicatedJobId)
96
+ return true
97
+ end
98
+ return false
99
+ end
85
100
  local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
86
101
  prefix)
87
102
  local deduplicationId = deduplicationOpts and deduplicationOpts['id']
88
103
  if deduplicationId then
89
104
  local ttl = deduplicationOpts['ttl']
90
- if deduplicationOpts['replace'] and ttl and ttl > 0 then
91
- local currentDebounceJobId = rcall('GET', deduplicationKey)
92
- if currentDebounceJobId then
93
- if rcall("ZREM", delayedKey, currentDebounceJobId) > 0 then
94
- removeJobKeys(prefix .. currentDebounceJobId)
95
- rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDebounceJobId,
96
- "prev", "delayed")
97
- if deduplicationOpts['extend'] then
98
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
105
+ if deduplicationOpts['replace'] then
106
+ if ttl and ttl > 0 then
107
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
108
+ if currentDebounceJobId then
109
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
110
+ currentDebounceJobId, jobId, deduplicationId, prefix)
111
+ if isRemoved then
112
+ if deduplicationOpts['extend'] then
113
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
114
+ else
115
+ rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
116
+ end
117
+ return
99
118
  else
100
- rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
119
+ return currentDebounceJobId
101
120
  end
102
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
103
- jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
104
- return
105
121
  else
106
- return currentDebounceJobId
122
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
123
+ return
107
124
  end
108
125
  else
109
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
110
- return
126
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
127
+ if currentDebounceJobId then
128
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
129
+ currentDebounceJobId, jobId, deduplicationId, prefix)
130
+ if isRemoved then
131
+ rcall('SET', deduplicationKey, jobId)
132
+ return
133
+ else
134
+ return currentDebounceJobId
135
+ end
136
+ else
137
+ rcall('SET', deduplicationKey, jobId)
138
+ return
139
+ end
111
140
  end
112
141
  else
113
- local ttl = deduplicationOpts['ttl']
114
142
  local deduplicationKeyExists
115
- if ttl then
143
+ if ttl and ttl > 0 then
116
144
  if deduplicationOpts['extend'] then
117
145
  local currentDebounceJobId = rcall('GET', deduplicationKey)
118
146
  if currentDebounceJobId then
@@ -134,6 +162,7 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
134
162
  end
135
163
  if deduplicationKeyExists then
136
164
  local currentDebounceJobId = rcall('GET', deduplicationKey)
165
+ -- TODO remove debounced event in next breaking change
137
166
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
138
167
  currentDebounceJobId, "debounceId", deduplicationId)
139
168
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
@@ -159,20 +188,20 @@ end
159
188
  (since an empty list and !EXISTS are not really the same).
160
189
  ]]
161
190
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
162
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
191
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
163
192
  if queueAttributes[1] then
164
- return pausedKey, true
193
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
165
194
  else
166
195
  if queueAttributes[2] then
167
196
  local activeCount = rcall("LLEN", activeKey)
168
197
  if activeCount >= tonumber(queueAttributes[2]) then
169
- return waitKey, true
198
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
170
199
  else
171
- return waitKey, false
200
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
172
201
  end
173
202
  end
174
203
  end
175
- return waitKey, false
204
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
176
205
  end
177
206
  --[[
178
207
  Function to handle the case when job is duplicated.
@@ -1 +1 @@
1
- {"version":3,"file":"addStandardJob-9.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgaf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addStandardJob-9.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6bf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -62,20 +62,20 @@ end
62
62
  (since an empty list and !EXISTS are not really the same).
63
63
  ]]
64
64
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
65
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
65
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
66
66
  if queueAttributes[1] then
67
- return pausedKey, true
67
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
68
68
  else
69
69
  if queueAttributes[2] then
70
70
  local activeCount = rcall("LLEN", activeKey)
71
71
  if activeCount >= tonumber(queueAttributes[2]) then
72
- return waitKey, true
72
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
73
73
  else
74
- return waitKey, false
74
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
75
75
  end
76
76
  end
77
77
  end
78
- return waitKey, false
78
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
79
79
  end
80
80
  --[[
81
81
  Function to push back job considering priority in front of same prioritized jobs.
@@ -126,20 +126,20 @@ end
126
126
  (since an empty list and !EXISTS are not really the same).
127
127
  ]]
128
128
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
129
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
129
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
130
130
  if queueAttributes[1] then
131
- return pausedKey, true
131
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
132
132
  else
133
133
  if queueAttributes[2] then
134
134
  local activeCount = rcall("LLEN", activeKey)
135
135
  if activeCount >= tonumber(queueAttributes[2]) then
136
- return waitKey, true
136
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
137
137
  else
138
- return waitKey, false
138
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
139
139
  end
140
140
  end
141
141
  end
142
- return waitKey, false
142
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
143
143
  end
144
144
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
145
145
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -98,20 +98,20 @@ end
98
98
  (since an empty list and !EXISTS are not really the same).
99
99
  ]]
100
100
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
101
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
101
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
102
102
  if queueAttributes[1] then
103
- return pausedKey, true
103
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
104
104
  else
105
105
  if queueAttributes[2] then
106
106
  local activeCount = rcall("LLEN", activeKey)
107
107
  if activeCount >= tonumber(queueAttributes[2]) then
108
- return waitKey, true
108
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
109
109
  else
110
- return waitKey, false
110
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
111
111
  end
112
112
  end
113
113
  end
114
- return waitKey, false
114
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
115
115
  end
116
116
  local function _moveParentToWait(parentPrefix, parentId, emitEvent)
117
117
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -5,6 +5,7 @@ const content = `--[[
5
5
  Get rate limit ttl
6
6
  Input:
7
7
  KEYS[1] 'limiter'
8
+ KEYS[2] 'meta'
8
9
  ARGV[1] maxJobs
9
10
  ]]
10
11
  local rcall = redis.call
@@ -28,12 +29,16 @@ local rateLimiterKey = KEYS[1]
28
29
  if ARGV[1] ~= "0" then
29
30
  return getRateLimitTTL(tonumber(ARGV[1]), rateLimiterKey)
30
31
  else
32
+ local rateLimitMax = rcall("HGET", KEYS[2], "max")
33
+ if rateLimitMax then
34
+ return getRateLimitTTL(tonumber(rateLimitMax), rateLimiterKey)
35
+ end
31
36
  return rcall("PTTL", rateLimiterKey)
32
37
  end
33
38
  `;
34
39
  exports.getRateLimitTtl = {
35
40
  name: 'getRateLimitTtl',
36
41
  content,
37
- keys: 1,
42
+ keys: 2,
38
43
  };
39
- //# sourceMappingURL=getRateLimitTtl-1.js.map
44
+ //# sourceMappingURL=getRateLimitTtl-2.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"getRateLimitTtl-2.js","sourceRoot":"","sources":["../../../src/scripts/getRateLimitTtl-2.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkCf,CAAC;AACW,QAAA,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -20,7 +20,7 @@ tslib_1.__exportStar(require("./getDependencyCounts-4"), exports);
20
20
  tslib_1.__exportStar(require("./getJobScheduler-1"), exports);
21
21
  tslib_1.__exportStar(require("./getMetrics-2"), exports);
22
22
  tslib_1.__exportStar(require("./getRanges-1"), exports);
23
- tslib_1.__exportStar(require("./getRateLimitTtl-1"), exports);
23
+ tslib_1.__exportStar(require("./getRateLimitTtl-2"), exports);
24
24
  tslib_1.__exportStar(require("./getState-8"), exports);
25
25
  tslib_1.__exportStar(require("./getStateV2-8"), exports);
26
26
  tslib_1.__exportStar(require("./isFinished-3"), exports);
@@ -12,7 +12,7 @@ const content = `--[[
12
12
  KEYS[6] limiter key
13
13
  KEYS[7] prioritized key
14
14
  KEYS[8] marker key
15
- KEYS[9] event key
15
+ KEYS[9] event key
16
16
  ARGV[1] job id
17
17
  ARGV[2] lock token
18
18
  ARGV[3] job id key
@@ -60,20 +60,20 @@ end
60
60
  (since an empty list and !EXISTS are not really the same).
61
61
  ]]
62
62
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
63
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
63
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
64
64
  if queueAttributes[1] then
65
- return pausedKey, true
65
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
66
66
  else
67
67
  if queueAttributes[2] then
68
68
  local activeCount = rcall("LLEN", activeKey)
69
69
  if activeCount >= tonumber(queueAttributes[2]) then
70
- return waitKey, true
70
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
71
71
  else
72
- return waitKey, false
72
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
73
73
  end
74
74
  end
75
75
  end
76
- return waitKey, false
76
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
77
77
  end
78
78
  local function removeLock(jobKey, stalledKey, token, jobId)
79
79
  if token ~= "0" then
@@ -64,20 +64,20 @@ end
64
64
  (since an empty list and !EXISTS are not really the same).
65
65
  ]]
66
66
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
67
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
67
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
68
68
  if queueAttributes[1] then
69
- return pausedKey, true
69
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
70
70
  else
71
71
  if queueAttributes[2] then
72
72
  local activeCount = rcall("LLEN", activeKey)
73
73
  if activeCount >= tonumber(queueAttributes[2]) then
74
- return waitKey, true
74
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
75
75
  else
76
- return waitKey, false
76
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
77
77
  end
78
78
  end
79
79
  end
80
- return waitKey, false
80
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
81
81
  end
82
82
  local metaKey = KEYS[6]
83
83
  local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[7], KEYS[4], KEYS[5])
@@ -53,30 +53,30 @@ local function batches(n, batchSize)
53
53
  end
54
54
  end
55
55
  end
56
+ --[[
57
+ Function to move job to wait to be picked up by a waiting worker.
58
+ ]]
59
+ -- Includes
56
60
  --[[
57
61
  Function to check for the meta.paused key to decide if we are paused or not
58
62
  (since an empty list and !EXISTS are not really the same).
59
63
  ]]
60
64
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
61
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
65
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
62
66
  if queueAttributes[1] then
63
- return pausedKey, true
67
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
64
68
  else
65
69
  if queueAttributes[2] then
66
70
  local activeCount = rcall("LLEN", activeKey)
67
71
  if activeCount >= tonumber(queueAttributes[2]) then
68
- return waitKey, true
72
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
69
73
  else
70
- return waitKey, false
74
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
71
75
  end
72
76
  end
73
77
  end
74
- return waitKey, false
78
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
75
79
  end
76
- --[[
77
- Function to move job to wait to be picked up by a waiting worker.
78
- ]]
79
- -- Includes
80
80
  local function moveJobToWait(metaKey, activeKey, waitKey, pausedKey, markerKey, eventStreamKey,
81
81
  jobId, pushCmd)
82
82
  local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
@@ -71,20 +71,20 @@ end
71
71
  (since an empty list and !EXISTS are not really the same).
72
72
  ]]
73
73
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
74
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
74
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
75
75
  if queueAttributes[1] then
76
- return pausedKey, true
76
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
77
77
  else
78
78
  if queueAttributes[2] then
79
79
  local activeCount = rcall("LLEN", activeKey)
80
80
  if activeCount >= tonumber(queueAttributes[2]) then
81
- return waitKey, true
81
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
82
82
  else
83
- return waitKey, false
83
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
84
84
  end
85
85
  end
86
86
  end
87
- return waitKey, false
87
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
88
88
  end
89
89
  --[[
90
90
  Function to move job from prioritized state to active.
@@ -115,20 +115,19 @@ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
115
115
  end
116
116
  end
117
117
  local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
118
- jobId, processedOn, maxJobs, markerKey, opts)
118
+ jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
119
119
  local jobKey = keyPrefix .. jobId
120
120
  -- Check if we need to perform rate limiting.
121
121
  if maxJobs then
122
122
  local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
123
123
  if jobCounter == 1 then
124
- local limiterDuration = opts['limiter'] and opts['limiter']['duration']
125
124
  local integerDuration = math.floor(math.abs(limiterDuration))
126
125
  rcall("PEXPIRE", rateLimiterKey, integerDuration)
127
126
  end
128
127
  end
129
- local lockKey = jobKey .. ':lock'
130
128
  -- get a lock
131
129
  if opts['token'] ~= "0" then
130
+ local lockKey = jobKey .. ':lock'
132
131
  rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
133
132
  end
134
133
  local optionalValues = {}
@@ -202,17 +201,19 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
202
201
  addBaseMarkerIfNeeded(markerKey, isPaused)
203
202
  end
204
203
  end
205
- local target, isPausedOrMaxed = getTargetQueueList(KEYS[9], activeKey, waitKey, KEYS[8])
204
+ local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(KEYS[9],
205
+ activeKey, waitKey, KEYS[8])
206
206
  -- Check if there are delayed jobs that we can move to wait.
207
207
  local markerKey = KEYS[11]
208
208
  promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
209
209
  ARGV[2], KEYS[10], isPausedOrMaxed)
210
- local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
210
+ local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
211
211
  local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
212
212
  -- Check if we are rate limited first.
213
213
  if expireTime > 0 then return {0, 0, expireTime, 0} end
214
214
  -- paused or maxed queue
215
215
  if isPausedOrMaxed then return {0, 0, 0, 0} end
216
+ local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
216
217
  -- no job ID, try non-blocking move from wait to active
217
218
  local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
218
219
  -- Markers in waitlist DEPRECATED in v5: Will be completely removed in v6.
@@ -222,12 +223,12 @@ if jobId and string.sub(jobId, 1, 2) == "0:" then
222
223
  end
223
224
  if jobId then
224
225
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
225
- maxJobs, markerKey, opts)
226
+ maxJobs, limiterDuration, markerKey, opts)
226
227
  else
227
228
  jobId = moveJobFromPrioritizedToActive(KEYS[3], activeKey, KEYS[10])
228
229
  if jobId then
229
230
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
230
- maxJobs, markerKey, opts)
231
+ maxJobs, limiterDuration, markerKey, opts)
231
232
  end
232
233
  end
233
234
  -- Return the timestamp for the next delayed job if any.
@@ -1 +1 @@
1
- {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyOf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Of,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -129,9 +129,9 @@ if rcall("EXISTS", jobKey) == 1 then
129
129
  local delayedKey = KEYS[4]
130
130
  local jobId = ARGV[3]
131
131
  local delay = tonumber(ARGV[5])
132
- local score, delayedTimestamp = getDelayedScore(delayedKey, ARGV[2], delay)
133
132
  local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
134
133
  if numRemovedElements < 1 then return -3 end
134
+ local score, delayedTimestamp = getDelayedScore(delayedKey, ARGV[2], delay)
135
135
  if ARGV[6] == "0" then
136
136
  rcall("HINCRBY", jobKey, "atm", 1)
137
137
  end