bullmq 5.44.1 → 5.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/dist/cjs/classes/async-fifo-queue.js.map +1 -1
  2. package/dist/cjs/classes/child-processor.js.map +1 -1
  3. package/dist/cjs/classes/child.js.map +1 -1
  4. package/dist/cjs/classes/flow-producer.js +15 -4
  5. package/dist/cjs/classes/flow-producer.js.map +1 -1
  6. package/dist/cjs/classes/job-scheduler.js +1 -1
  7. package/dist/cjs/classes/job-scheduler.js.map +1 -1
  8. package/dist/cjs/classes/job.js +137 -54
  9. package/dist/cjs/classes/job.js.map +1 -1
  10. package/dist/cjs/classes/queue-base.js +1 -4
  11. package/dist/cjs/classes/queue-base.js.map +1 -1
  12. package/dist/cjs/classes/queue-events-producer.js.map +1 -1
  13. package/dist/cjs/classes/queue-events.js.map +1 -1
  14. package/dist/cjs/classes/queue-getters.js +12 -15
  15. package/dist/cjs/classes/queue-getters.js.map +1 -1
  16. package/dist/cjs/classes/queue-keys.js.map +1 -1
  17. package/dist/cjs/classes/queue.js +12 -13
  18. package/dist/cjs/classes/queue.js.map +1 -1
  19. package/dist/cjs/classes/redis-connection.js +1 -1
  20. package/dist/cjs/classes/redis-connection.js.map +1 -1
  21. package/dist/cjs/classes/repeat.js +1 -1
  22. package/dist/cjs/classes/repeat.js.map +1 -1
  23. package/dist/cjs/classes/scripts.js +18 -7
  24. package/dist/cjs/classes/scripts.js.map +1 -1
  25. package/dist/cjs/classes/worker.js +4 -0
  26. package/dist/cjs/classes/worker.js.map +1 -1
  27. package/dist/cjs/commands/addJobScheduler-11.lua +52 -33
  28. package/dist/cjs/commands/includes/deduplicateJob.lua +16 -17
  29. package/dist/cjs/commands/includes/isLocked.lua +1 -0
  30. package/dist/cjs/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
  31. package/dist/cjs/commands/includes/moveParentToWait.lua +48 -0
  32. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
  33. package/dist/cjs/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
  34. package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
  35. package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
  36. package/dist/cjs/commands/includes/removeJob.lua +2 -2
  37. package/dist/cjs/commands/includes/removeJobWithChildren.lua +94 -0
  38. package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
  39. package/dist/cjs/commands/includes/storeJob.lua +1 -1
  40. package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +2 -2
  41. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +18 -53
  42. package/dist/cjs/commands/moveToFinished-14.lua +43 -51
  43. package/dist/cjs/commands/moveToWaitingChildren-8.lua +5 -32
  44. package/dist/cjs/commands/removeJob-3.lua +14 -72
  45. package/dist/cjs/commands/removeUnprocessedChildren-2.lua +31 -0
  46. package/dist/cjs/commands/updateJobScheduler-12.lua +29 -23
  47. package/dist/cjs/scripts/addDelayedJob-6.js +73 -55
  48. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  49. package/dist/cjs/scripts/addJobScheduler-11.js +59 -40
  50. package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
  51. package/dist/cjs/scripts/addParentJob-4.js +73 -55
  52. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  53. package/dist/cjs/scripts/addPrioritizedJob-8.js +73 -55
  54. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
  55. package/dist/cjs/scripts/addRepeatableJob-2.js +14 -9
  56. package/dist/cjs/scripts/addRepeatableJob-2.js.map +1 -1
  57. package/dist/cjs/scripts/addStandardJob-8.js +73 -55
  58. package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
  59. package/dist/cjs/scripts/cleanJobsInSet-3.js +14 -9
  60. package/dist/cjs/scripts/cleanJobsInSet-3.js.map +1 -1
  61. package/dist/cjs/scripts/drain-5.js +14 -9
  62. package/dist/cjs/scripts/drain-5.js.map +1 -1
  63. package/dist/cjs/scripts/index.js +1 -0
  64. package/dist/cjs/scripts/index.js.map +1 -1
  65. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +157 -155
  66. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  67. package/dist/cjs/scripts/moveToFinished-14.js +183 -157
  68. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  69. package/dist/cjs/scripts/moveToWaitingChildren-8.js +145 -137
  70. package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
  71. package/dist/cjs/scripts/obliterate-2.js +14 -9
  72. package/dist/cjs/scripts/obliterate-2.js.map +1 -1
  73. package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
  74. package/dist/cjs/scripts/removeJob-3.js +97 -63
  75. package/dist/cjs/scripts/removeJob-3.js.map +1 -1
  76. package/dist/cjs/scripts/removeUnprocessedChildren-2.js +338 -0
  77. package/dist/cjs/scripts/removeUnprocessedChildren-2.js.map +1 -0
  78. package/dist/cjs/scripts/updateJobScheduler-12.js +27 -21
  79. package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
  80. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  81. package/dist/cjs/types/index.js +1 -0
  82. package/dist/cjs/types/index.js.map +1 -1
  83. package/dist/cjs/types/job-progress.js +3 -0
  84. package/dist/cjs/types/job-progress.js.map +1 -0
  85. package/dist/cjs/utils.js +12 -1
  86. package/dist/cjs/utils.js.map +1 -1
  87. package/dist/cjs/version.js +1 -1
  88. package/dist/esm/classes/async-fifo-queue.js.map +1 -1
  89. package/dist/esm/classes/child-processor.js.map +1 -1
  90. package/dist/esm/classes/child.js.map +1 -1
  91. package/dist/esm/classes/flow-producer.js +15 -4
  92. package/dist/esm/classes/flow-producer.js.map +1 -1
  93. package/dist/esm/classes/job-scheduler.js +1 -1
  94. package/dist/esm/classes/job-scheduler.js.map +1 -1
  95. package/dist/esm/classes/job.d.ts +49 -14
  96. package/dist/esm/classes/job.js +138 -55
  97. package/dist/esm/classes/job.js.map +1 -1
  98. package/dist/esm/classes/queue-base.d.ts +1 -4
  99. package/dist/esm/classes/queue-base.js +1 -4
  100. package/dist/esm/classes/queue-base.js.map +1 -1
  101. package/dist/esm/classes/queue-events-producer.d.ts +2 -2
  102. package/dist/esm/classes/queue-events-producer.js.map +1 -1
  103. package/dist/esm/classes/queue-events.d.ts +118 -29
  104. package/dist/esm/classes/queue-events.js.map +1 -1
  105. package/dist/esm/classes/queue-getters.d.ts +11 -14
  106. package/dist/esm/classes/queue-getters.js +12 -15
  107. package/dist/esm/classes/queue-getters.js.map +1 -1
  108. package/dist/esm/classes/queue-keys.js.map +1 -1
  109. package/dist/esm/classes/queue.d.ts +14 -15
  110. package/dist/esm/classes/queue.js +12 -13
  111. package/dist/esm/classes/queue.js.map +1 -1
  112. package/dist/esm/classes/redis-connection.js +1 -1
  113. package/dist/esm/classes/redis-connection.js.map +1 -1
  114. package/dist/esm/classes/repeat.js +1 -1
  115. package/dist/esm/classes/repeat.js.map +1 -1
  116. package/dist/esm/classes/scripts.d.ts +4 -3
  117. package/dist/esm/classes/scripts.js +18 -7
  118. package/dist/esm/classes/scripts.js.map +1 -1
  119. package/dist/esm/classes/worker.d.ts +2 -1
  120. package/dist/esm/classes/worker.js +5 -1
  121. package/dist/esm/classes/worker.js.map +1 -1
  122. package/dist/esm/commands/addJobScheduler-11.lua +52 -33
  123. package/dist/esm/commands/includes/deduplicateJob.lua +16 -17
  124. package/dist/esm/commands/includes/isLocked.lua +1 -0
  125. package/dist/esm/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
  126. package/dist/esm/commands/includes/moveParentToWait.lua +48 -0
  127. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
  128. package/dist/esm/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
  129. package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
  130. package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
  131. package/dist/esm/commands/includes/removeJob.lua +2 -2
  132. package/dist/esm/commands/includes/removeJobWithChildren.lua +94 -0
  133. package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
  134. package/dist/esm/commands/includes/storeJob.lua +1 -1
  135. package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +2 -2
  136. package/dist/esm/commands/moveStalledJobsToWait-9.lua +18 -53
  137. package/dist/esm/commands/moveToFinished-14.lua +43 -51
  138. package/dist/esm/commands/moveToWaitingChildren-8.lua +5 -32
  139. package/dist/esm/commands/removeJob-3.lua +14 -72
  140. package/dist/esm/commands/removeUnprocessedChildren-2.lua +31 -0
  141. package/dist/esm/commands/updateJobScheduler-12.lua +29 -23
  142. package/dist/esm/interfaces/job-json.d.ts +5 -2
  143. package/dist/esm/interfaces/minimal-job.d.ts +29 -11
  144. package/dist/esm/interfaces/queue-options.d.ts +6 -2
  145. package/dist/esm/interfaces/repeat-options.d.ts +1 -1
  146. package/dist/esm/interfaces/sandboxed-job.d.ts +2 -2
  147. package/dist/esm/interfaces/sandboxed-options.d.ts +1 -1
  148. package/dist/esm/interfaces/telemetry.d.ts +18 -15
  149. package/dist/esm/interfaces/worker-options.d.ts +10 -10
  150. package/dist/esm/scripts/addDelayedJob-6.js +73 -55
  151. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  152. package/dist/esm/scripts/addJobScheduler-11.js +59 -40
  153. package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
  154. package/dist/esm/scripts/addParentJob-4.js +73 -55
  155. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  156. package/dist/esm/scripts/addPrioritizedJob-8.js +73 -55
  157. package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
  158. package/dist/esm/scripts/addRepeatableJob-2.js +14 -9
  159. package/dist/esm/scripts/addRepeatableJob-2.js.map +1 -1
  160. package/dist/esm/scripts/addStandardJob-8.js +73 -55
  161. package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
  162. package/dist/esm/scripts/cleanJobsInSet-3.js +14 -9
  163. package/dist/esm/scripts/cleanJobsInSet-3.js.map +1 -1
  164. package/dist/esm/scripts/drain-5.js +14 -9
  165. package/dist/esm/scripts/drain-5.js.map +1 -1
  166. package/dist/esm/scripts/index.d.ts +1 -0
  167. package/dist/esm/scripts/index.js +1 -0
  168. package/dist/esm/scripts/index.js.map +1 -1
  169. package/dist/esm/scripts/moveStalledJobsToWait-9.js +157 -155
  170. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  171. package/dist/esm/scripts/moveToFinished-14.js +183 -157
  172. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  173. package/dist/esm/scripts/moveToWaitingChildren-8.js +145 -137
  174. package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
  175. package/dist/esm/scripts/obliterate-2.js +14 -9
  176. package/dist/esm/scripts/obliterate-2.js.map +1 -1
  177. package/dist/esm/scripts/removeChildDependency-1.js +5 -5
  178. package/dist/esm/scripts/removeJob-3.js +97 -63
  179. package/dist/esm/scripts/removeJob-3.js.map +1 -1
  180. package/dist/esm/scripts/removeUnprocessedChildren-2.d.ts +5 -0
  181. package/dist/esm/scripts/removeUnprocessedChildren-2.js +335 -0
  182. package/dist/esm/scripts/removeUnprocessedChildren-2.js.map +1 -0
  183. package/dist/esm/scripts/updateJobScheduler-12.js +27 -21
  184. package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
  185. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  186. package/dist/esm/types/index.d.ts +1 -0
  187. package/dist/esm/types/index.js +1 -0
  188. package/dist/esm/types/index.js.map +1 -1
  189. package/dist/esm/types/job-options.d.ts +13 -2
  190. package/dist/esm/types/job-progress.d.ts +1 -0
  191. package/dist/esm/types/job-progress.js +2 -0
  192. package/dist/esm/types/job-progress.js.map +1 -0
  193. package/dist/esm/utils.d.ts +17 -0
  194. package/dist/esm/utils.js +11 -0
  195. package/dist/esm/utils.js.map +1 -1
  196. package/dist/esm/version.d.ts +1 -1
  197. package/dist/esm/version.js +1 -1
  198. package/package.json +26 -21
  199. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
  200. package/dist/cjs/commands/includes/removeDeduplicationKey.lua +0 -11
  201. package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
  202. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
  203. package/dist/esm/commands/includes/removeDeduplicationKey.lua +0 -11
  204. package/dist/esm/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
@@ -37,7 +37,16 @@ local jobId = ARGV[4]
37
37
  ]]
38
38
  -- Includes
39
39
  --[[
40
- Validate and move parent to active if needed.
40
+ Validate and move parent to a wait status (waiting, delayed or prioritized)
41
+ if no pending dependencies.
42
+ ]]
43
+ -- Includes
44
+ --[[
45
+ Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
46
+ ]]
47
+ -- Includes
48
+ --[[
49
+ Move parent to a wait status (wait, prioritized or delayed)
41
50
  ]]
42
51
  -- Includes
43
52
  --[[
@@ -133,58 +142,55 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
133
142
  end
134
143
  return waitKey, false
135
144
  end
136
- local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
137
- parentKey, parentId, timestamp)
138
- local isParentActive = rcall("ZSCORE",
139
- parentQueueKey .. ":waiting-children", parentId)
140
- if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
141
- rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
142
- local parentWaitKey = parentQueueKey .. ":wait"
143
- local parentPausedKey = parentQueueKey .. ":paused"
144
- local parentActiveKey = parentQueueKey .. ":active"
145
- local parentMetaKey = parentQueueKey .. ":meta"
146
- local parentMarkerKey = parentQueueKey .. ":marker"
147
- local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
148
- local priority = tonumber(jobAttributes[1]) or 0
149
- local delay = tonumber(jobAttributes[2]) or 0
150
- if delay > 0 then
151
- local delayedTimestamp = tonumber(timestamp) + delay
152
- local score = delayedTimestamp * 0x1000
153
- local parentDelayedKey = parentQueueKey .. ":delayed"
154
- rcall("ZADD", parentDelayedKey, score, parentId)
155
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
156
- "jobId", parentId, "delay", delayedTimestamp)
157
- addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
145
+ local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
146
+ local parentWaitKey = parentQueueKey .. ":wait"
147
+ local parentPausedKey = parentQueueKey .. ":paused"
148
+ local parentActiveKey = parentQueueKey .. ":active"
149
+ local parentMetaKey = parentQueueKey .. ":meta"
150
+ local parentMarkerKey = parentQueueKey .. ":marker"
151
+ local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
152
+ local priority = tonumber(jobAttributes[1]) or 0
153
+ local delay = tonumber(jobAttributes[2]) or 0
154
+ -- ignore dependencies if any left
155
+ rcall("HSET", parentKey, "igdp", 1)
156
+ if delay > 0 then
157
+ local delayedTimestamp = tonumber(timestamp) + delay
158
+ local score = delayedTimestamp * 0x1000
159
+ local parentDelayedKey = parentQueueKey .. ":delayed"
160
+ rcall("ZADD", parentDelayedKey, score, parentId)
161
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
162
+ delayedTimestamp)
163
+ addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
164
+ else
165
+ if priority == 0 then
166
+ local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
167
+ parentWaitKey, parentPausedKey)
168
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
158
169
  else
159
- if priority == 0 then
160
- local parentTarget, isParentPausedOrMaxed =
161
- getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
162
- parentPausedKey)
163
- addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
164
- parentId)
165
- else
166
- local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
167
- addJobWithPriority(parentMarkerKey,
168
- parentQueueKey .. ":prioritized", priority,
169
- parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
170
- end
171
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
172
- "jobId", parentId, "prev", "waiting-children")
170
+ local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
171
+ addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
172
+ parentQueueKey .. ":pc", isPausedOrMaxed)
173
173
  end
174
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
175
+ "waiting-children")
174
176
  end
175
177
  end
176
- --[[
177
- Function to remove deduplication key if needed.
178
- ]]
179
- local function removeDeduplicationKeyIfNeeded(prefixKey, deduplicationId)
180
- if deduplicationId then
181
- local deduplicationKey = prefixKey .. "de:" .. deduplicationId
182
- local pttl = rcall("PTTL", deduplicationKey)
183
- if pttl == 0 or pttl == -1 then
184
- rcall("DEL", deduplicationKey)
178
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
179
+ if rcall("EXISTS", parentKey) == 1 then
180
+ local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
181
+ if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
182
+ rcall("ZREM", parentWaitingChildrenKey, parentId)
183
+ moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
185
184
  end
186
185
  end
187
186
  end
187
+ local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
188
+ parentId, timestamp)
189
+ local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
190
+ if doNotHavePendingDependencies then
191
+ moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
192
+ end
193
+ end
188
194
  --[[
189
195
  Functions to remove jobs when removeOnFail option is provided.
190
196
  ]]
@@ -194,13 +200,18 @@ end
194
200
  ]]
195
201
  -- Includes
196
202
  --[[
197
- Function to remove deduplication key.
203
+ Function to remove deduplication key if needed
204
+ when a job is being removed.
198
205
  ]]
199
- local function removeDeduplicationKey(prefixKey, jobKey)
206
+ local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
207
+ jobKey, jobId)
200
208
  local deduplicationId = rcall("HGET", jobKey, "deid")
201
209
  if deduplicationId then
202
210
  local deduplicationKey = prefixKey .. "de:" .. deduplicationId
203
- rcall("DEL", deduplicationKey)
211
+ local currentJobId = rcall('GET', deduplicationKey)
212
+ if currentJobId and currentJobId == jobId then
213
+ return rcall("DEL", deduplicationKey)
214
+ end
204
215
  end
205
216
  end
206
217
  --[[
@@ -226,7 +237,7 @@ end
226
237
  local getJobKeyPrefix = function (jobKey, jobId)
227
238
  return string.sub(jobKey, 0, #jobKey - #jobId)
228
239
  end
229
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
240
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
230
241
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
231
242
  parentPrefix .. "wait", parentPrefix .. "paused")
232
243
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -254,10 +265,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
254
265
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
255
266
  end
256
267
  else
257
- moveParentToWait(parentPrefix, parentId)
268
+ _moveParentToWait(parentPrefix, parentId)
258
269
  end
259
270
  else
260
- moveParentToWait(parentPrefix, parentId, true)
271
+ _moveParentToWait(parentPrefix, parentId, true)
261
272
  end
262
273
  end
263
274
  end
@@ -285,10 +296,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
285
296
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
286
297
  end
287
298
  else
288
- moveParentToWait(parentPrefix, parentId)
299
+ _moveParentToWait(parentPrefix, parentId)
289
300
  end
290
301
  else
291
- moveParentToWait(parentPrefix, parentId, true)
302
+ _moveParentToWait(parentPrefix, parentId, true)
292
303
  end
293
304
  end
294
305
  end
@@ -302,7 +313,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
302
313
  local jobKey = baseKey .. jobId
303
314
  removeParentDependencyKey(jobKey, hard, nil, baseKey)
304
315
  if shouldRemoveDeduplicationKey then
305
- removeDeduplicationKey(baseKey, jobKey)
316
+ removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
306
317
  end
307
318
  removeJobKeys(jobKey)
308
319
  end
@@ -355,67 +366,90 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
355
366
  end
356
367
  end
357
368
  end
358
- local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
359
- local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
360
- local parentDelayedKey = parentQueueKey .. ":delayed"
361
- local parentPrioritizedKey = parentQueueKey .. ":prioritized"
362
- local parentWaitingChildrenOrDelayedOrPrioritizedKey
363
- local prevState
364
- if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
365
- parentWaitingChildrenOrDelayedOrPrioritizedKey = parentWaitingChildrenKey
366
- prevState = "waiting-children"
367
- elseif rcall("ZSCORE", parentDelayedKey, parentId) then
368
- parentWaitingChildrenOrDelayedOrPrioritizedKey = parentDelayedKey
369
- prevState = "delayed"
370
- elseif rcall("ZSCORE", parentPrioritizedKey, parentId) then
371
- parentWaitingChildrenOrDelayedOrPrioritizedKey = parentPrioritizedKey
372
- prevState = "prioritized"
369
+ local moveParentToFailedIfNeeded
370
+ local moveChildFromDependenciesIfNeeded
371
+ moveParentToFailedIfNeeded = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
372
+ if rcall("EXISTS", parentKey) == 1 then
373
+ local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
374
+ local parentDelayedKey = parentQueueKey .. ":delayed"
375
+ local parentPrioritizedKey = parentQueueKey .. ":prioritized"
376
+ local parentWaitingChildrenOrDelayedKey
377
+ local prevState
378
+ if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
379
+ parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
380
+ prevState = "waiting-children"
381
+ elseif rcall("ZSCORE", parentDelayedKey, parentId) then
382
+ parentWaitingChildrenOrDelayedKey = parentDelayedKey
383
+ prevState = "delayed"
384
+ rcall("HSET", parentKey, "delay", 0)
385
+ end
386
+ if parentWaitingChildrenOrDelayedKey then
387
+ rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
388
+ local parentQueuePrefix = parentQueueKey .. ":"
389
+ local parentFailedKey = parentQueueKey .. ":failed"
390
+ local deferredFailure = "child " .. jobIdKey .. " failed"
391
+ rcall("HSET", parentKey, "defa", deferredFailure)
392
+ moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
393
+ else
394
+ if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then
395
+ local deferredFailure = "child " .. jobIdKey .. " failed"
396
+ rcall("HSET", parentKey, "defa", deferredFailure)
397
+ end
398
+ end
373
399
  end
374
- if parentWaitingChildrenOrDelayedOrPrioritizedKey then
375
- rcall("ZREM", parentWaitingChildrenOrDelayedOrPrioritizedKey, parentId)
376
- local parentQueuePrefix = parentQueueKey .. ":"
377
- local parentFailedKey = parentQueueKey .. ":failed"
378
- rcall("ZADD", parentFailedKey, timestamp, parentId)
379
- local failedReason = "child " .. jobIdKey .. " failed"
380
- rcall("HSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
381
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
382
- failedReason, "prev", prevState)
383
- local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
384
- removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
385
- if jobAttributes[1] then
386
- local parentData = cjson.decode(jobAttributes[1])
387
- local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
388
- local grandParentUnsuccesssful = grandParentKey .. ":unsuccessful"
389
- rcall("ZADD", grandParentUnsuccesssful, timestamp, parentKey)
390
- if parentData['fpof'] then
391
- moveParentFromWaitingChildrenToFailed(
400
+ end
401
+ moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp)
402
+ if rawParentData then
403
+ local parentData = cjson.decode(rawParentData)
404
+ local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
405
+ local parentDependenciesChildrenKey = parentKey .. ":dependencies"
406
+ if parentData['fpof'] then
407
+ if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
408
+ local parentUnsuccesssfulChildrenKey = parentKey .. ":unsuccessful"
409
+ rcall("ZADD", parentUnsuccesssfulChildrenKey, timestamp, childKey)
410
+ moveParentToFailedIfNeeded(
392
411
  parentData['queueKey'],
393
- parentData['queueKey'] .. ':' .. parentData['id'],
394
- parentData['id'],
395
412
  parentKey,
413
+ parentData['id'],
414
+ childKey,
396
415
  timestamp
397
416
  )
398
- elseif parentData['idof'] or parentData['rdof'] then
399
- local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
400
- local grandParentDependenciesSet = grandParentKey .. ":dependencies"
401
- if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
402
- moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
403
- grandParentKey, parentData['id'], timestamp)
404
- if parentData['idof'] then
405
- local grandParentFailedSet = grandParentKey .. ":failed"
406
- rcall("HSET", grandParentFailedSet, parentKey, failedReason)
407
- end
417
+ end
418
+ elseif parentData['cpof'] then
419
+ if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
420
+ local parentFailedChildrenKey = parentKey .. ":failed"
421
+ rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
422
+ moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp)
423
+ end
424
+ elseif parentData['idof'] or parentData['rdof'] then
425
+ if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
426
+ moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey,
427
+ parentKey, parentData['id'], timestamp)
428
+ if parentData['idof'] then
429
+ local parentFailedChildrenKey = parentKey .. ":failed"
430
+ rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
408
431
  end
409
432
  end
410
433
  end
411
- local parentRawOpts = jobAttributes[3]
412
- local parentOpts = cjson.decode(parentRawOpts)
413
- removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
414
- else
415
- local grandParentKey = rcall("HGET", parentKey, "parentKey")
416
- if grandParentKey then
417
- local grandParentUnsuccesssfulSet = grandParentKey .. ":unsuccessful"
418
- rcall("ZADD", grandParentUnsuccesssfulSet, timestamp, parentKey)
434
+ end
435
+ end
436
+ --[[
437
+ Function to remove deduplication key if needed
438
+ when a job is moved to completed or failed states.
439
+ ]]
440
+ local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
441
+ deduplicationId, jobId)
442
+ if deduplicationId then
443
+ local deduplicationKey = prefixKey .. "de:" .. deduplicationId
444
+ local pttl = rcall("PTTL", deduplicationKey)
445
+ if pttl == 0 then
446
+ return rcall("DEL", deduplicationKey)
447
+ end
448
+ if pttl == -1 then
449
+ local currentJobId = rcall('GET', deduplicationKey)
450
+ if currentJobId and currentJobId == jobId then
451
+ return rcall("DEL", deduplicationKey)
452
+ end
419
453
  end
420
454
  end
421
455
  end
@@ -438,7 +472,7 @@ local function removeLock(jobKey, stalledKey, token, jobId)
438
472
  end
439
473
  return 0
440
474
  end
441
- local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId,
475
+ local function moveToWaitingChildren(activeKey, waitingChildrenKey, jobId,
442
476
  timestamp)
443
477
  local score = tonumber(timestamp)
444
478
  local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
@@ -452,7 +486,7 @@ if rcall("EXISTS", jobKey) == 1 then
452
486
  if rcall("ZCARD", jobUnsuccessfulKey) ~= 0 then
453
487
  -- TODO: refactor this logic in an include later
454
488
  local jobAttributes = rcall("HMGET", jobKey, "parent", "deid", "opts")
455
- removeDeduplicationKeyIfNeeded(ARGV[5], jobAttributes[2])
489
+ removeDeduplicationKeyIfNeededOnFinalization(ARGV[5], jobAttributes[2], jobId)
456
490
  local failedReason = "children are failed"
457
491
  rcall("ZADD", failedKey, timestamp, jobId)
458
492
  rcall("HSET", jobKey, "finishedOn", timestamp)
@@ -461,33 +495,7 @@ if rcall("EXISTS", jobKey) == 1 then
461
495
  local rawParentData = jobAttributes[1]
462
496
  local rawOpts = jobAttributes[3]
463
497
  local opts = cjson.decode(rawOpts)
464
- if rawParentData ~= false then
465
- if opts['fpof'] then
466
- local parentData = cjson.decode(rawParentData)
467
- local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
468
- local parentUnsuccesssful = parentKey .. ":unsuccessful"
469
- rcall("ZADD", parentUnsuccesssful, timestamp, jobKey)
470
- moveParentFromWaitingChildrenToFailed(
471
- parentData['queueKey'],
472
- parentData['queueKey'] .. ':' .. parentData['id'],
473
- parentData['id'],
474
- jobKey,
475
- timestamp
476
- )
477
- elseif opts['idof'] or opts['rdof'] then
478
- local parentData = cjson.decode(rawParentData)
479
- local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
480
- local dependenciesSet = parentKey .. ":dependencies"
481
- if rcall("SREM", dependenciesSet, jobKey) == 1 then
482
- moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
483
- parentKey, parentData['id'], timestamp)
484
- if opts['idof'] then
485
- local failedSet = parentKey .. ":failed"
486
- rcall("HSET", failedSet, jobKey, failedReason)
487
- end
488
- end
489
- end
490
- end
498
+ moveChildFromDependenciesIfNeeded(rawParentData, jobKey, failedReason, timestamp)
491
499
  removeJobsOnFail(ARGV[5], failedKey, jobId, opts, timestamp)
492
500
  return 0
493
501
  else
@@ -1 +1 @@
1
- {"version":3,"file":"moveToWaitingChildren-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToWaitingChildren-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmgBf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveToWaitingChildren-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToWaitingChildren-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2gBf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -25,13 +25,18 @@ local rcall = redis.call
25
25
  ]]
26
26
  -- Includes
27
27
  --[[
28
- Function to remove deduplication key.
28
+ Function to remove deduplication key if needed
29
+ when a job is being removed.
29
30
  ]]
30
- local function removeDeduplicationKey(prefixKey, jobKey)
31
+ local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
32
+ jobKey, jobId)
31
33
  local deduplicationId = rcall("HGET", jobKey, "deid")
32
34
  if deduplicationId then
33
35
  local deduplicationKey = prefixKey .. "de:" .. deduplicationId
34
- rcall("DEL", deduplicationKey)
36
+ local currentJobId = rcall('GET', deduplicationKey)
37
+ if currentJobId and currentJobId == jobId then
38
+ return rcall("DEL", deduplicationKey)
39
+ end
35
40
  end
36
41
  end
37
42
  --[[
@@ -93,7 +98,7 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
93
98
  end
94
99
  return waitKey, false
95
100
  end
96
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
101
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
97
102
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
98
103
  parentPrefix .. "wait", parentPrefix .. "paused")
99
104
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -121,10 +126,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
121
126
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
122
127
  end
123
128
  else
124
- moveParentToWait(parentPrefix, parentId)
129
+ _moveParentToWait(parentPrefix, parentId)
125
130
  end
126
131
  else
127
- moveParentToWait(parentPrefix, parentId, true)
132
+ _moveParentToWait(parentPrefix, parentId, true)
128
133
  end
129
134
  end
130
135
  end
@@ -152,10 +157,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
152
157
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
153
158
  end
154
159
  else
155
- moveParentToWait(parentPrefix, parentId)
160
+ _moveParentToWait(parentPrefix, parentId)
156
161
  end
157
162
  else
158
- moveParentToWait(parentPrefix, parentId, true)
163
+ _moveParentToWait(parentPrefix, parentId, true)
159
164
  end
160
165
  end
161
166
  end
@@ -169,7 +174,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
169
174
  local jobKey = baseKey .. jobId
170
175
  removeParentDependencyKey(jobKey, hard, nil, baseKey)
171
176
  if shouldRemoveDeduplicationKey then
172
- removeDeduplicationKey(baseKey, jobKey)
177
+ removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
173
178
  end
174
179
  removeJobKeys(jobKey)
175
180
  end
@@ -1 +1 @@
1
- {"version":3,"file":"obliterate-2.js","sourceRoot":"","sources":["../../../src/scripts/obliterate-2.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqUf,CAAC;AACF,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"obliterate-2.js","sourceRoot":"","sources":["../../../src/scripts/obliterate-2.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Uf,CAAC;AACF,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -74,7 +74,7 @@ local function removeJobKeys(jobKey)
74
74
  return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
75
75
  jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
76
76
  end
77
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
77
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
78
78
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
79
79
  parentPrefix .. "wait", parentPrefix .. "paused")
80
80
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -102,10 +102,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
102
102
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
103
103
  end
104
104
  else
105
- moveParentToWait(parentPrefix, parentId)
105
+ _moveParentToWait(parentPrefix, parentId)
106
106
  end
107
107
  else
108
- moveParentToWait(parentPrefix, parentId, true)
108
+ _moveParentToWait(parentPrefix, parentId, true)
109
109
  end
110
110
  end
111
111
  end
@@ -133,10 +133,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
133
133
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
134
134
  end
135
135
  else
136
- moveParentToWait(parentPrefix, parentId)
136
+ _moveParentToWait(parentPrefix, parentId)
137
137
  end
138
138
  else
139
- moveParentToWait(parentPrefix, parentId, true)
139
+ _moveParentToWait(parentPrefix, parentId, true)
140
140
  end
141
141
  end
142
142
  end