bullmq 5.44.1 → 5.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/dist/cjs/classes/async-fifo-queue.js.map +1 -1
  2. package/dist/cjs/classes/child-processor.js.map +1 -1
  3. package/dist/cjs/classes/child.js.map +1 -1
  4. package/dist/cjs/classes/flow-producer.js +15 -4
  5. package/dist/cjs/classes/flow-producer.js.map +1 -1
  6. package/dist/cjs/classes/job-scheduler.js +1 -1
  7. package/dist/cjs/classes/job-scheduler.js.map +1 -1
  8. package/dist/cjs/classes/job.js +137 -54
  9. package/dist/cjs/classes/job.js.map +1 -1
  10. package/dist/cjs/classes/queue-base.js +1 -4
  11. package/dist/cjs/classes/queue-base.js.map +1 -1
  12. package/dist/cjs/classes/queue-events-producer.js.map +1 -1
  13. package/dist/cjs/classes/queue-events.js.map +1 -1
  14. package/dist/cjs/classes/queue-getters.js +12 -15
  15. package/dist/cjs/classes/queue-getters.js.map +1 -1
  16. package/dist/cjs/classes/queue-keys.js.map +1 -1
  17. package/dist/cjs/classes/queue.js +12 -13
  18. package/dist/cjs/classes/queue.js.map +1 -1
  19. package/dist/cjs/classes/redis-connection.js +1 -1
  20. package/dist/cjs/classes/redis-connection.js.map +1 -1
  21. package/dist/cjs/classes/repeat.js +1 -1
  22. package/dist/cjs/classes/repeat.js.map +1 -1
  23. package/dist/cjs/classes/scripts.js +18 -7
  24. package/dist/cjs/classes/scripts.js.map +1 -1
  25. package/dist/cjs/classes/worker.js +4 -0
  26. package/dist/cjs/classes/worker.js.map +1 -1
  27. package/dist/cjs/commands/addJobScheduler-11.lua +52 -33
  28. package/dist/cjs/commands/includes/deduplicateJob.lua +16 -17
  29. package/dist/cjs/commands/includes/isLocked.lua +1 -0
  30. package/dist/cjs/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
  31. package/dist/cjs/commands/includes/moveParentToWait.lua +48 -0
  32. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
  33. package/dist/cjs/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
  34. package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
  35. package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
  36. package/dist/cjs/commands/includes/removeJob.lua +2 -2
  37. package/dist/cjs/commands/includes/removeJobWithChildren.lua +94 -0
  38. package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
  39. package/dist/cjs/commands/includes/storeJob.lua +1 -1
  40. package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +2 -2
  41. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +18 -53
  42. package/dist/cjs/commands/moveToFinished-14.lua +43 -51
  43. package/dist/cjs/commands/moveToWaitingChildren-8.lua +5 -32
  44. package/dist/cjs/commands/removeJob-3.lua +14 -72
  45. package/dist/cjs/commands/removeUnprocessedChildren-2.lua +31 -0
  46. package/dist/cjs/commands/updateJobScheduler-12.lua +29 -23
  47. package/dist/cjs/scripts/addDelayedJob-6.js +73 -55
  48. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  49. package/dist/cjs/scripts/addJobScheduler-11.js +59 -40
  50. package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
  51. package/dist/cjs/scripts/addParentJob-4.js +73 -55
  52. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  53. package/dist/cjs/scripts/addPrioritizedJob-8.js +73 -55
  54. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
  55. package/dist/cjs/scripts/addRepeatableJob-2.js +14 -9
  56. package/dist/cjs/scripts/addRepeatableJob-2.js.map +1 -1
  57. package/dist/cjs/scripts/addStandardJob-8.js +73 -55
  58. package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
  59. package/dist/cjs/scripts/cleanJobsInSet-3.js +14 -9
  60. package/dist/cjs/scripts/cleanJobsInSet-3.js.map +1 -1
  61. package/dist/cjs/scripts/drain-5.js +14 -9
  62. package/dist/cjs/scripts/drain-5.js.map +1 -1
  63. package/dist/cjs/scripts/index.js +1 -0
  64. package/dist/cjs/scripts/index.js.map +1 -1
  65. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +157 -155
  66. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  67. package/dist/cjs/scripts/moveToFinished-14.js +183 -157
  68. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  69. package/dist/cjs/scripts/moveToWaitingChildren-8.js +145 -137
  70. package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
  71. package/dist/cjs/scripts/obliterate-2.js +14 -9
  72. package/dist/cjs/scripts/obliterate-2.js.map +1 -1
  73. package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
  74. package/dist/cjs/scripts/removeJob-3.js +97 -63
  75. package/dist/cjs/scripts/removeJob-3.js.map +1 -1
  76. package/dist/cjs/scripts/removeUnprocessedChildren-2.js +338 -0
  77. package/dist/cjs/scripts/removeUnprocessedChildren-2.js.map +1 -0
  78. package/dist/cjs/scripts/updateJobScheduler-12.js +27 -21
  79. package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
  80. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  81. package/dist/cjs/types/index.js +1 -0
  82. package/dist/cjs/types/index.js.map +1 -1
  83. package/dist/cjs/types/job-progress.js +3 -0
  84. package/dist/cjs/types/job-progress.js.map +1 -0
  85. package/dist/cjs/utils.js +12 -1
  86. package/dist/cjs/utils.js.map +1 -1
  87. package/dist/cjs/version.js +1 -1
  88. package/dist/esm/classes/async-fifo-queue.js.map +1 -1
  89. package/dist/esm/classes/child-processor.js.map +1 -1
  90. package/dist/esm/classes/child.js.map +1 -1
  91. package/dist/esm/classes/flow-producer.js +15 -4
  92. package/dist/esm/classes/flow-producer.js.map +1 -1
  93. package/dist/esm/classes/job-scheduler.js +1 -1
  94. package/dist/esm/classes/job-scheduler.js.map +1 -1
  95. package/dist/esm/classes/job.d.ts +49 -14
  96. package/dist/esm/classes/job.js +138 -55
  97. package/dist/esm/classes/job.js.map +1 -1
  98. package/dist/esm/classes/queue-base.d.ts +1 -4
  99. package/dist/esm/classes/queue-base.js +1 -4
  100. package/dist/esm/classes/queue-base.js.map +1 -1
  101. package/dist/esm/classes/queue-events-producer.d.ts +2 -2
  102. package/dist/esm/classes/queue-events-producer.js.map +1 -1
  103. package/dist/esm/classes/queue-events.d.ts +118 -29
  104. package/dist/esm/classes/queue-events.js.map +1 -1
  105. package/dist/esm/classes/queue-getters.d.ts +11 -14
  106. package/dist/esm/classes/queue-getters.js +12 -15
  107. package/dist/esm/classes/queue-getters.js.map +1 -1
  108. package/dist/esm/classes/queue-keys.js.map +1 -1
  109. package/dist/esm/classes/queue.d.ts +14 -15
  110. package/dist/esm/classes/queue.js +12 -13
  111. package/dist/esm/classes/queue.js.map +1 -1
  112. package/dist/esm/classes/redis-connection.js +1 -1
  113. package/dist/esm/classes/redis-connection.js.map +1 -1
  114. package/dist/esm/classes/repeat.js +1 -1
  115. package/dist/esm/classes/repeat.js.map +1 -1
  116. package/dist/esm/classes/scripts.d.ts +4 -3
  117. package/dist/esm/classes/scripts.js +18 -7
  118. package/dist/esm/classes/scripts.js.map +1 -1
  119. package/dist/esm/classes/worker.d.ts +2 -1
  120. package/dist/esm/classes/worker.js +5 -1
  121. package/dist/esm/classes/worker.js.map +1 -1
  122. package/dist/esm/commands/addJobScheduler-11.lua +52 -33
  123. package/dist/esm/commands/includes/deduplicateJob.lua +16 -17
  124. package/dist/esm/commands/includes/isLocked.lua +1 -0
  125. package/dist/esm/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
  126. package/dist/esm/commands/includes/moveParentToWait.lua +48 -0
  127. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
  128. package/dist/esm/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
  129. package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
  130. package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
  131. package/dist/esm/commands/includes/removeJob.lua +2 -2
  132. package/dist/esm/commands/includes/removeJobWithChildren.lua +94 -0
  133. package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
  134. package/dist/esm/commands/includes/storeJob.lua +1 -1
  135. package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +2 -2
  136. package/dist/esm/commands/moveStalledJobsToWait-9.lua +18 -53
  137. package/dist/esm/commands/moveToFinished-14.lua +43 -51
  138. package/dist/esm/commands/moveToWaitingChildren-8.lua +5 -32
  139. package/dist/esm/commands/removeJob-3.lua +14 -72
  140. package/dist/esm/commands/removeUnprocessedChildren-2.lua +31 -0
  141. package/dist/esm/commands/updateJobScheduler-12.lua +29 -23
  142. package/dist/esm/interfaces/job-json.d.ts +5 -2
  143. package/dist/esm/interfaces/minimal-job.d.ts +29 -11
  144. package/dist/esm/interfaces/queue-options.d.ts +6 -2
  145. package/dist/esm/interfaces/repeat-options.d.ts +1 -1
  146. package/dist/esm/interfaces/sandboxed-job.d.ts +2 -2
  147. package/dist/esm/interfaces/sandboxed-options.d.ts +1 -1
  148. package/dist/esm/interfaces/telemetry.d.ts +18 -15
  149. package/dist/esm/interfaces/worker-options.d.ts +10 -10
  150. package/dist/esm/scripts/addDelayedJob-6.js +73 -55
  151. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  152. package/dist/esm/scripts/addJobScheduler-11.js +59 -40
  153. package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
  154. package/dist/esm/scripts/addParentJob-4.js +73 -55
  155. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  156. package/dist/esm/scripts/addPrioritizedJob-8.js +73 -55
  157. package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
  158. package/dist/esm/scripts/addRepeatableJob-2.js +14 -9
  159. package/dist/esm/scripts/addRepeatableJob-2.js.map +1 -1
  160. package/dist/esm/scripts/addStandardJob-8.js +73 -55
  161. package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
  162. package/dist/esm/scripts/cleanJobsInSet-3.js +14 -9
  163. package/dist/esm/scripts/cleanJobsInSet-3.js.map +1 -1
  164. package/dist/esm/scripts/drain-5.js +14 -9
  165. package/dist/esm/scripts/drain-5.js.map +1 -1
  166. package/dist/esm/scripts/index.d.ts +1 -0
  167. package/dist/esm/scripts/index.js +1 -0
  168. package/dist/esm/scripts/index.js.map +1 -1
  169. package/dist/esm/scripts/moveStalledJobsToWait-9.js +157 -155
  170. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  171. package/dist/esm/scripts/moveToFinished-14.js +183 -157
  172. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  173. package/dist/esm/scripts/moveToWaitingChildren-8.js +145 -137
  174. package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
  175. package/dist/esm/scripts/obliterate-2.js +14 -9
  176. package/dist/esm/scripts/obliterate-2.js.map +1 -1
  177. package/dist/esm/scripts/removeChildDependency-1.js +5 -5
  178. package/dist/esm/scripts/removeJob-3.js +97 -63
  179. package/dist/esm/scripts/removeJob-3.js.map +1 -1
  180. package/dist/esm/scripts/removeUnprocessedChildren-2.d.ts +5 -0
  181. package/dist/esm/scripts/removeUnprocessedChildren-2.js +335 -0
  182. package/dist/esm/scripts/removeUnprocessedChildren-2.js.map +1 -0
  183. package/dist/esm/scripts/updateJobScheduler-12.js +27 -21
  184. package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
  185. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  186. package/dist/esm/types/index.d.ts +1 -0
  187. package/dist/esm/types/index.js +1 -0
  188. package/dist/esm/types/index.js.map +1 -1
  189. package/dist/esm/types/job-options.d.ts +13 -2
  190. package/dist/esm/types/job-progress.d.ts +1 -0
  191. package/dist/esm/types/job-progress.js +2 -0
  192. package/dist/esm/types/job-progress.js.map +1 -0
  193. package/dist/esm/utils.d.ts +17 -0
  194. package/dist/esm/utils.js +11 -0
  195. package/dist/esm/utils.js.map +1 -1
  196. package/dist/esm/version.d.ts +1 -1
  197. package/dist/esm/version.js +1 -1
  198. package/package.json +26 -21
  199. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
  200. package/dist/cjs/commands/includes/removeDeduplicationKey.lua +0 -11
  201. package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
  202. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
  203. package/dist/esm/commands/includes/removeDeduplicationKey.lua +0 -11
  204. package/dist/esm/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
@@ -40,7 +40,16 @@ local jobId = ARGV[4]
40
40
  ]]
41
41
  -- Includes
42
42
  --[[
43
- Validate and move parent to active if needed.
43
+ Validate and move parent to a wait status (waiting, delayed or prioritized)
44
+ if no pending dependencies.
45
+ ]]
46
+ -- Includes
47
+ --[[
48
+ Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
49
+ ]]
50
+ -- Includes
51
+ --[[
52
+ Move parent to a wait status (wait, prioritized or delayed)
44
53
  ]]
45
54
  -- Includes
46
55
  --[[
@@ -136,58 +145,55 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
136
145
  end
137
146
  return waitKey, false
138
147
  end
139
- local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
140
- parentKey, parentId, timestamp)
141
- local isParentActive = rcall("ZSCORE",
142
- parentQueueKey .. ":waiting-children", parentId)
143
- if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
144
- rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
145
- local parentWaitKey = parentQueueKey .. ":wait"
146
- local parentPausedKey = parentQueueKey .. ":paused"
147
- local parentActiveKey = parentQueueKey .. ":active"
148
- local parentMetaKey = parentQueueKey .. ":meta"
149
- local parentMarkerKey = parentQueueKey .. ":marker"
150
- local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
151
- local priority = tonumber(jobAttributes[1]) or 0
152
- local delay = tonumber(jobAttributes[2]) or 0
153
- if delay > 0 then
154
- local delayedTimestamp = tonumber(timestamp) + delay
155
- local score = delayedTimestamp * 0x1000
156
- local parentDelayedKey = parentQueueKey .. ":delayed"
157
- rcall("ZADD", parentDelayedKey, score, parentId)
158
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
159
- "jobId", parentId, "delay", delayedTimestamp)
160
- addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
148
+ local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
149
+ local parentWaitKey = parentQueueKey .. ":wait"
150
+ local parentPausedKey = parentQueueKey .. ":paused"
151
+ local parentActiveKey = parentQueueKey .. ":active"
152
+ local parentMetaKey = parentQueueKey .. ":meta"
153
+ local parentMarkerKey = parentQueueKey .. ":marker"
154
+ local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
155
+ local priority = tonumber(jobAttributes[1]) or 0
156
+ local delay = tonumber(jobAttributes[2]) or 0
157
+ -- ignore dependencies if any left
158
+ rcall("HSET", parentKey, "igdp", 1)
159
+ if delay > 0 then
160
+ local delayedTimestamp = tonumber(timestamp) + delay
161
+ local score = delayedTimestamp * 0x1000
162
+ local parentDelayedKey = parentQueueKey .. ":delayed"
163
+ rcall("ZADD", parentDelayedKey, score, parentId)
164
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
165
+ delayedTimestamp)
166
+ addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
167
+ else
168
+ if priority == 0 then
169
+ local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
170
+ parentWaitKey, parentPausedKey)
171
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
161
172
  else
162
- if priority == 0 then
163
- local parentTarget, isParentPausedOrMaxed =
164
- getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
165
- parentPausedKey)
166
- addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
167
- parentId)
168
- else
169
- local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
170
- addJobWithPriority(parentMarkerKey,
171
- parentQueueKey .. ":prioritized", priority,
172
- parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
173
- end
174
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
175
- "jobId", parentId, "prev", "waiting-children")
173
+ local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
174
+ addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
175
+ parentQueueKey .. ":pc", isPausedOrMaxed)
176
176
  end
177
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
178
+ "waiting-children")
177
179
  end
178
180
  end
179
- --[[
180
- Function to remove deduplication key if needed.
181
- ]]
182
- local function removeDeduplicationKeyIfNeeded(prefixKey, deduplicationId)
183
- if deduplicationId then
184
- local deduplicationKey = prefixKey .. "de:" .. deduplicationId
185
- local pttl = rcall("PTTL", deduplicationKey)
186
- if pttl == 0 or pttl == -1 then
187
- rcall("DEL", deduplicationKey)
181
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
182
+ if rcall("EXISTS", parentKey) == 1 then
183
+ local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
184
+ if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
185
+ rcall("ZREM", parentWaitingChildrenKey, parentId)
186
+ moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
188
187
  end
189
188
  end
190
189
  end
190
+ local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
191
+ parentId, timestamp)
192
+ local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
193
+ if doNotHavePendingDependencies then
194
+ moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
195
+ end
196
+ end
191
197
  --[[
192
198
  Functions to remove jobs when removeOnFail option is provided.
193
199
  ]]
@@ -197,13 +203,18 @@ end
197
203
  ]]
198
204
  -- Includes
199
205
  --[[
200
- Function to remove deduplication key.
206
+ Function to remove deduplication key if needed
207
+ when a job is being removed.
201
208
  ]]
202
- local function removeDeduplicationKey(prefixKey, jobKey)
209
+ local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
210
+ jobKey, jobId)
203
211
  local deduplicationId = rcall("HGET", jobKey, "deid")
204
212
  if deduplicationId then
205
213
  local deduplicationKey = prefixKey .. "de:" .. deduplicationId
206
- rcall("DEL", deduplicationKey)
214
+ local currentJobId = rcall('GET', deduplicationKey)
215
+ if currentJobId and currentJobId == jobId then
216
+ return rcall("DEL", deduplicationKey)
217
+ end
207
218
  end
208
219
  end
209
220
  --[[
@@ -229,7 +240,7 @@ end
229
240
  local getJobKeyPrefix = function (jobKey, jobId)
230
241
  return string.sub(jobKey, 0, #jobKey - #jobId)
231
242
  end
232
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
243
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
233
244
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
234
245
  parentPrefix .. "wait", parentPrefix .. "paused")
235
246
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -257,10 +268,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
257
268
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
258
269
  end
259
270
  else
260
- moveParentToWait(parentPrefix, parentId)
271
+ _moveParentToWait(parentPrefix, parentId)
261
272
  end
262
273
  else
263
- moveParentToWait(parentPrefix, parentId, true)
274
+ _moveParentToWait(parentPrefix, parentId, true)
264
275
  end
265
276
  end
266
277
  end
@@ -288,10 +299,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
288
299
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
289
300
  end
290
301
  else
291
- moveParentToWait(parentPrefix, parentId)
302
+ _moveParentToWait(parentPrefix, parentId)
292
303
  end
293
304
  else
294
- moveParentToWait(parentPrefix, parentId, true)
305
+ _moveParentToWait(parentPrefix, parentId, true)
295
306
  end
296
307
  end
297
308
  end
@@ -305,7 +316,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
305
316
  local jobKey = baseKey .. jobId
306
317
  removeParentDependencyKey(jobKey, hard, nil, baseKey)
307
318
  if shouldRemoveDeduplicationKey then
308
- removeDeduplicationKey(baseKey, jobKey)
319
+ removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
309
320
  end
310
321
  removeJobKeys(jobKey)
311
322
  end
@@ -358,67 +369,90 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
358
369
  end
359
370
  end
360
371
  end
361
- local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
362
- local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
363
- local parentDelayedKey = parentQueueKey .. ":delayed"
364
- local parentPrioritizedKey = parentQueueKey .. ":prioritized"
365
- local parentWaitingChildrenOrDelayedOrPrioritizedKey
366
- local prevState
367
- if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
368
- parentWaitingChildrenOrDelayedOrPrioritizedKey = parentWaitingChildrenKey
369
- prevState = "waiting-children"
370
- elseif rcall("ZSCORE", parentDelayedKey, parentId) then
371
- parentWaitingChildrenOrDelayedOrPrioritizedKey = parentDelayedKey
372
- prevState = "delayed"
373
- elseif rcall("ZSCORE", parentPrioritizedKey, parentId) then
374
- parentWaitingChildrenOrDelayedOrPrioritizedKey = parentPrioritizedKey
375
- prevState = "prioritized"
372
+ local moveParentToFailedIfNeeded
373
+ local moveChildFromDependenciesIfNeeded
374
+ moveParentToFailedIfNeeded = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
375
+ if rcall("EXISTS", parentKey) == 1 then
376
+ local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
377
+ local parentDelayedKey = parentQueueKey .. ":delayed"
378
+ local parentPrioritizedKey = parentQueueKey .. ":prioritized"
379
+ local parentWaitingChildrenOrDelayedKey
380
+ local prevState
381
+ if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
382
+ parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
383
+ prevState = "waiting-children"
384
+ elseif rcall("ZSCORE", parentDelayedKey, parentId) then
385
+ parentWaitingChildrenOrDelayedKey = parentDelayedKey
386
+ prevState = "delayed"
387
+ rcall("HSET", parentKey, "delay", 0)
388
+ end
389
+ if parentWaitingChildrenOrDelayedKey then
390
+ rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
391
+ local parentQueuePrefix = parentQueueKey .. ":"
392
+ local parentFailedKey = parentQueueKey .. ":failed"
393
+ local deferredFailure = "child " .. jobIdKey .. " failed"
394
+ rcall("HSET", parentKey, "defa", deferredFailure)
395
+ moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
396
+ else
397
+ if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then
398
+ local deferredFailure = "child " .. jobIdKey .. " failed"
399
+ rcall("HSET", parentKey, "defa", deferredFailure)
400
+ end
401
+ end
376
402
  end
377
- if parentWaitingChildrenOrDelayedOrPrioritizedKey then
378
- rcall("ZREM", parentWaitingChildrenOrDelayedOrPrioritizedKey, parentId)
379
- local parentQueuePrefix = parentQueueKey .. ":"
380
- local parentFailedKey = parentQueueKey .. ":failed"
381
- rcall("ZADD", parentFailedKey, timestamp, parentId)
382
- local failedReason = "child " .. jobIdKey .. " failed"
383
- rcall("HSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
384
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
385
- failedReason, "prev", prevState)
386
- local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
387
- removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
388
- if jobAttributes[1] then
389
- local parentData = cjson.decode(jobAttributes[1])
390
- local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
391
- local grandParentUnsuccesssful = grandParentKey .. ":unsuccessful"
392
- rcall("ZADD", grandParentUnsuccesssful, timestamp, parentKey)
393
- if parentData['fpof'] then
394
- moveParentFromWaitingChildrenToFailed(
403
+ end
404
+ moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp)
405
+ if rawParentData then
406
+ local parentData = cjson.decode(rawParentData)
407
+ local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
408
+ local parentDependenciesChildrenKey = parentKey .. ":dependencies"
409
+ if parentData['fpof'] then
410
+ if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
411
+ local parentUnsuccesssfulChildrenKey = parentKey .. ":unsuccessful"
412
+ rcall("ZADD", parentUnsuccesssfulChildrenKey, timestamp, childKey)
413
+ moveParentToFailedIfNeeded(
395
414
  parentData['queueKey'],
396
- parentData['queueKey'] .. ':' .. parentData['id'],
397
- parentData['id'],
398
415
  parentKey,
416
+ parentData['id'],
417
+ childKey,
399
418
  timestamp
400
419
  )
401
- elseif parentData['idof'] or parentData['rdof'] then
402
- local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
403
- local grandParentDependenciesSet = grandParentKey .. ":dependencies"
404
- if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
405
- moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
406
- grandParentKey, parentData['id'], timestamp)
407
- if parentData['idof'] then
408
- local grandParentFailedSet = grandParentKey .. ":failed"
409
- rcall("HSET", grandParentFailedSet, parentKey, failedReason)
410
- end
420
+ end
421
+ elseif parentData['cpof'] then
422
+ if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
423
+ local parentFailedChildrenKey = parentKey .. ":failed"
424
+ rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
425
+ moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp)
426
+ end
427
+ elseif parentData['idof'] or parentData['rdof'] then
428
+ if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
429
+ moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey,
430
+ parentKey, parentData['id'], timestamp)
431
+ if parentData['idof'] then
432
+ local parentFailedChildrenKey = parentKey .. ":failed"
433
+ rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
411
434
  end
412
435
  end
413
436
  end
414
- local parentRawOpts = jobAttributes[3]
415
- local parentOpts = cjson.decode(parentRawOpts)
416
- removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
417
- else
418
- local grandParentKey = rcall("HGET", parentKey, "parentKey")
419
- if grandParentKey then
420
- local grandParentUnsuccesssfulSet = grandParentKey .. ":unsuccessful"
421
- rcall("ZADD", grandParentUnsuccesssfulSet, timestamp, parentKey)
437
+ end
438
+ end
439
+ --[[
440
+ Function to remove deduplication key if needed
441
+ when a job is moved to completed or failed states.
442
+ ]]
443
+ local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
444
+ deduplicationId, jobId)
445
+ if deduplicationId then
446
+ local deduplicationKey = prefixKey .. "de:" .. deduplicationId
447
+ local pttl = rcall("PTTL", deduplicationKey)
448
+ if pttl == 0 then
449
+ return rcall("DEL", deduplicationKey)
450
+ end
451
+ if pttl == -1 then
452
+ local currentJobId = rcall('GET', deduplicationKey)
453
+ if currentJobId and currentJobId == jobId then
454
+ return rcall("DEL", deduplicationKey)
455
+ end
422
456
  end
423
457
  end
424
458
  end
@@ -441,7 +475,7 @@ local function removeLock(jobKey, stalledKey, token, jobId)
441
475
  end
442
476
  return 0
443
477
  end
444
- local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId,
478
+ local function moveToWaitingChildren(activeKey, waitingChildrenKey, jobId,
445
479
  timestamp)
446
480
  local score = tonumber(timestamp)
447
481
  local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
@@ -455,7 +489,7 @@ if rcall("EXISTS", jobKey) == 1 then
455
489
  if rcall("ZCARD", jobUnsuccessfulKey) ~= 0 then
456
490
  -- TODO: refactor this logic in an include later
457
491
  local jobAttributes = rcall("HMGET", jobKey, "parent", "deid", "opts")
458
- removeDeduplicationKeyIfNeeded(ARGV[5], jobAttributes[2])
492
+ removeDeduplicationKeyIfNeededOnFinalization(ARGV[5], jobAttributes[2], jobId)
459
493
  local failedReason = "children are failed"
460
494
  rcall("ZADD", failedKey, timestamp, jobId)
461
495
  rcall("HSET", jobKey, "finishedOn", timestamp)
@@ -464,33 +498,7 @@ if rcall("EXISTS", jobKey) == 1 then
464
498
  local rawParentData = jobAttributes[1]
465
499
  local rawOpts = jobAttributes[3]
466
500
  local opts = cjson.decode(rawOpts)
467
- if rawParentData ~= false then
468
- if opts['fpof'] then
469
- local parentData = cjson.decode(rawParentData)
470
- local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
471
- local parentUnsuccesssful = parentKey .. ":unsuccessful"
472
- rcall("ZADD", parentUnsuccesssful, timestamp, jobKey)
473
- moveParentFromWaitingChildrenToFailed(
474
- parentData['queueKey'],
475
- parentData['queueKey'] .. ':' .. parentData['id'],
476
- parentData['id'],
477
- jobKey,
478
- timestamp
479
- )
480
- elseif opts['idof'] or opts['rdof'] then
481
- local parentData = cjson.decode(rawParentData)
482
- local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
483
- local dependenciesSet = parentKey .. ":dependencies"
484
- if rcall("SREM", dependenciesSet, jobKey) == 1 then
485
- moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
486
- parentKey, parentData['id'], timestamp)
487
- if opts['idof'] then
488
- local failedSet = parentKey .. ":failed"
489
- rcall("HSET", failedSet, jobKey, failedReason)
490
- end
491
- end
492
- end
493
- end
501
+ moveChildFromDependenciesIfNeeded(rawParentData, jobKey, failedReason, timestamp)
494
502
  removeJobsOnFail(ARGV[5], failedKey, jobId, opts, timestamp)
495
503
  return 0
496
504
  else
@@ -1 +1 @@
1
- {"version":3,"file":"moveToWaitingChildren-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToWaitingChildren-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmgBf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveToWaitingChildren-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToWaitingChildren-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2gBf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -28,13 +28,18 @@ local rcall = redis.call
28
28
  ]]
29
29
  -- Includes
30
30
  --[[
31
- Function to remove deduplication key.
31
+ Function to remove deduplication key if needed
32
+ when a job is being removed.
32
33
  ]]
33
- local function removeDeduplicationKey(prefixKey, jobKey)
34
+ local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
35
+ jobKey, jobId)
34
36
  local deduplicationId = rcall("HGET", jobKey, "deid")
35
37
  if deduplicationId then
36
38
  local deduplicationKey = prefixKey .. "de:" .. deduplicationId
37
- rcall("DEL", deduplicationKey)
39
+ local currentJobId = rcall('GET', deduplicationKey)
40
+ if currentJobId and currentJobId == jobId then
41
+ return rcall("DEL", deduplicationKey)
42
+ end
38
43
  end
39
44
  end
40
45
  --[[
@@ -96,7 +101,7 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
96
101
  end
97
102
  return waitKey, false
98
103
  end
99
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
104
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
100
105
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
101
106
  parentPrefix .. "wait", parentPrefix .. "paused")
102
107
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -124,10 +129,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
124
129
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
125
130
  end
126
131
  else
127
- moveParentToWait(parentPrefix, parentId)
132
+ _moveParentToWait(parentPrefix, parentId)
128
133
  end
129
134
  else
130
- moveParentToWait(parentPrefix, parentId, true)
135
+ _moveParentToWait(parentPrefix, parentId, true)
131
136
  end
132
137
  end
133
138
  end
@@ -155,10 +160,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
155
160
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
156
161
  end
157
162
  else
158
- moveParentToWait(parentPrefix, parentId)
163
+ _moveParentToWait(parentPrefix, parentId)
159
164
  end
160
165
  else
161
- moveParentToWait(parentPrefix, parentId, true)
166
+ _moveParentToWait(parentPrefix, parentId, true)
162
167
  end
163
168
  end
164
169
  end
@@ -172,7 +177,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
172
177
  local jobKey = baseKey .. jobId
173
178
  removeParentDependencyKey(jobKey, hard, nil, baseKey)
174
179
  if shouldRemoveDeduplicationKey then
175
- removeDeduplicationKey(baseKey, jobKey)
180
+ removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
176
181
  end
177
182
  removeJobKeys(jobKey)
178
183
  end
@@ -1 +1 @@
1
- {"version":3,"file":"obliterate-2.js","sourceRoot":"","sources":["../../../src/scripts/obliterate-2.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqUf,CAAC;AACW,QAAA,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"obliterate-2.js","sourceRoot":"","sources":["../../../src/scripts/obliterate-2.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Uf,CAAC;AACW,QAAA,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -77,7 +77,7 @@ local function removeJobKeys(jobKey)
77
77
  return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
78
78
  jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
79
79
  end
80
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
80
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
81
81
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
82
82
  parentPrefix .. "wait", parentPrefix .. "paused")
83
83
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -105,10 +105,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
105
105
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
106
106
  end
107
107
  else
108
- moveParentToWait(parentPrefix, parentId)
108
+ _moveParentToWait(parentPrefix, parentId)
109
109
  end
110
110
  else
111
- moveParentToWait(parentPrefix, parentId, true)
111
+ _moveParentToWait(parentPrefix, parentId, true)
112
112
  end
113
113
  end
114
114
  end
@@ -136,10 +136,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
136
136
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
137
137
  end
138
138
  else
139
- moveParentToWait(parentPrefix, parentId)
139
+ _moveParentToWait(parentPrefix, parentId)
140
140
  end
141
141
  else
142
- moveParentToWait(parentPrefix, parentId, true)
142
+ _moveParentToWait(parentPrefix, parentId, true)
143
143
  end
144
144
  end
145
145
  end