bullmq 5.44.1 → 5.52.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/dist/cjs/classes/async-fifo-queue.js.map +1 -1
  2. package/dist/cjs/classes/child-processor.js.map +1 -1
  3. package/dist/cjs/classes/child.js.map +1 -1
  4. package/dist/cjs/classes/flow-producer.js +15 -4
  5. package/dist/cjs/classes/flow-producer.js.map +1 -1
  6. package/dist/cjs/classes/job-scheduler.js +1 -1
  7. package/dist/cjs/classes/job-scheduler.js.map +1 -1
  8. package/dist/cjs/classes/job.js +137 -54
  9. package/dist/cjs/classes/job.js.map +1 -1
  10. package/dist/cjs/classes/queue-base.js +1 -4
  11. package/dist/cjs/classes/queue-base.js.map +1 -1
  12. package/dist/cjs/classes/queue-events-producer.js.map +1 -1
  13. package/dist/cjs/classes/queue-events.js.map +1 -1
  14. package/dist/cjs/classes/queue-getters.js +12 -15
  15. package/dist/cjs/classes/queue-getters.js.map +1 -1
  16. package/dist/cjs/classes/queue-keys.js.map +1 -1
  17. package/dist/cjs/classes/queue.js +12 -13
  18. package/dist/cjs/classes/queue.js.map +1 -1
  19. package/dist/cjs/classes/redis-connection.js +1 -1
  20. package/dist/cjs/classes/redis-connection.js.map +1 -1
  21. package/dist/cjs/classes/repeat.js +1 -1
  22. package/dist/cjs/classes/repeat.js.map +1 -1
  23. package/dist/cjs/classes/scripts.js +18 -7
  24. package/dist/cjs/classes/scripts.js.map +1 -1
  25. package/dist/cjs/classes/worker.js +4 -0
  26. package/dist/cjs/classes/worker.js.map +1 -1
  27. package/dist/cjs/commands/addJobScheduler-11.lua +52 -33
  28. package/dist/cjs/commands/includes/deduplicateJob.lua +16 -17
  29. package/dist/cjs/commands/includes/isLocked.lua +1 -0
  30. package/dist/cjs/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
  31. package/dist/cjs/commands/includes/moveParentToWait.lua +48 -0
  32. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
  33. package/dist/cjs/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
  34. package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
  35. package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
  36. package/dist/cjs/commands/includes/removeJob.lua +2 -2
  37. package/dist/cjs/commands/includes/removeJobWithChildren.lua +94 -0
  38. package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
  39. package/dist/cjs/commands/includes/storeJob.lua +1 -1
  40. package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +2 -2
  41. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +18 -53
  42. package/dist/cjs/commands/moveToFinished-14.lua +43 -51
  43. package/dist/cjs/commands/moveToWaitingChildren-8.lua +5 -32
  44. package/dist/cjs/commands/removeJob-3.lua +14 -72
  45. package/dist/cjs/commands/removeUnprocessedChildren-2.lua +31 -0
  46. package/dist/cjs/commands/updateJobScheduler-12.lua +29 -23
  47. package/dist/cjs/scripts/addDelayedJob-6.js +73 -55
  48. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  49. package/dist/cjs/scripts/addJobScheduler-11.js +59 -40
  50. package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
  51. package/dist/cjs/scripts/addParentJob-4.js +73 -55
  52. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  53. package/dist/cjs/scripts/addPrioritizedJob-8.js +73 -55
  54. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
  55. package/dist/cjs/scripts/addRepeatableJob-2.js +14 -9
  56. package/dist/cjs/scripts/addRepeatableJob-2.js.map +1 -1
  57. package/dist/cjs/scripts/addStandardJob-8.js +73 -55
  58. package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
  59. package/dist/cjs/scripts/cleanJobsInSet-3.js +14 -9
  60. package/dist/cjs/scripts/cleanJobsInSet-3.js.map +1 -1
  61. package/dist/cjs/scripts/drain-5.js +14 -9
  62. package/dist/cjs/scripts/drain-5.js.map +1 -1
  63. package/dist/cjs/scripts/index.js +1 -0
  64. package/dist/cjs/scripts/index.js.map +1 -1
  65. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +157 -155
  66. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  67. package/dist/cjs/scripts/moveToFinished-14.js +183 -157
  68. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  69. package/dist/cjs/scripts/moveToWaitingChildren-8.js +145 -137
  70. package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
  71. package/dist/cjs/scripts/obliterate-2.js +14 -9
  72. package/dist/cjs/scripts/obliterate-2.js.map +1 -1
  73. package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
  74. package/dist/cjs/scripts/removeJob-3.js +97 -63
  75. package/dist/cjs/scripts/removeJob-3.js.map +1 -1
  76. package/dist/cjs/scripts/removeUnprocessedChildren-2.js +338 -0
  77. package/dist/cjs/scripts/removeUnprocessedChildren-2.js.map +1 -0
  78. package/dist/cjs/scripts/updateJobScheduler-12.js +27 -21
  79. package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
  80. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  81. package/dist/cjs/types/index.js +1 -0
  82. package/dist/cjs/types/index.js.map +1 -1
  83. package/dist/cjs/types/job-progress.js +3 -0
  84. package/dist/cjs/types/job-progress.js.map +1 -0
  85. package/dist/cjs/utils.js +12 -1
  86. package/dist/cjs/utils.js.map +1 -1
  87. package/dist/cjs/version.js +1 -1
  88. package/dist/esm/classes/async-fifo-queue.js.map +1 -1
  89. package/dist/esm/classes/child-processor.js.map +1 -1
  90. package/dist/esm/classes/child.js.map +1 -1
  91. package/dist/esm/classes/flow-producer.js +15 -4
  92. package/dist/esm/classes/flow-producer.js.map +1 -1
  93. package/dist/esm/classes/job-scheduler.js +1 -1
  94. package/dist/esm/classes/job-scheduler.js.map +1 -1
  95. package/dist/esm/classes/job.d.ts +49 -14
  96. package/dist/esm/classes/job.js +138 -55
  97. package/dist/esm/classes/job.js.map +1 -1
  98. package/dist/esm/classes/queue-base.d.ts +1 -4
  99. package/dist/esm/classes/queue-base.js +1 -4
  100. package/dist/esm/classes/queue-base.js.map +1 -1
  101. package/dist/esm/classes/queue-events-producer.d.ts +2 -2
  102. package/dist/esm/classes/queue-events-producer.js.map +1 -1
  103. package/dist/esm/classes/queue-events.d.ts +118 -29
  104. package/dist/esm/classes/queue-events.js.map +1 -1
  105. package/dist/esm/classes/queue-getters.d.ts +11 -14
  106. package/dist/esm/classes/queue-getters.js +12 -15
  107. package/dist/esm/classes/queue-getters.js.map +1 -1
  108. package/dist/esm/classes/queue-keys.js.map +1 -1
  109. package/dist/esm/classes/queue.d.ts +14 -15
  110. package/dist/esm/classes/queue.js +12 -13
  111. package/dist/esm/classes/queue.js.map +1 -1
  112. package/dist/esm/classes/redis-connection.js +1 -1
  113. package/dist/esm/classes/redis-connection.js.map +1 -1
  114. package/dist/esm/classes/repeat.js +1 -1
  115. package/dist/esm/classes/repeat.js.map +1 -1
  116. package/dist/esm/classes/scripts.d.ts +4 -3
  117. package/dist/esm/classes/scripts.js +18 -7
  118. package/dist/esm/classes/scripts.js.map +1 -1
  119. package/dist/esm/classes/worker.d.ts +2 -1
  120. package/dist/esm/classes/worker.js +5 -1
  121. package/dist/esm/classes/worker.js.map +1 -1
  122. package/dist/esm/commands/addJobScheduler-11.lua +52 -33
  123. package/dist/esm/commands/includes/deduplicateJob.lua +16 -17
  124. package/dist/esm/commands/includes/isLocked.lua +1 -0
  125. package/dist/esm/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
  126. package/dist/esm/commands/includes/moveParentToWait.lua +48 -0
  127. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
  128. package/dist/esm/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
  129. package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
  130. package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
  131. package/dist/esm/commands/includes/removeJob.lua +2 -2
  132. package/dist/esm/commands/includes/removeJobWithChildren.lua +94 -0
  133. package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
  134. package/dist/esm/commands/includes/storeJob.lua +1 -1
  135. package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +2 -2
  136. package/dist/esm/commands/moveStalledJobsToWait-9.lua +18 -53
  137. package/dist/esm/commands/moveToFinished-14.lua +43 -51
  138. package/dist/esm/commands/moveToWaitingChildren-8.lua +5 -32
  139. package/dist/esm/commands/removeJob-3.lua +14 -72
  140. package/dist/esm/commands/removeUnprocessedChildren-2.lua +31 -0
  141. package/dist/esm/commands/updateJobScheduler-12.lua +29 -23
  142. package/dist/esm/interfaces/job-json.d.ts +5 -2
  143. package/dist/esm/interfaces/minimal-job.d.ts +29 -11
  144. package/dist/esm/interfaces/queue-options.d.ts +6 -2
  145. package/dist/esm/interfaces/repeat-options.d.ts +1 -1
  146. package/dist/esm/interfaces/sandboxed-job.d.ts +2 -2
  147. package/dist/esm/interfaces/sandboxed-options.d.ts +1 -1
  148. package/dist/esm/interfaces/telemetry.d.ts +18 -15
  149. package/dist/esm/interfaces/worker-options.d.ts +10 -10
  150. package/dist/esm/scripts/addDelayedJob-6.js +73 -55
  151. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  152. package/dist/esm/scripts/addJobScheduler-11.js +59 -40
  153. package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
  154. package/dist/esm/scripts/addParentJob-4.js +73 -55
  155. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  156. package/dist/esm/scripts/addPrioritizedJob-8.js +73 -55
  157. package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
  158. package/dist/esm/scripts/addRepeatableJob-2.js +14 -9
  159. package/dist/esm/scripts/addRepeatableJob-2.js.map +1 -1
  160. package/dist/esm/scripts/addStandardJob-8.js +73 -55
  161. package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
  162. package/dist/esm/scripts/cleanJobsInSet-3.js +14 -9
  163. package/dist/esm/scripts/cleanJobsInSet-3.js.map +1 -1
  164. package/dist/esm/scripts/drain-5.js +14 -9
  165. package/dist/esm/scripts/drain-5.js.map +1 -1
  166. package/dist/esm/scripts/index.d.ts +1 -0
  167. package/dist/esm/scripts/index.js +1 -0
  168. package/dist/esm/scripts/index.js.map +1 -1
  169. package/dist/esm/scripts/moveStalledJobsToWait-9.js +157 -155
  170. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  171. package/dist/esm/scripts/moveToFinished-14.js +183 -157
  172. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  173. package/dist/esm/scripts/moveToWaitingChildren-8.js +145 -137
  174. package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
  175. package/dist/esm/scripts/obliterate-2.js +14 -9
  176. package/dist/esm/scripts/obliterate-2.js.map +1 -1
  177. package/dist/esm/scripts/removeChildDependency-1.js +5 -5
  178. package/dist/esm/scripts/removeJob-3.js +97 -63
  179. package/dist/esm/scripts/removeJob-3.js.map +1 -1
  180. package/dist/esm/scripts/removeUnprocessedChildren-2.d.ts +5 -0
  181. package/dist/esm/scripts/removeUnprocessedChildren-2.js +335 -0
  182. package/dist/esm/scripts/removeUnprocessedChildren-2.js.map +1 -0
  183. package/dist/esm/scripts/updateJobScheduler-12.js +27 -21
  184. package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
  185. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  186. package/dist/esm/types/index.d.ts +1 -0
  187. package/dist/esm/types/index.js +1 -0
  188. package/dist/esm/types/index.js.map +1 -1
  189. package/dist/esm/types/job-options.d.ts +13 -2
  190. package/dist/esm/types/job-progress.d.ts +1 -0
  191. package/dist/esm/types/job-progress.js +2 -0
  192. package/dist/esm/types/job-progress.js.map +1 -0
  193. package/dist/esm/utils.d.ts +17 -0
  194. package/dist/esm/utils.js +11 -0
  195. package/dist/esm/utils.js.map +1 -1
  196. package/dist/esm/version.d.ts +1 -1
  197. package/dist/esm/version.js +1 -1
  198. package/package.json +26 -21
  199. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
  200. package/dist/cjs/commands/includes/removeDeduplicationKey.lua +0 -11
  201. package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
  202. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
  203. package/dist/esm/commands/includes/removeDeduplicationKey.lua +0 -11
  204. package/dist/esm/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
@@ -76,7 +76,16 @@ end
76
76
  ]]
77
77
  -- Includes
78
78
  --[[
79
- Validate and move parent to active if needed.
79
+ Validate and move parent to a wait status (waiting, delayed or prioritized)
80
+ if no pending dependencies.
81
+ ]]
82
+ -- Includes
83
+ --[[
84
+ Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
85
+ ]]
86
+ -- Includes
87
+ --[[
88
+ Move parent to a wait status (wait, prioritized or delayed)
80
89
  ]]
81
90
  -- Includes
82
91
  --[[
@@ -136,58 +145,55 @@ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
136
145
  end
137
146
  return false
138
147
  end
139
- local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
140
- parentKey, parentId, timestamp)
141
- local isParentActive = rcall("ZSCORE",
142
- parentQueueKey .. ":waiting-children", parentId)
143
- if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
144
- rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
145
- local parentWaitKey = parentQueueKey .. ":wait"
146
- local parentPausedKey = parentQueueKey .. ":paused"
147
- local parentActiveKey = parentQueueKey .. ":active"
148
- local parentMetaKey = parentQueueKey .. ":meta"
149
- local parentMarkerKey = parentQueueKey .. ":marker"
150
- local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
151
- local priority = tonumber(jobAttributes[1]) or 0
152
- local delay = tonumber(jobAttributes[2]) or 0
153
- if delay > 0 then
154
- local delayedTimestamp = tonumber(timestamp) + delay
155
- local score = delayedTimestamp * 0x1000
156
- local parentDelayedKey = parentQueueKey .. ":delayed"
157
- rcall("ZADD", parentDelayedKey, score, parentId)
158
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
159
- "jobId", parentId, "delay", delayedTimestamp)
160
- addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
148
+ local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
149
+ local parentWaitKey = parentQueueKey .. ":wait"
150
+ local parentPausedKey = parentQueueKey .. ":paused"
151
+ local parentActiveKey = parentQueueKey .. ":active"
152
+ local parentMetaKey = parentQueueKey .. ":meta"
153
+ local parentMarkerKey = parentQueueKey .. ":marker"
154
+ local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
155
+ local priority = tonumber(jobAttributes[1]) or 0
156
+ local delay = tonumber(jobAttributes[2]) or 0
157
+ -- ignore dependencies if any left
158
+ rcall("HSET", parentKey, "igdp", 1)
159
+ if delay > 0 then
160
+ local delayedTimestamp = tonumber(timestamp) + delay
161
+ local score = delayedTimestamp * 0x1000
162
+ local parentDelayedKey = parentQueueKey .. ":delayed"
163
+ rcall("ZADD", parentDelayedKey, score, parentId)
164
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
165
+ delayedTimestamp)
166
+ addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
167
+ else
168
+ if priority == 0 then
169
+ local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
170
+ parentWaitKey, parentPausedKey)
171
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
161
172
  else
162
- if priority == 0 then
163
- local parentTarget, isParentPausedOrMaxed =
164
- getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
165
- parentPausedKey)
166
- addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
167
- parentId)
168
- else
169
- local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
170
- addJobWithPriority(parentMarkerKey,
171
- parentQueueKey .. ":prioritized", priority,
172
- parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
173
- end
174
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
175
- "jobId", parentId, "prev", "waiting-children")
173
+ local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
174
+ addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
175
+ parentQueueKey .. ":pc", isPausedOrMaxed)
176
176
  end
177
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
178
+ "waiting-children")
177
179
  end
178
180
  end
179
- --[[
180
- Function to remove deduplication key if needed.
181
- ]]
182
- local function removeDeduplicationKeyIfNeeded(prefixKey, deduplicationId)
183
- if deduplicationId then
184
- local deduplicationKey = prefixKey .. "de:" .. deduplicationId
185
- local pttl = rcall("PTTL", deduplicationKey)
186
- if pttl == 0 or pttl == -1 then
187
- rcall("DEL", deduplicationKey)
181
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
182
+ if rcall("EXISTS", parentKey) == 1 then
183
+ local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
184
+ if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
185
+ rcall("ZREM", parentWaitingChildrenKey, parentId)
186
+ moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
188
187
  end
189
188
  end
190
189
  end
190
+ local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
191
+ parentId, timestamp)
192
+ local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
193
+ if doNotHavePendingDependencies then
194
+ moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
195
+ end
196
+ end
191
197
  --[[
192
198
  Functions to remove jobs when removeOnFail option is provided.
193
199
  ]]
@@ -197,13 +203,18 @@ end
197
203
  ]]
198
204
  -- Includes
199
205
  --[[
200
- Function to remove deduplication key.
206
+ Function to remove deduplication key if needed
207
+ when a job is being removed.
201
208
  ]]
202
- local function removeDeduplicationKey(prefixKey, jobKey)
209
+ local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
210
+ jobKey, jobId)
203
211
  local deduplicationId = rcall("HGET", jobKey, "deid")
204
212
  if deduplicationId then
205
213
  local deduplicationKey = prefixKey .. "de:" .. deduplicationId
206
- rcall("DEL", deduplicationKey)
214
+ local currentJobId = rcall('GET', deduplicationKey)
215
+ if currentJobId and currentJobId == jobId then
216
+ return rcall("DEL", deduplicationKey)
217
+ end
207
218
  end
208
219
  end
209
220
  --[[
@@ -229,7 +240,7 @@ end
229
240
  local getJobKeyPrefix = function (jobKey, jobId)
230
241
  return string.sub(jobKey, 0, #jobKey - #jobId)
231
242
  end
232
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
243
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
233
244
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
234
245
  parentPrefix .. "wait", parentPrefix .. "paused")
235
246
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -257,10 +268,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
257
268
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
258
269
  end
259
270
  else
260
- moveParentToWait(parentPrefix, parentId)
271
+ _moveParentToWait(parentPrefix, parentId)
261
272
  end
262
273
  else
263
- moveParentToWait(parentPrefix, parentId, true)
274
+ _moveParentToWait(parentPrefix, parentId, true)
264
275
  end
265
276
  end
266
277
  end
@@ -288,10 +299,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
288
299
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
289
300
  end
290
301
  else
291
- moveParentToWait(parentPrefix, parentId)
302
+ _moveParentToWait(parentPrefix, parentId)
292
303
  end
293
304
  else
294
- moveParentToWait(parentPrefix, parentId, true)
305
+ _moveParentToWait(parentPrefix, parentId, true)
295
306
  end
296
307
  end
297
308
  end
@@ -305,7 +316,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
305
316
  local jobKey = baseKey .. jobId
306
317
  removeParentDependencyKey(jobKey, hard, nil, baseKey)
307
318
  if shouldRemoveDeduplicationKey then
308
- removeDeduplicationKey(baseKey, jobKey)
319
+ removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
309
320
  end
310
321
  removeJobKeys(jobKey)
311
322
  end
@@ -358,67 +369,90 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
358
369
  end
359
370
  end
360
371
  end
361
- local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
362
- local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
363
- local parentDelayedKey = parentQueueKey .. ":delayed"
364
- local parentPrioritizedKey = parentQueueKey .. ":prioritized"
365
- local parentWaitingChildrenOrDelayedOrPrioritizedKey
366
- local prevState
367
- if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
368
- parentWaitingChildrenOrDelayedOrPrioritizedKey = parentWaitingChildrenKey
369
- prevState = "waiting-children"
370
- elseif rcall("ZSCORE", parentDelayedKey, parentId) then
371
- parentWaitingChildrenOrDelayedOrPrioritizedKey = parentDelayedKey
372
- prevState = "delayed"
373
- elseif rcall("ZSCORE", parentPrioritizedKey, parentId) then
374
- parentWaitingChildrenOrDelayedOrPrioritizedKey = parentPrioritizedKey
375
- prevState = "prioritized"
372
+ local moveParentToFailedIfNeeded
373
+ local moveChildFromDependenciesIfNeeded
374
+ moveParentToFailedIfNeeded = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
375
+ if rcall("EXISTS", parentKey) == 1 then
376
+ local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
377
+ local parentDelayedKey = parentQueueKey .. ":delayed"
378
+ local parentPrioritizedKey = parentQueueKey .. ":prioritized"
379
+ local parentWaitingChildrenOrDelayedKey
380
+ local prevState
381
+ if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
382
+ parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
383
+ prevState = "waiting-children"
384
+ elseif rcall("ZSCORE", parentDelayedKey, parentId) then
385
+ parentWaitingChildrenOrDelayedKey = parentDelayedKey
386
+ prevState = "delayed"
387
+ rcall("HSET", parentKey, "delay", 0)
388
+ end
389
+ if parentWaitingChildrenOrDelayedKey then
390
+ rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
391
+ local parentQueuePrefix = parentQueueKey .. ":"
392
+ local parentFailedKey = parentQueueKey .. ":failed"
393
+ local deferredFailure = "child " .. jobIdKey .. " failed"
394
+ rcall("HSET", parentKey, "defa", deferredFailure)
395
+ moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
396
+ else
397
+ if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then
398
+ local deferredFailure = "child " .. jobIdKey .. " failed"
399
+ rcall("HSET", parentKey, "defa", deferredFailure)
400
+ end
401
+ end
376
402
  end
377
- if parentWaitingChildrenOrDelayedOrPrioritizedKey then
378
- rcall("ZREM", parentWaitingChildrenOrDelayedOrPrioritizedKey, parentId)
379
- local parentQueuePrefix = parentQueueKey .. ":"
380
- local parentFailedKey = parentQueueKey .. ":failed"
381
- rcall("ZADD", parentFailedKey, timestamp, parentId)
382
- local failedReason = "child " .. jobIdKey .. " failed"
383
- rcall("HSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
384
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
385
- failedReason, "prev", prevState)
386
- local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
387
- removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
388
- if jobAttributes[1] then
389
- local parentData = cjson.decode(jobAttributes[1])
390
- local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
391
- local grandParentUnsuccesssful = grandParentKey .. ":unsuccessful"
392
- rcall("ZADD", grandParentUnsuccesssful, timestamp, parentKey)
393
- if parentData['fpof'] then
394
- moveParentFromWaitingChildrenToFailed(
403
+ end
404
+ moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp)
405
+ if rawParentData then
406
+ local parentData = cjson.decode(rawParentData)
407
+ local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
408
+ local parentDependenciesChildrenKey = parentKey .. ":dependencies"
409
+ if parentData['fpof'] then
410
+ if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
411
+ local parentUnsuccesssfulChildrenKey = parentKey .. ":unsuccessful"
412
+ rcall("ZADD", parentUnsuccesssfulChildrenKey, timestamp, childKey)
413
+ moveParentToFailedIfNeeded(
395
414
  parentData['queueKey'],
396
- parentData['queueKey'] .. ':' .. parentData['id'],
397
- parentData['id'],
398
415
  parentKey,
416
+ parentData['id'],
417
+ childKey,
399
418
  timestamp
400
419
  )
401
- elseif parentData['idof'] or parentData['rdof'] then
402
- local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
403
- local grandParentDependenciesSet = grandParentKey .. ":dependencies"
404
- if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
405
- moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
406
- grandParentKey, parentData['id'], timestamp)
407
- if parentData['idof'] then
408
- local grandParentFailedSet = grandParentKey .. ":failed"
409
- rcall("HSET", grandParentFailedSet, parentKey, failedReason)
410
- end
420
+ end
421
+ elseif parentData['cpof'] then
422
+ if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
423
+ local parentFailedChildrenKey = parentKey .. ":failed"
424
+ rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
425
+ moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp)
426
+ end
427
+ elseif parentData['idof'] or parentData['rdof'] then
428
+ if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
429
+ moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey,
430
+ parentKey, parentData['id'], timestamp)
431
+ if parentData['idof'] then
432
+ local parentFailedChildrenKey = parentKey .. ":failed"
433
+ rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
411
434
  end
412
435
  end
413
436
  end
414
- local parentRawOpts = jobAttributes[3]
415
- local parentOpts = cjson.decode(parentRawOpts)
416
- removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
417
- else
418
- local grandParentKey = rcall("HGET", parentKey, "parentKey")
419
- if grandParentKey then
420
- local grandParentUnsuccesssfulSet = grandParentKey .. ":unsuccessful"
421
- rcall("ZADD", grandParentUnsuccesssfulSet, timestamp, parentKey)
437
+ end
438
+ end
439
+ --[[
440
+ Function to remove deduplication key if needed
441
+ when a job is moved to completed or failed states.
442
+ ]]
443
+ local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
444
+ deduplicationId, jobId)
445
+ if deduplicationId then
446
+ local deduplicationKey = prefixKey .. "de:" .. deduplicationId
447
+ local pttl = rcall("PTTL", deduplicationKey)
448
+ if pttl == 0 then
449
+ return rcall("DEL", deduplicationKey)
450
+ end
451
+ if pttl == -1 then
452
+ local currentJobId = rcall('GET', deduplicationKey)
453
+ if currentJobId and currentJobId == jobId then
454
+ return rcall("DEL", deduplicationKey)
455
+ end
422
456
  end
423
457
  end
424
458
  end
@@ -458,7 +492,9 @@ local maxStalledJobCount = tonumber(ARGV[1])
458
492
  local queueKeyPrefix = ARGV[2]
459
493
  local timestamp = ARGV[3]
460
494
  local maxCheckTime = ARGV[4]
461
- if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
495
+ if rcall("EXISTS", stalledCheckKey) == 1 then
496
+ return {{}, {}}
497
+ end
462
498
  rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
463
499
  -- Trim events before emiting them to avoid trimming events emitted in this script
464
500
  trimEvents(metaKey, eventStreamKey)
@@ -482,63 +518,28 @@ if (#stalling > 0) then
482
518
  local removed = rcall("LREM", activeKey, 1, jobId)
483
519
  if (removed > 0) then
484
520
  -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
485
- local stalledCount =
486
- rcall("HINCRBY", jobKey, "stalledCounter", 1)
521
+ local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
487
522
  if (stalledCount > maxStalledJobCount) then
488
523
  local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
489
524
  local rawOpts = jobAttributes[1]
490
525
  local rawParentData = jobAttributes[2]
491
526
  local opts = cjson.decode(rawOpts)
492
527
  rcall("ZADD", failedKey, timestamp, jobId)
493
- removeDeduplicationKeyIfNeeded(queueKeyPrefix, jobAttributes[3])
494
- local failedReason =
495
- "job stalled more than allowable limit"
496
- rcall("HMSET", jobKey, "failedReason", failedReason,
497
- "finishedOn", timestamp)
498
- rcall("XADD", eventStreamKey, "*", "event",
499
- "failed", "jobId", jobId, 'prev', 'active',
500
- 'failedReason', failedReason)
501
- if rawParentData then
502
- if opts['fpof'] then
503
- local parentData = cjson.decode(rawParentData)
504
- -- TODO: need to remove this job from dependencies set in next breaking change
505
- -- no for now as it would imply a breaking change
506
- local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
507
- local unsuccesssfulSet = parentKey .. ":unsuccessful"
508
- rcall("ZADD", unsuccesssfulSet, timestamp, jobKey)
509
- moveParentFromWaitingChildrenToFailed(
510
- parentData['queueKey'],
511
- parentData['queueKey'] .. ':' .. parentData['id'],
512
- parentData['id'],
513
- jobKey,
514
- timestamp
515
- )
516
- elseif opts['idof'] or opts['rdof'] then
517
- local parentData = cjson.decode(rawParentData)
518
- local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
519
- local dependenciesSet = parentKey .. ":dependencies"
520
- if rcall("SREM", dependenciesSet, jobKey) == 1 then
521
- moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
522
- parentKey, parentData['id'], timestamp)
523
- if opts['idof'] then
524
- local failedSet = parentKey .. ":failed"
525
- rcall("HSET", failedSet, jobKey, failedReason)
526
- end
527
- end
528
- end
529
- end
528
+ removeDeduplicationKeyIfNeededOnFinalization(queueKeyPrefix, jobAttributes[3], jobId)
529
+ local failedReason = "job stalled more than allowable limit"
530
+ rcall("HMSET", jobKey, "failedReason", failedReason, "finishedOn", timestamp)
531
+ rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId", jobId, 'prev', 'active',
532
+ 'failedReason', failedReason)
533
+ moveChildFromDependenciesIfNeeded(rawParentData, jobKey, failedReason, timestamp)
530
534
  removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
531
535
  table.insert(failed, jobId)
532
536
  else
533
- local target, isPausedOrMaxed =
534
- getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
537
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
535
538
  -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
536
539
  addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
537
- rcall("XADD", eventStreamKey, "*", "event",
538
- "waiting", "jobId", jobId, 'prev', 'active')
540
+ rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, 'prev', 'active')
539
541
  -- Emit the stalled event
540
- rcall("XADD", eventStreamKey, "*", "event",
541
- "stalled", "jobId", jobId)
542
+ rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId", jobId)
542
543
  table.insert(stalled, jobId)
543
544
  end
544
545
  end
@@ -553,7 +554,8 @@ if (#active > 0) then
553
554
  rcall('SADD', stalledKey, unpack(active, from, to))
554
555
  end
555
556
  end
556
- return {failed, stalled}`;
557
+ return {failed, stalled}
558
+ `;
557
559
  export const moveStalledJobsToWait = {
558
560
  name: 'moveStalledJobsToWait',
559
561
  content,
@@ -1 +1 @@
1
- {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBA2iBS,CAAC;AAC1B,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6iBf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}