bullmq 5.46.1 → 5.47.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/dist/cjs/classes/flow-producer.js +1 -1
  2. package/dist/cjs/classes/flow-producer.js.map +1 -1
  3. package/dist/cjs/classes/job.js +11 -9
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/scripts.js +4 -3
  6. package/dist/cjs/classes/scripts.js.map +1 -1
  7. package/dist/cjs/classes/worker.js.map +1 -1
  8. package/dist/cjs/commands/includes/moveParentToFailedIfNeeded.lua +3 -0
  9. package/dist/cjs/commands/includes/moveParentToWait.lua +49 -0
  10. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +6 -51
  11. package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
  12. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +26 -30
  13. package/dist/cjs/commands/moveToFinished-14.lua +38 -30
  14. package/dist/cjs/commands/moveToWaitingChildren-8.lua +13 -8
  15. package/dist/cjs/scripts/addDelayedJob-6.js +24 -20
  16. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  17. package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
  18. package/dist/cjs/scripts/addParentJob-4.js +24 -20
  19. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  20. package/dist/cjs/scripts/addPrioritizedJob-8.js +24 -20
  21. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
  22. package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
  23. package/dist/cjs/scripts/addStandardJob-8.js +24 -20
  24. package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
  25. package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
  26. package/dist/cjs/scripts/drain-5.js +5 -5
  27. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +56 -54
  28. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  29. package/dist/cjs/scripts/moveToFinished-14.js +67 -55
  30. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  31. package/dist/cjs/scripts/moveToWaitingChildren-8.js +43 -33
  32. package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
  33. package/dist/cjs/scripts/obliterate-2.js +5 -5
  34. package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
  35. package/dist/cjs/scripts/removeJob-3.js +5 -5
  36. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  37. package/dist/cjs/version.js +1 -1
  38. package/dist/esm/classes/flow-producer.js +1 -1
  39. package/dist/esm/classes/flow-producer.js.map +1 -1
  40. package/dist/esm/classes/job.js +11 -9
  41. package/dist/esm/classes/job.js.map +1 -1
  42. package/dist/esm/classes/scripts.js +4 -3
  43. package/dist/esm/classes/scripts.js.map +1 -1
  44. package/dist/esm/classes/worker.js.map +1 -1
  45. package/dist/esm/commands/includes/moveParentToFailedIfNeeded.lua +3 -0
  46. package/dist/esm/commands/includes/moveParentToWait.lua +49 -0
  47. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +6 -51
  48. package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
  49. package/dist/esm/commands/moveStalledJobsToWait-9.lua +26 -30
  50. package/dist/esm/commands/moveToFinished-14.lua +38 -30
  51. package/dist/esm/commands/moveToWaitingChildren-8.lua +13 -8
  52. package/dist/esm/scripts/addDelayedJob-6.js +24 -20
  53. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  54. package/dist/esm/scripts/addJobScheduler-11.js +5 -5
  55. package/dist/esm/scripts/addParentJob-4.js +24 -20
  56. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  57. package/dist/esm/scripts/addPrioritizedJob-8.js +24 -20
  58. package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
  59. package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
  60. package/dist/esm/scripts/addStandardJob-8.js +24 -20
  61. package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
  62. package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
  63. package/dist/esm/scripts/drain-5.js +5 -5
  64. package/dist/esm/scripts/moveStalledJobsToWait-9.js +56 -54
  65. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  66. package/dist/esm/scripts/moveToFinished-14.js +67 -55
  67. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  68. package/dist/esm/scripts/moveToWaitingChildren-8.js +43 -33
  69. package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
  70. package/dist/esm/scripts/obliterate-2.js +5 -5
  71. package/dist/esm/scripts/removeChildDependency-1.js +5 -5
  72. package/dist/esm/scripts/removeJob-3.js +5 -5
  73. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  74. package/dist/esm/types/job-options.d.ts +12 -1
  75. package/dist/esm/version.d.ts +1 -1
  76. package/dist/esm/version.js +1 -1
  77. package/package.json +1 -1
@@ -156,9 +156,11 @@ end
156
156
  ]]
157
157
  -- Includes
158
158
  --[[
159
- Validate and move parent to active if needed.
159
+ Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
160
+ ]]
161
+ --[[
162
+ Validate and move parent to a wait status (wait, prioritized or delayed)
160
163
  ]]
161
- -- Includes
162
164
  --[[
163
165
  Add delay marker if needed.
164
166
  ]]
@@ -227,12 +229,9 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
227
229
  end
228
230
  return waitKey, false
229
231
  end
230
- local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
231
- parentKey, parentId, timestamp)
232
- local isParentActive = rcall("ZSCORE",
233
- parentQueueKey .. ":waiting-children", parentId)
234
- if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
235
- rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
232
+ local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
233
+ local isParentWaitingChildren = rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
234
+ if isParentWaitingChildren > 0 then
236
235
  local parentWaitKey = parentQueueKey .. ":wait"
237
236
  local parentPausedKey = parentQueueKey .. ":paused"
238
237
  local parentActiveKey = parentQueueKey .. ":active"
@@ -241,32 +240,37 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
241
240
  local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
242
241
  local priority = tonumber(jobAttributes[1]) or 0
243
242
  local delay = tonumber(jobAttributes[2]) or 0
243
+ -- ignore dependencies if any left
244
+ rcall("HSET", parentKey, "igdp", 1)
244
245
  if delay > 0 then
245
246
  local delayedTimestamp = tonumber(timestamp) + delay
246
247
  local score = delayedTimestamp * 0x1000
247
248
  local parentDelayedKey = parentQueueKey .. ":delayed"
248
249
  rcall("ZADD", parentDelayedKey, score, parentId)
249
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
250
- "jobId", parentId, "delay", delayedTimestamp)
250
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
251
+ delayedTimestamp)
251
252
  addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
252
253
  else
253
254
  if priority == 0 then
254
- local parentTarget, isParentPausedOrMaxed =
255
- getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
256
- parentPausedKey)
257
- addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
258
- parentId)
255
+ local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
256
+ parentWaitKey, parentPausedKey)
257
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
259
258
  else
260
259
  local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
261
- addJobWithPriority(parentMarkerKey,
262
- parentQueueKey .. ":prioritized", priority,
263
- parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
260
+ addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
261
+ parentQueueKey .. ":pc", isPausedOrMaxed)
264
262
  end
265
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
266
- "jobId", parentId, "prev", "waiting-children")
263
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
264
+ "waiting-children")
267
265
  end
268
266
  end
269
267
  end
268
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
269
+ local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
270
+ if hasPendingDependencies then
271
+ moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
272
+ end
273
+ end
270
274
  local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
271
275
  parentId, jobIdKey, returnvalue, timestamp )
272
276
  local processedSet = parentKey .. ":processed"
@@ -1 +1 @@
1
- {"version":3,"file":"addPrioritizedJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Vf,CAAC;AACW,QAAA,iBAAiB,GAAG;IAC/B,IAAI,EAAE,mBAAmB;IACzB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addPrioritizedJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Vf,CAAC;AACW,QAAA,iBAAiB,GAAG;IAC/B,IAAI,EAAE,mBAAmB;IACzB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -100,7 +100,7 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
100
100
  end
101
101
  return waitKey, false
102
102
  end
103
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
103
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
104
104
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
105
105
  parentPrefix .. "wait", parentPrefix .. "paused")
106
106
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -128,10 +128,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
128
128
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
129
129
  end
130
130
  else
131
- moveParentToWait(parentPrefix, parentId)
131
+ _moveParentToWait(parentPrefix, parentId)
132
132
  end
133
133
  else
134
- moveParentToWait(parentPrefix, parentId, true)
134
+ _moveParentToWait(parentPrefix, parentId, true)
135
135
  end
136
136
  end
137
137
  end
@@ -159,10 +159,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
159
159
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
160
160
  end
161
161
  else
162
- moveParentToWait(parentPrefix, parentId)
162
+ _moveParentToWait(parentPrefix, parentId)
163
163
  end
164
164
  else
165
- moveParentToWait(parentPrefix, parentId, true)
165
+ _moveParentToWait(parentPrefix, parentId, true)
166
166
  end
167
167
  end
168
168
  end
@@ -140,9 +140,11 @@ end
140
140
  ]]
141
141
  -- Includes
142
142
  --[[
143
- Validate and move parent to active if needed.
143
+ Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
144
+ ]]
145
+ --[[
146
+ Validate and move parent to a wait status (wait, prioritized or delayed)
144
147
  ]]
145
- -- Includes
146
148
  --[[
147
149
  Add delay marker if needed.
148
150
  ]]
@@ -200,12 +202,9 @@ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
200
202
  end
201
203
  return false
202
204
  end
203
- local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
204
- parentKey, parentId, timestamp)
205
- local isParentActive = rcall("ZSCORE",
206
- parentQueueKey .. ":waiting-children", parentId)
207
- if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
208
- rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
205
+ local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
206
+ local isParentWaitingChildren = rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
207
+ if isParentWaitingChildren > 0 then
209
208
  local parentWaitKey = parentQueueKey .. ":wait"
210
209
  local parentPausedKey = parentQueueKey .. ":paused"
211
210
  local parentActiveKey = parentQueueKey .. ":active"
@@ -214,32 +213,37 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
214
213
  local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
215
214
  local priority = tonumber(jobAttributes[1]) or 0
216
215
  local delay = tonumber(jobAttributes[2]) or 0
216
+ -- ignore dependencies if any left
217
+ rcall("HSET", parentKey, "igdp", 1)
217
218
  if delay > 0 then
218
219
  local delayedTimestamp = tonumber(timestamp) + delay
219
220
  local score = delayedTimestamp * 0x1000
220
221
  local parentDelayedKey = parentQueueKey .. ":delayed"
221
222
  rcall("ZADD", parentDelayedKey, score, parentId)
222
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
223
- "jobId", parentId, "delay", delayedTimestamp)
223
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
224
+ delayedTimestamp)
224
225
  addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
225
226
  else
226
227
  if priority == 0 then
227
- local parentTarget, isParentPausedOrMaxed =
228
- getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
229
- parentPausedKey)
230
- addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
231
- parentId)
228
+ local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
229
+ parentWaitKey, parentPausedKey)
230
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
232
231
  else
233
232
  local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
234
- addJobWithPriority(parentMarkerKey,
235
- parentQueueKey .. ":prioritized", priority,
236
- parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
233
+ addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
234
+ parentQueueKey .. ":pc", isPausedOrMaxed)
237
235
  end
238
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
239
- "jobId", parentId, "prev", "waiting-children")
236
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
237
+ "waiting-children")
240
238
  end
241
239
  end
242
240
  end
241
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
242
+ local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
243
+ if hasPendingDependencies then
244
+ moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
245
+ end
246
+ end
243
247
  local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
244
248
  parentId, jobIdKey, returnvalue, timestamp )
245
249
  local processedSet = parentKey .. ":processed"
@@ -1 +1 @@
1
- {"version":3,"file":"addStandardJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Vf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addStandardJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkWf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -136,7 +136,7 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
136
136
  end
137
137
  return waitKey, false
138
138
  end
139
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
139
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
140
140
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
141
141
  parentPrefix .. "wait", parentPrefix .. "paused")
142
142
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -164,10 +164,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
164
164
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
165
165
  end
166
166
  else
167
- moveParentToWait(parentPrefix, parentId)
167
+ _moveParentToWait(parentPrefix, parentId)
168
168
  end
169
169
  else
170
- moveParentToWait(parentPrefix, parentId, true)
170
+ _moveParentToWait(parentPrefix, parentId, true)
171
171
  end
172
172
  end
173
173
  end
@@ -195,10 +195,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
195
195
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
196
196
  end
197
197
  else
198
- moveParentToWait(parentPrefix, parentId)
198
+ _moveParentToWait(parentPrefix, parentId)
199
199
  end
200
200
  else
201
- moveParentToWait(parentPrefix, parentId, true)
201
+ _moveParentToWait(parentPrefix, parentId, true)
202
202
  end
203
203
  end
204
204
  end
@@ -108,7 +108,7 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
108
108
  end
109
109
  return waitKey, false
110
110
  end
111
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
111
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
112
112
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
113
113
  parentPrefix .. "wait", parentPrefix .. "paused")
114
114
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -136,10 +136,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
136
136
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
137
137
  end
138
138
  else
139
- moveParentToWait(parentPrefix, parentId)
139
+ _moveParentToWait(parentPrefix, parentId)
140
140
  end
141
141
  else
142
- moveParentToWait(parentPrefix, parentId, true)
142
+ _moveParentToWait(parentPrefix, parentId, true)
143
143
  end
144
144
  end
145
145
  end
@@ -167,10 +167,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
167
167
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
168
168
  end
169
169
  else
170
- moveParentToWait(parentPrefix, parentId)
170
+ _moveParentToWait(parentPrefix, parentId)
171
171
  end
172
172
  else
173
- moveParentToWait(parentPrefix, parentId, true)
173
+ _moveParentToWait(parentPrefix, parentId, true)
174
174
  end
175
175
  end
176
176
  end
@@ -79,9 +79,11 @@ end
79
79
  ]]
80
80
  -- Includes
81
81
  --[[
82
- Validate and move parent to active if needed.
82
+ Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
83
+ ]]
84
+ --[[
85
+ Validate and move parent to a wait status (wait, prioritized or delayed)
83
86
  ]]
84
- -- Includes
85
87
  --[[
86
88
  Add delay marker if needed.
87
89
  ]]
@@ -139,12 +141,9 @@ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
139
141
  end
140
142
  return false
141
143
  end
142
- local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
143
- parentKey, parentId, timestamp)
144
- local isParentActive = rcall("ZSCORE",
145
- parentQueueKey .. ":waiting-children", parentId)
146
- if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
147
- rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
144
+ local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
145
+ local isParentWaitingChildren = rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
146
+ if isParentWaitingChildren > 0 then
148
147
  local parentWaitKey = parentQueueKey .. ":wait"
149
148
  local parentPausedKey = parentQueueKey .. ":paused"
150
149
  local parentActiveKey = parentQueueKey .. ":active"
@@ -153,32 +152,37 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
153
152
  local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
154
153
  local priority = tonumber(jobAttributes[1]) or 0
155
154
  local delay = tonumber(jobAttributes[2]) or 0
155
+ -- ignore dependencies if any left
156
+ rcall("HSET", parentKey, "igdp", 1)
156
157
  if delay > 0 then
157
158
  local delayedTimestamp = tonumber(timestamp) + delay
158
159
  local score = delayedTimestamp * 0x1000
159
160
  local parentDelayedKey = parentQueueKey .. ":delayed"
160
161
  rcall("ZADD", parentDelayedKey, score, parentId)
161
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
162
- "jobId", parentId, "delay", delayedTimestamp)
162
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
163
+ delayedTimestamp)
163
164
  addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
164
165
  else
165
166
  if priority == 0 then
166
- local parentTarget, isParentPausedOrMaxed =
167
- getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
168
- parentPausedKey)
169
- addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
170
- parentId)
167
+ local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
168
+ parentWaitKey, parentPausedKey)
169
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
171
170
  else
172
171
  local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
173
- addJobWithPriority(parentMarkerKey,
174
- parentQueueKey .. ":prioritized", priority,
175
- parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
172
+ addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
173
+ parentQueueKey .. ":pc", isPausedOrMaxed)
176
174
  end
177
- rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
178
- "jobId", parentId, "prev", "waiting-children")
175
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
176
+ "waiting-children")
179
177
  end
180
178
  end
181
179
  end
180
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
181
+ local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
182
+ if hasPendingDependencies then
183
+ moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
184
+ end
185
+ end
182
186
  --[[
183
187
  Function to remove deduplication key if needed.
184
188
  ]]
@@ -232,7 +236,7 @@ end
232
236
  local getJobKeyPrefix = function (jobKey, jobId)
233
237
  return string.sub(jobKey, 0, #jobKey - #jobId)
234
238
  end
235
- local function moveParentToWait(parentPrefix, parentId, emitEvent)
239
+ local function _moveParentToWait(parentPrefix, parentId, emitEvent)
236
240
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
237
241
  parentPrefix .. "wait", parentPrefix .. "paused")
238
242
  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
@@ -260,10 +264,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
260
264
  rcall("DEL", parentPrefix .. "de:" .. debounceId)
261
265
  end
262
266
  else
263
- moveParentToWait(parentPrefix, parentId)
267
+ _moveParentToWait(parentPrefix, parentId)
264
268
  end
265
269
  else
266
- moveParentToWait(parentPrefix, parentId, true)
270
+ _moveParentToWait(parentPrefix, parentId, true)
267
271
  end
268
272
  end
269
273
  end
@@ -291,10 +295,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
291
295
  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
292
296
  end
293
297
  else
294
- moveParentToWait(parentPrefix, parentId)
298
+ _moveParentToWait(parentPrefix, parentId)
295
299
  end
296
300
  else
297
- moveParentToWait(parentPrefix, parentId, true)
301
+ _moveParentToWait(parentPrefix, parentId, true)
298
302
  end
299
303
  end
300
304
  end
@@ -402,6 +406,8 @@ local function moveParentToFailedIfNeeded(parentQueueKey, parentKey, parentId, j
402
406
  parentKey,
403
407
  timestamp
404
408
  )
409
+ elseif parentData['cpof'] then
410
+ moveParentToWait(parentData['queueKey'], parentKey, parentData['id'], timestamp)
405
411
  elseif parentData['idof'] or parentData['rdof'] then
406
412
  local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
407
413
  local grandParentDependenciesSet = grandParentKey .. ":dependencies"
@@ -463,7 +469,9 @@ local maxStalledJobCount = tonumber(ARGV[1])
463
469
  local queueKeyPrefix = ARGV[2]
464
470
  local timestamp = ARGV[3]
465
471
  local maxCheckTime = ARGV[4]
466
- if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
472
+ if rcall("EXISTS", stalledCheckKey) == 1 then
473
+ return {{}, {}}
474
+ end
467
475
  rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
468
476
  -- Trim events before emiting them to avoid trimming events emitted in this script
469
477
  trimEvents(metaKey, eventStreamKey)
@@ -487,8 +495,7 @@ if (#stalling > 0) then
487
495
  local removed = rcall("LREM", activeKey, 1, jobId)
488
496
  if (removed > 0) then
489
497
  -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
490
- local stalledCount =
491
- rcall("HINCRBY", jobKey, "stalledCounter", 1)
498
+ local stalledCount = rcall("HINCRBY", jobKey, "stalledCounter", 1)
492
499
  if (stalledCount > maxStalledJobCount) then
493
500
  local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
494
501
  local rawOpts = jobAttributes[1]
@@ -496,38 +503,35 @@ if (#stalling > 0) then
496
503
  local opts = cjson.decode(rawOpts)
497
504
  rcall("ZADD", failedKey, timestamp, jobId)
498
505
  removeDeduplicationKeyIfNeeded(queueKeyPrefix, jobAttributes[3])
499
- local failedReason =
500
- "job stalled more than allowable limit"
501
- rcall("HMSET", jobKey, "failedReason", failedReason,
502
- "finishedOn", timestamp)
503
- rcall("XADD", eventStreamKey, "*", "event",
504
- "failed", "jobId", jobId, 'prev', 'active',
505
- 'failedReason', failedReason)
506
+ local failedReason = "job stalled more than allowable limit"
507
+ rcall("HMSET", jobKey, "failedReason", failedReason, "finishedOn", timestamp)
508
+ rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId", jobId, 'prev', 'active',
509
+ 'failedReason', failedReason)
506
510
  if rawParentData then
507
- if opts['fpof'] then
511
+ if opts['fpof'] or opts['cpof'] then
508
512
  local parentData = cjson.decode(rawParentData)
509
513
  -- TODO: need to remove this job from dependencies set in next breaking change
510
514
  -- no for now as it would imply a breaking change
511
515
  local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
512
516
  local unsuccesssfulSet = parentKey .. ":unsuccessful"
513
517
  rcall("ZADD", unsuccesssfulSet, timestamp, jobKey)
514
- moveParentToFailedIfNeeded(
515
- parentData['queueKey'],
516
- parentData['queueKey'] .. ':' .. parentData['id'],
517
- parentData['id'],
518
- jobKey,
519
- timestamp
520
- )
518
+ if opts['fpof'] then
519
+ moveParentToFailedIfNeeded(parentData['queueKey'],
520
+ parentData['queueKey'] .. ':' .. parentData['id'], parentData['id'], jobKey,
521
+ timestamp)
522
+ elseif opts['cpof'] then
523
+ moveParentToWait(parentData['queueKey'], parentKey, parentData['id'], timestamp)
524
+ end
521
525
  elseif opts['idof'] or opts['rdof'] then
522
526
  local parentData = cjson.decode(rawParentData)
523
527
  local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
524
528
  local dependenciesSet = parentKey .. ":dependencies"
525
529
  if rcall("SREM", dependenciesSet, jobKey) == 1 then
526
- moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
527
- parentKey, parentData['id'], timestamp)
530
+ moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet, parentKey,
531
+ parentData['id'], timestamp)
528
532
  if opts['idof'] then
529
- local failedSet = parentKey .. ":failed"
530
- rcall("HSET", failedSet, jobKey, failedReason)
533
+ local failedSet = parentKey .. ":failed"
534
+ rcall("HSET", failedSet, jobKey, failedReason)
531
535
  end
532
536
  end
533
537
  end
@@ -535,15 +539,12 @@ if (#stalling > 0) then
535
539
  removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
536
540
  table.insert(failed, jobId)
537
541
  else
538
- local target, isPausedOrMaxed =
539
- getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
542
+ local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
540
543
  -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
541
544
  addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
542
- rcall("XADD", eventStreamKey, "*", "event",
543
- "waiting", "jobId", jobId, 'prev', 'active')
545
+ rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, 'prev', 'active')
544
546
  -- Emit the stalled event
545
- rcall("XADD", eventStreamKey, "*", "event",
546
- "stalled", "jobId", jobId)
547
+ rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId", jobId)
547
548
  table.insert(stalled, jobId)
548
549
  end
549
550
  end
@@ -558,7 +559,8 @@ if (#active > 0) then
558
559
  rcall('SADD', stalledKey, unpack(active, from, to))
559
560
  end
560
561
  end
561
- return {failed, stalled}`;
562
+ return {failed, stalled}
563
+ `;
562
564
  exports.moveStalledJobsToWait = {
563
565
  name: 'moveStalledJobsToWait',
564
566
  content,
@@ -1 +1 @@
1
- {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBA6iBS,CAAC;AACb,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+iBf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}