bullmq 5.41.8 → 5.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/dist/cjs/classes/job-scheduler.js +14 -18
  2. package/dist/cjs/classes/job-scheduler.js.map +1 -1
  3. package/dist/cjs/classes/job.js +20 -13
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/scripts.js +33 -3
  6. package/dist/cjs/classes/scripts.js.map +1 -1
  7. package/dist/cjs/classes/worker.js +4 -8
  8. package/dist/cjs/classes/worker.js.map +1 -1
  9. package/dist/cjs/commands/addJobScheduler-10.lua +9 -36
  10. package/dist/cjs/commands/includes/addJobFromScheduler.lua +41 -0
  11. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +11 -1
  12. package/dist/cjs/commands/includes/removeJobKeys.lua +2 -2
  13. package/dist/cjs/commands/includes/storeJobScheduler.lua +1 -1
  14. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +5 -0
  15. package/dist/cjs/commands/moveToFinished-14.lua +2 -0
  16. package/dist/cjs/commands/moveToWaitingChildren-8.lua +133 -0
  17. package/dist/cjs/commands/removeJob-3.lua +10 -0
  18. package/dist/{esm/commands/updateJobScheduler-7.lua → cjs/commands/updateJobScheduler-11.lua} +28 -21
  19. package/dist/cjs/scripts/addJobScheduler-10.js +80 -70
  20. package/dist/cjs/scripts/addJobScheduler-10.js.map +1 -1
  21. package/dist/cjs/scripts/addRepeatableJob-2.js +2 -2
  22. package/dist/cjs/scripts/cleanJobsInSet-3.js +2 -2
  23. package/dist/cjs/scripts/drain-5.js +2 -2
  24. package/dist/cjs/scripts/index.js +2 -2
  25. package/dist/cjs/scripts/index.js.map +1 -1
  26. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +17 -3
  27. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  28. package/dist/cjs/scripts/moveToFinished-14.js +14 -3
  29. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  30. package/dist/cjs/scripts/moveToWaitingChildren-8.js +549 -0
  31. package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -0
  32. package/dist/cjs/scripts/obliterate-2.js +2 -2
  33. package/dist/cjs/scripts/removeChildDependency-1.js +2 -2
  34. package/dist/cjs/scripts/removeJob-3.js +10 -2
  35. package/dist/cjs/scripts/removeJob-3.js.map +1 -1
  36. package/dist/cjs/scripts/removeJobScheduler-3.js +2 -2
  37. package/dist/cjs/scripts/removeRepeatable-3.js +2 -2
  38. package/dist/cjs/scripts/updateJobScheduler-11.js +246 -0
  39. package/dist/cjs/scripts/updateJobScheduler-11.js.map +1 -0
  40. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  41. package/dist/cjs/version.js +1 -1
  42. package/dist/esm/classes/job-scheduler.js +14 -18
  43. package/dist/esm/classes/job-scheduler.js.map +1 -1
  44. package/dist/esm/classes/job.js +20 -13
  45. package/dist/esm/classes/job.js.map +1 -1
  46. package/dist/esm/classes/scripts.d.ts +1 -1
  47. package/dist/esm/classes/scripts.js +33 -3
  48. package/dist/esm/classes/scripts.js.map +1 -1
  49. package/dist/esm/classes/worker.js +4 -8
  50. package/dist/esm/classes/worker.js.map +1 -1
  51. package/dist/esm/commands/addJobScheduler-10.lua +9 -36
  52. package/dist/esm/commands/includes/addJobFromScheduler.lua +41 -0
  53. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +11 -1
  54. package/dist/esm/commands/includes/removeJobKeys.lua +2 -2
  55. package/dist/esm/commands/includes/storeJobScheduler.lua +1 -1
  56. package/dist/esm/commands/moveStalledJobsToWait-9.lua +5 -0
  57. package/dist/esm/commands/moveToFinished-14.lua +2 -0
  58. package/dist/esm/commands/moveToWaitingChildren-8.lua +133 -0
  59. package/dist/esm/commands/removeJob-3.lua +10 -0
  60. package/dist/{cjs/commands/updateJobScheduler-7.lua → esm/commands/updateJobScheduler-11.lua} +28 -21
  61. package/dist/esm/scripts/addJobScheduler-10.js +80 -70
  62. package/dist/esm/scripts/addJobScheduler-10.js.map +1 -1
  63. package/dist/esm/scripts/addRepeatableJob-2.js +2 -2
  64. package/dist/esm/scripts/cleanJobsInSet-3.js +2 -2
  65. package/dist/esm/scripts/drain-5.js +2 -2
  66. package/dist/esm/scripts/index.d.ts +2 -2
  67. package/dist/esm/scripts/index.js +2 -2
  68. package/dist/esm/scripts/index.js.map +1 -1
  69. package/dist/esm/scripts/moveStalledJobsToWait-9.js +17 -3
  70. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  71. package/dist/esm/scripts/moveToFinished-14.js +14 -3
  72. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  73. package/dist/esm/scripts/moveToWaitingChildren-8.js +546 -0
  74. package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -0
  75. package/dist/esm/scripts/obliterate-2.js +2 -2
  76. package/dist/esm/scripts/removeChildDependency-1.js +2 -2
  77. package/dist/esm/scripts/removeJob-3.js +10 -2
  78. package/dist/esm/scripts/removeJob-3.js.map +1 -1
  79. package/dist/esm/scripts/removeJobScheduler-3.js +2 -2
  80. package/dist/esm/scripts/removeRepeatable-3.js +2 -2
  81. package/dist/esm/scripts/updateJobScheduler-11.js +243 -0
  82. package/dist/esm/scripts/updateJobScheduler-11.js.map +1 -0
  83. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  84. package/dist/esm/version.d.ts +1 -1
  85. package/dist/esm/version.js +1 -1
  86. package/package.json +1 -1
  87. package/dist/cjs/commands/moveToWaitingChildren-5.lua +0 -68
  88. package/dist/cjs/scripts/moveToWaitingChildren-5.js +0 -83
  89. package/dist/cjs/scripts/moveToWaitingChildren-5.js.map +0 -1
  90. package/dist/cjs/scripts/updateJobScheduler-7.js +0 -144
  91. package/dist/cjs/scripts/updateJobScheduler-7.js.map +0 -1
  92. package/dist/esm/commands/moveToWaitingChildren-5.lua +0 -68
  93. package/dist/esm/scripts/moveToWaitingChildren-5.js +0 -80
  94. package/dist/esm/scripts/moveToWaitingChildren-5.js.map +0 -1
  95. package/dist/esm/scripts/updateJobScheduler-7.js +0 -141
  96. package/dist/esm/scripts/updateJobScheduler-7.js.map +0 -1
  97. /package/dist/esm/scripts/{moveToWaitingChildren-5.d.ts → moveToWaitingChildren-8.d.ts} +0 -0
  98. /package/dist/esm/scripts/{updateJobScheduler-7.d.ts → updateJobScheduler-11.d.ts} +0 -0
@@ -0,0 +1,546 @@
1
+ const content = `--[[
2
+ Moves job from active to waiting children set.
3
+ Input:
4
+ KEYS[1] active key
5
+ KEYS[2] wait-children key
6
+ KEYS[3] job key
7
+ KEYS[4] job dependencies key
8
+ KEYS[5] job unsuccessful key
9
+ KEYS[6] stalled key
10
+ KEYS[7] failed key
11
+ KEYS[8] events key
12
+ ARGV[1] token
13
+ ARGV[2] child key
14
+ ARGV[3] timestamp
15
+ ARGV[4] jobId
16
+ ARGV[5] prefix
17
+ Output:
18
+ 0 - OK
19
+ 1 - There are not pending dependencies.
20
+ -1 - Missing job.
21
+ -2 - Missing lock
22
+ -3 - Job not in active set
23
+ ]]
24
+ local rcall = redis.call
25
+ local activeKey = KEYS[1]
26
+ local waitingChildrenKey = KEYS[2]
27
+ local jobKey = KEYS[3]
28
+ local jobDependenciesKey = KEYS[4]
29
+ local jobUnsuccessfulKey = KEYS[5]
30
+ local stalledKey = KEYS[6]
31
+ local failedKey = KEYS[7]
32
+ local timestamp = ARGV[3]
33
+ local jobId = ARGV[4]
34
+ --- Includes
35
+ --[[
36
+ Function to recursively move from waitingChildren to failed.
37
+ ]]
38
+ -- Includes
39
+ --[[
40
+ Validate and move parent to active if needed.
41
+ ]]
42
+ -- Includes
43
+ --[[
44
+ Add delay marker if needed.
45
+ ]]
46
+ -- Includes
47
+ --[[
48
+ Function to return the next delayed job timestamp.
49
+ ]]
50
+ local function getNextDelayedTimestamp(delayedKey)
51
+ local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
52
+ if #result then
53
+ local nextTimestamp = tonumber(result[2])
54
+ if nextTimestamp ~= nil then
55
+ return nextTimestamp / 0x1000
56
+ end
57
+ end
58
+ end
59
+ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
60
+ local nextTimestamp = getNextDelayedTimestamp(delayedKey)
61
+ if nextTimestamp ~= nil then
62
+ -- Replace the score of the marker with the newest known
63
+ -- next timestamp.
64
+ rcall("ZADD", markerKey, nextTimestamp, "1")
65
+ end
66
+ end
67
+ --[[
68
+ Function to add job in target list and add marker if needed.
69
+ ]]
70
+ -- Includes
71
+ --[[
72
+ Add marker if needed when a job is available.
73
+ ]]
74
+ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
75
+ if not isPausedOrMaxed then
76
+ rcall("ZADD", markerKey, 0, "0")
77
+ end
78
+ end
79
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
80
+ rcall(pushCmd, targetKey, jobId)
81
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
82
+ end
83
+ --[[
84
+ Function to add job considering priority.
85
+ ]]
86
+ -- Includes
87
+ --[[
88
+ Function to get priority score.
89
+ ]]
90
+ local function getPriorityScore(priority, priorityCounterKey)
91
+ local prioCounter = rcall("INCR", priorityCounterKey)
92
+ return priority * 0x100000000 + prioCounter % 0x100000000
93
+ end
94
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
95
+ isPausedOrMaxed)
96
+ local score = getPriorityScore(priority, priorityCounterKey)
97
+ rcall("ZADD", prioritizedKey, score, jobId)
98
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
99
+ end
100
+ --[[
101
+ Function to check if queue is paused or maxed
102
+ (since an empty list and !EXISTS are not really the same).
103
+ ]]
104
+ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
105
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
106
+ if queueAttributes[1] then
107
+ return true
108
+ else
109
+ if queueAttributes[2] then
110
+ local activeCount = rcall("LLEN", activeKey)
111
+ return activeCount >= tonumber(queueAttributes[2])
112
+ end
113
+ end
114
+ return false
115
+ end
116
+ --[[
117
+ Function to check for the meta.paused key to decide if we are paused or not
118
+ (since an empty list and !EXISTS are not really the same).
119
+ ]]
120
+ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
121
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
122
+ if queueAttributes[1] then
123
+ return pausedKey, true
124
+ else
125
+ if queueAttributes[2] then
126
+ local activeCount = rcall("LLEN", activeKey)
127
+ if activeCount >= tonumber(queueAttributes[2]) then
128
+ return waitKey, true
129
+ else
130
+ return waitKey, false
131
+ end
132
+ end
133
+ end
134
+ return waitKey, false
135
+ end
136
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
137
+ parentKey, parentId, timestamp)
138
+ local isParentActive = rcall("ZSCORE",
139
+ parentQueueKey .. ":waiting-children", parentId)
140
+ if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
141
+ rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
142
+ local parentWaitKey = parentQueueKey .. ":wait"
143
+ local parentPausedKey = parentQueueKey .. ":paused"
144
+ local parentActiveKey = parentQueueKey .. ":active"
145
+ local parentMetaKey = parentQueueKey .. ":meta"
146
+ local parentMarkerKey = parentQueueKey .. ":marker"
147
+ local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
148
+ local priority = tonumber(jobAttributes[1]) or 0
149
+ local delay = tonumber(jobAttributes[2]) or 0
150
+ if delay > 0 then
151
+ local delayedTimestamp = tonumber(timestamp) + delay
152
+ local score = delayedTimestamp * 0x1000
153
+ local parentDelayedKey = parentQueueKey .. ":delayed"
154
+ rcall("ZADD", parentDelayedKey, score, parentId)
155
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
156
+ "jobId", parentId, "delay", delayedTimestamp)
157
+ addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
158
+ else
159
+ if priority == 0 then
160
+ local parentTarget, isParentPausedOrMaxed =
161
+ getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
162
+ parentPausedKey)
163
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
164
+ parentId)
165
+ else
166
+ local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
167
+ addJobWithPriority(parentMarkerKey,
168
+ parentQueueKey .. ":prioritized", priority,
169
+ parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
170
+ end
171
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
172
+ "jobId", parentId, "prev", "waiting-children")
173
+ end
174
+ end
175
+ end
176
+ --[[
177
+ Function to remove deduplication key if needed.
178
+ ]]
179
+ local function removeDeduplicationKeyIfNeeded(prefixKey, deduplicationId)
180
+ if deduplicationId then
181
+ local deduplicationKey = prefixKey .. "de:" .. deduplicationId
182
+ local pttl = rcall("PTTL", deduplicationKey)
183
+ if pttl == 0 or pttl == -1 then
184
+ rcall("DEL", deduplicationKey)
185
+ end
186
+ end
187
+ end
188
+ --[[
189
+ Functions to remove jobs when removeOnFail option is provided.
190
+ ]]
191
+ -- Includes
192
+ --[[
193
+ Function to remove job.
194
+ ]]
195
+ -- Includes
196
+ --[[
197
+ Function to remove deduplication key.
198
+ ]]
199
+ local function removeDeduplicationKey(prefixKey, jobKey)
200
+ local deduplicationId = rcall("HGET", jobKey, "deid")
201
+ if deduplicationId then
202
+ local deduplicationKey = prefixKey .. "de:" .. deduplicationId
203
+ rcall("DEL", deduplicationKey)
204
+ end
205
+ end
206
+ --[[
207
+ Function to remove job keys.
208
+ ]]
209
+ local function removeJobKeys(jobKey)
210
+ return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
211
+ jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
212
+ end
213
+ --[[
214
+ Check if this job has a parent. If so we will just remove it from
215
+ the parent child list, but if it is the last child we should move the parent to "wait/paused"
216
+ which requires code from "moveToFinished"
217
+ ]]
218
+ -- Includes
219
+ --[[
220
+ Functions to destructure job key.
221
+ Just a bit of warning, these functions may be a bit slow and affect performance significantly.
222
+ ]]
223
+ local getJobIdFromKey = function (jobKey)
224
+ return string.match(jobKey, ".*:(.*)")
225
+ end
226
+ local getJobKeyPrefix = function (jobKey, jobId)
227
+ return string.sub(jobKey, 0, #jobKey - #jobId)
228
+ end
229
+ local function moveParentToWait(parentPrefix, parentId, emitEvent)
230
+ local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
231
+ parentPrefix .. "wait", parentPrefix .. "paused")
232
+ addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
233
+ if emitEvent then
234
+ local parentEventStream = parentPrefix .. "events"
235
+ rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
236
+ end
237
+ end
238
+ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
239
+ if parentKey then
240
+ local parentDependenciesKey = parentKey .. ":dependencies"
241
+ local result = rcall("SREM", parentDependenciesKey, jobKey)
242
+ if result > 0 then
243
+ local pendingDependencies = rcall("SCARD", parentDependenciesKey)
244
+ if pendingDependencies == 0 then
245
+ local parentId = getJobIdFromKey(parentKey)
246
+ local parentPrefix = getJobKeyPrefix(parentKey, parentId)
247
+ local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
248
+ if numRemovedElements == 1 then
249
+ if hard then -- remove parent in same queue
250
+ if parentPrefix == baseKey then
251
+ removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
252
+ removeJobKeys(parentKey)
253
+ if debounceId then
254
+ rcall("DEL", parentPrefix .. "de:" .. debounceId)
255
+ end
256
+ else
257
+ moveParentToWait(parentPrefix, parentId)
258
+ end
259
+ else
260
+ moveParentToWait(parentPrefix, parentId, true)
261
+ end
262
+ end
263
+ end
264
+ return true
265
+ end
266
+ else
267
+ local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
268
+ local missedParentKey = parentAttributes[1]
269
+ if( (type(missedParentKey) == "string") and missedParentKey ~= ""
270
+ and (rcall("EXISTS", missedParentKey) == 1)) then
271
+ local parentDependenciesKey = missedParentKey .. ":dependencies"
272
+ local result = rcall("SREM", parentDependenciesKey, jobKey)
273
+ if result > 0 then
274
+ local pendingDependencies = rcall("SCARD", parentDependenciesKey)
275
+ if pendingDependencies == 0 then
276
+ local parentId = getJobIdFromKey(missedParentKey)
277
+ local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
278
+ local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
279
+ if numRemovedElements == 1 then
280
+ if hard then
281
+ if parentPrefix == baseKey then
282
+ removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
283
+ removeJobKeys(missedParentKey)
284
+ if parentAttributes[2] then
285
+ rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
286
+ end
287
+ else
288
+ moveParentToWait(parentPrefix, parentId)
289
+ end
290
+ else
291
+ moveParentToWait(parentPrefix, parentId, true)
292
+ end
293
+ end
294
+ end
295
+ return true
296
+ end
297
+ end
298
+ end
299
+ return false
300
+ end
301
+ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
302
+ local jobKey = baseKey .. jobId
303
+ removeParentDependencyKey(jobKey, hard, nil, baseKey)
304
+ if shouldRemoveDeduplicationKey then
305
+ removeDeduplicationKey(baseKey, jobKey)
306
+ end
307
+ removeJobKeys(jobKey)
308
+ end
309
+ --[[
310
+ Functions to remove jobs by max age.
311
+ ]]
312
+ -- Includes
313
+ local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix,
314
+ shouldRemoveDebounceKey)
315
+ local start = timestamp - maxAge * 1000
316
+ local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
317
+ for i, jobId in ipairs(jobIds) do
318
+ removeJob(jobId, false, prefix, false --[[remove debounce key]])
319
+ end
320
+ rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
321
+ end
322
+ --[[
323
+ Functions to remove jobs by max count.
324
+ ]]
325
+ -- Includes
326
+ local function removeJobsByMaxCount(maxCount, targetSet, prefix)
327
+ local start = maxCount
328
+ local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
329
+ for i, jobId in ipairs(jobIds) do
330
+ removeJob(jobId, false, prefix, false --[[remove debounce key]])
331
+ end
332
+ rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
333
+ end
334
+ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
335
+ local removeOnFailType = type(opts["removeOnFail"])
336
+ if removeOnFailType == "number" then
337
+ removeJobsByMaxCount(opts["removeOnFail"],
338
+ failedKey, queueKeyPrefix)
339
+ elseif removeOnFailType == "boolean" then
340
+ if opts["removeOnFail"] then
341
+ removeJob(jobId, false, queueKeyPrefix,
342
+ false --[[remove debounce key]])
343
+ rcall("ZREM", failedKey, jobId)
344
+ end
345
+ elseif removeOnFailType ~= "nil" then
346
+ local maxAge = opts["removeOnFail"]["age"]
347
+ local maxCount = opts["removeOnFail"]["count"]
348
+ if maxAge ~= nil then
349
+ removeJobsByMaxAge(timestamp, maxAge,
350
+ failedKey, queueKeyPrefix)
351
+ end
352
+ if maxCount ~= nil and maxCount > 0 then
353
+ removeJobsByMaxCount(maxCount, failedKey,
354
+ queueKeyPrefix)
355
+ end
356
+ end
357
+ end
358
+ local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
359
+ local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
360
+ local parentDelayedKey = parentQueueKey .. ":delayed"
361
+ if rcall("ZSCORE", parentWaitingChildrenKey, parentId) ~= false then
362
+ rcall("ZREM", parentWaitingChildrenKey, parentId)
363
+ local parentQueuePrefix = parentQueueKey .. ":"
364
+ local parentFailedKey = parentQueueKey .. ":failed"
365
+ rcall("ZADD", parentFailedKey, timestamp, parentId)
366
+ local failedReason = "child " .. jobIdKey .. " failed"
367
+ rcall("HSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
368
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
369
+ failedReason, "prev", "waiting-children")
370
+ local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
371
+ removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
372
+ if jobAttributes[1] then
373
+ local parentData = cjson.decode(jobAttributes[1])
374
+ local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
375
+ local grandParentUnsuccesssful = grandParentKey .. ":unsuccessful"
376
+ rcall("ZADD", grandParentUnsuccesssful, timestamp, parentKey)
377
+ if parentData['fpof'] then
378
+ moveParentFromWaitingChildrenToFailed(
379
+ parentData['queueKey'],
380
+ parentData['queueKey'] .. ':' .. parentData['id'],
381
+ parentData['id'],
382
+ parentKey,
383
+ timestamp
384
+ )
385
+ elseif parentData['idof'] or parentData['rdof'] then
386
+ local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
387
+ local grandParentDependenciesSet = grandParentKey .. ":dependencies"
388
+ if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
389
+ moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
390
+ grandParentKey, parentData['id'], timestamp)
391
+ if parentData['idof'] then
392
+ local grandParentFailedSet = grandParentKey .. ":failed"
393
+ rcall("HSET", grandParentFailedSet, parentKey, failedReason)
394
+ end
395
+ end
396
+ end
397
+ end
398
+ local parentRawOpts = jobAttributes[3]
399
+ local parentOpts = cjson.decode(parentRawOpts)
400
+ removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
401
+ elseif rcall("ZSCORE", parentDelayedKey, parentId) ~= false then
402
+ rcall("ZREM", parentDelayedKey, parentId)
403
+ local parentQueuePrefix = parentQueueKey .. ":"
404
+ local parentFailedKey = parentQueueKey .. ":failed"
405
+ rcall("ZADD", parentFailedKey, timestamp, parentId)
406
+ local failedReason = "child " .. jobIdKey .. " failed"
407
+ rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
408
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
409
+ failedReason, "prev", "delayed")
410
+ local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
411
+ removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
412
+ if jobAttributes[1] then
413
+ local parentData = cjson.decode(jobAttributes[1])
414
+ if parentData['fpof'] then
415
+ moveParentFromWaitingChildrenToFailed(
416
+ parentData['queueKey'],
417
+ parentData['queueKey'] .. ':' .. parentData['id'],
418
+ parentData['id'],
419
+ parentKey,
420
+ timestamp
421
+ )
422
+ elseif parentData['idof'] or parentData['rdof'] then
423
+ local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
424
+ local grandParentDependenciesSet = grandParentKey .. ":dependencies"
425
+ if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
426
+ moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
427
+ grandParentKey, parentData['id'], timestamp)
428
+ if parentData['idof'] then
429
+ local grandParentFailedSet = grandParentKey .. ":failed"
430
+ rcall("HSET", grandParentFailedSet, parentKey, failedReason)
431
+ end
432
+ end
433
+ end
434
+ end
435
+ local parentRawOpts = jobAttributes[3]
436
+ local parentOpts = cjson.decode(parentRawOpts)
437
+ removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
438
+ else
439
+ local grandParentKey = rcall("HGET", parentKey, "parentKey")
440
+ if grandParentKey then
441
+ local grandParentUnsuccesssfulSet = grandParentKey .. ":unsuccessful"
442
+ rcall("ZADD", grandParentUnsuccesssfulSet, timestamp, parentKey)
443
+ end
444
+ end
445
+ end
446
+ local function removeLock(jobKey, stalledKey, token, jobId)
447
+ if token ~= "0" then
448
+ local lockKey = jobKey .. ':lock'
449
+ local lockToken = rcall("GET", lockKey)
450
+ if lockToken == token then
451
+ rcall("DEL", lockKey)
452
+ rcall("SREM", stalledKey, jobId)
453
+ else
454
+ if lockToken then
455
+ -- Lock exists but token does not match
456
+ return -6
457
+ else
458
+ -- Lock is missing completely
459
+ return -2
460
+ end
461
+ end
462
+ end
463
+ return 0
464
+ end
465
+ local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId,
466
+ timestamp)
467
+ local score = tonumber(timestamp)
468
+ local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
469
+ if(numRemovedElements < 1) then
470
+ return -3
471
+ end
472
+ rcall("ZADD", waitingChildrenKey, score, jobId)
473
+ return 0
474
+ end
475
+ if rcall("EXISTS", jobKey) == 1 then
476
+ if rcall("ZCARD", jobUnsuccessfulKey) ~= 0 then
477
+ -- TODO: refactor this logic in an include later
478
+ local jobAttributes = rcall("HMGET", jobKey, "parent", "deid", "opts")
479
+ removeDeduplicationKeyIfNeeded(ARGV[5], jobAttributes[2])
480
+ local failedReason = "children are failed"
481
+ rcall("ZADD", failedKey, timestamp, jobId)
482
+ rcall("HSET", jobKey, "finishedOn", timestamp)
483
+ rcall("XADD", KEYS[8], "*", "event", "failed", "jobId", jobId, "failedReason",
484
+ failedReason, "prev", "active")
485
+ local rawParentData = jobAttributes[1]
486
+ local rawOpts = jobAttributes[3]
487
+ local opts = cjson.decode(rawOpts)
488
+ if rawParentData ~= false then
489
+ if opts['fpof'] then
490
+ local parentData = cjson.decode(rawParentData)
491
+ local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
492
+ local parentUnsuccesssful = parentKey .. ":unsuccessful"
493
+ rcall("ZADD", parentUnsuccesssful, timestamp, jobKey)
494
+ moveParentFromWaitingChildrenToFailed(
495
+ parentData['queueKey'],
496
+ parentData['queueKey'] .. ':' .. parentData['id'],
497
+ parentData['id'],
498
+ jobKey,
499
+ timestamp
500
+ )
501
+ elseif opts['idof'] or opts['rdof'] then
502
+ local parentData = cjson.decode(rawParentData)
503
+ local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
504
+ local dependenciesSet = parentKey .. ":dependencies"
505
+ if rcall("SREM", dependenciesSet, jobKey) == 1 then
506
+ moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
507
+ parentKey, parentData['id'], timestamp)
508
+ if opts['idof'] then
509
+ local failedSet = parentKey .. ":failed"
510
+ rcall("HSET", failedSet, jobKey, failedReason)
511
+ end
512
+ end
513
+ end
514
+ end
515
+ removeJobsOnFail(ARGV[5], failedKey, jobId, opts, timestamp)
516
+ return 0
517
+ else
518
+ if ARGV[2] ~= "" then
519
+ if rcall("SISMEMBER", jobDependenciesKey, ARGV[2]) ~= 0 then
520
+ local errorCode = removeLock(jobKey, stalledKey, ARGV[1], jobId)
521
+ if errorCode < 0 then
522
+ return errorCode
523
+ end
524
+ return moveToWaitingChildren(activeKey, waitingChildrenKey, jobId, timestamp)
525
+ end
526
+ return 1
527
+ else
528
+ if rcall("SCARD", jobDependenciesKey) ~= 0 then
529
+ local errorCode = removeLock(jobKey, stalledKey, ARGV[1], jobId)
530
+ if errorCode < 0 then
531
+ return errorCode
532
+ end
533
+ return moveToWaitingChildren(activeKey, waitingChildrenKey, jobId, timestamp)
534
+ end
535
+ return 1
536
+ end
537
+ end
538
+ end
539
+ return -1
540
+ `;
541
+ export const moveToWaitingChildren = {
542
+ name: 'moveToWaitingChildren',
543
+ content,
544
+ keys: 8,
545
+ };
546
+ //# sourceMappingURL=moveToWaitingChildren-8.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"moveToWaitingChildren-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToWaitingChildren-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2hBf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -38,8 +38,8 @@ end
38
38
  Function to remove job keys.
39
39
  ]]
40
40
  local function removeJobKeys(jobKey)
41
- return rcall("DEL", jobKey, jobKey .. ':logs',
42
- jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
41
+ return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
42
+ jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
43
43
  end
44
44
  --[[
45
45
  Check if this job has a parent. If so we will just remove it from
@@ -71,8 +71,8 @@ end
71
71
  Function to remove job keys.
72
72
  ]]
73
73
  local function removeJobKeys(jobKey)
74
- return rcall("DEL", jobKey, jobKey .. ':logs',
75
- jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
74
+ return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
75
+ jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
76
76
  end
77
77
  local function moveParentToWait(parentPrefix, parentId, emitEvent)
78
78
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
@@ -124,8 +124,8 @@ end
124
124
  Function to remove job keys.
125
125
  ]]
126
126
  local function removeJobKeys(jobKey)
127
- return rcall("DEL", jobKey, jobKey .. ':logs',
128
- jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
127
+ return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
128
+ jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
129
129
  end
130
130
  --[[
131
131
  Check if this job has a parent. If so we will just remove it from
@@ -274,6 +274,14 @@ local function removeJob(prefix, jobId, parentKey, removeChildren)
274
274
  removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
275
275
  end
276
276
  end
277
+ local unsuccessful = rcall("ZRANGE", jobKey .. ":unsuccessful", 0, -1)
278
+ if (#unsuccessful > 0) then
279
+ for i = 1, #unsuccessful, 1 do
280
+ local childJobId = getJobIdFromKey(unsuccessful[i])
281
+ local childJobPrefix = getJobKeyPrefix(unsuccessful[i], childJobId)
282
+ removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
283
+ end
284
+ end
277
285
  end
278
286
  local prev = removeJobFromAnyState(prefix, jobId)
279
287
  removeDeduplicationKey(prefix, jobKey)
@@ -1 +1 @@
1
- {"version":3,"file":"removeJob-3.js","sourceRoot":"","sources":["../../../src/scripts/removeJob-3.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Sf,CAAC;AACF,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"removeJob-3.js","sourceRoot":"","sources":["../../../src/scripts/removeJob-3.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkTf,CAAC;AACF,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -18,8 +18,8 @@ local rcall = redis.call
18
18
  Function to remove job keys.
19
19
  ]]
20
20
  local function removeJobKeys(jobKey)
21
- return rcall("DEL", jobKey, jobKey .. ':logs',
22
- jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
21
+ return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
22
+ jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
23
23
  end
24
24
  local jobSchedulerId = ARGV[1]
25
25
  local prefix = ARGV[2]
@@ -21,8 +21,8 @@ local millis = rcall("ZSCORE", KEYS[1], ARGV[2])
21
21
  Function to remove job keys.
22
22
  ]]
23
23
  local function removeJobKeys(jobKey)
24
- return rcall("DEL", jobKey, jobKey .. ':logs',
25
- jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
24
+ return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
25
+ jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
26
26
  end
27
27
  -- legacy removal TODO: remove in next breaking change
28
28
  if millis then