bullmq 5.44.1 → 5.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/async-fifo-queue.js.map +1 -1
- package/dist/cjs/classes/child-processor.js.map +1 -1
- package/dist/cjs/classes/child.js.map +1 -1
- package/dist/cjs/classes/flow-producer.js +15 -4
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job-scheduler.js +1 -1
- package/dist/cjs/classes/job-scheduler.js.map +1 -1
- package/dist/cjs/classes/job.js +137 -54
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-base.js +1 -4
- package/dist/cjs/classes/queue-base.js.map +1 -1
- package/dist/cjs/classes/queue-events-producer.js.map +1 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-getters.js +12 -15
- package/dist/cjs/classes/queue-getters.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/queue.js +12 -13
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/redis-connection.js +1 -1
- package/dist/cjs/classes/redis-connection.js.map +1 -1
- package/dist/cjs/classes/repeat.js +1 -1
- package/dist/cjs/classes/repeat.js.map +1 -1
- package/dist/cjs/classes/scripts.js +18 -7
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +4 -0
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addJobScheduler-11.lua +52 -33
- package/dist/cjs/commands/includes/deduplicateJob.lua +16 -17
- package/dist/cjs/commands/includes/isLocked.lua +1 -0
- package/dist/cjs/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
- package/dist/cjs/commands/includes/moveParentToWait.lua +48 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
- package/dist/cjs/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/cjs/commands/includes/removeJob.lua +2 -2
- package/dist/cjs/commands/includes/removeJobWithChildren.lua +94 -0
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/cjs/commands/includes/storeJob.lua +1 -1
- package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +18 -53
- package/dist/cjs/commands/moveToFinished-14.lua +43 -51
- package/dist/cjs/commands/moveToWaitingChildren-8.lua +5 -32
- package/dist/cjs/commands/removeJob-3.lua +14 -72
- package/dist/cjs/commands/removeUnprocessedChildren-2.lua +31 -0
- package/dist/cjs/commands/updateJobScheduler-12.lua +29 -23
- package/dist/cjs/scripts/addDelayedJob-6.js +73 -55
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +59 -40
- package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +73 -55
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js +73 -55
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +14 -9
- package/dist/cjs/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/cjs/scripts/addStandardJob-8.js +73 -55
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
- package/dist/cjs/scripts/cleanJobsInSet-3.js +14 -9
- package/dist/cjs/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/cjs/scripts/drain-5.js +14 -9
- package/dist/cjs/scripts/drain-5.js.map +1 -1
- package/dist/cjs/scripts/index.js +1 -0
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +157 -155
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +183 -157
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-8.js +145 -137
- package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +14 -9
- package/dist/cjs/scripts/obliterate-2.js.map +1 -1
- package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
- package/dist/cjs/scripts/removeJob-3.js +97 -63
- package/dist/cjs/scripts/removeJob-3.js.map +1 -1
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js +338 -0
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js.map +1 -0
- package/dist/cjs/scripts/updateJobScheduler-12.js +27 -21
- package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/types/index.js +1 -0
- package/dist/cjs/types/index.js.map +1 -1
- package/dist/cjs/types/job-progress.js +3 -0
- package/dist/cjs/types/job-progress.js.map +1 -0
- package/dist/cjs/utils.js +12 -1
- package/dist/cjs/utils.js.map +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/async-fifo-queue.js.map +1 -1
- package/dist/esm/classes/child-processor.js.map +1 -1
- package/dist/esm/classes/child.js.map +1 -1
- package/dist/esm/classes/flow-producer.js +15 -4
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job-scheduler.js +1 -1
- package/dist/esm/classes/job-scheduler.js.map +1 -1
- package/dist/esm/classes/job.d.ts +49 -14
- package/dist/esm/classes/job.js +138 -55
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-base.d.ts +1 -4
- package/dist/esm/classes/queue-base.js +1 -4
- package/dist/esm/classes/queue-base.js.map +1 -1
- package/dist/esm/classes/queue-events-producer.d.ts +2 -2
- package/dist/esm/classes/queue-events-producer.js.map +1 -1
- package/dist/esm/classes/queue-events.d.ts +118 -29
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-getters.d.ts +11 -14
- package/dist/esm/classes/queue-getters.js +12 -15
- package/dist/esm/classes/queue-getters.js.map +1 -1
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +14 -15
- package/dist/esm/classes/queue.js +12 -13
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/redis-connection.js +1 -1
- package/dist/esm/classes/redis-connection.js.map +1 -1
- package/dist/esm/classes/repeat.js +1 -1
- package/dist/esm/classes/repeat.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +4 -3
- package/dist/esm/classes/scripts.js +18 -7
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +2 -1
- package/dist/esm/classes/worker.js +5 -1
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addJobScheduler-11.lua +52 -33
- package/dist/esm/commands/includes/deduplicateJob.lua +16 -17
- package/dist/esm/commands/includes/isLocked.lua +1 -0
- package/dist/esm/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
- package/dist/esm/commands/includes/moveParentToWait.lua +48 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
- package/dist/esm/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/esm/commands/includes/removeJob.lua +2 -2
- package/dist/esm/commands/includes/removeJobWithChildren.lua +94 -0
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/esm/commands/includes/storeJob.lua +1 -1
- package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +18 -53
- package/dist/esm/commands/moveToFinished-14.lua +43 -51
- package/dist/esm/commands/moveToWaitingChildren-8.lua +5 -32
- package/dist/esm/commands/removeJob-3.lua +14 -72
- package/dist/esm/commands/removeUnprocessedChildren-2.lua +31 -0
- package/dist/esm/commands/updateJobScheduler-12.lua +29 -23
- package/dist/esm/interfaces/job-json.d.ts +5 -2
- package/dist/esm/interfaces/minimal-job.d.ts +29 -11
- package/dist/esm/interfaces/queue-options.d.ts +6 -2
- package/dist/esm/interfaces/repeat-options.d.ts +1 -1
- package/dist/esm/interfaces/sandboxed-job.d.ts +2 -2
- package/dist/esm/interfaces/sandboxed-options.d.ts +1 -1
- package/dist/esm/interfaces/telemetry.d.ts +18 -15
- package/dist/esm/interfaces/worker-options.d.ts +10 -10
- package/dist/esm/scripts/addDelayedJob-6.js +73 -55
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addJobScheduler-11.js +59 -40
- package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/esm/scripts/addParentJob-4.js +73 -55
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js +73 -55
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +14 -9
- package/dist/esm/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/esm/scripts/addStandardJob-8.js +73 -55
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
- package/dist/esm/scripts/cleanJobsInSet-3.js +14 -9
- package/dist/esm/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/esm/scripts/drain-5.js +14 -9
- package/dist/esm/scripts/drain-5.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +1 -0
- package/dist/esm/scripts/index.js +1 -0
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +157 -155
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +183 -157
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-8.js +145 -137
- package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +14 -9
- package/dist/esm/scripts/obliterate-2.js.map +1 -1
- package/dist/esm/scripts/removeChildDependency-1.js +5 -5
- package/dist/esm/scripts/removeJob-3.js +97 -63
- package/dist/esm/scripts/removeJob-3.js.map +1 -1
- package/dist/esm/scripts/removeUnprocessedChildren-2.d.ts +5 -0
- package/dist/esm/scripts/removeUnprocessedChildren-2.js +335 -0
- package/dist/esm/scripts/removeUnprocessedChildren-2.js.map +1 -0
- package/dist/esm/scripts/updateJobScheduler-12.js +27 -21
- package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/index.d.ts +1 -0
- package/dist/esm/types/index.js +1 -0
- package/dist/esm/types/index.js.map +1 -1
- package/dist/esm/types/job-options.d.ts +13 -2
- package/dist/esm/types/job-progress.d.ts +1 -0
- package/dist/esm/types/job-progress.js +2 -0
- package/dist/esm/types/job-progress.js.map +1 -0
- package/dist/esm/utils.d.ts +17 -0
- package/dist/esm/utils.js +11 -0
- package/dist/esm/utils.js.map +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +26 -21
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
- package/dist/cjs/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
- package/dist/esm/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
@@ -76,7 +76,16 @@ end
|
|
76
76
|
]]
|
77
77
|
-- Includes
|
78
78
|
--[[
|
79
|
-
Validate and move parent to
|
79
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
80
|
+
if no pending dependencies.
|
81
|
+
]]
|
82
|
+
-- Includes
|
83
|
+
--[[
|
84
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
85
|
+
]]
|
86
|
+
-- Includes
|
87
|
+
--[[
|
88
|
+
Move parent to a wait status (wait, prioritized or delayed)
|
80
89
|
]]
|
81
90
|
-- Includes
|
82
91
|
--[[
|
@@ -136,58 +145,55 @@ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
|
136
145
|
end
|
137
146
|
return false
|
138
147
|
end
|
139
|
-
local function
|
140
|
-
|
141
|
-
local
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
local
|
152
|
-
local
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
148
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
149
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
150
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
151
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
152
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
153
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
154
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
155
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
156
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
157
|
+
-- ignore dependencies if any left
|
158
|
+
rcall("HSET", parentKey, "igdp", 1)
|
159
|
+
if delay > 0 then
|
160
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
161
|
+
local score = delayedTimestamp * 0x1000
|
162
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
163
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
164
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
165
|
+
delayedTimestamp)
|
166
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
167
|
+
else
|
168
|
+
if priority == 0 then
|
169
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
170
|
+
parentWaitKey, parentPausedKey)
|
171
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
161
172
|
else
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
parentPausedKey)
|
166
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
167
|
-
parentId)
|
168
|
-
else
|
169
|
-
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
170
|
-
addJobWithPriority(parentMarkerKey,
|
171
|
-
parentQueueKey .. ":prioritized", priority,
|
172
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
173
|
-
end
|
174
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
175
|
-
"jobId", parentId, "prev", "waiting-children")
|
173
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
174
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
175
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
176
176
|
end
|
177
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
178
|
+
"waiting-children")
|
177
179
|
end
|
178
180
|
end
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
local pttl = rcall("PTTL", deduplicationKey)
|
186
|
-
if pttl == 0 or pttl == -1 then
|
187
|
-
rcall("DEL", deduplicationKey)
|
181
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
182
|
+
if rcall("EXISTS", parentKey) == 1 then
|
183
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
184
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
185
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
186
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
188
187
|
end
|
189
188
|
end
|
190
189
|
end
|
190
|
+
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
191
|
+
parentId, timestamp)
|
192
|
+
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
193
|
+
if doNotHavePendingDependencies then
|
194
|
+
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
195
|
+
end
|
196
|
+
end
|
191
197
|
--[[
|
192
198
|
Functions to remove jobs when removeOnFail option is provided.
|
193
199
|
]]
|
@@ -197,13 +203,18 @@ end
|
|
197
203
|
]]
|
198
204
|
-- Includes
|
199
205
|
--[[
|
200
|
-
Function to remove deduplication key
|
206
|
+
Function to remove deduplication key if needed
|
207
|
+
when a job is being removed.
|
201
208
|
]]
|
202
|
-
local function
|
209
|
+
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
210
|
+
jobKey, jobId)
|
203
211
|
local deduplicationId = rcall("HGET", jobKey, "deid")
|
204
212
|
if deduplicationId then
|
205
213
|
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
206
|
-
rcall(
|
214
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
215
|
+
if currentJobId and currentJobId == jobId then
|
216
|
+
return rcall("DEL", deduplicationKey)
|
217
|
+
end
|
207
218
|
end
|
208
219
|
end
|
209
220
|
--[[
|
@@ -229,7 +240,7 @@ end
|
|
229
240
|
local getJobKeyPrefix = function (jobKey, jobId)
|
230
241
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
231
242
|
end
|
232
|
-
local function
|
243
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
233
244
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
234
245
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
235
246
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -257,10 +268,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
257
268
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
258
269
|
end
|
259
270
|
else
|
260
|
-
|
271
|
+
_moveParentToWait(parentPrefix, parentId)
|
261
272
|
end
|
262
273
|
else
|
263
|
-
|
274
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
264
275
|
end
|
265
276
|
end
|
266
277
|
end
|
@@ -288,10 +299,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
288
299
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
289
300
|
end
|
290
301
|
else
|
291
|
-
|
302
|
+
_moveParentToWait(parentPrefix, parentId)
|
292
303
|
end
|
293
304
|
else
|
294
|
-
|
305
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
295
306
|
end
|
296
307
|
end
|
297
308
|
end
|
@@ -305,7 +316,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
305
316
|
local jobKey = baseKey .. jobId
|
306
317
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
307
318
|
if shouldRemoveDeduplicationKey then
|
308
|
-
|
319
|
+
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
|
309
320
|
end
|
310
321
|
removeJobKeys(jobKey)
|
311
322
|
end
|
@@ -358,67 +369,90 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
|
|
358
369
|
end
|
359
370
|
end
|
360
371
|
end
|
361
|
-
local
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
prevState
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
372
|
+
local moveParentToFailedIfNeeded
|
373
|
+
local moveChildFromDependenciesIfNeeded
|
374
|
+
moveParentToFailedIfNeeded = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
375
|
+
if rcall("EXISTS", parentKey) == 1 then
|
376
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
377
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
378
|
+
local parentPrioritizedKey = parentQueueKey .. ":prioritized"
|
379
|
+
local parentWaitingChildrenOrDelayedKey
|
380
|
+
local prevState
|
381
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
382
|
+
parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
|
383
|
+
prevState = "waiting-children"
|
384
|
+
elseif rcall("ZSCORE", parentDelayedKey, parentId) then
|
385
|
+
parentWaitingChildrenOrDelayedKey = parentDelayedKey
|
386
|
+
prevState = "delayed"
|
387
|
+
rcall("HSET", parentKey, "delay", 0)
|
388
|
+
end
|
389
|
+
if parentWaitingChildrenOrDelayedKey then
|
390
|
+
rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
|
391
|
+
local parentQueuePrefix = parentQueueKey .. ":"
|
392
|
+
local parentFailedKey = parentQueueKey .. ":failed"
|
393
|
+
local deferredFailure = "child " .. jobIdKey .. " failed"
|
394
|
+
rcall("HSET", parentKey, "defa", deferredFailure)
|
395
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
396
|
+
else
|
397
|
+
if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then
|
398
|
+
local deferredFailure = "child " .. jobIdKey .. " failed"
|
399
|
+
rcall("HSET", parentKey, "defa", deferredFailure)
|
400
|
+
end
|
401
|
+
end
|
376
402
|
end
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
local
|
381
|
-
|
382
|
-
local
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
if jobAttributes[1] then
|
389
|
-
local parentData = cjson.decode(jobAttributes[1])
|
390
|
-
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
391
|
-
local grandParentUnsuccesssful = grandParentKey .. ":unsuccessful"
|
392
|
-
rcall("ZADD", grandParentUnsuccesssful, timestamp, parentKey)
|
393
|
-
if parentData['fpof'] then
|
394
|
-
moveParentFromWaitingChildrenToFailed(
|
403
|
+
end
|
404
|
+
moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp)
|
405
|
+
if rawParentData then
|
406
|
+
local parentData = cjson.decode(rawParentData)
|
407
|
+
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
408
|
+
local parentDependenciesChildrenKey = parentKey .. ":dependencies"
|
409
|
+
if parentData['fpof'] then
|
410
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
411
|
+
local parentUnsuccesssfulChildrenKey = parentKey .. ":unsuccessful"
|
412
|
+
rcall("ZADD", parentUnsuccesssfulChildrenKey, timestamp, childKey)
|
413
|
+
moveParentToFailedIfNeeded(
|
395
414
|
parentData['queueKey'],
|
396
|
-
parentData['queueKey'] .. ':' .. parentData['id'],
|
397
|
-
parentData['id'],
|
398
415
|
parentKey,
|
416
|
+
parentData['id'],
|
417
|
+
childKey,
|
399
418
|
timestamp
|
400
419
|
)
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
420
|
+
end
|
421
|
+
elseif parentData['cpof'] then
|
422
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
423
|
+
local parentFailedChildrenKey = parentKey .. ":failed"
|
424
|
+
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
425
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
426
|
+
end
|
427
|
+
elseif parentData['idof'] or parentData['rdof'] then
|
428
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
429
|
+
moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey,
|
430
|
+
parentKey, parentData['id'], timestamp)
|
431
|
+
if parentData['idof'] then
|
432
|
+
local parentFailedChildrenKey = parentKey .. ":failed"
|
433
|
+
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
411
434
|
end
|
412
435
|
end
|
413
436
|
end
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
437
|
+
end
|
438
|
+
end
|
439
|
+
--[[
|
440
|
+
Function to remove deduplication key if needed
|
441
|
+
when a job is moved to completed or failed states.
|
442
|
+
]]
|
443
|
+
local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
|
444
|
+
deduplicationId, jobId)
|
445
|
+
if deduplicationId then
|
446
|
+
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
447
|
+
local pttl = rcall("PTTL", deduplicationKey)
|
448
|
+
if pttl == 0 then
|
449
|
+
return rcall("DEL", deduplicationKey)
|
450
|
+
end
|
451
|
+
if pttl == -1 then
|
452
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
453
|
+
if currentJobId and currentJobId == jobId then
|
454
|
+
return rcall("DEL", deduplicationKey)
|
455
|
+
end
|
422
456
|
end
|
423
457
|
end
|
424
458
|
end
|
@@ -458,7 +492,9 @@ local maxStalledJobCount = tonumber(ARGV[1])
|
|
458
492
|
local queueKeyPrefix = ARGV[2]
|
459
493
|
local timestamp = ARGV[3]
|
460
494
|
local maxCheckTime = ARGV[4]
|
461
|
-
if rcall("EXISTS", stalledCheckKey) == 1 then
|
495
|
+
if rcall("EXISTS", stalledCheckKey) == 1 then
|
496
|
+
return {{}, {}}
|
497
|
+
end
|
462
498
|
rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
|
463
499
|
-- Trim events before emiting them to avoid trimming events emitted in this script
|
464
500
|
trimEvents(metaKey, eventStreamKey)
|
@@ -482,63 +518,28 @@ if (#stalling > 0) then
|
|
482
518
|
local removed = rcall("LREM", activeKey, 1, jobId)
|
483
519
|
if (removed > 0) then
|
484
520
|
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
485
|
-
local stalledCount =
|
486
|
-
rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
521
|
+
local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
|
487
522
|
if (stalledCount > maxStalledJobCount) then
|
488
523
|
local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
|
489
524
|
local rawOpts = jobAttributes[1]
|
490
525
|
local rawParentData = jobAttributes[2]
|
491
526
|
local opts = cjson.decode(rawOpts)
|
492
527
|
rcall("ZADD", failedKey, timestamp, jobId)
|
493
|
-
|
494
|
-
local failedReason =
|
495
|
-
|
496
|
-
rcall("
|
497
|
-
|
498
|
-
|
499
|
-
"failed", "jobId", jobId, 'prev', 'active',
|
500
|
-
'failedReason', failedReason)
|
501
|
-
if rawParentData then
|
502
|
-
if opts['fpof'] then
|
503
|
-
local parentData = cjson.decode(rawParentData)
|
504
|
-
-- TODO: need to remove this job from dependencies set in next breaking change
|
505
|
-
-- no for now as it would imply a breaking change
|
506
|
-
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
507
|
-
local unsuccesssfulSet = parentKey .. ":unsuccessful"
|
508
|
-
rcall("ZADD", unsuccesssfulSet, timestamp, jobKey)
|
509
|
-
moveParentFromWaitingChildrenToFailed(
|
510
|
-
parentData['queueKey'],
|
511
|
-
parentData['queueKey'] .. ':' .. parentData['id'],
|
512
|
-
parentData['id'],
|
513
|
-
jobKey,
|
514
|
-
timestamp
|
515
|
-
)
|
516
|
-
elseif opts['idof'] or opts['rdof'] then
|
517
|
-
local parentData = cjson.decode(rawParentData)
|
518
|
-
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
519
|
-
local dependenciesSet = parentKey .. ":dependencies"
|
520
|
-
if rcall("SREM", dependenciesSet, jobKey) == 1 then
|
521
|
-
moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
|
522
|
-
parentKey, parentData['id'], timestamp)
|
523
|
-
if opts['idof'] then
|
524
|
-
local failedSet = parentKey .. ":failed"
|
525
|
-
rcall("HSET", failedSet, jobKey, failedReason)
|
526
|
-
end
|
527
|
-
end
|
528
|
-
end
|
529
|
-
end
|
528
|
+
removeDeduplicationKeyIfNeededOnFinalization(queueKeyPrefix, jobAttributes[3], jobId)
|
529
|
+
local failedReason = "job stalled more than allowable limit"
|
530
|
+
rcall("HMSET", jobKey, "failedReason", failedReason, "finishedOn", timestamp)
|
531
|
+
rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId", jobId, 'prev', 'active',
|
532
|
+
'failedReason', failedReason)
|
533
|
+
moveChildFromDependenciesIfNeeded(rawParentData, jobKey, failedReason, timestamp)
|
530
534
|
removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
|
531
535
|
table.insert(failed, jobId)
|
532
536
|
else
|
533
|
-
local target, isPausedOrMaxed =
|
534
|
-
getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
|
537
|
+
local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
|
535
538
|
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
|
536
539
|
addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
|
537
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
538
|
-
"waiting", "jobId", jobId, 'prev', 'active')
|
540
|
+
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, 'prev', 'active')
|
539
541
|
-- Emit the stalled event
|
540
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
541
|
-
"stalled", "jobId", jobId)
|
542
|
+
rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId", jobId)
|
542
543
|
table.insert(stalled, jobId)
|
543
544
|
end
|
544
545
|
end
|
@@ -553,7 +554,8 @@ if (#active > 0) then
|
|
553
554
|
rcall('SADD', stalledKey, unpack(active, from, to))
|
554
555
|
end
|
555
556
|
end
|
556
|
-
return {failed, stalled}
|
557
|
+
return {failed, stalled}
|
558
|
+
`;
|
557
559
|
export const moveStalledJobsToWait = {
|
558
560
|
name: 'moveStalledJobsToWait',
|
559
561
|
content,
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6iBf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|