bullmq 5.44.1 → 5.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/async-fifo-queue.js.map +1 -1
- package/dist/cjs/classes/child-processor.js.map +1 -1
- package/dist/cjs/classes/child.js.map +1 -1
- package/dist/cjs/classes/flow-producer.js +15 -4
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job-scheduler.js +1 -1
- package/dist/cjs/classes/job-scheduler.js.map +1 -1
- package/dist/cjs/classes/job.js +137 -54
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-base.js +1 -4
- package/dist/cjs/classes/queue-base.js.map +1 -1
- package/dist/cjs/classes/queue-events-producer.js.map +1 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-getters.js +12 -15
- package/dist/cjs/classes/queue-getters.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/queue.js +12 -13
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/redis-connection.js +1 -1
- package/dist/cjs/classes/redis-connection.js.map +1 -1
- package/dist/cjs/classes/repeat.js +1 -1
- package/dist/cjs/classes/repeat.js.map +1 -1
- package/dist/cjs/classes/scripts.js +18 -7
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +4 -0
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addJobScheduler-11.lua +52 -33
- package/dist/cjs/commands/includes/deduplicateJob.lua +16 -17
- package/dist/cjs/commands/includes/isLocked.lua +1 -0
- package/dist/cjs/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
- package/dist/cjs/commands/includes/moveParentToWait.lua +48 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
- package/dist/cjs/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/cjs/commands/includes/removeJob.lua +2 -2
- package/dist/cjs/commands/includes/removeJobWithChildren.lua +94 -0
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/cjs/commands/includes/storeJob.lua +1 -1
- package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +18 -53
- package/dist/cjs/commands/moveToFinished-14.lua +43 -51
- package/dist/cjs/commands/moveToWaitingChildren-8.lua +5 -32
- package/dist/cjs/commands/removeJob-3.lua +14 -72
- package/dist/cjs/commands/removeUnprocessedChildren-2.lua +31 -0
- package/dist/cjs/commands/updateJobScheduler-12.lua +29 -23
- package/dist/cjs/scripts/addDelayedJob-6.js +73 -55
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +59 -40
- package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +73 -55
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js +73 -55
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +14 -9
- package/dist/cjs/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/cjs/scripts/addStandardJob-8.js +73 -55
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
- package/dist/cjs/scripts/cleanJobsInSet-3.js +14 -9
- package/dist/cjs/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/cjs/scripts/drain-5.js +14 -9
- package/dist/cjs/scripts/drain-5.js.map +1 -1
- package/dist/cjs/scripts/index.js +1 -0
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +157 -155
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +183 -157
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-8.js +145 -137
- package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +14 -9
- package/dist/cjs/scripts/obliterate-2.js.map +1 -1
- package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
- package/dist/cjs/scripts/removeJob-3.js +97 -63
- package/dist/cjs/scripts/removeJob-3.js.map +1 -1
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js +338 -0
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js.map +1 -0
- package/dist/cjs/scripts/updateJobScheduler-12.js +27 -21
- package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/types/index.js +1 -0
- package/dist/cjs/types/index.js.map +1 -1
- package/dist/cjs/types/job-progress.js +3 -0
- package/dist/cjs/types/job-progress.js.map +1 -0
- package/dist/cjs/utils.js +12 -1
- package/dist/cjs/utils.js.map +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/async-fifo-queue.js.map +1 -1
- package/dist/esm/classes/child-processor.js.map +1 -1
- package/dist/esm/classes/child.js.map +1 -1
- package/dist/esm/classes/flow-producer.js +15 -4
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job-scheduler.js +1 -1
- package/dist/esm/classes/job-scheduler.js.map +1 -1
- package/dist/esm/classes/job.d.ts +49 -14
- package/dist/esm/classes/job.js +138 -55
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-base.d.ts +1 -4
- package/dist/esm/classes/queue-base.js +1 -4
- package/dist/esm/classes/queue-base.js.map +1 -1
- package/dist/esm/classes/queue-events-producer.d.ts +2 -2
- package/dist/esm/classes/queue-events-producer.js.map +1 -1
- package/dist/esm/classes/queue-events.d.ts +118 -29
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-getters.d.ts +11 -14
- package/dist/esm/classes/queue-getters.js +12 -15
- package/dist/esm/classes/queue-getters.js.map +1 -1
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +14 -15
- package/dist/esm/classes/queue.js +12 -13
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/redis-connection.js +1 -1
- package/dist/esm/classes/redis-connection.js.map +1 -1
- package/dist/esm/classes/repeat.js +1 -1
- package/dist/esm/classes/repeat.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +4 -3
- package/dist/esm/classes/scripts.js +18 -7
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +2 -1
- package/dist/esm/classes/worker.js +5 -1
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addJobScheduler-11.lua +52 -33
- package/dist/esm/commands/includes/deduplicateJob.lua +16 -17
- package/dist/esm/commands/includes/isLocked.lua +1 -0
- package/dist/esm/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
- package/dist/esm/commands/includes/moveParentToWait.lua +48 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
- package/dist/esm/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/esm/commands/includes/removeJob.lua +2 -2
- package/dist/esm/commands/includes/removeJobWithChildren.lua +94 -0
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/esm/commands/includes/storeJob.lua +1 -1
- package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +18 -53
- package/dist/esm/commands/moveToFinished-14.lua +43 -51
- package/dist/esm/commands/moveToWaitingChildren-8.lua +5 -32
- package/dist/esm/commands/removeJob-3.lua +14 -72
- package/dist/esm/commands/removeUnprocessedChildren-2.lua +31 -0
- package/dist/esm/commands/updateJobScheduler-12.lua +29 -23
- package/dist/esm/interfaces/job-json.d.ts +5 -2
- package/dist/esm/interfaces/minimal-job.d.ts +29 -11
- package/dist/esm/interfaces/queue-options.d.ts +6 -2
- package/dist/esm/interfaces/repeat-options.d.ts +1 -1
- package/dist/esm/interfaces/sandboxed-job.d.ts +2 -2
- package/dist/esm/interfaces/sandboxed-options.d.ts +1 -1
- package/dist/esm/interfaces/telemetry.d.ts +18 -15
- package/dist/esm/interfaces/worker-options.d.ts +10 -10
- package/dist/esm/scripts/addDelayedJob-6.js +73 -55
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addJobScheduler-11.js +59 -40
- package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/esm/scripts/addParentJob-4.js +73 -55
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js +73 -55
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +14 -9
- package/dist/esm/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/esm/scripts/addStandardJob-8.js +73 -55
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
- package/dist/esm/scripts/cleanJobsInSet-3.js +14 -9
- package/dist/esm/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/esm/scripts/drain-5.js +14 -9
- package/dist/esm/scripts/drain-5.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +1 -0
- package/dist/esm/scripts/index.js +1 -0
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +157 -155
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +183 -157
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-8.js +145 -137
- package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +14 -9
- package/dist/esm/scripts/obliterate-2.js.map +1 -1
- package/dist/esm/scripts/removeChildDependency-1.js +5 -5
- package/dist/esm/scripts/removeJob-3.js +97 -63
- package/dist/esm/scripts/removeJob-3.js.map +1 -1
- package/dist/esm/scripts/removeUnprocessedChildren-2.d.ts +5 -0
- package/dist/esm/scripts/removeUnprocessedChildren-2.js +335 -0
- package/dist/esm/scripts/removeUnprocessedChildren-2.js.map +1 -0
- package/dist/esm/scripts/updateJobScheduler-12.js +27 -21
- package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/index.d.ts +1 -0
- package/dist/esm/types/index.js +1 -0
- package/dist/esm/types/index.js.map +1 -1
- package/dist/esm/types/job-options.d.ts +13 -2
- package/dist/esm/types/job-progress.d.ts +1 -0
- package/dist/esm/types/job-progress.js +2 -0
- package/dist/esm/types/job-progress.js.map +1 -0
- package/dist/esm/utils.d.ts +17 -0
- package/dist/esm/utils.js +11 -0
- package/dist/esm/utils.js.map +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +26 -21
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
- package/dist/cjs/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
- package/dist/esm/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
@@ -37,7 +37,16 @@ local jobId = ARGV[4]
|
|
37
37
|
]]
|
38
38
|
-- Includes
|
39
39
|
--[[
|
40
|
-
Validate and move parent to
|
40
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
41
|
+
if no pending dependencies.
|
42
|
+
]]
|
43
|
+
-- Includes
|
44
|
+
--[[
|
45
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
46
|
+
]]
|
47
|
+
-- Includes
|
48
|
+
--[[
|
49
|
+
Move parent to a wait status (wait, prioritized or delayed)
|
41
50
|
]]
|
42
51
|
-- Includes
|
43
52
|
--[[
|
@@ -133,58 +142,55 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
133
142
|
end
|
134
143
|
return waitKey, false
|
135
144
|
end
|
136
|
-
local function
|
137
|
-
|
138
|
-
local
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
local
|
149
|
-
local
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
145
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
146
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
147
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
148
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
149
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
150
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
151
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
152
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
153
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
154
|
+
-- ignore dependencies if any left
|
155
|
+
rcall("HSET", parentKey, "igdp", 1)
|
156
|
+
if delay > 0 then
|
157
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
158
|
+
local score = delayedTimestamp * 0x1000
|
159
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
160
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
161
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
162
|
+
delayedTimestamp)
|
163
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
164
|
+
else
|
165
|
+
if priority == 0 then
|
166
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
167
|
+
parentWaitKey, parentPausedKey)
|
168
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
158
169
|
else
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
parentPausedKey)
|
163
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
164
|
-
parentId)
|
165
|
-
else
|
166
|
-
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
167
|
-
addJobWithPriority(parentMarkerKey,
|
168
|
-
parentQueueKey .. ":prioritized", priority,
|
169
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
170
|
-
end
|
171
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
172
|
-
"jobId", parentId, "prev", "waiting-children")
|
170
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
171
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
172
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
173
173
|
end
|
174
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
175
|
+
"waiting-children")
|
174
176
|
end
|
175
177
|
end
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
local pttl = rcall("PTTL", deduplicationKey)
|
183
|
-
if pttl == 0 or pttl == -1 then
|
184
|
-
rcall("DEL", deduplicationKey)
|
178
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
179
|
+
if rcall("EXISTS", parentKey) == 1 then
|
180
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
181
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
182
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
183
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
185
184
|
end
|
186
185
|
end
|
187
186
|
end
|
187
|
+
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
188
|
+
parentId, timestamp)
|
189
|
+
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
190
|
+
if doNotHavePendingDependencies then
|
191
|
+
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
192
|
+
end
|
193
|
+
end
|
188
194
|
--[[
|
189
195
|
Functions to remove jobs when removeOnFail option is provided.
|
190
196
|
]]
|
@@ -194,13 +200,18 @@ end
|
|
194
200
|
]]
|
195
201
|
-- Includes
|
196
202
|
--[[
|
197
|
-
Function to remove deduplication key
|
203
|
+
Function to remove deduplication key if needed
|
204
|
+
when a job is being removed.
|
198
205
|
]]
|
199
|
-
local function
|
206
|
+
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
207
|
+
jobKey, jobId)
|
200
208
|
local deduplicationId = rcall("HGET", jobKey, "deid")
|
201
209
|
if deduplicationId then
|
202
210
|
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
203
|
-
rcall(
|
211
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
212
|
+
if currentJobId and currentJobId == jobId then
|
213
|
+
return rcall("DEL", deduplicationKey)
|
214
|
+
end
|
204
215
|
end
|
205
216
|
end
|
206
217
|
--[[
|
@@ -226,7 +237,7 @@ end
|
|
226
237
|
local getJobKeyPrefix = function (jobKey, jobId)
|
227
238
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
228
239
|
end
|
229
|
-
local function
|
240
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
230
241
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
231
242
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
232
243
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -254,10 +265,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
254
265
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
255
266
|
end
|
256
267
|
else
|
257
|
-
|
268
|
+
_moveParentToWait(parentPrefix, parentId)
|
258
269
|
end
|
259
270
|
else
|
260
|
-
|
271
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
261
272
|
end
|
262
273
|
end
|
263
274
|
end
|
@@ -285,10 +296,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
285
296
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
286
297
|
end
|
287
298
|
else
|
288
|
-
|
299
|
+
_moveParentToWait(parentPrefix, parentId)
|
289
300
|
end
|
290
301
|
else
|
291
|
-
|
302
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
292
303
|
end
|
293
304
|
end
|
294
305
|
end
|
@@ -302,7 +313,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
302
313
|
local jobKey = baseKey .. jobId
|
303
314
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
304
315
|
if shouldRemoveDeduplicationKey then
|
305
|
-
|
316
|
+
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
|
306
317
|
end
|
307
318
|
removeJobKeys(jobKey)
|
308
319
|
end
|
@@ -355,67 +366,90 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
|
|
355
366
|
end
|
356
367
|
end
|
357
368
|
end
|
358
|
-
local
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
prevState
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
369
|
+
local moveParentToFailedIfNeeded
|
370
|
+
local moveChildFromDependenciesIfNeeded
|
371
|
+
moveParentToFailedIfNeeded = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
372
|
+
if rcall("EXISTS", parentKey) == 1 then
|
373
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
374
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
375
|
+
local parentPrioritizedKey = parentQueueKey .. ":prioritized"
|
376
|
+
local parentWaitingChildrenOrDelayedKey
|
377
|
+
local prevState
|
378
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
379
|
+
parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
|
380
|
+
prevState = "waiting-children"
|
381
|
+
elseif rcall("ZSCORE", parentDelayedKey, parentId) then
|
382
|
+
parentWaitingChildrenOrDelayedKey = parentDelayedKey
|
383
|
+
prevState = "delayed"
|
384
|
+
rcall("HSET", parentKey, "delay", 0)
|
385
|
+
end
|
386
|
+
if parentWaitingChildrenOrDelayedKey then
|
387
|
+
rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
|
388
|
+
local parentQueuePrefix = parentQueueKey .. ":"
|
389
|
+
local parentFailedKey = parentQueueKey .. ":failed"
|
390
|
+
local deferredFailure = "child " .. jobIdKey .. " failed"
|
391
|
+
rcall("HSET", parentKey, "defa", deferredFailure)
|
392
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
393
|
+
else
|
394
|
+
if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then
|
395
|
+
local deferredFailure = "child " .. jobIdKey .. " failed"
|
396
|
+
rcall("HSET", parentKey, "defa", deferredFailure)
|
397
|
+
end
|
398
|
+
end
|
373
399
|
end
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
local
|
378
|
-
|
379
|
-
local
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
if jobAttributes[1] then
|
386
|
-
local parentData = cjson.decode(jobAttributes[1])
|
387
|
-
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
388
|
-
local grandParentUnsuccesssful = grandParentKey .. ":unsuccessful"
|
389
|
-
rcall("ZADD", grandParentUnsuccesssful, timestamp, parentKey)
|
390
|
-
if parentData['fpof'] then
|
391
|
-
moveParentFromWaitingChildrenToFailed(
|
400
|
+
end
|
401
|
+
moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp)
|
402
|
+
if rawParentData then
|
403
|
+
local parentData = cjson.decode(rawParentData)
|
404
|
+
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
405
|
+
local parentDependenciesChildrenKey = parentKey .. ":dependencies"
|
406
|
+
if parentData['fpof'] then
|
407
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
408
|
+
local parentUnsuccesssfulChildrenKey = parentKey .. ":unsuccessful"
|
409
|
+
rcall("ZADD", parentUnsuccesssfulChildrenKey, timestamp, childKey)
|
410
|
+
moveParentToFailedIfNeeded(
|
392
411
|
parentData['queueKey'],
|
393
|
-
parentData['queueKey'] .. ':' .. parentData['id'],
|
394
|
-
parentData['id'],
|
395
412
|
parentKey,
|
413
|
+
parentData['id'],
|
414
|
+
childKey,
|
396
415
|
timestamp
|
397
416
|
)
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
417
|
+
end
|
418
|
+
elseif parentData['cpof'] then
|
419
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
420
|
+
local parentFailedChildrenKey = parentKey .. ":failed"
|
421
|
+
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
422
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
423
|
+
end
|
424
|
+
elseif parentData['idof'] or parentData['rdof'] then
|
425
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
426
|
+
moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey,
|
427
|
+
parentKey, parentData['id'], timestamp)
|
428
|
+
if parentData['idof'] then
|
429
|
+
local parentFailedChildrenKey = parentKey .. ":failed"
|
430
|
+
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
408
431
|
end
|
409
432
|
end
|
410
433
|
end
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
434
|
+
end
|
435
|
+
end
|
436
|
+
--[[
|
437
|
+
Function to remove deduplication key if needed
|
438
|
+
when a job is moved to completed or failed states.
|
439
|
+
]]
|
440
|
+
local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
|
441
|
+
deduplicationId, jobId)
|
442
|
+
if deduplicationId then
|
443
|
+
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
444
|
+
local pttl = rcall("PTTL", deduplicationKey)
|
445
|
+
if pttl == 0 then
|
446
|
+
return rcall("DEL", deduplicationKey)
|
447
|
+
end
|
448
|
+
if pttl == -1 then
|
449
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
450
|
+
if currentJobId and currentJobId == jobId then
|
451
|
+
return rcall("DEL", deduplicationKey)
|
452
|
+
end
|
419
453
|
end
|
420
454
|
end
|
421
455
|
end
|
@@ -438,7 +472,7 @@ local function removeLock(jobKey, stalledKey, token, jobId)
|
|
438
472
|
end
|
439
473
|
return 0
|
440
474
|
end
|
441
|
-
local function moveToWaitingChildren
|
475
|
+
local function moveToWaitingChildren(activeKey, waitingChildrenKey, jobId,
|
442
476
|
timestamp)
|
443
477
|
local score = tonumber(timestamp)
|
444
478
|
local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
|
@@ -452,7 +486,7 @@ if rcall("EXISTS", jobKey) == 1 then
|
|
452
486
|
if rcall("ZCARD", jobUnsuccessfulKey) ~= 0 then
|
453
487
|
-- TODO: refactor this logic in an include later
|
454
488
|
local jobAttributes = rcall("HMGET", jobKey, "parent", "deid", "opts")
|
455
|
-
|
489
|
+
removeDeduplicationKeyIfNeededOnFinalization(ARGV[5], jobAttributes[2], jobId)
|
456
490
|
local failedReason = "children are failed"
|
457
491
|
rcall("ZADD", failedKey, timestamp, jobId)
|
458
492
|
rcall("HSET", jobKey, "finishedOn", timestamp)
|
@@ -461,33 +495,7 @@ if rcall("EXISTS", jobKey) == 1 then
|
|
461
495
|
local rawParentData = jobAttributes[1]
|
462
496
|
local rawOpts = jobAttributes[3]
|
463
497
|
local opts = cjson.decode(rawOpts)
|
464
|
-
|
465
|
-
if opts['fpof'] then
|
466
|
-
local parentData = cjson.decode(rawParentData)
|
467
|
-
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
468
|
-
local parentUnsuccesssful = parentKey .. ":unsuccessful"
|
469
|
-
rcall("ZADD", parentUnsuccesssful, timestamp, jobKey)
|
470
|
-
moveParentFromWaitingChildrenToFailed(
|
471
|
-
parentData['queueKey'],
|
472
|
-
parentData['queueKey'] .. ':' .. parentData['id'],
|
473
|
-
parentData['id'],
|
474
|
-
jobKey,
|
475
|
-
timestamp
|
476
|
-
)
|
477
|
-
elseif opts['idof'] or opts['rdof'] then
|
478
|
-
local parentData = cjson.decode(rawParentData)
|
479
|
-
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
480
|
-
local dependenciesSet = parentKey .. ":dependencies"
|
481
|
-
if rcall("SREM", dependenciesSet, jobKey) == 1 then
|
482
|
-
moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
|
483
|
-
parentKey, parentData['id'], timestamp)
|
484
|
-
if opts['idof'] then
|
485
|
-
local failedSet = parentKey .. ":failed"
|
486
|
-
rcall("HSET", failedSet, jobKey, failedReason)
|
487
|
-
end
|
488
|
-
end
|
489
|
-
end
|
490
|
-
end
|
498
|
+
moveChildFromDependenciesIfNeeded(rawParentData, jobKey, failedReason, timestamp)
|
491
499
|
removeJobsOnFail(ARGV[5], failedKey, jobId, opts, timestamp)
|
492
500
|
return 0
|
493
501
|
else
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveToWaitingChildren-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToWaitingChildren-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveToWaitingChildren-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToWaitingChildren-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2gBf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -25,13 +25,18 @@ local rcall = redis.call
|
|
25
25
|
]]
|
26
26
|
-- Includes
|
27
27
|
--[[
|
28
|
-
Function to remove deduplication key
|
28
|
+
Function to remove deduplication key if needed
|
29
|
+
when a job is being removed.
|
29
30
|
]]
|
30
|
-
local function
|
31
|
+
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
32
|
+
jobKey, jobId)
|
31
33
|
local deduplicationId = rcall("HGET", jobKey, "deid")
|
32
34
|
if deduplicationId then
|
33
35
|
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
34
|
-
rcall(
|
36
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
37
|
+
if currentJobId and currentJobId == jobId then
|
38
|
+
return rcall("DEL", deduplicationKey)
|
39
|
+
end
|
35
40
|
end
|
36
41
|
end
|
37
42
|
--[[
|
@@ -93,7 +98,7 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
93
98
|
end
|
94
99
|
return waitKey, false
|
95
100
|
end
|
96
|
-
local function
|
101
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
97
102
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
98
103
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
99
104
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -121,10 +126,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
121
126
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
122
127
|
end
|
123
128
|
else
|
124
|
-
|
129
|
+
_moveParentToWait(parentPrefix, parentId)
|
125
130
|
end
|
126
131
|
else
|
127
|
-
|
132
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
128
133
|
end
|
129
134
|
end
|
130
135
|
end
|
@@ -152,10 +157,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
152
157
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
153
158
|
end
|
154
159
|
else
|
155
|
-
|
160
|
+
_moveParentToWait(parentPrefix, parentId)
|
156
161
|
end
|
157
162
|
else
|
158
|
-
|
163
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
159
164
|
end
|
160
165
|
end
|
161
166
|
end
|
@@ -169,7 +174,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
169
174
|
local jobKey = baseKey .. jobId
|
170
175
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
171
176
|
if shouldRemoveDeduplicationKey then
|
172
|
-
|
177
|
+
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
|
173
178
|
end
|
174
179
|
removeJobKeys(jobKey)
|
175
180
|
end
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"obliterate-2.js","sourceRoot":"","sources":["../../../src/scripts/obliterate-2.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"obliterate-2.js","sourceRoot":"","sources":["../../../src/scripts/obliterate-2.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Uf,CAAC;AACF,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -74,7 +74,7 @@ local function removeJobKeys(jobKey)
|
|
74
74
|
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
75
75
|
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
76
76
|
end
|
77
|
-
local function
|
77
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
78
78
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
79
79
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
80
80
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -102,10 +102,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
102
102
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
103
103
|
end
|
104
104
|
else
|
105
|
-
|
105
|
+
_moveParentToWait(parentPrefix, parentId)
|
106
106
|
end
|
107
107
|
else
|
108
|
-
|
108
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
109
109
|
end
|
110
110
|
end
|
111
111
|
end
|
@@ -133,10 +133,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
133
133
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
134
134
|
end
|
135
135
|
else
|
136
|
-
|
136
|
+
_moveParentToWait(parentPrefix, parentId)
|
137
137
|
end
|
138
138
|
else
|
139
|
-
|
139
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
140
140
|
end
|
141
141
|
end
|
142
142
|
end
|