bullmq 5.44.1 → 5.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/async-fifo-queue.js.map +1 -1
- package/dist/cjs/classes/child-processor.js.map +1 -1
- package/dist/cjs/classes/child.js.map +1 -1
- package/dist/cjs/classes/flow-producer.js +15 -4
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job-scheduler.js +1 -1
- package/dist/cjs/classes/job-scheduler.js.map +1 -1
- package/dist/cjs/classes/job.js +137 -54
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-base.js +1 -4
- package/dist/cjs/classes/queue-base.js.map +1 -1
- package/dist/cjs/classes/queue-events-producer.js.map +1 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-getters.js +12 -15
- package/dist/cjs/classes/queue-getters.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/queue.js +12 -13
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/redis-connection.js +1 -1
- package/dist/cjs/classes/redis-connection.js.map +1 -1
- package/dist/cjs/classes/repeat.js +1 -1
- package/dist/cjs/classes/repeat.js.map +1 -1
- package/dist/cjs/classes/scripts.js +18 -7
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +4 -0
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addJobScheduler-11.lua +52 -33
- package/dist/cjs/commands/includes/deduplicateJob.lua +16 -17
- package/dist/cjs/commands/includes/isLocked.lua +1 -0
- package/dist/cjs/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
- package/dist/cjs/commands/includes/moveParentToWait.lua +48 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
- package/dist/cjs/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/cjs/commands/includes/removeJob.lua +2 -2
- package/dist/cjs/commands/includes/removeJobWithChildren.lua +94 -0
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/cjs/commands/includes/storeJob.lua +1 -1
- package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +18 -53
- package/dist/cjs/commands/moveToFinished-14.lua +43 -51
- package/dist/cjs/commands/moveToWaitingChildren-8.lua +5 -32
- package/dist/cjs/commands/removeJob-3.lua +14 -72
- package/dist/cjs/commands/removeUnprocessedChildren-2.lua +31 -0
- package/dist/cjs/commands/updateJobScheduler-12.lua +29 -23
- package/dist/cjs/scripts/addDelayedJob-6.js +73 -55
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +59 -40
- package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +73 -55
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js +73 -55
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +14 -9
- package/dist/cjs/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/cjs/scripts/addStandardJob-8.js +73 -55
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
- package/dist/cjs/scripts/cleanJobsInSet-3.js +14 -9
- package/dist/cjs/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/cjs/scripts/drain-5.js +14 -9
- package/dist/cjs/scripts/drain-5.js.map +1 -1
- package/dist/cjs/scripts/index.js +1 -0
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +157 -155
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +183 -157
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-8.js +145 -137
- package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +14 -9
- package/dist/cjs/scripts/obliterate-2.js.map +1 -1
- package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
- package/dist/cjs/scripts/removeJob-3.js +97 -63
- package/dist/cjs/scripts/removeJob-3.js.map +1 -1
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js +338 -0
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js.map +1 -0
- package/dist/cjs/scripts/updateJobScheduler-12.js +27 -21
- package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/types/index.js +1 -0
- package/dist/cjs/types/index.js.map +1 -1
- package/dist/cjs/types/job-progress.js +3 -0
- package/dist/cjs/types/job-progress.js.map +1 -0
- package/dist/cjs/utils.js +12 -1
- package/dist/cjs/utils.js.map +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/async-fifo-queue.js.map +1 -1
- package/dist/esm/classes/child-processor.js.map +1 -1
- package/dist/esm/classes/child.js.map +1 -1
- package/dist/esm/classes/flow-producer.js +15 -4
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job-scheduler.js +1 -1
- package/dist/esm/classes/job-scheduler.js.map +1 -1
- package/dist/esm/classes/job.d.ts +49 -14
- package/dist/esm/classes/job.js +138 -55
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-base.d.ts +1 -4
- package/dist/esm/classes/queue-base.js +1 -4
- package/dist/esm/classes/queue-base.js.map +1 -1
- package/dist/esm/classes/queue-events-producer.d.ts +2 -2
- package/dist/esm/classes/queue-events-producer.js.map +1 -1
- package/dist/esm/classes/queue-events.d.ts +118 -29
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-getters.d.ts +11 -14
- package/dist/esm/classes/queue-getters.js +12 -15
- package/dist/esm/classes/queue-getters.js.map +1 -1
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +14 -15
- package/dist/esm/classes/queue.js +12 -13
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/redis-connection.js +1 -1
- package/dist/esm/classes/redis-connection.js.map +1 -1
- package/dist/esm/classes/repeat.js +1 -1
- package/dist/esm/classes/repeat.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +4 -3
- package/dist/esm/classes/scripts.js +18 -7
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +2 -1
- package/dist/esm/classes/worker.js +5 -1
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addJobScheduler-11.lua +52 -33
- package/dist/esm/commands/includes/deduplicateJob.lua +16 -17
- package/dist/esm/commands/includes/isLocked.lua +1 -0
- package/dist/esm/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
- package/dist/esm/commands/includes/moveParentToWait.lua +48 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
- package/dist/esm/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/esm/commands/includes/removeJob.lua +2 -2
- package/dist/esm/commands/includes/removeJobWithChildren.lua +94 -0
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/esm/commands/includes/storeJob.lua +1 -1
- package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +18 -53
- package/dist/esm/commands/moveToFinished-14.lua +43 -51
- package/dist/esm/commands/moveToWaitingChildren-8.lua +5 -32
- package/dist/esm/commands/removeJob-3.lua +14 -72
- package/dist/esm/commands/removeUnprocessedChildren-2.lua +31 -0
- package/dist/esm/commands/updateJobScheduler-12.lua +29 -23
- package/dist/esm/interfaces/job-json.d.ts +5 -2
- package/dist/esm/interfaces/minimal-job.d.ts +29 -11
- package/dist/esm/interfaces/queue-options.d.ts +6 -2
- package/dist/esm/interfaces/repeat-options.d.ts +1 -1
- package/dist/esm/interfaces/sandboxed-job.d.ts +2 -2
- package/dist/esm/interfaces/sandboxed-options.d.ts +1 -1
- package/dist/esm/interfaces/telemetry.d.ts +18 -15
- package/dist/esm/interfaces/worker-options.d.ts +10 -10
- package/dist/esm/scripts/addDelayedJob-6.js +73 -55
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addJobScheduler-11.js +59 -40
- package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/esm/scripts/addParentJob-4.js +73 -55
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js +73 -55
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +14 -9
- package/dist/esm/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/esm/scripts/addStandardJob-8.js +73 -55
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
- package/dist/esm/scripts/cleanJobsInSet-3.js +14 -9
- package/dist/esm/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/esm/scripts/drain-5.js +14 -9
- package/dist/esm/scripts/drain-5.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +1 -0
- package/dist/esm/scripts/index.js +1 -0
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +157 -155
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +183 -157
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-8.js +145 -137
- package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +14 -9
- package/dist/esm/scripts/obliterate-2.js.map +1 -1
- package/dist/esm/scripts/removeChildDependency-1.js +5 -5
- package/dist/esm/scripts/removeJob-3.js +97 -63
- package/dist/esm/scripts/removeJob-3.js.map +1 -1
- package/dist/esm/scripts/removeUnprocessedChildren-2.d.ts +5 -0
- package/dist/esm/scripts/removeUnprocessedChildren-2.js +335 -0
- package/dist/esm/scripts/removeUnprocessedChildren-2.js.map +1 -0
- package/dist/esm/scripts/updateJobScheduler-12.js +27 -21
- package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/index.d.ts +1 -0
- package/dist/esm/types/index.js +1 -0
- package/dist/esm/types/index.js.map +1 -1
- package/dist/esm/types/job-options.d.ts +13 -2
- package/dist/esm/types/job-progress.d.ts +1 -0
- package/dist/esm/types/job-progress.js +2 -0
- package/dist/esm/types/job-progress.js.map +1 -0
- package/dist/esm/utils.d.ts +17 -0
- package/dist/esm/utils.js +11 -0
- package/dist/esm/utils.js.map +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +26 -21
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
- package/dist/cjs/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
- package/dist/esm/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
@@ -40,7 +40,16 @@ local jobId = ARGV[4]
|
|
40
40
|
]]
|
41
41
|
-- Includes
|
42
42
|
--[[
|
43
|
-
Validate and move parent to
|
43
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
44
|
+
if no pending dependencies.
|
45
|
+
]]
|
46
|
+
-- Includes
|
47
|
+
--[[
|
48
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
49
|
+
]]
|
50
|
+
-- Includes
|
51
|
+
--[[
|
52
|
+
Move parent to a wait status (wait, prioritized or delayed)
|
44
53
|
]]
|
45
54
|
-- Includes
|
46
55
|
--[[
|
@@ -136,58 +145,55 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
136
145
|
end
|
137
146
|
return waitKey, false
|
138
147
|
end
|
139
|
-
local function
|
140
|
-
|
141
|
-
local
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
local
|
152
|
-
local
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
148
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
149
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
150
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
151
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
152
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
153
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
154
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
155
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
156
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
157
|
+
-- ignore dependencies if any left
|
158
|
+
rcall("HSET", parentKey, "igdp", 1)
|
159
|
+
if delay > 0 then
|
160
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
161
|
+
local score = delayedTimestamp * 0x1000
|
162
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
163
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
164
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
165
|
+
delayedTimestamp)
|
166
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
167
|
+
else
|
168
|
+
if priority == 0 then
|
169
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
170
|
+
parentWaitKey, parentPausedKey)
|
171
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
161
172
|
else
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
parentPausedKey)
|
166
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
167
|
-
parentId)
|
168
|
-
else
|
169
|
-
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
170
|
-
addJobWithPriority(parentMarkerKey,
|
171
|
-
parentQueueKey .. ":prioritized", priority,
|
172
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
173
|
-
end
|
174
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
175
|
-
"jobId", parentId, "prev", "waiting-children")
|
173
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
174
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
175
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
176
176
|
end
|
177
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
178
|
+
"waiting-children")
|
177
179
|
end
|
178
180
|
end
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
local pttl = rcall("PTTL", deduplicationKey)
|
186
|
-
if pttl == 0 or pttl == -1 then
|
187
|
-
rcall("DEL", deduplicationKey)
|
181
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
182
|
+
if rcall("EXISTS", parentKey) == 1 then
|
183
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
184
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
185
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
186
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
188
187
|
end
|
189
188
|
end
|
190
189
|
end
|
190
|
+
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
191
|
+
parentId, timestamp)
|
192
|
+
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
193
|
+
if doNotHavePendingDependencies then
|
194
|
+
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
195
|
+
end
|
196
|
+
end
|
191
197
|
--[[
|
192
198
|
Functions to remove jobs when removeOnFail option is provided.
|
193
199
|
]]
|
@@ -197,13 +203,18 @@ end
|
|
197
203
|
]]
|
198
204
|
-- Includes
|
199
205
|
--[[
|
200
|
-
Function to remove deduplication key
|
206
|
+
Function to remove deduplication key if needed
|
207
|
+
when a job is being removed.
|
201
208
|
]]
|
202
|
-
local function
|
209
|
+
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
210
|
+
jobKey, jobId)
|
203
211
|
local deduplicationId = rcall("HGET", jobKey, "deid")
|
204
212
|
if deduplicationId then
|
205
213
|
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
206
|
-
rcall(
|
214
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
215
|
+
if currentJobId and currentJobId == jobId then
|
216
|
+
return rcall("DEL", deduplicationKey)
|
217
|
+
end
|
207
218
|
end
|
208
219
|
end
|
209
220
|
--[[
|
@@ -229,7 +240,7 @@ end
|
|
229
240
|
local getJobKeyPrefix = function (jobKey, jobId)
|
230
241
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
231
242
|
end
|
232
|
-
local function
|
243
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
233
244
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
234
245
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
235
246
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -257,10 +268,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
257
268
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
258
269
|
end
|
259
270
|
else
|
260
|
-
|
271
|
+
_moveParentToWait(parentPrefix, parentId)
|
261
272
|
end
|
262
273
|
else
|
263
|
-
|
274
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
264
275
|
end
|
265
276
|
end
|
266
277
|
end
|
@@ -288,10 +299,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
288
299
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
289
300
|
end
|
290
301
|
else
|
291
|
-
|
302
|
+
_moveParentToWait(parentPrefix, parentId)
|
292
303
|
end
|
293
304
|
else
|
294
|
-
|
305
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
295
306
|
end
|
296
307
|
end
|
297
308
|
end
|
@@ -305,7 +316,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
305
316
|
local jobKey = baseKey .. jobId
|
306
317
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
307
318
|
if shouldRemoveDeduplicationKey then
|
308
|
-
|
319
|
+
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
|
309
320
|
end
|
310
321
|
removeJobKeys(jobKey)
|
311
322
|
end
|
@@ -358,67 +369,90 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
|
|
358
369
|
end
|
359
370
|
end
|
360
371
|
end
|
361
|
-
local
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
prevState
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
372
|
+
local moveParentToFailedIfNeeded
|
373
|
+
local moveChildFromDependenciesIfNeeded
|
374
|
+
moveParentToFailedIfNeeded = function (parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
375
|
+
if rcall("EXISTS", parentKey) == 1 then
|
376
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
377
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
378
|
+
local parentPrioritizedKey = parentQueueKey .. ":prioritized"
|
379
|
+
local parentWaitingChildrenOrDelayedKey
|
380
|
+
local prevState
|
381
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
382
|
+
parentWaitingChildrenOrDelayedKey = parentWaitingChildrenKey
|
383
|
+
prevState = "waiting-children"
|
384
|
+
elseif rcall("ZSCORE", parentDelayedKey, parentId) then
|
385
|
+
parentWaitingChildrenOrDelayedKey = parentDelayedKey
|
386
|
+
prevState = "delayed"
|
387
|
+
rcall("HSET", parentKey, "delay", 0)
|
388
|
+
end
|
389
|
+
if parentWaitingChildrenOrDelayedKey then
|
390
|
+
rcall("ZREM", parentWaitingChildrenOrDelayedKey, parentId)
|
391
|
+
local parentQueuePrefix = parentQueueKey .. ":"
|
392
|
+
local parentFailedKey = parentQueueKey .. ":failed"
|
393
|
+
local deferredFailure = "child " .. jobIdKey .. " failed"
|
394
|
+
rcall("HSET", parentKey, "defa", deferredFailure)
|
395
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
396
|
+
else
|
397
|
+
if not rcall("ZSCORE", parentQueueKey .. ":failed", parentId) then
|
398
|
+
local deferredFailure = "child " .. jobIdKey .. " failed"
|
399
|
+
rcall("HSET", parentKey, "defa", deferredFailure)
|
400
|
+
end
|
401
|
+
end
|
376
402
|
end
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
local
|
381
|
-
|
382
|
-
local
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
if jobAttributes[1] then
|
389
|
-
local parentData = cjson.decode(jobAttributes[1])
|
390
|
-
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
391
|
-
local grandParentUnsuccesssful = grandParentKey .. ":unsuccessful"
|
392
|
-
rcall("ZADD", grandParentUnsuccesssful, timestamp, parentKey)
|
393
|
-
if parentData['fpof'] then
|
394
|
-
moveParentFromWaitingChildrenToFailed(
|
403
|
+
end
|
404
|
+
moveChildFromDependenciesIfNeeded = function (rawParentData, childKey, failedReason, timestamp)
|
405
|
+
if rawParentData then
|
406
|
+
local parentData = cjson.decode(rawParentData)
|
407
|
+
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
408
|
+
local parentDependenciesChildrenKey = parentKey .. ":dependencies"
|
409
|
+
if parentData['fpof'] then
|
410
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
411
|
+
local parentUnsuccesssfulChildrenKey = parentKey .. ":unsuccessful"
|
412
|
+
rcall("ZADD", parentUnsuccesssfulChildrenKey, timestamp, childKey)
|
413
|
+
moveParentToFailedIfNeeded(
|
395
414
|
parentData['queueKey'],
|
396
|
-
parentData['queueKey'] .. ':' .. parentData['id'],
|
397
|
-
parentData['id'],
|
398
415
|
parentKey,
|
416
|
+
parentData['id'],
|
417
|
+
childKey,
|
399
418
|
timestamp
|
400
419
|
)
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
420
|
+
end
|
421
|
+
elseif parentData['cpof'] then
|
422
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
423
|
+
local parentFailedChildrenKey = parentKey .. ":failed"
|
424
|
+
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
425
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
426
|
+
end
|
427
|
+
elseif parentData['idof'] or parentData['rdof'] then
|
428
|
+
if rcall("SREM", parentDependenciesChildrenKey, childKey) == 1 then
|
429
|
+
moveParentToWaitIfNoPendingDependencies(parentData['queueKey'], parentDependenciesChildrenKey,
|
430
|
+
parentKey, parentData['id'], timestamp)
|
431
|
+
if parentData['idof'] then
|
432
|
+
local parentFailedChildrenKey = parentKey .. ":failed"
|
433
|
+
rcall("HSET", parentFailedChildrenKey, childKey, failedReason)
|
411
434
|
end
|
412
435
|
end
|
413
436
|
end
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
437
|
+
end
|
438
|
+
end
|
439
|
+
--[[
|
440
|
+
Function to remove deduplication key if needed
|
441
|
+
when a job is moved to completed or failed states.
|
442
|
+
]]
|
443
|
+
local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
|
444
|
+
deduplicationId, jobId)
|
445
|
+
if deduplicationId then
|
446
|
+
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
447
|
+
local pttl = rcall("PTTL", deduplicationKey)
|
448
|
+
if pttl == 0 then
|
449
|
+
return rcall("DEL", deduplicationKey)
|
450
|
+
end
|
451
|
+
if pttl == -1 then
|
452
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
453
|
+
if currentJobId and currentJobId == jobId then
|
454
|
+
return rcall("DEL", deduplicationKey)
|
455
|
+
end
|
422
456
|
end
|
423
457
|
end
|
424
458
|
end
|
@@ -441,7 +475,7 @@ local function removeLock(jobKey, stalledKey, token, jobId)
|
|
441
475
|
end
|
442
476
|
return 0
|
443
477
|
end
|
444
|
-
local function moveToWaitingChildren
|
478
|
+
local function moveToWaitingChildren(activeKey, waitingChildrenKey, jobId,
|
445
479
|
timestamp)
|
446
480
|
local score = tonumber(timestamp)
|
447
481
|
local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
|
@@ -455,7 +489,7 @@ if rcall("EXISTS", jobKey) == 1 then
|
|
455
489
|
if rcall("ZCARD", jobUnsuccessfulKey) ~= 0 then
|
456
490
|
-- TODO: refactor this logic in an include later
|
457
491
|
local jobAttributes = rcall("HMGET", jobKey, "parent", "deid", "opts")
|
458
|
-
|
492
|
+
removeDeduplicationKeyIfNeededOnFinalization(ARGV[5], jobAttributes[2], jobId)
|
459
493
|
local failedReason = "children are failed"
|
460
494
|
rcall("ZADD", failedKey, timestamp, jobId)
|
461
495
|
rcall("HSET", jobKey, "finishedOn", timestamp)
|
@@ -464,33 +498,7 @@ if rcall("EXISTS", jobKey) == 1 then
|
|
464
498
|
local rawParentData = jobAttributes[1]
|
465
499
|
local rawOpts = jobAttributes[3]
|
466
500
|
local opts = cjson.decode(rawOpts)
|
467
|
-
|
468
|
-
if opts['fpof'] then
|
469
|
-
local parentData = cjson.decode(rawParentData)
|
470
|
-
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
471
|
-
local parentUnsuccesssful = parentKey .. ":unsuccessful"
|
472
|
-
rcall("ZADD", parentUnsuccesssful, timestamp, jobKey)
|
473
|
-
moveParentFromWaitingChildrenToFailed(
|
474
|
-
parentData['queueKey'],
|
475
|
-
parentData['queueKey'] .. ':' .. parentData['id'],
|
476
|
-
parentData['id'],
|
477
|
-
jobKey,
|
478
|
-
timestamp
|
479
|
-
)
|
480
|
-
elseif opts['idof'] or opts['rdof'] then
|
481
|
-
local parentData = cjson.decode(rawParentData)
|
482
|
-
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
483
|
-
local dependenciesSet = parentKey .. ":dependencies"
|
484
|
-
if rcall("SREM", dependenciesSet, jobKey) == 1 then
|
485
|
-
moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
|
486
|
-
parentKey, parentData['id'], timestamp)
|
487
|
-
if opts['idof'] then
|
488
|
-
local failedSet = parentKey .. ":failed"
|
489
|
-
rcall("HSET", failedSet, jobKey, failedReason)
|
490
|
-
end
|
491
|
-
end
|
492
|
-
end
|
493
|
-
end
|
501
|
+
moveChildFromDependenciesIfNeeded(rawParentData, jobKey, failedReason, timestamp)
|
494
502
|
removeJobsOnFail(ARGV[5], failedKey, jobId, opts, timestamp)
|
495
503
|
return 0
|
496
504
|
else
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveToWaitingChildren-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToWaitingChildren-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveToWaitingChildren-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToWaitingChildren-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2gBf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -28,13 +28,18 @@ local rcall = redis.call
|
|
28
28
|
]]
|
29
29
|
-- Includes
|
30
30
|
--[[
|
31
|
-
Function to remove deduplication key
|
31
|
+
Function to remove deduplication key if needed
|
32
|
+
when a job is being removed.
|
32
33
|
]]
|
33
|
-
local function
|
34
|
+
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
35
|
+
jobKey, jobId)
|
34
36
|
local deduplicationId = rcall("HGET", jobKey, "deid")
|
35
37
|
if deduplicationId then
|
36
38
|
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
37
|
-
rcall(
|
39
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
40
|
+
if currentJobId and currentJobId == jobId then
|
41
|
+
return rcall("DEL", deduplicationKey)
|
42
|
+
end
|
38
43
|
end
|
39
44
|
end
|
40
45
|
--[[
|
@@ -96,7 +101,7 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
96
101
|
end
|
97
102
|
return waitKey, false
|
98
103
|
end
|
99
|
-
local function
|
104
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
100
105
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
101
106
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
102
107
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -124,10 +129,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
124
129
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
125
130
|
end
|
126
131
|
else
|
127
|
-
|
132
|
+
_moveParentToWait(parentPrefix, parentId)
|
128
133
|
end
|
129
134
|
else
|
130
|
-
|
135
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
131
136
|
end
|
132
137
|
end
|
133
138
|
end
|
@@ -155,10 +160,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
155
160
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
156
161
|
end
|
157
162
|
else
|
158
|
-
|
163
|
+
_moveParentToWait(parentPrefix, parentId)
|
159
164
|
end
|
160
165
|
else
|
161
|
-
|
166
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
162
167
|
end
|
163
168
|
end
|
164
169
|
end
|
@@ -172,7 +177,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
172
177
|
local jobKey = baseKey .. jobId
|
173
178
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
174
179
|
if shouldRemoveDeduplicationKey then
|
175
|
-
|
180
|
+
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
|
176
181
|
end
|
177
182
|
removeJobKeys(jobKey)
|
178
183
|
end
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"obliterate-2.js","sourceRoot":"","sources":["../../../src/scripts/obliterate-2.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"obliterate-2.js","sourceRoot":"","sources":["../../../src/scripts/obliterate-2.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Uf,CAAC;AACW,QAAA,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -77,7 +77,7 @@ local function removeJobKeys(jobKey)
|
|
77
77
|
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
78
78
|
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
79
79
|
end
|
80
|
-
local function
|
80
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
81
81
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
82
82
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
83
83
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -105,10 +105,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
105
105
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
106
106
|
end
|
107
107
|
else
|
108
|
-
|
108
|
+
_moveParentToWait(parentPrefix, parentId)
|
109
109
|
end
|
110
110
|
else
|
111
|
-
|
111
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
112
112
|
end
|
113
113
|
end
|
114
114
|
end
|
@@ -136,10 +136,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
136
136
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
137
137
|
end
|
138
138
|
else
|
139
|
-
|
139
|
+
_moveParentToWait(parentPrefix, parentId)
|
140
140
|
end
|
141
141
|
else
|
142
|
-
|
142
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
143
143
|
end
|
144
144
|
end
|
145
145
|
end
|