bullmq 5.49.1 → 5.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/job.js +23 -18
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue.js +0 -1
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/worker.js +4 -0
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/includes/moveChildFromDependenciesIfNeeded.lua +15 -32
- package/dist/cjs/commands/includes/moveParentToWait.lua +32 -33
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +8 -4
- package/dist/cjs/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/cjs/commands/includes/removeJob.lua +2 -2
- package/dist/cjs/commands/includes/removeJobWithChildren.lua +2 -2
- package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +2 -4
- package/dist/cjs/commands/moveToFinished-14.lua +7 -5
- package/dist/cjs/commands/moveToWaitingChildren-8.lua +2 -4
- package/dist/cjs/scripts/addDelayedJob-6.js +51 -37
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +9 -4
- package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +51 -37
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js +51 -37
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +9 -4
- package/dist/cjs/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/cjs/scripts/addStandardJob-8.js +51 -37
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
- package/dist/cjs/scripts/cleanJobsInSet-3.js +9 -4
- package/dist/cjs/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/cjs/scripts/drain-5.js +9 -4
- package/dist/cjs/scripts/drain-5.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +92 -76
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +97 -78
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-8.js +92 -76
- package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +9 -4
- package/dist/cjs/scripts/obliterate-2.js.map +1 -1
- package/dist/cjs/scripts/removeJob-3.js +9 -4
- package/dist/cjs/scripts/removeJob-3.js.map +1 -1
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js +9 -4
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/utils.js +11 -1
- package/dist/cjs/utils.js.map +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/job.d.ts +21 -4
- package/dist/esm/classes/job.js +24 -19
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +0 -1
- package/dist/esm/classes/queue.js +0 -1
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/worker.js +5 -1
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/includes/moveChildFromDependenciesIfNeeded.lua +15 -32
- package/dist/esm/commands/includes/moveParentToWait.lua +32 -33
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +8 -4
- package/dist/esm/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/esm/commands/includes/removeJob.lua +2 -2
- package/dist/esm/commands/includes/removeJobWithChildren.lua +2 -2
- package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +2 -4
- package/dist/esm/commands/moveToFinished-14.lua +7 -5
- package/dist/esm/commands/moveToWaitingChildren-8.lua +2 -4
- package/dist/esm/interfaces/job-json.d.ts +1 -0
- package/dist/esm/scripts/addDelayedJob-6.js +51 -37
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addJobScheduler-11.js +9 -4
- package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/esm/scripts/addParentJob-4.js +51 -37
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js +51 -37
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +9 -4
- package/dist/esm/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/esm/scripts/addStandardJob-8.js +51 -37
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
- package/dist/esm/scripts/cleanJobsInSet-3.js +9 -4
- package/dist/esm/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/esm/scripts/drain-5.js +9 -4
- package/dist/esm/scripts/drain-5.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +92 -76
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +97 -78
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-8.js +92 -76
- package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +9 -4
- package/dist/esm/scripts/obliterate-2.js.map +1 -1
- package/dist/esm/scripts/removeJob-3.js +9 -4
- package/dist/esm/scripts/removeJob-3.js.map +1 -1
- package/dist/esm/scripts/removeUnprocessedChildren-2.js +9 -4
- package/dist/esm/scripts/removeUnprocessedChildren-2.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/utils.d.ts +17 -0
- package/dist/esm/utils.js +10 -0
- package/dist/esm/utils.js.map +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
- package/dist/esm/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
@@ -1,10 +1,14 @@
|
|
1
1
|
--[[
|
2
2
|
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
3
3
|
]]
|
4
|
+
-- Includes
|
4
5
|
--- @include "moveParentToWait"
|
5
|
-
local function moveParentToWaitIfNeeded(parentQueueKey,
|
6
|
-
|
7
|
-
|
8
|
-
|
6
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
7
|
+
if rcall("EXISTS", parentKey) == 1 then
|
8
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
9
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
10
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
11
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
9
12
|
end
|
13
|
+
end
|
10
14
|
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
--[[
|
2
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
3
|
+
if no pending dependencies.
|
4
|
+
]]
|
5
|
+
-- Includes
|
6
|
+
--- @include "moveParentToWaitIfNeeded"
|
7
|
+
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
8
|
+
parentId, timestamp)
|
9
|
+
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
10
|
+
if doNotHavePendingDependencies then
|
11
|
+
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
--[[
|
2
|
+
Function to remove deduplication key if needed
|
3
|
+
when a job is moved to completed or failed states.
|
4
|
+
]]
|
5
|
+
|
6
|
+
local function removeDeduplicationKeyIfNeededOnFinalization(prefixKey,
|
7
|
+
deduplicationId, jobId)
|
8
|
+
if deduplicationId then
|
9
|
+
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
10
|
+
local pttl = rcall("PTTL", deduplicationKey)
|
11
|
+
|
12
|
+
if pttl == 0 then
|
13
|
+
return rcall("DEL", deduplicationKey)
|
14
|
+
end
|
15
|
+
|
16
|
+
if pttl == -1 then
|
17
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
18
|
+
if currentJobId and currentJobId == jobId then
|
19
|
+
return rcall("DEL", deduplicationKey)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
--[[
|
2
|
+
Function to remove deduplication key if needed
|
3
|
+
when a job is being removed.
|
4
|
+
]]
|
5
|
+
|
6
|
+
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
7
|
+
jobKey, jobId)
|
8
|
+
local deduplicationId = rcall("HGET", jobKey, "deid")
|
9
|
+
if deduplicationId then
|
10
|
+
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
11
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
12
|
+
if currentJobId and currentJobId == jobId then
|
13
|
+
return rcall("DEL", deduplicationKey)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -3,7 +3,7 @@
|
|
3
3
|
]]
|
4
4
|
|
5
5
|
-- Includes
|
6
|
-
--- @include "
|
6
|
+
--- @include "removeDeduplicationKeyIfNeededOnRemoval"
|
7
7
|
--- @include "removeJobKeys"
|
8
8
|
--- @include "removeParentDependencyKey"
|
9
9
|
|
@@ -11,7 +11,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
11
11
|
local jobKey = baseKey .. jobId
|
12
12
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
13
13
|
if shouldRemoveDeduplicationKey then
|
14
|
-
|
14
|
+
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
|
15
15
|
end
|
16
16
|
removeJobKeys(jobKey)
|
17
17
|
end
|
@@ -12,7 +12,7 @@ local rcall = redis.call
|
|
12
12
|
--- @include "destructureJobKey"
|
13
13
|
--- @include "getOrSetMaxEvents"
|
14
14
|
--- @include "isJobSchedulerJob"
|
15
|
-
--- @include "
|
15
|
+
--- @include "removeDeduplicationKeyIfNeededOnRemoval"
|
16
16
|
--- @include "removeJobFromAnyState"
|
17
17
|
--- @include "removeJobKeys"
|
18
18
|
--- @include "removeParentDependencyKey"
|
@@ -84,7 +84,7 @@ removeJobWithChildren = function(prefix, meta, jobId, parentKey, options)
|
|
84
84
|
end
|
85
85
|
|
86
86
|
local prev = removeJobFromAnyState(prefix, jobId)
|
87
|
-
|
87
|
+
removeDeduplicationKeyIfNeededOnRemoval(prefix, jobKey, jobId)
|
88
88
|
if removeJobKeys(jobKey) > 0 then
|
89
89
|
local maxEvents = getOrSetMaxEvents(meta)
|
90
90
|
rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
|
@@ -3,11 +3,11 @@
|
|
3
3
|
]]
|
4
4
|
|
5
5
|
-- Includes
|
6
|
-
--- @include "
|
6
|
+
--- @include "moveParentToWaitIfNoPendingDependencies"
|
7
7
|
|
8
8
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
9
9
|
parentId, jobIdKey, returnvalue, timestamp )
|
10
10
|
local processedSet = parentKey .. ":processed"
|
11
11
|
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
12
|
-
|
12
|
+
moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
13
13
|
end
|
@@ -27,9 +27,7 @@ local rcall = redis.call
|
|
27
27
|
--- @include "includes/batches"
|
28
28
|
--- @include "includes/getTargetQueueList"
|
29
29
|
--- @include "includes/moveChildFromDependenciesIfNeeded"
|
30
|
-
--- @include "includes/
|
31
|
-
--- @include "includes/moveParentToWait"
|
32
|
-
--- @include "includes/removeDeduplicationKeyIfNeeded"
|
30
|
+
--- @include "includes/removeDeduplicationKeyIfNeededOnFinalization"
|
33
31
|
--- @include "includes/removeJobsOnFail"
|
34
32
|
--- @include "includes/trimEvents"
|
35
33
|
|
@@ -86,7 +84,7 @@ if (#stalling > 0) then
|
|
86
84
|
local rawParentData = jobAttributes[2]
|
87
85
|
local opts = cjson.decode(rawOpts)
|
88
86
|
rcall("ZADD", failedKey, timestamp, jobId)
|
89
|
-
|
87
|
+
removeDeduplicationKeyIfNeededOnFinalization(queueKeyPrefix, jobAttributes[3], jobId)
|
90
88
|
|
91
89
|
local failedReason = "job stalled more than allowable limit"
|
92
90
|
rcall("HMSET", jobKey, "failedReason", failedReason, "finishedOn", timestamp)
|
@@ -65,11 +65,9 @@ local rcall = redis.call
|
|
65
65
|
--- @include "includes/getTargetQueueList"
|
66
66
|
--- @include "includes/moveJobFromPriorityToActive"
|
67
67
|
--- @include "includes/moveChildFromDependenciesIfNeeded"
|
68
|
-
--- @include "includes/moveParentToWait"
|
69
|
-
--- @include "includes/moveParentToWaitIfNeeded"
|
70
68
|
--- @include "includes/prepareJobForProcessing"
|
71
69
|
--- @include "includes/promoteDelayedJobs"
|
72
|
-
--- @include "includes/
|
70
|
+
--- @include "includes/removeDeduplicationKeyIfNeededOnFinalization"
|
73
71
|
--- @include "includes/removeJobKeys"
|
74
72
|
--- @include "includes/removeJobsByMaxAge"
|
75
73
|
--- @include "includes/removeJobsByMaxCount"
|
@@ -132,7 +130,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
132
130
|
|
133
131
|
local prefix = ARGV[7]
|
134
132
|
|
135
|
-
|
133
|
+
removeDeduplicationKeyIfNeededOnFinalization(prefix, jobAttributes[3], jobId)
|
136
134
|
|
137
135
|
-- If job has a parent we need to
|
138
136
|
-- 1) remove this job id from parents dependencies
|
@@ -163,9 +161,13 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
|
|
163
161
|
local targetSet = KEYS[11]
|
164
162
|
-- Add to complete/failed set
|
165
163
|
rcall("ZADD", targetSet, timestamp, jobId)
|
166
|
-
rcall("
|
164
|
+
rcall("HSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
|
167
165
|
-- "returnvalue" / "failedReason" and "finishedOn"
|
168
166
|
|
167
|
+
if ARGV[5] == "failed" then
|
168
|
+
rcall("HDEL", jobIdKey, "defa")
|
169
|
+
end
|
170
|
+
|
169
171
|
-- Remove old jobs?
|
170
172
|
if maxAge ~= nil then
|
171
173
|
removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
|
@@ -37,9 +37,7 @@ local jobId = ARGV[4]
|
|
37
37
|
|
38
38
|
--- Includes
|
39
39
|
--- @include "includes/moveChildFromDependenciesIfNeeded"
|
40
|
-
--- @include "includes/
|
41
|
-
--- @include "includes/moveParentToWaitIfNeeded"
|
42
|
-
--- @include "includes/removeDeduplicationKeyIfNeeded"
|
40
|
+
--- @include "includes/removeDeduplicationKeyIfNeededOnFinalization"
|
43
41
|
--- @include "includes/removeJobsOnFail"
|
44
42
|
--- @include "includes/removeLock"
|
45
43
|
|
@@ -63,7 +61,7 @@ if rcall("EXISTS", jobKey) == 1 then
|
|
63
61
|
-- TODO: refactor this logic in an include later
|
64
62
|
local jobAttributes = rcall("HMGET", jobKey, "parent", "deid", "opts")
|
65
63
|
|
66
|
-
|
64
|
+
removeDeduplicationKeyIfNeededOnFinalization(ARGV[5], jobAttributes[2], jobId)
|
67
65
|
|
68
66
|
local failedReason = "children are failed"
|
69
67
|
rcall("ZADD", failedKey, timestamp, jobId)
|
@@ -158,12 +158,19 @@ end
|
|
158
158
|
Validate and move or add dependencies to parent.
|
159
159
|
]]
|
160
160
|
-- Includes
|
161
|
+
--[[
|
162
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
163
|
+
if no pending dependencies.
|
164
|
+
]]
|
165
|
+
-- Includes
|
161
166
|
--[[
|
162
167
|
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
163
168
|
]]
|
169
|
+
-- Includes
|
164
170
|
--[[
|
165
|
-
|
171
|
+
Move parent to a wait status (wait, prioritized or delayed)
|
166
172
|
]]
|
173
|
+
-- Includes
|
167
174
|
--[[
|
168
175
|
Function to add job in target list and add marker if needed.
|
169
176
|
]]
|
@@ -234,52 +241,59 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
234
241
|
return waitKey, false
|
235
242
|
end
|
236
243
|
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
237
|
-
local
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
244
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
245
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
246
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
247
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
248
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
249
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
250
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
251
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
252
|
+
-- ignore dependencies if any left
|
253
|
+
rcall("HSET", parentKey, "igdp", 1)
|
254
|
+
if delay > 0 then
|
255
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
256
|
+
local score = delayedTimestamp * 0x1000
|
257
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
258
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
259
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
260
|
+
delayedTimestamp)
|
261
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
262
|
+
else
|
263
|
+
if priority == 0 then
|
264
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
265
|
+
parentWaitKey, parentPausedKey)
|
266
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
257
267
|
else
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
262
|
-
else
|
263
|
-
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
264
|
-
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
265
|
-
parentQueueKey .. ":pc", isPausedOrMaxed)
|
266
|
-
end
|
267
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
268
|
-
"waiting-children")
|
268
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
269
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
270
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
269
271
|
end
|
272
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
273
|
+
"waiting-children")
|
270
274
|
end
|
271
275
|
end
|
272
|
-
local function moveParentToWaitIfNeeded(parentQueueKey,
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
277
|
+
if rcall("EXISTS", parentKey) == 1 then
|
278
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
279
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
280
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
281
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
276
282
|
end
|
283
|
+
end
|
284
|
+
end
|
285
|
+
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
286
|
+
parentId, timestamp)
|
287
|
+
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
288
|
+
if doNotHavePendingDependencies then
|
289
|
+
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
290
|
+
end
|
277
291
|
end
|
278
292
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
279
293
|
parentId, jobIdKey, returnvalue, timestamp )
|
280
294
|
local processedSet = parentKey .. ":processed"
|
281
295
|
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
282
|
-
|
296
|
+
moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
283
297
|
end
|
284
298
|
local function updateExistingJobsParent(parentKey, parent, parentData,
|
285
299
|
parentDependenciesKey, completedKey,
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2Yf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -238,13 +238,18 @@ end
|
|
238
238
|
]]
|
239
239
|
-- Includes
|
240
240
|
--[[
|
241
|
-
Function to remove deduplication key
|
241
|
+
Function to remove deduplication key if needed
|
242
|
+
when a job is being removed.
|
242
243
|
]]
|
243
|
-
local function
|
244
|
+
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
245
|
+
jobKey, jobId)
|
244
246
|
local deduplicationId = rcall("HGET", jobKey, "deid")
|
245
247
|
if deduplicationId then
|
246
248
|
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
247
|
-
rcall(
|
249
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
250
|
+
if currentJobId and currentJobId == jobId then
|
251
|
+
return rcall("DEL", deduplicationKey)
|
252
|
+
end
|
248
253
|
end
|
249
254
|
end
|
250
255
|
--[[
|
@@ -346,7 +351,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
346
351
|
local jobKey = baseKey .. jobId
|
347
352
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
348
353
|
if shouldRemoveDeduplicationKey then
|
349
|
-
|
354
|
+
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
|
350
355
|
end
|
351
356
|
removeJobKeys(jobKey)
|
352
357
|
end
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAucf,CAAC;AACF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|
@@ -90,12 +90,19 @@ end
|
|
90
90
|
Validate and move or add dependencies to parent.
|
91
91
|
]]
|
92
92
|
-- Includes
|
93
|
+
--[[
|
94
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
95
|
+
if no pending dependencies.
|
96
|
+
]]
|
97
|
+
-- Includes
|
93
98
|
--[[
|
94
99
|
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
95
100
|
]]
|
101
|
+
-- Includes
|
96
102
|
--[[
|
97
|
-
|
103
|
+
Move parent to a wait status (wait, prioritized or delayed)
|
98
104
|
]]
|
105
|
+
-- Includes
|
99
106
|
--[[
|
100
107
|
Add delay marker if needed.
|
101
108
|
]]
|
@@ -190,52 +197,59 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
190
197
|
return waitKey, false
|
191
198
|
end
|
192
199
|
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
193
|
-
local
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
200
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
201
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
202
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
203
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
204
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
205
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
206
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
207
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
208
|
+
-- ignore dependencies if any left
|
209
|
+
rcall("HSET", parentKey, "igdp", 1)
|
210
|
+
if delay > 0 then
|
211
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
212
|
+
local score = delayedTimestamp * 0x1000
|
213
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
214
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
215
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
216
|
+
delayedTimestamp)
|
217
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
218
|
+
else
|
219
|
+
if priority == 0 then
|
220
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
221
|
+
parentWaitKey, parentPausedKey)
|
222
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
213
223
|
else
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
218
|
-
else
|
219
|
-
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
220
|
-
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
221
|
-
parentQueueKey .. ":pc", isPausedOrMaxed)
|
222
|
-
end
|
223
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
224
|
-
"waiting-children")
|
224
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
225
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
226
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
225
227
|
end
|
228
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
229
|
+
"waiting-children")
|
226
230
|
end
|
227
231
|
end
|
228
|
-
local function moveParentToWaitIfNeeded(parentQueueKey,
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
233
|
+
if rcall("EXISTS", parentKey) == 1 then
|
234
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
235
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
236
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
237
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
232
238
|
end
|
239
|
+
end
|
240
|
+
end
|
241
|
+
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
242
|
+
parentId, timestamp)
|
243
|
+
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
244
|
+
if doNotHavePendingDependencies then
|
245
|
+
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
246
|
+
end
|
233
247
|
end
|
234
248
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
235
249
|
parentId, jobIdKey, returnvalue, timestamp )
|
236
250
|
local processedSet = parentKey .. ":processed"
|
237
251
|
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
238
|
-
|
252
|
+
moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
239
253
|
end
|
240
254
|
local function updateExistingJobsParent(parentKey, parent, parentData,
|
241
255
|
parentDependenciesKey, completedKey,
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkWf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -152,12 +152,19 @@ end
|
|
152
152
|
Validate and move or add dependencies to parent.
|
153
153
|
]]
|
154
154
|
-- Includes
|
155
|
+
--[[
|
156
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
157
|
+
if no pending dependencies.
|
158
|
+
]]
|
159
|
+
-- Includes
|
155
160
|
--[[
|
156
161
|
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
157
162
|
]]
|
163
|
+
-- Includes
|
158
164
|
--[[
|
159
|
-
|
165
|
+
Move parent to a wait status (wait, prioritized or delayed)
|
160
166
|
]]
|
167
|
+
-- Includes
|
161
168
|
--[[
|
162
169
|
Add delay marker if needed.
|
163
170
|
]]
|
@@ -227,52 +234,59 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
227
234
|
return waitKey, false
|
228
235
|
end
|
229
236
|
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
230
|
-
local
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
237
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
238
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
239
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
240
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
241
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
242
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
243
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
244
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
245
|
+
-- ignore dependencies if any left
|
246
|
+
rcall("HSET", parentKey, "igdp", 1)
|
247
|
+
if delay > 0 then
|
248
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
249
|
+
local score = delayedTimestamp * 0x1000
|
250
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
251
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
252
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
253
|
+
delayedTimestamp)
|
254
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
255
|
+
else
|
256
|
+
if priority == 0 then
|
257
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
258
|
+
parentWaitKey, parentPausedKey)
|
259
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
250
260
|
else
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
255
|
-
else
|
256
|
-
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
257
|
-
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
258
|
-
parentQueueKey .. ":pc", isPausedOrMaxed)
|
259
|
-
end
|
260
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
261
|
-
"waiting-children")
|
261
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
262
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
263
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
262
264
|
end
|
265
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
266
|
+
"waiting-children")
|
263
267
|
end
|
264
268
|
end
|
265
|
-
local function moveParentToWaitIfNeeded(parentQueueKey,
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
270
|
+
if rcall("EXISTS", parentKey) == 1 then
|
271
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
272
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
273
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
274
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
269
275
|
end
|
276
|
+
end
|
277
|
+
end
|
278
|
+
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
279
|
+
parentId, timestamp)
|
280
|
+
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
281
|
+
if doNotHavePendingDependencies then
|
282
|
+
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
283
|
+
end
|
270
284
|
end
|
271
285
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
272
286
|
parentId, jobIdKey, returnvalue, timestamp )
|
273
287
|
local processedSet = parentKey .. ":processed"
|
274
288
|
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
275
|
-
|
289
|
+
moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
276
290
|
end
|
277
291
|
local function updateExistingJobsParent(parentKey, parent, parentData,
|
278
292
|
parentDependenciesKey, completedKey,
|