bullmq 4.16.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/flow-producer.js +3 -10
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job.js +21 -11
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-base.js +2 -9
- package/dist/cjs/classes/queue-base.js.map +1 -1
- package/dist/cjs/classes/queue-events.js +3 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js +1 -0
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/redis-connection.js +9 -7
- package/dist/cjs/classes/redis-connection.js.map +1 -1
- package/dist/cjs/classes/scripts.js +34 -34
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +41 -37
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
- package/dist/cjs/commands/addParentJob-4.lua +0 -2
- package/dist/{esm/commands/addPrioritizedJob-8.lua → cjs/commands/addPrioritizedJob-7.lua} +18 -24
- package/dist/cjs/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
- package/dist/cjs/commands/changePriority-6.lua +57 -0
- package/dist/cjs/commands/getCounts-1.lua +1 -0
- package/dist/cjs/commands/getRanges-1.lua +1 -0
- package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
- package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -7
- package/dist/cjs/commands/includes/checkStalledJobs.lua +1 -1
- package/dist/cjs/commands/includes/isQueuePaused.lua +7 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
- package/dist/cjs/commands/includes/prepareJobForProcessing.lua +1 -1
- package/dist/cjs/commands/includes/promoteDelayedJobs.lua +7 -4
- package/dist/{esm/commands/moveToActive-10.lua → cjs/commands/moveToActive-11.lua} +18 -29
- package/dist/cjs/commands/moveToDelayed-7.lua +77 -0
- package/dist/cjs/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
- package/dist/cjs/commands/moveToWaitingChildren-4.lua +7 -4
- package/dist/cjs/commands/pause-7.lua +41 -0
- package/dist/cjs/commands/promote-8.lua +60 -0
- package/dist/cjs/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
- package/dist/cjs/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
- package/dist/cjs/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +51 -61
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
- package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -0
- package/dist/cjs/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
- package/dist/cjs/scripts/addStandardJob-7.js.map +1 -0
- package/dist/cjs/scripts/changePriority-6.js +85 -0
- package/dist/cjs/scripts/changePriority-6.js.map +1 -0
- package/dist/cjs/scripts/getCounts-1.js +1 -0
- package/dist/cjs/scripts/getCounts-1.js.map +1 -1
- package/dist/cjs/scripts/getRanges-1.js +1 -0
- package/dist/cjs/scripts/getRanges-1.js.map +1 -1
- package/dist/cjs/scripts/index.js +10 -10
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js +1 -0
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/cjs/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
- package/dist/cjs/scripts/moveToActive-11.js.map +1 -0
- package/dist/cjs/scripts/moveToDelayed-7.js +103 -0
- package/dist/cjs/scripts/moveToDelayed-7.js.map +1 -0
- package/dist/cjs/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
- package/dist/cjs/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-4.js +6 -3
- package/dist/cjs/scripts/moveToWaitingChildren-4.js.map +1 -1
- package/dist/cjs/scripts/pause-7.js +67 -0
- package/dist/cjs/scripts/pause-7.js.map +1 -0
- package/dist/cjs/scripts/promote-8.js +79 -0
- package/dist/cjs/scripts/promote-8.js.map +1 -0
- package/dist/cjs/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
- package/dist/cjs/scripts/retryJob-10.js.map +1 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/esm/classes/flow-producer.js +3 -10
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job.d.ts +5 -0
- package/dist/esm/classes/job.js +21 -11
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-base.js +2 -9
- package/dist/esm/classes/queue-base.js.map +1 -1
- package/dist/esm/classes/queue-events.js +3 -1
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-keys.js +1 -0
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/redis-connection.js +9 -7
- package/dist/esm/classes/redis-connection.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +4 -4
- package/dist/esm/classes/scripts.js +34 -34
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +4 -2
- package/dist/esm/classes/worker.js +41 -37
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
- package/dist/esm/commands/addParentJob-4.lua +0 -2
- package/dist/{cjs/commands/addPrioritizedJob-8.lua → esm/commands/addPrioritizedJob-7.lua} +18 -24
- package/dist/esm/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
- package/dist/esm/commands/changePriority-6.lua +57 -0
- package/dist/esm/commands/getCounts-1.lua +1 -0
- package/dist/esm/commands/getRanges-1.lua +1 -0
- package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
- package/dist/esm/commands/includes/addJobWithPriority.lua +3 -7
- package/dist/esm/commands/includes/checkStalledJobs.lua +1 -1
- package/dist/esm/commands/includes/isQueuePaused.lua +7 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
- package/dist/esm/commands/includes/prepareJobForProcessing.lua +1 -1
- package/dist/esm/commands/includes/promoteDelayedJobs.lua +7 -4
- package/dist/{cjs/commands/moveToActive-10.lua → esm/commands/moveToActive-11.lua} +18 -29
- package/dist/esm/commands/moveToDelayed-7.lua +77 -0
- package/dist/esm/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
- package/dist/esm/commands/moveToWaitingChildren-4.lua +7 -4
- package/dist/esm/commands/pause-7.lua +41 -0
- package/dist/esm/commands/promote-8.lua +60 -0
- package/dist/esm/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
- package/dist/esm/interfaces/job-json.d.ts +4 -1
- package/dist/esm/interfaces/minimal-job.d.ts +3 -0
- package/dist/esm/interfaces/queue-options.d.ts +1 -1
- package/dist/esm/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
- package/dist/esm/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
- package/dist/esm/scripts/addParentJob-4.js +51 -61
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
- package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -0
- package/dist/esm/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
- package/dist/esm/scripts/addStandardJob-7.js.map +1 -0
- package/dist/esm/scripts/changePriority-6.js +82 -0
- package/dist/esm/scripts/changePriority-6.js.map +1 -0
- package/dist/esm/scripts/getCounts-1.js +1 -0
- package/dist/esm/scripts/getCounts-1.js.map +1 -1
- package/dist/esm/scripts/getRanges-1.js +1 -0
- package/dist/esm/scripts/getRanges-1.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +10 -10
- package/dist/esm/scripts/index.js +10 -10
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-8.js +1 -0
- package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/esm/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
- package/dist/esm/scripts/moveToActive-11.js.map +1 -0
- package/dist/esm/scripts/moveToDelayed-7.js +100 -0
- package/dist/esm/scripts/moveToDelayed-7.js.map +1 -0
- package/dist/esm/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
- package/dist/esm/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-4.js +6 -3
- package/dist/esm/scripts/moveToWaitingChildren-4.js.map +1 -1
- package/dist/esm/scripts/pause-7.js +64 -0
- package/dist/esm/scripts/pause-7.js.map +1 -0
- package/dist/esm/scripts/promote-8.js +76 -0
- package/dist/esm/scripts/promote-8.js.map +1 -0
- package/dist/esm/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
- package/dist/esm/scripts/retryJob-10.js.map +1 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/changePriority-5.lua +0 -52
- package/dist/cjs/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
- package/dist/cjs/commands/moveToDelayed-8.lua +0 -72
- package/dist/cjs/commands/pause-5.lua +0 -36
- package/dist/cjs/commands/promote-7.lua +0 -57
- package/dist/cjs/scripts/addDelayedJob-7.js.map +0 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +0 -1
- package/dist/cjs/scripts/addStandardJob-6.js.map +0 -1
- package/dist/cjs/scripts/changePriority-5.js +0 -87
- package/dist/cjs/scripts/changePriority-5.js.map +0 -1
- package/dist/cjs/scripts/moveToDelayed-8.js +0 -168
- package/dist/cjs/scripts/moveToDelayed-8.js.map +0 -1
- package/dist/cjs/scripts/pause-5.js +0 -48
- package/dist/cjs/scripts/pause-5.js.map +0 -1
- package/dist/cjs/scripts/promote-7.js +0 -87
- package/dist/cjs/scripts/promote-7.js.map +0 -1
- package/dist/cjs/scripts/retryJob-9.js.map +0 -1
- package/dist/esm/commands/changePriority-5.lua +0 -52
- package/dist/esm/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
- package/dist/esm/commands/moveToDelayed-8.lua +0 -72
- package/dist/esm/commands/pause-5.lua +0 -36
- package/dist/esm/commands/promote-7.lua +0 -57
- package/dist/esm/scripts/addDelayedJob-7.js.map +0 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +0 -1
- package/dist/esm/scripts/addStandardJob-6.js.map +0 -1
- package/dist/esm/scripts/changePriority-5.js +0 -84
- package/dist/esm/scripts/changePriority-5.js.map +0 -1
- package/dist/esm/scripts/moveToDelayed-8.js +0 -165
- package/dist/esm/scripts/moveToDelayed-8.js.map +0 -1
- package/dist/esm/scripts/pause-5.js +0 -45
- package/dist/esm/scripts/pause-5.js.map +0 -1
- package/dist/esm/scripts/promote-7.js +0 -84
- package/dist/esm/scripts/promote-7.js.map +0 -1
- package/dist/esm/scripts/retryJob-9.js.map +0 -1
- /package/dist/esm/scripts/{addDelayedJob-7.d.ts → addDelayedJob-6.d.ts} +0 -0
- /package/dist/esm/scripts/{addPrioritizedJob-8.d.ts → addPrioritizedJob-7.d.ts} +0 -0
- /package/dist/esm/scripts/{addStandardJob-6.d.ts → addStandardJob-7.d.ts} +0 -0
- /package/dist/esm/scripts/{changePriority-5.d.ts → changePriority-6.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToActive-10.d.ts → moveToActive-11.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToDelayed-8.d.ts → moveToDelayed-7.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToFinished-13.d.ts → moveToFinished-14.d.ts} +0 -0
- /package/dist/esm/scripts/{pause-5.d.ts → pause-7.d.ts} +0 -0
- /package/dist/esm/scripts/{promote-7.d.ts → promote-8.d.ts} +0 -0
- /package/dist/esm/scripts/{retryJob-9.d.ts → retryJob-10.d.ts} +0 -0
@@ -74,7 +74,7 @@ end
|
|
74
74
|
-- Includes
|
75
75
|
--[[
|
76
76
|
Add delay marker if needed.
|
77
|
-
]]
|
77
|
+
]]
|
78
78
|
-- Includes
|
79
79
|
--[[
|
80
80
|
Function to return the next delayed job timestamp.
|
@@ -89,47 +89,30 @@ local function getNextDelayedTimestamp(delayedKey)
|
|
89
89
|
return nextTimestamp
|
90
90
|
end
|
91
91
|
end
|
92
|
-
local function addDelayMarkerIfNeeded(
|
93
|
-
local waitLen = rcall("LLEN", targetKey)
|
94
|
-
if waitLen <= 1 then
|
92
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
95
93
|
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
96
94
|
if nextTimestamp ~= nil then
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
local marker = rcall("LINDEX", targetKey, 0)
|
101
|
-
local oldTimestamp = tonumber(marker:sub(3))
|
102
|
-
if oldTimestamp and oldTimestamp > nextTimestamp then
|
103
|
-
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
104
|
-
end
|
105
|
-
else
|
106
|
-
-- if there is no marker, then we need to add one
|
107
|
-
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
108
|
-
end
|
95
|
+
-- Replace the score of the marker with the newest known
|
96
|
+
-- next timestamp.
|
97
|
+
rcall("ZADD", markerKey, nextTimestamp, "0")
|
109
98
|
end
|
110
|
-
end
|
111
99
|
end
|
112
100
|
--[[
|
113
|
-
Function to
|
101
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
102
|
+
(since an empty list and !EXISTS are not really the same).
|
114
103
|
]]
|
115
|
-
|
104
|
+
local function isQueuePaused(queueMetaKey)
|
105
|
+
return rcall("HEXISTS", queueMetaKey, "paused") == 1
|
106
|
+
end
|
116
107
|
--[[
|
117
|
-
Function
|
118
|
-
in order to wake up our workers and to respect priority
|
119
|
-
order as much as possible
|
108
|
+
Function to add job considering priority.
|
120
109
|
]]
|
121
|
-
local function
|
122
|
-
local waitLen = rcall("LLEN", waitKey)
|
123
|
-
if waitLen == 0 then
|
124
|
-
rcall("LPUSH", waitKey, "0:0")
|
125
|
-
end
|
126
|
-
end
|
127
|
-
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
110
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
128
111
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
129
112
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
130
113
|
rcall("ZADD", prioritizedKey, score, jobId)
|
131
|
-
if not
|
132
|
-
|
114
|
+
if not isPaused then
|
115
|
+
rcall("ZADD", markerKey, 0, "0")
|
133
116
|
end
|
134
117
|
end
|
135
118
|
--[[
|
@@ -143,35 +126,44 @@ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
|
143
126
|
return pausedKey, true
|
144
127
|
end
|
145
128
|
end
|
146
|
-
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
129
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
130
|
+
parentKey, parentId, timestamp)
|
131
|
+
local isParentActive = rcall("ZSCORE",
|
132
|
+
parentQueueKey .. ":waiting-children", parentId)
|
133
|
+
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
134
|
+
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
135
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
136
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
137
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
138
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
139
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
140
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
141
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
142
|
+
if delay > 0 then
|
143
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
144
|
+
local score = delayedTimestamp * 0x1000
|
145
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
146
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
147
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
148
|
+
"jobId", parentId, "delay", delayedTimestamp)
|
149
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
150
|
+
else
|
151
|
+
if priority == 0 then
|
152
|
+
local parentTarget, _paused =
|
153
|
+
getTargetQueueList(parentMetaKey, parentWaitKey,
|
154
|
+
parentPausedKey)
|
155
|
+
rcall("RPUSH", parentTarget, parentId)
|
156
|
+
rcall("ZADD", parentMarkerKey, 0, "0")
|
157
|
+
else
|
158
|
+
local isPaused = isQueuePaused(parentMetaKey)
|
159
|
+
addJobWithPriority(parentMarkerKey,
|
160
|
+
parentQueueKey .. ":prioritized", priority,
|
161
|
+
parentId, parentQueueKey .. ":pc", isPaused)
|
162
|
+
end
|
163
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
164
|
+
"jobId", parentId, "prev", "waiting-children")
|
165
|
+
end
|
173
166
|
end
|
174
|
-
end
|
175
167
|
end
|
176
168
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
177
169
|
parentId, jobIdKey, returnvalue, timestamp )
|
@@ -241,8 +233,6 @@ rcall("ZADD", waitChildrenKey, timestamp, jobId)
|
|
241
233
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
242
234
|
"waiting-children", "jobId", jobId)
|
243
235
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
244
|
-
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
245
|
-
-- fail in this case.
|
246
236
|
if parentDependenciesKey ~= nil then
|
247
237
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
248
238
|
end
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Of,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -4,14 +4,13 @@ const content = `--[[
|
|
4
4
|
- Creates a new job key with the job data.
|
5
5
|
- Adds the job to the "added" list so that workers gets notified.
|
6
6
|
Input:
|
7
|
-
KEYS[1] '
|
8
|
-
KEYS[2] '
|
9
|
-
KEYS[3] '
|
10
|
-
KEYS[4] '
|
11
|
-
KEYS[5] '
|
12
|
-
KEYS[6]
|
13
|
-
KEYS[7]
|
14
|
-
KEYS[8] 'pc' priority counter
|
7
|
+
KEYS[1] 'marker',
|
8
|
+
KEYS[2] 'meta'
|
9
|
+
KEYS[3] 'id'
|
10
|
+
KEYS[4] 'prioritized'
|
11
|
+
KEYS[5] 'completed'
|
12
|
+
KEYS[6] events stream key
|
13
|
+
KEYS[7] 'pc' priority counter
|
15
14
|
ARGV[1] msgpacked arguments array
|
16
15
|
[1] key prefix,
|
17
16
|
[2] custom id (will not generate one automatically)
|
@@ -27,15 +26,13 @@ const content = `--[[
|
|
27
26
|
Output:
|
28
27
|
jobId - OK
|
29
28
|
-5 - Missing parent key
|
30
|
-
]]
|
31
|
-
local
|
32
|
-
local
|
33
|
-
local
|
34
|
-
local
|
35
|
-
local
|
36
|
-
local
|
37
|
-
local eventsKey = KEYS[7]
|
38
|
-
local priorityCounterKey = KEYS[8]
|
29
|
+
]]
|
30
|
+
local metaKey = KEYS[2]
|
31
|
+
local idKey = KEYS[3]
|
32
|
+
local priorityKey = KEYS[4]
|
33
|
+
local completedKey = KEYS[5]
|
34
|
+
local eventsKey = KEYS[6]
|
35
|
+
local priorityCounterKey = KEYS[7]
|
39
36
|
local jobId
|
40
37
|
local jobIdKey
|
41
38
|
local rcall = redis.call
|
@@ -73,37 +70,21 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
73
70
|
return delay, priority
|
74
71
|
end
|
75
72
|
--[[
|
76
|
-
Function to
|
73
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
74
|
+
(since an empty list and !EXISTS are not really the same).
|
77
75
|
]]
|
78
|
-
|
76
|
+
local function isQueuePaused(queueMetaKey)
|
77
|
+
return rcall("HEXISTS", queueMetaKey, "paused") == 1
|
78
|
+
end
|
79
79
|
--[[
|
80
|
-
Function
|
81
|
-
in order to wake up our workers and to respect priority
|
82
|
-
order as much as possible
|
80
|
+
Function to add job considering priority.
|
83
81
|
]]
|
84
|
-
local function
|
85
|
-
local waitLen = rcall("LLEN", waitKey)
|
86
|
-
if waitLen == 0 then
|
87
|
-
rcall("LPUSH", waitKey, "0:0")
|
88
|
-
end
|
89
|
-
end
|
90
|
-
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
82
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
91
83
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
92
84
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
93
85
|
rcall("ZADD", prioritizedKey, score, jobId)
|
94
|
-
if not
|
95
|
-
|
96
|
-
end
|
97
|
-
end
|
98
|
-
--[[
|
99
|
-
Function to check for the meta.paused key to decide if we are paused or not
|
100
|
-
(since an empty list and !EXISTS are not really the same).
|
101
|
-
]]
|
102
|
-
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
103
|
-
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
104
|
-
return waitKey, false
|
105
|
-
else
|
106
|
-
return pausedKey, true
|
86
|
+
if not isPaused then
|
87
|
+
rcall("ZADD", markerKey, 0, "0")
|
107
88
|
end
|
108
89
|
end
|
109
90
|
--[[
|
@@ -116,7 +97,7 @@ end
|
|
116
97
|
-- Includes
|
117
98
|
--[[
|
118
99
|
Add delay marker if needed.
|
119
|
-
]]
|
100
|
+
]]
|
120
101
|
-- Includes
|
121
102
|
--[[
|
122
103
|
Function to return the next delayed job timestamp.
|
@@ -131,55 +112,63 @@ local function getNextDelayedTimestamp(delayedKey)
|
|
131
112
|
return nextTimestamp
|
132
113
|
end
|
133
114
|
end
|
134
|
-
local function addDelayMarkerIfNeeded(
|
135
|
-
local waitLen = rcall("LLEN", targetKey)
|
136
|
-
if waitLen <= 1 then
|
115
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
137
116
|
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
138
117
|
if nextTimestamp ~= nil then
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
local marker = rcall("LINDEX", targetKey, 0)
|
143
|
-
local oldTimestamp = tonumber(marker:sub(3))
|
144
|
-
if oldTimestamp and oldTimestamp > nextTimestamp then
|
145
|
-
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
146
|
-
end
|
147
|
-
else
|
148
|
-
-- if there is no marker, then we need to add one
|
149
|
-
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
150
|
-
end
|
118
|
+
-- Replace the score of the marker with the newest known
|
119
|
+
-- next timestamp.
|
120
|
+
rcall("ZADD", markerKey, nextTimestamp, "0")
|
151
121
|
end
|
122
|
+
end
|
123
|
+
--[[
|
124
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
125
|
+
(since an empty list and !EXISTS are not really the same).
|
126
|
+
]]
|
127
|
+
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
128
|
+
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
129
|
+
return waitKey, false
|
130
|
+
else
|
131
|
+
return pausedKey, true
|
152
132
|
end
|
153
133
|
end
|
154
|
-
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
134
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
135
|
+
parentKey, parentId, timestamp)
|
136
|
+
local isParentActive = rcall("ZSCORE",
|
137
|
+
parentQueueKey .. ":waiting-children", parentId)
|
138
|
+
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
139
|
+
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
140
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
141
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
142
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
143
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
144
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
145
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
146
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
147
|
+
if delay > 0 then
|
148
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
149
|
+
local score = delayedTimestamp * 0x1000
|
150
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
151
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
152
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
153
|
+
"jobId", parentId, "delay", delayedTimestamp)
|
154
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
155
|
+
else
|
156
|
+
if priority == 0 then
|
157
|
+
local parentTarget, _paused =
|
158
|
+
getTargetQueueList(parentMetaKey, parentWaitKey,
|
159
|
+
parentPausedKey)
|
160
|
+
rcall("RPUSH", parentTarget, parentId)
|
161
|
+
rcall("ZADD", parentMarkerKey, 0, "0")
|
162
|
+
else
|
163
|
+
local isPaused = isQueuePaused(parentMetaKey)
|
164
|
+
addJobWithPriority(parentMarkerKey,
|
165
|
+
parentQueueKey .. ":prioritized", priority,
|
166
|
+
parentId, parentQueueKey .. ":pc", isPaused)
|
167
|
+
end
|
168
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
169
|
+
"jobId", parentId, "prev", "waiting-children")
|
170
|
+
end
|
181
171
|
end
|
182
|
-
end
|
183
172
|
end
|
184
173
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
185
174
|
parentId, jobIdKey, returnvalue, timestamp )
|
@@ -245,15 +234,13 @@ end
|
|
245
234
|
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
246
235
|
opts, timestamp, parentKey, parentData,
|
247
236
|
repeatJobKey)
|
248
|
-
|
249
|
-
|
250
|
-
|
237
|
+
-- Add the job to the prioritized set
|
238
|
+
local isPause = isQueuePaused(metaKey)
|
239
|
+
addJobWithPriority( KEYS[1], priorityKey, priority, jobId, priorityCounterKey, isPause)
|
251
240
|
-- Emit waiting event
|
252
241
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
253
242
|
"jobId", jobId)
|
254
243
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
255
|
-
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
256
|
-
-- fail in this case.
|
257
244
|
if parentDependenciesKey ~= nil then
|
258
245
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
259
246
|
end
|
@@ -262,6 +249,6 @@ return jobId .. "" -- convert to string
|
|
262
249
|
export const addPrioritizedJob = {
|
263
250
|
name: 'addPrioritizedJob',
|
264
251
|
content,
|
265
|
-
keys:
|
252
|
+
keys: 7,
|
266
253
|
};
|
267
|
-
//# sourceMappingURL=addPrioritizedJob-
|
254
|
+
//# sourceMappingURL=addPrioritizedJob-7.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"addPrioritizedJob-7.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuPf,CAAC;AACF,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,IAAI,EAAE,mBAAmB;IACzB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -19,6 +19,7 @@ const content = `--[[
|
|
19
19
|
KEYS[4] 'id'
|
20
20
|
KEYS[5] 'completed'
|
21
21
|
KEYS[6] events stream key
|
22
|
+
KEYS[7] marker key
|
22
23
|
ARGV[1] msgpacked arguments array
|
23
24
|
[1] key prefix,
|
24
25
|
[2] custom id (will not generate one automatically)
|
@@ -82,7 +83,7 @@ end
|
|
82
83
|
-- Includes
|
83
84
|
--[[
|
84
85
|
Add delay marker if needed.
|
85
|
-
]]
|
86
|
+
]]
|
86
87
|
-- Includes
|
87
88
|
--[[
|
88
89
|
Function to return the next delayed job timestamp.
|
@@ -97,47 +98,30 @@ local function getNextDelayedTimestamp(delayedKey)
|
|
97
98
|
return nextTimestamp
|
98
99
|
end
|
99
100
|
end
|
100
|
-
local function addDelayMarkerIfNeeded(
|
101
|
-
local waitLen = rcall("LLEN", targetKey)
|
102
|
-
if waitLen <= 1 then
|
101
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
103
102
|
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
104
103
|
if nextTimestamp ~= nil then
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
local marker = rcall("LINDEX", targetKey, 0)
|
109
|
-
local oldTimestamp = tonumber(marker:sub(3))
|
110
|
-
if oldTimestamp and oldTimestamp > nextTimestamp then
|
111
|
-
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
112
|
-
end
|
113
|
-
else
|
114
|
-
-- if there is no marker, then we need to add one
|
115
|
-
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
116
|
-
end
|
104
|
+
-- Replace the score of the marker with the newest known
|
105
|
+
-- next timestamp.
|
106
|
+
rcall("ZADD", markerKey, nextTimestamp, "0")
|
117
107
|
end
|
118
|
-
end
|
119
108
|
end
|
120
109
|
--[[
|
121
|
-
Function to
|
110
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
111
|
+
(since an empty list and !EXISTS are not really the same).
|
122
112
|
]]
|
123
|
-
|
113
|
+
local function isQueuePaused(queueMetaKey)
|
114
|
+
return rcall("HEXISTS", queueMetaKey, "paused") == 1
|
115
|
+
end
|
124
116
|
--[[
|
125
|
-
Function
|
126
|
-
in order to wake up our workers and to respect priority
|
127
|
-
order as much as possible
|
117
|
+
Function to add job considering priority.
|
128
118
|
]]
|
129
|
-
local function
|
130
|
-
local waitLen = rcall("LLEN", waitKey)
|
131
|
-
if waitLen == 0 then
|
132
|
-
rcall("LPUSH", waitKey, "0:0")
|
133
|
-
end
|
134
|
-
end
|
135
|
-
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
119
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
136
120
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
137
121
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
138
122
|
rcall("ZADD", prioritizedKey, score, jobId)
|
139
|
-
if not
|
140
|
-
|
123
|
+
if not isPaused then
|
124
|
+
rcall("ZADD", markerKey, 0, "0")
|
141
125
|
end
|
142
126
|
end
|
143
127
|
--[[
|
@@ -151,35 +135,44 @@ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
|
151
135
|
return pausedKey, true
|
152
136
|
end
|
153
137
|
end
|
154
|
-
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
138
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
139
|
+
parentKey, parentId, timestamp)
|
140
|
+
local isParentActive = rcall("ZSCORE",
|
141
|
+
parentQueueKey .. ":waiting-children", parentId)
|
142
|
+
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
143
|
+
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
144
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
145
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
146
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
147
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
148
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
149
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
150
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
151
|
+
if delay > 0 then
|
152
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
153
|
+
local score = delayedTimestamp * 0x1000
|
154
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
155
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
156
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
157
|
+
"jobId", parentId, "delay", delayedTimestamp)
|
158
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
159
|
+
else
|
160
|
+
if priority == 0 then
|
161
|
+
local parentTarget, _paused =
|
162
|
+
getTargetQueueList(parentMetaKey, parentWaitKey,
|
163
|
+
parentPausedKey)
|
164
|
+
rcall("RPUSH", parentTarget, parentId)
|
165
|
+
rcall("ZADD", parentMarkerKey, 0, "0")
|
166
|
+
else
|
167
|
+
local isPaused = isQueuePaused(parentMetaKey)
|
168
|
+
addJobWithPriority(parentMarkerKey,
|
169
|
+
parentQueueKey .. ":prioritized", priority,
|
170
|
+
parentId, parentQueueKey .. ":pc", isPaused)
|
171
|
+
end
|
172
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
173
|
+
"jobId", parentId, "prev", "waiting-children")
|
174
|
+
end
|
181
175
|
end
|
182
|
-
end
|
183
176
|
end
|
184
177
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
185
178
|
parentId, jobIdKey, returnvalue, timestamp )
|
@@ -246,6 +239,10 @@ end
|
|
246
239
|
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
247
240
|
parentKey, parentData, repeatJobKey)
|
248
241
|
local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
|
242
|
+
if not paused then
|
243
|
+
-- mark that a job is available
|
244
|
+
rcall("ZADD", KEYS[7], 0, "0")
|
245
|
+
end
|
249
246
|
-- LIFO or FIFO
|
250
247
|
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
|
251
248
|
rcall(pushCmd, target, jobId)
|
@@ -253,8 +250,6 @@ rcall(pushCmd, target, jobId)
|
|
253
250
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
254
251
|
"jobId", jobId)
|
255
252
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
256
|
-
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
257
|
-
-- fail in this case.
|
258
253
|
if parentDependenciesKey ~= nil then
|
259
254
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
260
255
|
end
|
@@ -263,6 +258,6 @@ return jobId .. "" -- convert to string
|
|
263
258
|
export const addStandardJob = {
|
264
259
|
name: 'addStandardJob',
|
265
260
|
content,
|
266
|
-
keys:
|
261
|
+
keys: 7,
|
267
262
|
};
|
268
|
-
//# sourceMappingURL=addStandardJob-
|
263
|
+
//# sourceMappingURL=addStandardJob-7.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"addStandardJob-7.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgQf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|