bullmq 5.1.3 → 5.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/scripts.js +1 -0
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/commands/addStandardJob-7.lua +4 -8
- package/dist/cjs/commands/changeDelay-3.lua +1 -0
- package/dist/cjs/commands/changePriority-6.lua +0 -1
- package/dist/cjs/commands/includes/addBaseMarkerIfNeeded.lua +9 -0
- package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +3 -1
- package/dist/cjs/commands/includes/addJobInTargetList.lua +11 -0
- package/dist/cjs/commands/includes/addJobWithPriority.lua +5 -3
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +6 -4
- package/dist/cjs/commands/includes/prepareJobForProcessing.lua +3 -39
- package/dist/cjs/commands/includes/promoteDelayedJobs.lua +2 -4
- package/dist/cjs/commands/moveJobsToWait-7.lua +2 -3
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +150 -0
- package/dist/cjs/commands/moveToActive-11.lua +6 -5
- package/dist/cjs/commands/moveToFinished-14.lua +13 -11
- package/dist/cjs/commands/pause-7.lua +1 -0
- package/dist/cjs/commands/promote-8.lua +2 -2
- package/dist/cjs/commands/retryJob-10.lua +1 -0
- package/dist/cjs/scripts/addDelayedJob-6.js +21 -7
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +25 -11
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-7.js +21 -7
- package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -1
- package/dist/cjs/scripts/addStandardJob-7.js +46 -36
- package/dist/cjs/scripts/addStandardJob-7.js.map +1 -1
- package/dist/cjs/scripts/changeDelay-3.js +1 -0
- package/dist/cjs/scripts/changeDelay-3.js.map +1 -1
- package/dist/cjs/scripts/changePriority-6.js +10 -14
- package/dist/cjs/scripts/changePriority-6.js.map +1 -1
- package/dist/cjs/scripts/index.js +1 -1
- package/dist/cjs/scripts/moveJobsToWait-7.js +9 -3
- package/dist/cjs/scripts/moveJobsToWait-7.js.map +1 -1
- package/dist/cjs/scripts/{moveStalledJobsToWait-8.js → moveStalledJobsToWait-9.js} +109 -106
- package/dist/cjs/scripts/{moveStalledJobsToWait-8.js.map → moveStalledJobsToWait-9.js.map} +1 -1
- package/dist/cjs/scripts/moveToActive-11.js +28 -50
- package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
- package/dist/cjs/scripts/moveToDelayed-7.js +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +47 -70
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/pause-7.js +2 -1
- package/dist/cjs/scripts/pause-7.js.map +1 -1
- package/dist/cjs/scripts/promote-8.js +19 -5
- package/dist/cjs/scripts/promote-8.js.map +1 -1
- package/dist/cjs/scripts/retryJob-10.js +20 -7
- package/dist/cjs/scripts/retryJob-10.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/esm/classes/scripts.js +1 -0
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/commands/addStandardJob-7.lua +4 -8
- package/dist/esm/commands/changeDelay-3.lua +1 -0
- package/dist/esm/commands/changePriority-6.lua +0 -1
- package/dist/esm/commands/includes/addBaseMarkerIfNeeded.lua +9 -0
- package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +3 -1
- package/dist/esm/commands/includes/addJobInTargetList.lua +11 -0
- package/dist/esm/commands/includes/addJobWithPriority.lua +5 -3
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +6 -4
- package/dist/esm/commands/includes/prepareJobForProcessing.lua +3 -39
- package/dist/esm/commands/includes/promoteDelayedJobs.lua +2 -4
- package/dist/esm/commands/moveJobsToWait-7.lua +2 -3
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +150 -0
- package/dist/esm/commands/moveToActive-11.lua +6 -5
- package/dist/esm/commands/moveToFinished-14.lua +13 -11
- package/dist/esm/commands/pause-7.lua +1 -0
- package/dist/esm/commands/promote-8.lua +2 -2
- package/dist/esm/commands/retryJob-10.lua +1 -0
- package/dist/esm/scripts/addDelayedJob-6.js +21 -7
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addParentJob-4.js +25 -11
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-7.js +21 -7
- package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -1
- package/dist/esm/scripts/addStandardJob-7.js +46 -36
- package/dist/esm/scripts/addStandardJob-7.js.map +1 -1
- package/dist/esm/scripts/changeDelay-3.js +1 -0
- package/dist/esm/scripts/changeDelay-3.js.map +1 -1
- package/dist/esm/scripts/changePriority-6.js +10 -14
- package/dist/esm/scripts/changePriority-6.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +1 -1
- package/dist/esm/scripts/index.js +1 -1
- package/dist/esm/scripts/moveJobsToWait-7.js +9 -3
- package/dist/esm/scripts/moveJobsToWait-7.js.map +1 -1
- package/dist/esm/scripts/{moveStalledJobsToWait-8.js → moveStalledJobsToWait-9.js} +109 -106
- package/dist/esm/scripts/{moveStalledJobsToWait-8.js.map → moveStalledJobsToWait-9.js.map} +1 -1
- package/dist/esm/scripts/moveToActive-11.js +28 -50
- package/dist/esm/scripts/moveToActive-11.js.map +1 -1
- package/dist/esm/scripts/moveToDelayed-7.js +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +47 -70
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/pause-7.js +2 -1
- package/dist/esm/scripts/pause-7.js.map +1 -1
- package/dist/esm/scripts/promote-8.js +19 -5
- package/dist/esm/scripts/promote-8.js.map +1 -1
- package/dist/esm/scripts/retryJob-10.js +20 -7
- package/dist/esm/scripts/retryJob-10.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/includes/checkStalledJobs.lua +0 -139
- package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -24
- package/dist/esm/commands/includes/checkStalledJobs.lua +0 -139
- package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -24
- /package/dist/esm/scripts/{moveStalledJobsToWait-8.d.ts → moveStalledJobsToWait-9.d.ts} +0 -0
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveStalledJobsToWait-
|
1
|
+
{"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAuSS,CAAC;AAC1B,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -30,6 +30,7 @@ const content = `--[[
|
|
30
30
|
local rcall = redis.call
|
31
31
|
local waitKey = KEYS[1]
|
32
32
|
local activeKey = KEYS[2]
|
33
|
+
local eventStreamKey = KEYS[4]
|
33
34
|
local rateLimiterKey = KEYS[6]
|
34
35
|
local delayedKey = KEYS[7]
|
35
36
|
local opts = cmsgpack.unpack(ARGV[3])
|
@@ -88,50 +89,15 @@ end
|
|
88
89
|
--[[
|
89
90
|
Function to move job from wait state to active.
|
90
91
|
Input:
|
91
|
-
keys[1] wait key
|
92
|
-
keys[2] active key
|
93
|
-
keys[3] prioritized key
|
94
|
-
keys[4] stream events key
|
95
|
-
keys[5] stalled key
|
96
|
-
-- Rate limiting
|
97
|
-
keys[6] rate limiter key
|
98
|
-
keys[7] delayed key
|
99
|
-
keys[8] paused key
|
100
|
-
keys[9] meta key
|
101
|
-
keys[10] pc priority counter
|
102
92
|
opts - token - lock token
|
103
93
|
opts - lockDuration
|
104
94
|
opts - limiter
|
105
95
|
]]
|
106
|
-
|
107
|
-
|
108
|
-
Function to push back job considering priority in front of same prioritized jobs.
|
109
|
-
]]
|
110
|
-
local function pushBackJobWithPriority(prioritizedKey, priority, jobId)
|
111
|
-
-- in order to put it at front of same prioritized jobs
|
112
|
-
-- we consider prioritized counter as 0
|
113
|
-
local score = priority * 0x100000000
|
114
|
-
rcall("ZADD", prioritizedKey, score, jobId)
|
115
|
-
end
|
116
|
-
local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, processedOn,
|
117
|
-
maxJobs, expireTime, opts)
|
96
|
+
local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
|
97
|
+
jobId, processedOn, maxJobs, opts)
|
118
98
|
local jobKey = keyPrefix .. jobId
|
119
99
|
-- Check if we need to perform rate limiting.
|
120
100
|
if maxJobs then
|
121
|
-
local rateLimiterKey = keys[6];
|
122
|
-
-- check if we exceeded rate limit, we need to remove the job and return expireTime
|
123
|
-
if expireTime > 0 then
|
124
|
-
-- remove from active queue and add back to the wait list
|
125
|
-
rcall("LREM", keys[2], 1, jobId)
|
126
|
-
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
|
127
|
-
if priority == 0 then
|
128
|
-
rcall("RPUSH", targetKey, jobId)
|
129
|
-
else
|
130
|
-
pushBackJobWithPriority(keys[3], priority, jobId)
|
131
|
-
end
|
132
|
-
-- Return when we can process more jobs
|
133
|
-
return {0, 0, expireTime, 0}
|
134
|
-
end
|
135
101
|
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
136
102
|
if jobCounter == 1 then
|
137
103
|
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
|
@@ -144,7 +110,7 @@ local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, proces
|
|
144
110
|
if opts['token'] ~= "0" then
|
145
111
|
rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
|
146
112
|
end
|
147
|
-
rcall("XADD",
|
113
|
+
rcall("XADD", eventStreamKey, "*", "event", "active", "jobId", jobId, "prev", "waiting")
|
148
114
|
rcall("HSET", jobKey, "processedOn", processedOn)
|
149
115
|
rcall("HINCRBY", jobKey, "ats", 1)
|
150
116
|
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
|
@@ -156,16 +122,31 @@ end
|
|
156
122
|
'waiting'
|
157
123
|
]]
|
158
124
|
-- Includes
|
125
|
+
--[[
|
126
|
+
Function to add job in target list and add marker if needed.
|
127
|
+
]]
|
128
|
+
-- Includes
|
129
|
+
--[[
|
130
|
+
Add marker if needed when a job is available.
|
131
|
+
]]
|
132
|
+
local function addBaseMarkerIfNeeded(markerKey, isPaused)
|
133
|
+
if not isPaused then
|
134
|
+
rcall("ZADD", markerKey, 0, "0")
|
135
|
+
end
|
136
|
+
end
|
137
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
|
138
|
+
rcall(pushCmd, targetKey, jobId)
|
139
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
140
|
+
end
|
159
141
|
--[[
|
160
142
|
Function to add job considering priority.
|
161
143
|
]]
|
144
|
+
-- Includes
|
162
145
|
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
163
146
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
164
147
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
165
148
|
rcall("ZADD", prioritizedKey, score, jobId)
|
166
|
-
|
167
|
-
rcall("ZADD", markerKey, 0, "0")
|
168
|
-
end
|
149
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
169
150
|
end
|
170
151
|
-- Try to get as much as 1000 jobs at once
|
171
152
|
local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
|
@@ -179,10 +160,7 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
|
|
179
160
|
tonumber(rcall("HGET", jobKey, "priority")) or 0
|
180
161
|
if priority == 0 then
|
181
162
|
-- LIFO or FIFO
|
182
|
-
|
183
|
-
if not isPaused then
|
184
|
-
rcall("ZADD", markerKey, 0, "0")
|
185
|
-
end
|
163
|
+
addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
|
186
164
|
else
|
187
165
|
addJobWithPriority(markerKey, prioritizedKey, priority,
|
188
166
|
jobId, priorityCounterKey, isPaused)
|
@@ -197,7 +175,7 @@ end
|
|
197
175
|
local target, paused = getTargetQueueList(KEYS[9], waitKey, KEYS[8])
|
198
176
|
-- Check if there are delayed jobs that we can move to wait.
|
199
177
|
local markerKey = KEYS[11]
|
200
|
-
promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3],
|
178
|
+
promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
|
201
179
|
ARGV[2], KEYS[10], paused)
|
202
180
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
203
181
|
local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
|
@@ -213,13 +191,13 @@ if jobId and string.sub(jobId, 1, 2) == "0:" then
|
|
213
191
|
jobId = rcall("RPOPLPUSH", waitKey, activeKey)
|
214
192
|
end
|
215
193
|
if jobId then
|
216
|
-
return prepareJobForProcessing(
|
217
|
-
maxJobs,
|
194
|
+
return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
|
195
|
+
maxJobs, opts)
|
218
196
|
else
|
219
197
|
jobId = moveJobFromPriorityToActive(KEYS[3], activeKey, KEYS[10])
|
220
198
|
if jobId then
|
221
|
-
return prepareJobForProcessing(
|
222
|
-
maxJobs,
|
199
|
+
return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
|
200
|
+
maxJobs, opts)
|
223
201
|
end
|
224
202
|
end
|
225
203
|
-- Return the timestamp for the next delayed job if any.
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Mf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|
@@ -113,6 +113,17 @@ local function getNextDelayedTimestamp(delayedKey)
|
|
113
113
|
return nextTimestamp
|
114
114
|
end
|
115
115
|
end
|
116
|
+
--[[
|
117
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
118
|
+
(since an empty list and !EXISTS are not really the same).
|
119
|
+
]]
|
120
|
+
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
121
|
+
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
122
|
+
return waitKey, false
|
123
|
+
else
|
124
|
+
return pausedKey, true
|
125
|
+
end
|
126
|
+
end
|
116
127
|
--[[
|
117
128
|
Function to move job from prioritized state to active.
|
118
129
|
]]
|
@@ -128,50 +139,15 @@ end
|
|
128
139
|
--[[
|
129
140
|
Function to move job from wait state to active.
|
130
141
|
Input:
|
131
|
-
keys[1] wait key
|
132
|
-
keys[2] active key
|
133
|
-
keys[3] prioritized key
|
134
|
-
keys[4] stream events key
|
135
|
-
keys[5] stalled key
|
136
|
-
-- Rate limiting
|
137
|
-
keys[6] rate limiter key
|
138
|
-
keys[7] delayed key
|
139
|
-
keys[8] paused key
|
140
|
-
keys[9] meta key
|
141
|
-
keys[10] pc priority counter
|
142
142
|
opts - token - lock token
|
143
143
|
opts - lockDuration
|
144
144
|
opts - limiter
|
145
145
|
]]
|
146
|
-
|
147
|
-
|
148
|
-
Function to push back job considering priority in front of same prioritized jobs.
|
149
|
-
]]
|
150
|
-
local function pushBackJobWithPriority(prioritizedKey, priority, jobId)
|
151
|
-
-- in order to put it at front of same prioritized jobs
|
152
|
-
-- we consider prioritized counter as 0
|
153
|
-
local score = priority * 0x100000000
|
154
|
-
rcall("ZADD", prioritizedKey, score, jobId)
|
155
|
-
end
|
156
|
-
local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, processedOn,
|
157
|
-
maxJobs, expireTime, opts)
|
146
|
+
local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
|
147
|
+
jobId, processedOn, maxJobs, opts)
|
158
148
|
local jobKey = keyPrefix .. jobId
|
159
149
|
-- Check if we need to perform rate limiting.
|
160
150
|
if maxJobs then
|
161
|
-
local rateLimiterKey = keys[6];
|
162
|
-
-- check if we exceeded rate limit, we need to remove the job and return expireTime
|
163
|
-
if expireTime > 0 then
|
164
|
-
-- remove from active queue and add back to the wait list
|
165
|
-
rcall("LREM", keys[2], 1, jobId)
|
166
|
-
local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
|
167
|
-
if priority == 0 then
|
168
|
-
rcall("RPUSH", targetKey, jobId)
|
169
|
-
else
|
170
|
-
pushBackJobWithPriority(keys[3], priority, jobId)
|
171
|
-
end
|
172
|
-
-- Return when we can process more jobs
|
173
|
-
return {0, 0, expireTime, 0}
|
174
|
-
end
|
175
151
|
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
176
152
|
if jobCounter == 1 then
|
177
153
|
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
|
@@ -184,7 +160,7 @@ local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, proces
|
|
184
160
|
if opts['token'] ~= "0" then
|
185
161
|
rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
|
186
162
|
end
|
187
|
-
rcall("XADD",
|
163
|
+
rcall("XADD", eventStreamKey, "*", "event", "active", "jobId", jobId, "prev", "waiting")
|
188
164
|
rcall("HSET", jobKey, "processedOn", processedOn)
|
189
165
|
rcall("HINCRBY", jobKey, "ats", 1)
|
190
166
|
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
|
@@ -199,7 +175,7 @@ end
|
|
199
175
|
-- Includes
|
200
176
|
--[[
|
201
177
|
Add delay marker if needed.
|
202
|
-
]]
|
178
|
+
]]
|
203
179
|
-- Includes
|
204
180
|
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
205
181
|
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
@@ -210,33 +186,37 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
210
186
|
end
|
211
187
|
end
|
212
188
|
--[[
|
213
|
-
Function to
|
214
|
-
(since an empty list and !EXISTS are not really the same).
|
189
|
+
Function to add job in target list and add marker if needed.
|
215
190
|
]]
|
216
|
-
|
217
|
-
|
191
|
+
-- Includes
|
192
|
+
--[[
|
193
|
+
Add marker if needed when a job is available.
|
194
|
+
]]
|
195
|
+
local function addBaseMarkerIfNeeded(markerKey, isPaused)
|
196
|
+
if not isPaused then
|
197
|
+
rcall("ZADD", markerKey, 0, "0")
|
198
|
+
end
|
199
|
+
end
|
200
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
|
201
|
+
rcall(pushCmd, targetKey, jobId)
|
202
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
218
203
|
end
|
219
204
|
--[[
|
220
205
|
Function to add job considering priority.
|
221
206
|
]]
|
207
|
+
-- Includes
|
222
208
|
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
223
209
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
224
210
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
225
211
|
rcall("ZADD", prioritizedKey, score, jobId)
|
226
|
-
|
227
|
-
rcall("ZADD", markerKey, 0, "0")
|
228
|
-
end
|
212
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
229
213
|
end
|
230
214
|
--[[
|
231
215
|
Function to check for the meta.paused key to decide if we are paused or not
|
232
216
|
(since an empty list and !EXISTS are not really the same).
|
233
217
|
]]
|
234
|
-
local function
|
235
|
-
|
236
|
-
return waitKey, false
|
237
|
-
else
|
238
|
-
return pausedKey, true
|
239
|
-
end
|
218
|
+
local function isQueuePaused(queueMetaKey)
|
219
|
+
return rcall("HEXISTS", queueMetaKey, "paused") == 1
|
240
220
|
end
|
241
221
|
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
242
222
|
parentKey, parentId, timestamp)
|
@@ -261,11 +241,10 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
261
241
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
262
242
|
else
|
263
243
|
if priority == 0 then
|
264
|
-
local parentTarget,
|
244
|
+
local parentTarget, isParentPaused =
|
265
245
|
getTargetQueueList(parentMetaKey, parentWaitKey,
|
266
246
|
parentPausedKey)
|
267
|
-
|
268
|
-
rcall("ZADD", parentMarkerKey, 0, "0")
|
247
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPaused, parentId)
|
269
248
|
else
|
270
249
|
local isPaused = isQueuePaused(parentMetaKey)
|
271
250
|
addJobWithPriority(parentMarkerKey,
|
@@ -325,10 +304,7 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
|
|
325
304
|
tonumber(rcall("HGET", jobKey, "priority")) or 0
|
326
305
|
if priority == 0 then
|
327
306
|
-- LIFO or FIFO
|
328
|
-
|
329
|
-
if not isPaused then
|
330
|
-
rcall("ZADD", markerKey, 0, "0")
|
331
|
-
end
|
307
|
+
addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
|
332
308
|
else
|
333
309
|
addJobWithPriority(markerKey, prioritizedKey, priority,
|
334
310
|
jobId, priorityCounterKey, isPaused)
|
@@ -540,9 +516,10 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
540
516
|
-- Remove from active list (if not active we shall return error)
|
541
517
|
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
542
518
|
if (numRemovedElements < 1) then return -3 end
|
519
|
+
local eventStreamKey = KEYS[4]
|
543
520
|
local metaKey = KEYS[9]
|
544
521
|
-- Trim events before emiting them to avoid trimming events emitted in this script
|
545
|
-
trimEvents(metaKey,
|
522
|
+
trimEvents(metaKey, eventStreamKey)
|
546
523
|
-- If job has a parent we need to
|
547
524
|
-- 1) remove this job id from parents dependencies
|
548
525
|
-- 2) move the job Id to parent "processed" set
|
@@ -596,11 +573,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
596
573
|
removeParentDependencyKey(jobIdKey, false, parentKey)
|
597
574
|
end
|
598
575
|
end
|
599
|
-
rcall("XADD",
|
576
|
+
rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
|
600
577
|
ARGV[4])
|
601
578
|
if ARGV[5] == "failed" then
|
602
579
|
if tonumber(attemptsMade) >= tonumber(attempts) then
|
603
|
-
rcall("XADD",
|
580
|
+
rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId",
|
604
581
|
jobId, "attemptsMade", attemptsMade)
|
605
582
|
end
|
606
583
|
end
|
@@ -613,7 +590,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
613
590
|
if (ARGV[6] == "1") then
|
614
591
|
local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[8])
|
615
592
|
-- Check if there are delayed jobs that can be promoted
|
616
|
-
promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3],
|
593
|
+
promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], eventStreamKey, ARGV[7],
|
617
594
|
timestamp, KEYS[10], paused)
|
618
595
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
619
596
|
-- Check if we are rate limited first.
|
@@ -631,20 +608,20 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
631
608
|
if jobId == "0:0" then
|
632
609
|
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
|
633
610
|
KEYS[10])
|
634
|
-
return prepareJobForProcessing(
|
611
|
+
return prepareJobForProcessing(ARGV[7], KEYS[6], eventStreamKey, jobId,
|
635
612
|
timestamp, maxJobs,
|
636
|
-
|
613
|
+
opts)
|
637
614
|
end
|
638
615
|
else
|
639
|
-
return prepareJobForProcessing(
|
640
|
-
timestamp, maxJobs,
|
616
|
+
return prepareJobForProcessing(ARGV[7], KEYS[6], eventStreamKey, jobId,
|
617
|
+
timestamp, maxJobs,
|
641
618
|
opts)
|
642
619
|
end
|
643
620
|
else
|
644
621
|
jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
|
645
622
|
if jobId then
|
646
|
-
return prepareJobForProcessing(
|
647
|
-
timestamp, maxJobs,
|
623
|
+
return prepareJobForProcessing(ARGV[7], KEYS[6], eventStreamKey, jobId,
|
624
|
+
timestamp, maxJobs,
|
648
625
|
opts)
|
649
626
|
end
|
650
627
|
end
|
@@ -662,7 +639,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
662
639
|
if activeLen == 0 then
|
663
640
|
local prioritizedLen = rcall("ZCARD", KEYS[3])
|
664
641
|
if prioritizedLen == 0 then
|
665
|
-
rcall("XADD",
|
642
|
+
rcall("XADD", eventStreamKey, "*", "event", "drained")
|
666
643
|
end
|
667
644
|
end
|
668
645
|
end
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyoBf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|
@@ -16,7 +16,7 @@ local rcall = redis.call
|
|
16
16
|
-- Includes
|
17
17
|
--[[
|
18
18
|
Add delay marker if needed.
|
19
|
-
]]
|
19
|
+
]]
|
20
20
|
-- Includes
|
21
21
|
--[[
|
22
22
|
Function to return the next delayed job timestamp.
|
@@ -41,6 +41,7 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
41
41
|
end
|
42
42
|
local markerKey = KEYS[7]
|
43
43
|
local hasJobs = rcall("EXISTS", KEYS[1]) == 1
|
44
|
+
--TODO: check this logic to be reused when changing a delay
|
44
45
|
if hasJobs then rcall("RENAME", KEYS[1], KEYS[2]) end
|
45
46
|
if ARGV[1] == "paused" then
|
46
47
|
rcall("HSET", KEYS[3], "paused", 1)
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"pause-7.js","sourceRoot":"","sources":["../../../src/scripts/pause-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"pause-7.js","sourceRoot":"","sources":["../../../src/scripts/pause-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Df,CAAC;AACF,MAAM,CAAC,MAAM,KAAK,GAAG;IACnB,IAAI,EAAE,OAAO;IACb,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -20,16 +20,31 @@ const content = `--[[
|
|
20
20
|
local rcall = redis.call
|
21
21
|
local jobId = ARGV[2]
|
22
22
|
-- Includes
|
23
|
+
--[[
|
24
|
+
Function to add job in target list and add marker if needed.
|
25
|
+
]]
|
26
|
+
-- Includes
|
27
|
+
--[[
|
28
|
+
Add marker if needed when a job is available.
|
29
|
+
]]
|
30
|
+
local function addBaseMarkerIfNeeded(markerKey, isPaused)
|
31
|
+
if not isPaused then
|
32
|
+
rcall("ZADD", markerKey, 0, "0")
|
33
|
+
end
|
34
|
+
end
|
35
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
|
36
|
+
rcall(pushCmd, targetKey, jobId)
|
37
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
38
|
+
end
|
23
39
|
--[[
|
24
40
|
Function to add job considering priority.
|
25
41
|
]]
|
42
|
+
-- Includes
|
26
43
|
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
27
44
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
28
45
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
29
46
|
rcall("ZADD", prioritizedKey, score, jobId)
|
30
|
-
|
31
|
-
rcall("ZADD", markerKey, 0, "0")
|
32
|
-
end
|
47
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
33
48
|
end
|
34
49
|
--[[
|
35
50
|
Function to check for the meta.paused key to decide if we are paused or not
|
@@ -54,8 +69,7 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then
|
|
54
69
|
if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
|
55
70
|
if priority == 0 then
|
56
71
|
-- LIFO or FIFO
|
57
|
-
|
58
|
-
if not paused then rcall("ZADD", KEYS[8], 0, "0") end
|
72
|
+
addJobInTargetList(target, KEYS[8], "LPUSH", paused, jobId)
|
59
73
|
else
|
60
74
|
addJobWithPriority(KEYS[8], KEYS[5], priority, jobId, KEYS[6], paused)
|
61
75
|
end
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"promote-8.js","sourceRoot":"","sources":["../../../src/scripts/promote-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"promote-8.js","sourceRoot":"","sources":["../../../src/scripts/promote-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmFf,CAAC;AACF,MAAM,CAAC,MAAM,OAAO,GAAG;IACrB,IAAI,EAAE,SAAS;IACf,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -28,13 +28,20 @@ local rcall = redis.call
|
|
28
28
|
--[[
|
29
29
|
Function to add job considering priority.
|
30
30
|
]]
|
31
|
+
-- Includes
|
32
|
+
--[[
|
33
|
+
Add marker if needed when a job is available.
|
34
|
+
]]
|
35
|
+
local function addBaseMarkerIfNeeded(markerKey, isPaused)
|
36
|
+
if not isPaused then
|
37
|
+
rcall("ZADD", markerKey, 0, "0")
|
38
|
+
end
|
39
|
+
end
|
31
40
|
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
32
41
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
33
42
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
34
43
|
rcall("ZADD", prioritizedKey, score, jobId)
|
35
|
-
|
36
|
-
rcall("ZADD", markerKey, 0, "0")
|
37
|
-
end
|
44
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
38
45
|
end
|
39
46
|
--[[
|
40
47
|
Function to get max events value or set by default 10000.
|
@@ -65,6 +72,14 @@ end
|
|
65
72
|
'waiting'
|
66
73
|
]]
|
67
74
|
-- Includes
|
75
|
+
--[[
|
76
|
+
Function to add job in target list and add marker if needed.
|
77
|
+
]]
|
78
|
+
-- Includes
|
79
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
|
80
|
+
rcall(pushCmd, targetKey, jobId)
|
81
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
82
|
+
end
|
68
83
|
-- Try to get as much as 1000 jobs at once
|
69
84
|
local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
|
70
85
|
eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
|
@@ -77,10 +92,7 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
|
|
77
92
|
tonumber(rcall("HGET", jobKey, "priority")) or 0
|
78
93
|
if priority == 0 then
|
79
94
|
-- LIFO or FIFO
|
80
|
-
|
81
|
-
if not isPaused then
|
82
|
-
rcall("ZADD", markerKey, 0, "0")
|
83
|
-
end
|
95
|
+
addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
|
84
96
|
else
|
85
97
|
addJobWithPriority(markerKey, prioritizedKey, priority,
|
86
98
|
jobId, priorityCounterKey, isPaused)
|
@@ -111,6 +123,7 @@ if rcall("EXISTS", KEYS[4]) == 1 then
|
|
111
123
|
-- Standard or priority add
|
112
124
|
if priority == 0 then
|
113
125
|
rcall(ARGV[3], target, ARGV[4])
|
126
|
+
-- TODO: check if we need to add marker in this case too
|
114
127
|
else
|
115
128
|
addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], paused)
|
116
129
|
end
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"retryJob-10.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-10.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"retryJob-10.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-10.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0If,CAAC;AACF,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|