bullmq 5.1.3 → 5.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/scripts.js +1 -0
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/commands/addStandardJob-7.lua +4 -8
- package/dist/cjs/commands/changeDelay-3.lua +1 -0
- package/dist/cjs/commands/changePriority-6.lua +0 -1
- package/dist/cjs/commands/includes/addBaseMarkerIfNeeded.lua +9 -0
- package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +3 -1
- package/dist/cjs/commands/includes/addJobInTargetList.lua +11 -0
- package/dist/cjs/commands/includes/addJobWithPriority.lua +5 -3
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +6 -4
- package/dist/cjs/commands/includes/prepareJobForProcessing.lua +3 -39
- package/dist/cjs/commands/includes/promoteDelayedJobs.lua +2 -4
- package/dist/cjs/commands/moveJobsToWait-7.lua +2 -3
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +150 -0
- package/dist/cjs/commands/moveToActive-11.lua +6 -5
- package/dist/cjs/commands/moveToFinished-14.lua +13 -11
- package/dist/cjs/commands/pause-7.lua +1 -0
- package/dist/cjs/commands/promote-8.lua +2 -2
- package/dist/cjs/commands/retryJob-10.lua +1 -0
- package/dist/cjs/scripts/addDelayedJob-6.js +21 -7
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +25 -11
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-7.js +21 -7
- package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -1
- package/dist/cjs/scripts/addStandardJob-7.js +46 -36
- package/dist/cjs/scripts/addStandardJob-7.js.map +1 -1
- package/dist/cjs/scripts/changeDelay-3.js +1 -0
- package/dist/cjs/scripts/changeDelay-3.js.map +1 -1
- package/dist/cjs/scripts/changePriority-6.js +10 -14
- package/dist/cjs/scripts/changePriority-6.js.map +1 -1
- package/dist/cjs/scripts/index.js +1 -1
- package/dist/cjs/scripts/moveJobsToWait-7.js +9 -3
- package/dist/cjs/scripts/moveJobsToWait-7.js.map +1 -1
- package/dist/cjs/scripts/{moveStalledJobsToWait-8.js → moveStalledJobsToWait-9.js} +109 -106
- package/dist/cjs/scripts/{moveStalledJobsToWait-8.js.map → moveStalledJobsToWait-9.js.map} +1 -1
- package/dist/cjs/scripts/moveToActive-11.js +28 -50
- package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
- package/dist/cjs/scripts/moveToDelayed-7.js +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +47 -70
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/pause-7.js +2 -1
- package/dist/cjs/scripts/pause-7.js.map +1 -1
- package/dist/cjs/scripts/promote-8.js +19 -5
- package/dist/cjs/scripts/promote-8.js.map +1 -1
- package/dist/cjs/scripts/retryJob-10.js +20 -7
- package/dist/cjs/scripts/retryJob-10.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/esm/classes/scripts.js +1 -0
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/commands/addStandardJob-7.lua +4 -8
- package/dist/esm/commands/changeDelay-3.lua +1 -0
- package/dist/esm/commands/changePriority-6.lua +0 -1
- package/dist/esm/commands/includes/addBaseMarkerIfNeeded.lua +9 -0
- package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +3 -1
- package/dist/esm/commands/includes/addJobInTargetList.lua +11 -0
- package/dist/esm/commands/includes/addJobWithPriority.lua +5 -3
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +6 -4
- package/dist/esm/commands/includes/prepareJobForProcessing.lua +3 -39
- package/dist/esm/commands/includes/promoteDelayedJobs.lua +2 -4
- package/dist/esm/commands/moveJobsToWait-7.lua +2 -3
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +150 -0
- package/dist/esm/commands/moveToActive-11.lua +6 -5
- package/dist/esm/commands/moveToFinished-14.lua +13 -11
- package/dist/esm/commands/pause-7.lua +1 -0
- package/dist/esm/commands/promote-8.lua +2 -2
- package/dist/esm/commands/retryJob-10.lua +1 -0
- package/dist/esm/scripts/addDelayedJob-6.js +21 -7
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addParentJob-4.js +25 -11
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-7.js +21 -7
- package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -1
- package/dist/esm/scripts/addStandardJob-7.js +46 -36
- package/dist/esm/scripts/addStandardJob-7.js.map +1 -1
- package/dist/esm/scripts/changeDelay-3.js +1 -0
- package/dist/esm/scripts/changeDelay-3.js.map +1 -1
- package/dist/esm/scripts/changePriority-6.js +10 -14
- package/dist/esm/scripts/changePriority-6.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +1 -1
- package/dist/esm/scripts/index.js +1 -1
- package/dist/esm/scripts/moveJobsToWait-7.js +9 -3
- package/dist/esm/scripts/moveJobsToWait-7.js.map +1 -1
- package/dist/esm/scripts/{moveStalledJobsToWait-8.js → moveStalledJobsToWait-9.js} +109 -106
- package/dist/esm/scripts/{moveStalledJobsToWait-8.js.map → moveStalledJobsToWait-9.js.map} +1 -1
- package/dist/esm/scripts/moveToActive-11.js +28 -50
- package/dist/esm/scripts/moveToActive-11.js.map +1 -1
- package/dist/esm/scripts/moveToDelayed-7.js +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +47 -70
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/pause-7.js +2 -1
- package/dist/esm/scripts/pause-7.js.map +1 -1
- package/dist/esm/scripts/promote-8.js +19 -5
- package/dist/esm/scripts/promote-8.js.map +1 -1
- package/dist/esm/scripts/retryJob-10.js +20 -7
- package/dist/esm/scripts/retryJob-10.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/includes/checkStalledJobs.lua +0 -139
- package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -24
- package/dist/esm/commands/includes/checkStalledJobs.lua +0 -139
- package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -24
- /package/dist/esm/scripts/{moveStalledJobsToWait-8.d.ts → moveStalledJobsToWait-9.d.ts} +0 -0
@@ -81,7 +81,7 @@ end
|
|
81
81
|
-- Includes
|
82
82
|
--[[
|
83
83
|
Add delay marker if needed.
|
84
|
-
]]
|
84
|
+
]]
|
85
85
|
-- Includes
|
86
86
|
--[[
|
87
87
|
Function to return the next delayed job timestamp.
|
@@ -105,22 +105,37 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
105
105
|
end
|
106
106
|
end
|
107
107
|
--[[
|
108
|
-
Function to
|
109
|
-
(since an empty list and !EXISTS are not really the same).
|
108
|
+
Function to add job in target list and add marker if needed.
|
110
109
|
]]
|
111
|
-
|
112
|
-
|
110
|
+
-- Includes
|
111
|
+
--[[
|
112
|
+
Add marker if needed when a job is available.
|
113
|
+
]]
|
114
|
+
local function addBaseMarkerIfNeeded(markerKey, isPaused)
|
115
|
+
if not isPaused then
|
116
|
+
rcall("ZADD", markerKey, 0, "0")
|
117
|
+
end
|
118
|
+
end
|
119
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
|
120
|
+
rcall(pushCmd, targetKey, jobId)
|
121
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
113
122
|
end
|
114
123
|
--[[
|
115
124
|
Function to add job considering priority.
|
116
125
|
]]
|
126
|
+
-- Includes
|
117
127
|
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
118
128
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
119
129
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
120
130
|
rcall("ZADD", prioritizedKey, score, jobId)
|
121
|
-
|
122
|
-
|
123
|
-
|
131
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
132
|
+
end
|
133
|
+
--[[
|
134
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
135
|
+
(since an empty list and !EXISTS are not really the same).
|
136
|
+
]]
|
137
|
+
local function isQueuePaused(queueMetaKey)
|
138
|
+
return rcall("HEXISTS", queueMetaKey, "paused") == 1
|
124
139
|
end
|
125
140
|
--[[
|
126
141
|
Function to check for the meta.paused key to decide if we are paused or not
|
@@ -156,11 +171,10 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
156
171
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
157
172
|
else
|
158
173
|
if priority == 0 then
|
159
|
-
local parentTarget,
|
174
|
+
local parentTarget, isParentPaused =
|
160
175
|
getTargetQueueList(parentMetaKey, parentWaitKey,
|
161
176
|
parentPausedKey)
|
162
|
-
|
163
|
-
rcall("ZADD", parentMarkerKey, 0, "0")
|
177
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPaused, parentId)
|
164
178
|
else
|
165
179
|
local isPaused = isQueuePaused(parentMetaKey)
|
166
180
|
addJobWithPriority(parentMarkerKey,
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiQf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -79,13 +79,20 @@ end
|
|
79
79
|
--[[
|
80
80
|
Function to add job considering priority.
|
81
81
|
]]
|
82
|
+
-- Includes
|
83
|
+
--[[
|
84
|
+
Add marker if needed when a job is available.
|
85
|
+
]]
|
86
|
+
local function addBaseMarkerIfNeeded(markerKey, isPaused)
|
87
|
+
if not isPaused then
|
88
|
+
rcall("ZADD", markerKey, 0, "0")
|
89
|
+
end
|
90
|
+
end
|
82
91
|
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
83
92
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
84
93
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
85
94
|
rcall("ZADD", prioritizedKey, score, jobId)
|
86
|
-
|
87
|
-
rcall("ZADD", markerKey, 0, "0")
|
88
|
-
end
|
95
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
89
96
|
end
|
90
97
|
--[[
|
91
98
|
This function is used to update the parent's dependencies if the job
|
@@ -104,7 +111,7 @@ end
|
|
104
111
|
-- Includes
|
105
112
|
--[[
|
106
113
|
Add delay marker if needed.
|
107
|
-
]]
|
114
|
+
]]
|
108
115
|
-- Includes
|
109
116
|
--[[
|
110
117
|
Function to return the next delayed job timestamp.
|
@@ -127,6 +134,14 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
127
134
|
rcall("ZADD", markerKey, nextTimestamp, "0")
|
128
135
|
end
|
129
136
|
end
|
137
|
+
--[[
|
138
|
+
Function to add job in target list and add marker if needed.
|
139
|
+
]]
|
140
|
+
-- Includes
|
141
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
|
142
|
+
rcall(pushCmd, targetKey, jobId)
|
143
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
144
|
+
end
|
130
145
|
--[[
|
131
146
|
Function to check for the meta.paused key to decide if we are paused or not
|
132
147
|
(since an empty list and !EXISTS are not really the same).
|
@@ -161,11 +176,10 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
161
176
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
162
177
|
else
|
163
178
|
if priority == 0 then
|
164
|
-
local parentTarget,
|
179
|
+
local parentTarget, isParentPaused =
|
165
180
|
getTargetQueueList(parentMetaKey, parentWaitKey,
|
166
181
|
parentPausedKey)
|
167
|
-
|
168
|
-
rcall("ZADD", parentMarkerKey, 0, "0")
|
182
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPaused, parentId)
|
169
183
|
else
|
170
184
|
local isPaused = isQueuePaused(parentMetaKey)
|
171
185
|
addJobWithPriority(parentMarkerKey,
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addPrioritizedJob-7.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addPrioritizedJob-7.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyQf,CAAC;AACF,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,IAAI,EAAE,mBAAmB;IACzB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -48,6 +48,44 @@ local repeatJobKey = args[9]
|
|
48
48
|
local parent = args[8]
|
49
49
|
local parentData
|
50
50
|
-- Includes
|
51
|
+
--[[
|
52
|
+
Function to add job in target list and add marker if needed.
|
53
|
+
]]
|
54
|
+
-- Includes
|
55
|
+
--[[
|
56
|
+
Add marker if needed when a job is available.
|
57
|
+
]]
|
58
|
+
local function addBaseMarkerIfNeeded(markerKey, isPaused)
|
59
|
+
if not isPaused then
|
60
|
+
rcall("ZADD", markerKey, 0, "0")
|
61
|
+
end
|
62
|
+
end
|
63
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
|
64
|
+
rcall(pushCmd, targetKey, jobId)
|
65
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
66
|
+
end
|
67
|
+
--[[
|
68
|
+
Function to get max events value or set by default 10000.
|
69
|
+
]]
|
70
|
+
local function getOrSetMaxEvents(metaKey)
|
71
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
72
|
+
if not maxEvents then
|
73
|
+
maxEvents = 10000
|
74
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
75
|
+
end
|
76
|
+
return maxEvents
|
77
|
+
end
|
78
|
+
--[[
|
79
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
80
|
+
(since an empty list and !EXISTS are not really the same).
|
81
|
+
]]
|
82
|
+
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
83
|
+
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
84
|
+
return waitKey, false
|
85
|
+
else
|
86
|
+
return pausedKey, true
|
87
|
+
end
|
88
|
+
end
|
51
89
|
--[[
|
52
90
|
Function to store a job
|
53
91
|
]]
|
@@ -90,7 +128,7 @@ end
|
|
90
128
|
-- Includes
|
91
129
|
--[[
|
92
130
|
Add delay marker if needed.
|
93
|
-
]]
|
131
|
+
]]
|
94
132
|
-- Includes
|
95
133
|
--[[
|
96
134
|
Function to return the next delayed job timestamp.
|
@@ -113,34 +151,22 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
|
113
151
|
rcall("ZADD", markerKey, nextTimestamp, "0")
|
114
152
|
end
|
115
153
|
end
|
116
|
-
--[[
|
117
|
-
Function to check for the meta.paused key to decide if we are paused or not
|
118
|
-
(since an empty list and !EXISTS are not really the same).
|
119
|
-
]]
|
120
|
-
local function isQueuePaused(queueMetaKey)
|
121
|
-
return rcall("HEXISTS", queueMetaKey, "paused") == 1
|
122
|
-
end
|
123
154
|
--[[
|
124
155
|
Function to add job considering priority.
|
125
156
|
]]
|
157
|
+
-- Includes
|
126
158
|
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
127
159
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
128
160
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
129
161
|
rcall("ZADD", prioritizedKey, score, jobId)
|
130
|
-
|
131
|
-
rcall("ZADD", markerKey, 0, "0")
|
132
|
-
end
|
162
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
133
163
|
end
|
134
164
|
--[[
|
135
165
|
Function to check for the meta.paused key to decide if we are paused or not
|
136
166
|
(since an empty list and !EXISTS are not really the same).
|
137
167
|
]]
|
138
|
-
local function
|
139
|
-
|
140
|
-
return waitKey, false
|
141
|
-
else
|
142
|
-
return pausedKey, true
|
143
|
-
end
|
168
|
+
local function isQueuePaused(queueMetaKey)
|
169
|
+
return rcall("HEXISTS", queueMetaKey, "paused") == 1
|
144
170
|
end
|
145
171
|
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
146
172
|
parentKey, parentId, timestamp)
|
@@ -165,11 +191,10 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
165
191
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
166
192
|
else
|
167
193
|
if priority == 0 then
|
168
|
-
local parentTarget,
|
194
|
+
local parentTarget, isParentPaused =
|
169
195
|
getTargetQueueList(parentMetaKey, parentWaitKey,
|
170
196
|
parentPausedKey)
|
171
|
-
|
172
|
-
rcall("ZADD", parentMarkerKey, 0, "0")
|
197
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPaused, parentId)
|
173
198
|
else
|
174
199
|
local isPaused = isQueuePaused(parentMetaKey)
|
175
200
|
addJobWithPriority(parentMarkerKey,
|
@@ -204,17 +229,6 @@ local function updateExistingJobsParent(parentKey, parent, parentData,
|
|
204
229
|
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
205
230
|
end
|
206
231
|
end
|
207
|
-
--[[
|
208
|
-
Function to get max events value or set by default 10000.
|
209
|
-
]]
|
210
|
-
local function getOrSetMaxEvents(metaKey)
|
211
|
-
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
212
|
-
if not maxEvents then
|
213
|
-
maxEvents = 10000
|
214
|
-
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
215
|
-
end
|
216
|
-
return maxEvents
|
217
|
-
end
|
218
232
|
if parentKey ~= nil then
|
219
233
|
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
220
234
|
parentData = cjson.encode(parent)
|
@@ -243,13 +257,9 @@ end
|
|
243
257
|
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
244
258
|
parentKey, parentData, repeatJobKey)
|
245
259
|
local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
|
246
|
-
if not paused then
|
247
|
-
-- mark that a job is available
|
248
|
-
rcall("ZADD", KEYS[7], 0, "0")
|
249
|
-
end
|
250
260
|
-- LIFO or FIFO
|
251
261
|
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
|
252
|
-
|
262
|
+
addJobInTargetList(target, KEYS[7], pushCmd, paused, jobId)
|
253
263
|
-- Emit waiting event
|
254
264
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
255
265
|
"jobId", jobId)
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addStandardJob-7.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addStandardJob-7.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Qf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -25,6 +25,7 @@ if rcall("EXISTS", KEYS[2]) == 1 then
|
|
25
25
|
end
|
26
26
|
rcall("HSET", KEYS[2], "delay", tonumber(ARGV[1]))
|
27
27
|
rcall("ZADD", KEYS[1], score, jobId)
|
28
|
+
-- TODO: check if we need to evaluate a new marker
|
28
29
|
rcall("XADD", KEYS[3], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp)
|
29
30
|
return 0
|
30
31
|
else
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"changeDelay-3.js","sourceRoot":"","sources":["../../../src/scripts/changeDelay-3.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"changeDelay-3.js","sourceRoot":"","sources":["../../../src/scripts/changeDelay-3.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAgCZ,CAAC;AACL,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,IAAI,EAAE,aAAa;IACnB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -30,24 +30,20 @@ end
|
|
30
30
|
--[[
|
31
31
|
Function to add job considering priority.
|
32
32
|
]]
|
33
|
+
-- Includes
|
34
|
+
--[[
|
35
|
+
Add marker if needed when a job is available.
|
36
|
+
]]
|
37
|
+
local function addBaseMarkerIfNeeded(markerKey, isPaused)
|
38
|
+
if not isPaused then
|
39
|
+
rcall("ZADD", markerKey, 0, "0")
|
40
|
+
end
|
41
|
+
end
|
33
42
|
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
34
43
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
35
44
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
36
45
|
rcall("ZADD", prioritizedKey, score, jobId)
|
37
|
-
|
38
|
-
rcall("ZADD", markerKey, 0, "0")
|
39
|
-
end
|
40
|
-
end
|
41
|
-
--[[
|
42
|
-
Function to check for the meta.paused key to decide if we are paused or not
|
43
|
-
(since an empty list and !EXISTS are not really the same).
|
44
|
-
]]
|
45
|
-
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
46
|
-
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
47
|
-
return waitKey, false
|
48
|
-
else
|
49
|
-
return pausedKey, true
|
50
|
-
end
|
46
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
51
47
|
end
|
52
48
|
if rcall("EXISTS", jobKey) == 1 then
|
53
49
|
local metaKey = KEYS[3]
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"changePriority-6.js","sourceRoot":"","sources":["../../../src/scripts/changePriority-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"changePriority-6.js","sourceRoot":"","sources":["../../../src/scripts/changePriority-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuEf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -15,7 +15,7 @@ export * from './isFinished-3';
|
|
15
15
|
export * from './isJobInList-1';
|
16
16
|
export * from './moveJobFromActiveToWait-9';
|
17
17
|
export * from './moveJobsToWait-7';
|
18
|
-
export * from './moveStalledJobsToWait-
|
18
|
+
export * from './moveStalledJobsToWait-9';
|
19
19
|
export * from './moveToActive-11';
|
20
20
|
export * from './moveToDelayed-7';
|
21
21
|
export * from './moveToFinished-14';
|
@@ -15,7 +15,7 @@ export * from './isFinished-3';
|
|
15
15
|
export * from './isJobInList-1';
|
16
16
|
export * from './moveJobFromActiveToWait-9';
|
17
17
|
export * from './moveJobsToWait-7';
|
18
|
-
export * from './moveStalledJobsToWait-
|
18
|
+
export * from './moveStalledJobsToWait-9';
|
19
19
|
export * from './moveToActive-11';
|
20
20
|
export * from './moveToDelayed-7';
|
21
21
|
export * from './moveToFinished-14';
|
@@ -20,6 +20,14 @@ local maxCount = tonumber(ARGV[1])
|
|
20
20
|
local timestamp = tonumber(ARGV[2])
|
21
21
|
local rcall = redis.call;
|
22
22
|
-- Includes
|
23
|
+
--[[
|
24
|
+
Add marker if needed when a job is available.
|
25
|
+
]]
|
26
|
+
local function addBaseMarkerIfNeeded(markerKey, isPaused)
|
27
|
+
if not isPaused then
|
28
|
+
rcall("ZADD", markerKey, 0, "0")
|
29
|
+
end
|
30
|
+
end
|
23
31
|
--[[
|
24
32
|
Function to loop in batches.
|
25
33
|
Just a bit of warning, some commands as ZREM
|
@@ -83,9 +91,7 @@ if (#jobs > 0) then
|
|
83
91
|
rcall("ZREM", KEYS[3], unpack(jobs, from, to))
|
84
92
|
rcall("LPUSH", target, unpack(jobs, from, to))
|
85
93
|
end
|
86
|
-
|
87
|
-
rcall("ZADD", KEYS[7], 0, "0")
|
88
|
-
end
|
94
|
+
addBaseMarkerIfNeeded(KEYS[7], paused)
|
89
95
|
end
|
90
96
|
maxCount = maxCount - #jobs
|
91
97
|
if (maxCount <= 0) then return 1 end
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveJobsToWait-7.js","sourceRoot":"","sources":["../../../src/scripts/moveJobsToWait-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveJobsToWait-7.js","sourceRoot":"","sources":["../../../src/scripts/moveJobsToWait-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkGf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -8,34 +8,33 @@ const content = `--[[
|
|
8
8
|
KEYS[5] 'stalled-check', (KEY)
|
9
9
|
KEYS[6] 'meta', (KEY)
|
10
10
|
KEYS[7] 'paused', (LIST)
|
11
|
-
KEYS[8] '
|
11
|
+
KEYS[8] 'marker'
|
12
|
+
KEYS[9] 'event stream' (STREAM)
|
12
13
|
ARGV[1] Max stalled job count
|
13
14
|
ARGV[2] queue.toKey('')
|
14
15
|
ARGV[3] timestamp
|
15
16
|
ARGV[4] max check time
|
16
17
|
Events:
|
17
18
|
'stalled' with stalled job id.
|
18
|
-
]] -- Includes
|
19
|
-
--[[
|
20
|
-
Move stalled jobs to wait.
|
21
|
-
Input:
|
22
|
-
stalledKey 'stalled' (SET)
|
23
|
-
waitKey 'wait', (LIST)
|
24
|
-
activeKey 'active', (LIST)
|
25
|
-
failedKey 'failed', (ZSET)
|
26
|
-
stalledCheckKey 'stalled-check', (KEY)
|
27
|
-
metaKey 'meta', (KEY)
|
28
|
-
pausedKey 'paused', (LIST)
|
29
|
-
eventStreamKey 'event stream' (STREAM)
|
30
|
-
maxStalledJobCount Max stalled job count
|
31
|
-
queueKeyPrefix queue.toKey('')
|
32
|
-
timestamp timestamp
|
33
|
-
maxCheckTime max check time
|
34
|
-
Events:
|
35
|
-
'stalled' with stalled job id.
|
36
19
|
]]
|
37
20
|
local rcall = redis.call
|
38
21
|
-- Includes
|
22
|
+
--[[
|
23
|
+
Function to add job in target list and add marker if needed.
|
24
|
+
]]
|
25
|
+
-- Includes
|
26
|
+
--[[
|
27
|
+
Add marker if needed when a job is available.
|
28
|
+
]]
|
29
|
+
local function addBaseMarkerIfNeeded(markerKey, isPaused)
|
30
|
+
if not isPaused then
|
31
|
+
rcall("ZADD", markerKey, 0, "0")
|
32
|
+
end
|
33
|
+
end
|
34
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
|
35
|
+
rcall(pushCmd, targetKey, jobId)
|
36
|
+
addBaseMarkerIfNeeded(markerKey, isPaused)
|
37
|
+
end
|
39
38
|
--[[
|
40
39
|
Function to loop in batches.
|
41
40
|
Just a bit of warning, some commands as ZREM
|
@@ -197,103 +196,107 @@ local function trimEvents(metaKey, eventStreamKey)
|
|
197
196
|
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
|
198
197
|
end
|
199
198
|
end
|
200
|
-
|
201
|
-
local
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
199
|
+
local stalledKey = KEYS[1]
|
200
|
+
local waitKey = KEYS[2]
|
201
|
+
local activeKey = KEYS[3]
|
202
|
+
local failedKey = KEYS[4]
|
203
|
+
local stalledCheckKey = KEYS[5]
|
204
|
+
local metaKey = KEYS[6]
|
205
|
+
local pausedKey = KEYS[7]
|
206
|
+
local markerKey = KEYS[8]
|
207
|
+
local eventStreamKey = KEYS[9]
|
208
|
+
local maxStalledJobCount = ARGV[1]
|
209
|
+
local queueKeyPrefix = ARGV[2]
|
210
|
+
local timestamp = ARGV[3]
|
211
|
+
local maxCheckTime = ARGV[4]
|
212
|
+
if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
|
213
|
+
rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
|
214
|
+
-- Trim events before emiting them to avoid trimming events emitted in this script
|
215
|
+
trimEvents(metaKey, eventStreamKey)
|
216
|
+
-- Move all stalled jobs to wait
|
217
|
+
local stalling = rcall('SMEMBERS', stalledKey)
|
218
|
+
local stalled = {}
|
219
|
+
local failed = {}
|
220
|
+
if (#stalling > 0) then
|
221
|
+
rcall('DEL', stalledKey)
|
222
|
+
local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
|
223
|
+
-- Remove from active list
|
224
|
+
for i, jobId in ipairs(stalling) do
|
225
|
+
-- Markers in waitlist DEPRECATED in v5: Remove in v6.
|
226
|
+
if string.sub(jobId, 1, 2) == "0:" then
|
227
|
+
-- If the jobId is a delay marker ID we just remove it.
|
228
|
+
rcall("LREM", activeKey, 1, jobId)
|
229
|
+
else
|
230
|
+
local jobKey = queueKeyPrefix .. jobId
|
231
|
+
-- Check that the lock is also missing, then we can handle this job as really stalled.
|
232
|
+
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
|
233
|
+
-- Remove from the active queue.
|
234
|
+
local removed = rcall("LREM", activeKey, 1, jobId)
|
235
|
+
if (removed > 0) then
|
236
|
+
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
237
|
+
local stalledCount =
|
238
|
+
rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
239
|
+
if (stalledCount > MAX_STALLED_JOB_COUNT) then
|
240
|
+
local rawOpts = rcall("HGET", jobKey, "opts")
|
241
|
+
local opts = cjson.decode(rawOpts)
|
242
|
+
local removeOnFailType = type(opts["removeOnFail"])
|
243
|
+
rcall("ZADD", failedKey, timestamp, jobId)
|
244
|
+
local failedReason =
|
245
|
+
"job stalled more than allowable limit"
|
246
|
+
rcall("HMSET", jobKey, "failedReason", failedReason,
|
247
|
+
"finishedOn", timestamp)
|
248
|
+
rcall("XADD", eventStreamKey, "*", "event",
|
249
|
+
"failed", "jobId", jobId, 'prev', 'active',
|
250
|
+
'failedReason', failedReason)
|
251
|
+
if removeOnFailType == "number" then
|
252
|
+
removeJobsByMaxCount(opts["removeOnFail"],
|
253
|
+
failedKey, queueKeyPrefix)
|
254
|
+
elseif removeOnFailType == "boolean" then
|
255
|
+
if opts["removeOnFail"] then
|
256
|
+
removeJob(jobId, false, queueKeyPrefix)
|
257
|
+
rcall("ZREM", failedKey, jobId)
|
258
|
+
end
|
259
|
+
elseif removeOnFailType ~= "nil" then
|
260
|
+
local maxAge = opts["removeOnFail"]["age"]
|
261
|
+
local maxCount = opts["removeOnFail"]["count"]
|
262
|
+
if maxAge ~= nil then
|
263
|
+
removeJobsByMaxAge(timestamp, maxAge,
|
264
|
+
failedKey, queueKeyPrefix)
|
265
|
+
end
|
266
|
+
if maxCount ~= nil and maxCount > 0 then
|
267
|
+
removeJobsByMaxCount(maxCount, failedKey,
|
268
|
+
queueKeyPrefix)
|
263
269
|
end
|
264
|
-
table.insert(failed, jobId)
|
265
|
-
else
|
266
|
-
local target =
|
267
|
-
getTargetQueueList(metaKey, waitKey, pausedKey)
|
268
|
-
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
|
269
|
-
rcall("RPUSH", target, jobId)
|
270
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
271
|
-
"waiting", "jobId", jobId, 'prev', 'active')
|
272
|
-
-- Emit the stalled event
|
273
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
274
|
-
"stalled", "jobId", jobId)
|
275
|
-
table.insert(stalled, jobId)
|
276
270
|
end
|
271
|
+
table.insert(failed, jobId)
|
272
|
+
else
|
273
|
+
local target, isPaused=
|
274
|
+
getTargetQueueList(metaKey, waitKey, pausedKey)
|
275
|
+
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
|
276
|
+
addJobInTargetList(target, markerKey, "RPUSH", isPaused, jobId)
|
277
|
+
rcall("XADD", eventStreamKey, "*", "event",
|
278
|
+
"waiting", "jobId", jobId, 'prev', 'active')
|
279
|
+
-- Emit the stalled event
|
280
|
+
rcall("XADD", eventStreamKey, "*", "event",
|
281
|
+
"stalled", "jobId", jobId)
|
282
|
+
table.insert(stalled, jobId)
|
277
283
|
end
|
278
284
|
end
|
279
285
|
end
|
280
286
|
end
|
281
287
|
end
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
+
end
|
289
|
+
-- Mark potentially stalled jobs
|
290
|
+
local active = rcall('LRANGE', activeKey, 0, -1)
|
291
|
+
if (#active > 0) then
|
292
|
+
for from, to in batches(#active, 7000) do
|
293
|
+
rcall('SADD', stalledKey, unpack(active, from, to))
|
288
294
|
end
|
289
|
-
return {failed, stalled}
|
290
295
|
end
|
291
|
-
return
|
292
|
-
KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])
|
293
|
-
`;
|
296
|
+
return {failed, stalled}`;
|
294
297
|
export const moveStalledJobsToWait = {
|
295
298
|
name: 'moveStalledJobsToWait',
|
296
299
|
content,
|
297
|
-
keys:
|
300
|
+
keys: 9,
|
298
301
|
};
|
299
|
-
//# sourceMappingURL=moveStalledJobsToWait-
|
302
|
+
//# sourceMappingURL=moveStalledJobsToWait-9.js.map
|