bullmq 5.46.1 → 5.47.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/flow-producer.js +1 -1
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job.js +11 -9
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/scripts.js +4 -3
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/includes/moveParentToFailedIfNeeded.lua +3 -0
- package/dist/cjs/commands/includes/moveParentToWait.lua +49 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +6 -51
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +26 -30
- package/dist/cjs/commands/moveToFinished-14.lua +38 -30
- package/dist/cjs/commands/moveToWaitingChildren-8.lua +13 -8
- package/dist/cjs/scripts/addDelayedJob-6.js +24 -20
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
- package/dist/cjs/scripts/addParentJob-4.js +24 -20
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js +24 -20
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
- package/dist/cjs/scripts/addStandardJob-8.js +24 -20
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
- package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
- package/dist/cjs/scripts/drain-5.js +5 -5
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +56 -54
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +67 -55
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-8.js +43 -33
- package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +5 -5
- package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
- package/dist/cjs/scripts/removeJob-3.js +5 -5
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/flow-producer.js +1 -1
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job.js +11 -9
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/scripts.js +4 -3
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/includes/moveParentToFailedIfNeeded.lua +3 -0
- package/dist/esm/commands/includes/moveParentToWait.lua +49 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +6 -51
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +26 -30
- package/dist/esm/commands/moveToFinished-14.lua +38 -30
- package/dist/esm/commands/moveToWaitingChildren-8.lua +13 -8
- package/dist/esm/scripts/addDelayedJob-6.js +24 -20
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addJobScheduler-11.js +5 -5
- package/dist/esm/scripts/addParentJob-4.js +24 -20
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js +24 -20
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
- package/dist/esm/scripts/addStandardJob-8.js +24 -20
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
- package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
- package/dist/esm/scripts/drain-5.js +5 -5
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +56 -54
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +67 -55
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-8.js +43 -33
- package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +5 -5
- package/dist/esm/scripts/removeChildDependency-1.js +5 -5
- package/dist/esm/scripts/removeJob-3.js +5 -5
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/job-options.d.ts +12 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
@@ -156,9 +156,11 @@ end
|
|
156
156
|
]]
|
157
157
|
-- Includes
|
158
158
|
--[[
|
159
|
-
Validate and move parent to
|
159
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
160
|
+
]]
|
161
|
+
--[[
|
162
|
+
Validate and move parent to a wait status (wait, prioritized or delayed)
|
160
163
|
]]
|
161
|
-
-- Includes
|
162
164
|
--[[
|
163
165
|
Add delay marker if needed.
|
164
166
|
]]
|
@@ -227,12 +229,9 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
227
229
|
end
|
228
230
|
return waitKey, false
|
229
231
|
end
|
230
|
-
local function
|
231
|
-
|
232
|
-
|
233
|
-
parentQueueKey .. ":waiting-children", parentId)
|
234
|
-
if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
|
235
|
-
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
232
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
233
|
+
local isParentWaitingChildren = rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
234
|
+
if isParentWaitingChildren > 0 then
|
236
235
|
local parentWaitKey = parentQueueKey .. ":wait"
|
237
236
|
local parentPausedKey = parentQueueKey .. ":paused"
|
238
237
|
local parentActiveKey = parentQueueKey .. ":active"
|
@@ -241,32 +240,37 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
241
240
|
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
242
241
|
local priority = tonumber(jobAttributes[1]) or 0
|
243
242
|
local delay = tonumber(jobAttributes[2]) or 0
|
243
|
+
-- ignore dependencies if any left
|
244
|
+
rcall("HSET", parentKey, "igdp", 1)
|
244
245
|
if delay > 0 then
|
245
246
|
local delayedTimestamp = tonumber(timestamp) + delay
|
246
247
|
local score = delayedTimestamp * 0x1000
|
247
248
|
local parentDelayedKey = parentQueueKey .. ":delayed"
|
248
249
|
rcall("ZADD", parentDelayedKey, score, parentId)
|
249
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
250
|
-
|
250
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
251
|
+
delayedTimestamp)
|
251
252
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
252
253
|
else
|
253
254
|
if priority == 0 then
|
254
|
-
local parentTarget, isParentPausedOrMaxed =
|
255
|
-
|
256
|
-
|
257
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
258
|
-
parentId)
|
255
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
256
|
+
parentWaitKey, parentPausedKey)
|
257
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
259
258
|
else
|
260
259
|
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
261
|
-
addJobWithPriority(parentMarkerKey,
|
262
|
-
|
263
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
260
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
261
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
264
262
|
end
|
265
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
266
|
-
|
263
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
264
|
+
"waiting-children")
|
267
265
|
end
|
268
266
|
end
|
269
267
|
end
|
268
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
269
|
+
local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
270
|
+
if hasPendingDependencies then
|
271
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
272
|
+
end
|
273
|
+
end
|
270
274
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
271
275
|
parentId, jobIdKey, returnvalue, timestamp )
|
272
276
|
local processedSet = parentKey .. ":processed"
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addPrioritizedJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addPrioritizedJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Vf,CAAC;AACW,QAAA,iBAAiB,GAAG;IAC/B,IAAI,EAAE,mBAAmB;IACzB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -100,7 +100,7 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
100
100
|
end
|
101
101
|
return waitKey, false
|
102
102
|
end
|
103
|
-
local function
|
103
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
104
104
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
105
105
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
106
106
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -128,10 +128,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
128
128
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
129
129
|
end
|
130
130
|
else
|
131
|
-
|
131
|
+
_moveParentToWait(parentPrefix, parentId)
|
132
132
|
end
|
133
133
|
else
|
134
|
-
|
134
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
135
135
|
end
|
136
136
|
end
|
137
137
|
end
|
@@ -159,10 +159,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
159
159
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
160
160
|
end
|
161
161
|
else
|
162
|
-
|
162
|
+
_moveParentToWait(parentPrefix, parentId)
|
163
163
|
end
|
164
164
|
else
|
165
|
-
|
165
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
166
166
|
end
|
167
167
|
end
|
168
168
|
end
|
@@ -140,9 +140,11 @@ end
|
|
140
140
|
]]
|
141
141
|
-- Includes
|
142
142
|
--[[
|
143
|
-
Validate and move parent to
|
143
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
144
|
+
]]
|
145
|
+
--[[
|
146
|
+
Validate and move parent to a wait status (wait, prioritized or delayed)
|
144
147
|
]]
|
145
|
-
-- Includes
|
146
148
|
--[[
|
147
149
|
Add delay marker if needed.
|
148
150
|
]]
|
@@ -200,12 +202,9 @@ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
|
200
202
|
end
|
201
203
|
return false
|
202
204
|
end
|
203
|
-
local function
|
204
|
-
|
205
|
-
|
206
|
-
parentQueueKey .. ":waiting-children", parentId)
|
207
|
-
if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
|
208
|
-
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
205
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
206
|
+
local isParentWaitingChildren = rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
207
|
+
if isParentWaitingChildren > 0 then
|
209
208
|
local parentWaitKey = parentQueueKey .. ":wait"
|
210
209
|
local parentPausedKey = parentQueueKey .. ":paused"
|
211
210
|
local parentActiveKey = parentQueueKey .. ":active"
|
@@ -214,32 +213,37 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
214
213
|
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
215
214
|
local priority = tonumber(jobAttributes[1]) or 0
|
216
215
|
local delay = tonumber(jobAttributes[2]) or 0
|
216
|
+
-- ignore dependencies if any left
|
217
|
+
rcall("HSET", parentKey, "igdp", 1)
|
217
218
|
if delay > 0 then
|
218
219
|
local delayedTimestamp = tonumber(timestamp) + delay
|
219
220
|
local score = delayedTimestamp * 0x1000
|
220
221
|
local parentDelayedKey = parentQueueKey .. ":delayed"
|
221
222
|
rcall("ZADD", parentDelayedKey, score, parentId)
|
222
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
223
|
-
|
223
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
224
|
+
delayedTimestamp)
|
224
225
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
225
226
|
else
|
226
227
|
if priority == 0 then
|
227
|
-
local parentTarget, isParentPausedOrMaxed =
|
228
|
-
|
229
|
-
|
230
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
231
|
-
parentId)
|
228
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
229
|
+
parentWaitKey, parentPausedKey)
|
230
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
232
231
|
else
|
233
232
|
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
234
|
-
addJobWithPriority(parentMarkerKey,
|
235
|
-
|
236
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
233
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
234
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
237
235
|
end
|
238
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
239
|
-
|
236
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
237
|
+
"waiting-children")
|
240
238
|
end
|
241
239
|
end
|
242
240
|
end
|
241
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
242
|
+
local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
243
|
+
if hasPendingDependencies then
|
244
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
245
|
+
end
|
246
|
+
end
|
243
247
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
244
248
|
parentId, jobIdKey, returnvalue, timestamp )
|
245
249
|
local processedSet = parentKey .. ":processed"
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addStandardJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addStandardJob-8.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkWf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -136,7 +136,7 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
136
136
|
end
|
137
137
|
return waitKey, false
|
138
138
|
end
|
139
|
-
local function
|
139
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
140
140
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
141
141
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
142
142
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -164,10 +164,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
164
164
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
165
165
|
end
|
166
166
|
else
|
167
|
-
|
167
|
+
_moveParentToWait(parentPrefix, parentId)
|
168
168
|
end
|
169
169
|
else
|
170
|
-
|
170
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
171
171
|
end
|
172
172
|
end
|
173
173
|
end
|
@@ -195,10 +195,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
195
195
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
196
196
|
end
|
197
197
|
else
|
198
|
-
|
198
|
+
_moveParentToWait(parentPrefix, parentId)
|
199
199
|
end
|
200
200
|
else
|
201
|
-
|
201
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
202
202
|
end
|
203
203
|
end
|
204
204
|
end
|
@@ -108,7 +108,7 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
108
108
|
end
|
109
109
|
return waitKey, false
|
110
110
|
end
|
111
|
-
local function
|
111
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
112
112
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
113
113
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
114
114
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -136,10 +136,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
136
136
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
137
137
|
end
|
138
138
|
else
|
139
|
-
|
139
|
+
_moveParentToWait(parentPrefix, parentId)
|
140
140
|
end
|
141
141
|
else
|
142
|
-
|
142
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
143
143
|
end
|
144
144
|
end
|
145
145
|
end
|
@@ -167,10 +167,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
167
167
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
168
168
|
end
|
169
169
|
else
|
170
|
-
|
170
|
+
_moveParentToWait(parentPrefix, parentId)
|
171
171
|
end
|
172
172
|
else
|
173
|
-
|
173
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
174
174
|
end
|
175
175
|
end
|
176
176
|
end
|
@@ -79,9 +79,11 @@ end
|
|
79
79
|
]]
|
80
80
|
-- Includes
|
81
81
|
--[[
|
82
|
-
Validate and move parent to
|
82
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
83
|
+
]]
|
84
|
+
--[[
|
85
|
+
Validate and move parent to a wait status (wait, prioritized or delayed)
|
83
86
|
]]
|
84
|
-
-- Includes
|
85
87
|
--[[
|
86
88
|
Add delay marker if needed.
|
87
89
|
]]
|
@@ -139,12 +141,9 @@ local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
|
|
139
141
|
end
|
140
142
|
return false
|
141
143
|
end
|
142
|
-
local function
|
143
|
-
|
144
|
-
|
145
|
-
parentQueueKey .. ":waiting-children", parentId)
|
146
|
-
if isParentActive and rcall("SCARD", parentDependenciesKey) == 0 then
|
147
|
-
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
144
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
145
|
+
local isParentWaitingChildren = rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
146
|
+
if isParentWaitingChildren > 0 then
|
148
147
|
local parentWaitKey = parentQueueKey .. ":wait"
|
149
148
|
local parentPausedKey = parentQueueKey .. ":paused"
|
150
149
|
local parentActiveKey = parentQueueKey .. ":active"
|
@@ -153,32 +152,37 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
|
153
152
|
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
154
153
|
local priority = tonumber(jobAttributes[1]) or 0
|
155
154
|
local delay = tonumber(jobAttributes[2]) or 0
|
155
|
+
-- ignore dependencies if any left
|
156
|
+
rcall("HSET", parentKey, "igdp", 1)
|
156
157
|
if delay > 0 then
|
157
158
|
local delayedTimestamp = tonumber(timestamp) + delay
|
158
159
|
local score = delayedTimestamp * 0x1000
|
159
160
|
local parentDelayedKey = parentQueueKey .. ":delayed"
|
160
161
|
rcall("ZADD", parentDelayedKey, score, parentId)
|
161
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
162
|
-
|
162
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
163
|
+
delayedTimestamp)
|
163
164
|
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
164
165
|
else
|
165
166
|
if priority == 0 then
|
166
|
-
local parentTarget, isParentPausedOrMaxed =
|
167
|
-
|
168
|
-
|
169
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
170
|
-
parentId)
|
167
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
168
|
+
parentWaitKey, parentPausedKey)
|
169
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
171
170
|
else
|
172
171
|
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
173
|
-
addJobWithPriority(parentMarkerKey,
|
174
|
-
|
175
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
172
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
173
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
176
174
|
end
|
177
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
178
|
-
|
175
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
176
|
+
"waiting-children")
|
179
177
|
end
|
180
178
|
end
|
181
179
|
end
|
180
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
181
|
+
local hasPendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
182
|
+
if hasPendingDependencies then
|
183
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
184
|
+
end
|
185
|
+
end
|
182
186
|
--[[
|
183
187
|
Function to remove deduplication key if needed.
|
184
188
|
]]
|
@@ -232,7 +236,7 @@ end
|
|
232
236
|
local getJobKeyPrefix = function (jobKey, jobId)
|
233
237
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
234
238
|
end
|
235
|
-
local function
|
239
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
236
240
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
237
241
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
238
242
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -260,10 +264,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
260
264
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
261
265
|
end
|
262
266
|
else
|
263
|
-
|
267
|
+
_moveParentToWait(parentPrefix, parentId)
|
264
268
|
end
|
265
269
|
else
|
266
|
-
|
270
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
267
271
|
end
|
268
272
|
end
|
269
273
|
end
|
@@ -291,10 +295,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
291
295
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
292
296
|
end
|
293
297
|
else
|
294
|
-
|
298
|
+
_moveParentToWait(parentPrefix, parentId)
|
295
299
|
end
|
296
300
|
else
|
297
|
-
|
301
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
298
302
|
end
|
299
303
|
end
|
300
304
|
end
|
@@ -402,6 +406,8 @@ local function moveParentToFailedIfNeeded(parentQueueKey, parentKey, parentId, j
|
|
402
406
|
parentKey,
|
403
407
|
timestamp
|
404
408
|
)
|
409
|
+
elseif parentData['cpof'] then
|
410
|
+
moveParentToWait(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
405
411
|
elseif parentData['idof'] or parentData['rdof'] then
|
406
412
|
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
407
413
|
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
|
@@ -463,7 +469,9 @@ local maxStalledJobCount = tonumber(ARGV[1])
|
|
463
469
|
local queueKeyPrefix = ARGV[2]
|
464
470
|
local timestamp = ARGV[3]
|
465
471
|
local maxCheckTime = ARGV[4]
|
466
|
-
if rcall("EXISTS", stalledCheckKey) == 1 then
|
472
|
+
if rcall("EXISTS", stalledCheckKey) == 1 then
|
473
|
+
return {{}, {}}
|
474
|
+
end
|
467
475
|
rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
|
468
476
|
-- Trim events before emiting them to avoid trimming events emitted in this script
|
469
477
|
trimEvents(metaKey, eventStreamKey)
|
@@ -487,8 +495,7 @@ if (#stalling > 0) then
|
|
487
495
|
local removed = rcall("LREM", activeKey, 1, jobId)
|
488
496
|
if (removed > 0) then
|
489
497
|
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
490
|
-
local stalledCount =
|
491
|
-
rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
498
|
+
local stalledCount = rcall("HINCRBY", jobKey, "stalledCounter", 1)
|
492
499
|
if (stalledCount > maxStalledJobCount) then
|
493
500
|
local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
|
494
501
|
local rawOpts = jobAttributes[1]
|
@@ -496,38 +503,35 @@ if (#stalling > 0) then
|
|
496
503
|
local opts = cjson.decode(rawOpts)
|
497
504
|
rcall("ZADD", failedKey, timestamp, jobId)
|
498
505
|
removeDeduplicationKeyIfNeeded(queueKeyPrefix, jobAttributes[3])
|
499
|
-
local failedReason =
|
500
|
-
|
501
|
-
rcall("
|
502
|
-
|
503
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
504
|
-
"failed", "jobId", jobId, 'prev', 'active',
|
505
|
-
'failedReason', failedReason)
|
506
|
+
local failedReason = "job stalled more than allowable limit"
|
507
|
+
rcall("HMSET", jobKey, "failedReason", failedReason, "finishedOn", timestamp)
|
508
|
+
rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId", jobId, 'prev', 'active',
|
509
|
+
'failedReason', failedReason)
|
506
510
|
if rawParentData then
|
507
|
-
if opts['fpof'] then
|
511
|
+
if opts['fpof'] or opts['cpof'] then
|
508
512
|
local parentData = cjson.decode(rawParentData)
|
509
513
|
-- TODO: need to remove this job from dependencies set in next breaking change
|
510
514
|
-- no for now as it would imply a breaking change
|
511
515
|
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
512
516
|
local unsuccesssfulSet = parentKey .. ":unsuccessful"
|
513
517
|
rcall("ZADD", unsuccesssfulSet, timestamp, jobKey)
|
514
|
-
|
515
|
-
parentData['queueKey'],
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
timestamp
|
520
|
-
|
518
|
+
if opts['fpof'] then
|
519
|
+
moveParentToFailedIfNeeded(parentData['queueKey'],
|
520
|
+
parentData['queueKey'] .. ':' .. parentData['id'], parentData['id'], jobKey,
|
521
|
+
timestamp)
|
522
|
+
elseif opts['cpof'] then
|
523
|
+
moveParentToWait(parentData['queueKey'], parentKey, parentData['id'], timestamp)
|
524
|
+
end
|
521
525
|
elseif opts['idof'] or opts['rdof'] then
|
522
526
|
local parentData = cjson.decode(rawParentData)
|
523
527
|
local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
524
528
|
local dependenciesSet = parentKey .. ":dependencies"
|
525
529
|
if rcall("SREM", dependenciesSet, jobKey) == 1 then
|
526
|
-
moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
|
527
|
-
|
530
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet, parentKey,
|
531
|
+
parentData['id'], timestamp)
|
528
532
|
if opts['idof'] then
|
529
|
-
|
530
|
-
|
533
|
+
local failedSet = parentKey .. ":failed"
|
534
|
+
rcall("HSET", failedSet, jobKey, failedReason)
|
531
535
|
end
|
532
536
|
end
|
533
537
|
end
|
@@ -535,15 +539,12 @@ if (#stalling > 0) then
|
|
535
539
|
removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
|
536
540
|
table.insert(failed, jobId)
|
537
541
|
else
|
538
|
-
local target, isPausedOrMaxed =
|
539
|
-
getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
|
542
|
+
local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
|
540
543
|
-- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
|
541
544
|
addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
|
542
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
543
|
-
"waiting", "jobId", jobId, 'prev', 'active')
|
545
|
+
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, 'prev', 'active')
|
544
546
|
-- Emit the stalled event
|
545
|
-
rcall("XADD", eventStreamKey, "*", "event",
|
546
|
-
"stalled", "jobId", jobId)
|
547
|
+
rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId", jobId)
|
547
548
|
table.insert(stalled, jobId)
|
548
549
|
end
|
549
550
|
end
|
@@ -558,7 +559,8 @@ if (#active > 0) then
|
|
558
559
|
rcall('SADD', stalledKey, unpack(active, from, to))
|
559
560
|
end
|
560
561
|
end
|
561
|
-
return {failed, stalled}
|
562
|
+
return {failed, stalled}
|
563
|
+
`;
|
562
564
|
exports.moveStalledJobsToWait = {
|
563
565
|
name: 'moveStalledJobsToWait',
|
564
566
|
content,
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+iBf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|