bullmq 5.41.4 → 5.41.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/job-scheduler.js +40 -15
- package/dist/cjs/classes/job-scheduler.js.map +1 -1
- package/dist/cjs/classes/job.js +9 -12
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/scripts.js +15 -1
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/commands/addDelayedJob-6.lua +5 -2
- package/dist/cjs/commands/addJobScheduler-10.lua +130 -0
- package/dist/cjs/commands/includes/addDelayedJob.lua +2 -7
- package/dist/cjs/commands/includes/getOrSetMaxEvents.lua +6 -6
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +1 -1
- package/dist/cjs/commands/moveToFinished-14.lua +1 -1
- package/dist/cjs/scripts/addDelayedJob-6.js +41 -43
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/{addJobScheduler-2.js → addJobScheduler-10.js} +234 -42
- package/dist/cjs/scripts/addJobScheduler-10.js.map +1 -0
- package/dist/cjs/scripts/addParentJob-4.js +6 -6
- package/dist/cjs/scripts/addPrioritizedJob-8.js +6 -6
- package/dist/cjs/scripts/addStandardJob-8.js +6 -6
- package/dist/cjs/scripts/changeDelay-4.js +6 -6
- package/dist/cjs/scripts/index.js +1 -1
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +6 -6
- package/dist/cjs/scripts/moveJobsToWait-8.js +6 -6
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +49 -9
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToDelayed-8.js +6 -6
- package/dist/cjs/scripts/moveToFinished-14.js +49 -9
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/removeJob-3.js +6 -6
- package/dist/cjs/scripts/reprocessJob-8.js +6 -6
- package/dist/cjs/scripts/retryJob-11.js +6 -6
- package/dist/cjs/scripts/updateJobScheduler-7.js +8 -41
- package/dist/cjs/scripts/updateJobScheduler-7.js.map +1 -1
- package/dist/cjs/scripts/updateProgress-3.js +6 -6
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/job-scheduler.d.ts +1 -0
- package/dist/esm/classes/job-scheduler.js +40 -15
- package/dist/esm/classes/job-scheduler.js.map +1 -1
- package/dist/esm/classes/job.js +9 -12
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +2 -2
- package/dist/esm/classes/scripts.js +15 -1
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/commands/addDelayedJob-6.lua +5 -2
- package/dist/esm/commands/addJobScheduler-10.lua +130 -0
- package/dist/esm/commands/includes/addDelayedJob.lua +2 -7
- package/dist/esm/commands/includes/getOrSetMaxEvents.lua +6 -6
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +1 -1
- package/dist/esm/commands/moveToFinished-14.lua +1 -1
- package/dist/esm/scripts/addDelayedJob-6.js +41 -43
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/{addJobScheduler-2.js → addJobScheduler-10.js} +234 -42
- package/dist/esm/scripts/addJobScheduler-10.js.map +1 -0
- package/dist/esm/scripts/addParentJob-4.js +6 -6
- package/dist/esm/scripts/addPrioritizedJob-8.js +6 -6
- package/dist/esm/scripts/addStandardJob-8.js +6 -6
- package/dist/esm/scripts/changeDelay-4.js +6 -6
- package/dist/esm/scripts/index.d.ts +1 -1
- package/dist/esm/scripts/index.js +1 -1
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveJobFromActiveToWait-9.js +6 -6
- package/dist/esm/scripts/moveJobsToWait-8.js +6 -6
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +49 -9
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToDelayed-8.js +6 -6
- package/dist/esm/scripts/moveToFinished-14.js +49 -9
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/removeJob-3.js +6 -6
- package/dist/esm/scripts/reprocessJob-8.js +6 -6
- package/dist/esm/scripts/retryJob-11.js +6 -6
- package/dist/esm/scripts/updateJobScheduler-7.js +8 -41
- package/dist/esm/scripts/updateJobScheduler-7.js.map +1 -1
- package/dist/esm/scripts/updateProgress-3.js +6 -6
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/addJobScheduler-2.lua +0 -52
- package/dist/cjs/scripts/addJobScheduler-2.js.map +0 -1
- package/dist/esm/commands/addJobScheduler-2.lua +0 -52
- package/dist/esm/scripts/addJobScheduler-2.js.map +0 -1
- /package/dist/esm/scripts/{addJobScheduler-2.d.ts → addJobScheduler-10.d.ts} +0 -0
@@ -4,8 +4,16 @@ exports.addJobScheduler = void 0;
|
|
4
4
|
const content = `--[[
|
5
5
|
Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
|
6
6
|
Input:
|
7
|
-
KEYS[1]
|
8
|
-
KEYS[2]
|
7
|
+
KEYS[1] 'repeat' key
|
8
|
+
KEYS[2] 'delayed' key
|
9
|
+
KEYS[3] 'wait' key
|
10
|
+
KEYS[4] 'paused' key
|
11
|
+
KEYS[5] 'meta' key
|
12
|
+
KEYS[6] 'prioritized' key
|
13
|
+
KEYS[7] 'marker' key
|
14
|
+
KEYS[8] 'id' key
|
15
|
+
KEYS[9] 'events' key
|
16
|
+
KEYS[10] 'pc' priority counter
|
9
17
|
ARGV[1] next milliseconds
|
10
18
|
ARGV[2] msgpacked options
|
11
19
|
[1] name
|
@@ -16,46 +24,88 @@ const content = `--[[
|
|
16
24
|
ARGV[3] jobs scheduler id
|
17
25
|
ARGV[4] Json stringified template data
|
18
26
|
ARGV[5] mspacked template opts
|
19
|
-
ARGV[6]
|
27
|
+
ARGV[6] msgpacked delayed opts
|
28
|
+
ARGV[7] timestamp
|
29
|
+
ARGV[8] prefix key
|
30
|
+
ARGV[9] producer key
|
20
31
|
Output:
|
21
32
|
repeatableKey - OK
|
22
|
-
]]
|
33
|
+
]]
|
34
|
+
local rcall = redis.call
|
23
35
|
local repeatKey = KEYS[1]
|
24
36
|
local delayedKey = KEYS[2]
|
37
|
+
local prioritizedKey = KEYS[6]
|
25
38
|
local nextMillis = ARGV[1]
|
26
39
|
local jobSchedulerId = ARGV[3]
|
27
40
|
local templateOpts = cmsgpack.unpack(ARGV[5])
|
28
|
-
local prefixKey = ARGV[
|
41
|
+
local prefixKey = ARGV[8]
|
29
42
|
-- Includes
|
30
43
|
--[[
|
31
|
-
|
44
|
+
Adds a delayed job to the queue by doing the following:
|
45
|
+
- Creates a new job key with the job data.
|
46
|
+
- adds to delayed zset.
|
47
|
+
- Emits a global event 'delayed' if the job is delayed.
|
32
48
|
]]
|
33
49
|
-- Includes
|
34
50
|
--[[
|
35
|
-
|
51
|
+
Add delay marker if needed.
|
36
52
|
]]
|
37
|
-
|
38
|
-
local deduplicationId = rcall("HGET", jobKey, "deid")
|
39
|
-
if deduplicationId then
|
40
|
-
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
41
|
-
rcall("DEL", deduplicationKey)
|
42
|
-
end
|
43
|
-
end
|
53
|
+
-- Includes
|
44
54
|
--[[
|
45
|
-
Function to
|
55
|
+
Function to return the next delayed job timestamp.
|
46
56
|
]]
|
47
|
-
local function
|
48
|
-
|
49
|
-
|
57
|
+
local function getNextDelayedTimestamp(delayedKey)
|
58
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
59
|
+
if #result then
|
60
|
+
local nextTimestamp = tonumber(result[2])
|
61
|
+
if nextTimestamp ~= nil then
|
62
|
+
return nextTimestamp / 0x1000
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
67
|
+
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
68
|
+
if nextTimestamp ~= nil then
|
69
|
+
-- Replace the score of the marker with the newest known
|
70
|
+
-- next timestamp.
|
71
|
+
rcall("ZADD", markerKey, nextTimestamp, "1")
|
72
|
+
end
|
50
73
|
end
|
51
74
|
--[[
|
52
|
-
|
53
|
-
|
54
|
-
|
75
|
+
Bake in the job id first 12 bits into the timestamp
|
76
|
+
to guarantee correct execution order of delayed jobs
|
77
|
+
(up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
|
78
|
+
WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
|
55
79
|
]]
|
56
|
-
|
80
|
+
local function getDelayedScore(delayedKey, timestamp, delay)
|
81
|
+
local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
|
82
|
+
local minScore = delayedTimestamp * 0x1000
|
83
|
+
local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
|
84
|
+
local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
|
85
|
+
minScore, "WITHSCORES","LIMIT", 0, 1)
|
86
|
+
if #result then
|
87
|
+
local currentMaxScore = tonumber(result[2])
|
88
|
+
if currentMaxScore ~= nil then
|
89
|
+
if currentMaxScore >= maxScore then
|
90
|
+
return maxScore, delayedTimestamp
|
91
|
+
else
|
92
|
+
return currentMaxScore + 1, delayedTimestamp
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
return minScore, delayedTimestamp
|
97
|
+
end
|
98
|
+
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
99
|
+
maxEvents, markerKey, delay)
|
100
|
+
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
|
101
|
+
rcall("ZADD", delayedKey, score, jobId)
|
102
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
103
|
+
"jobId", jobId, "delay", delayedTimestamp)
|
104
|
+
-- mark that a delayed job is available
|
105
|
+
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
106
|
+
end
|
57
107
|
--[[
|
58
|
-
Function to add job
|
108
|
+
Function to add job considering priority.
|
59
109
|
]]
|
60
110
|
-- Includes
|
61
111
|
--[[
|
@@ -66,19 +116,29 @@ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
66
116
|
rcall("ZADD", markerKey, 0, "0")
|
67
117
|
end
|
68
118
|
end
|
69
|
-
|
70
|
-
|
119
|
+
--[[
|
120
|
+
Function to get priority score.
|
121
|
+
]]
|
122
|
+
local function getPriorityScore(priority, priorityCounterKey)
|
123
|
+
local prioCounter = rcall("INCR", priorityCounterKey)
|
124
|
+
return priority * 0x100000000 + prioCounter % 0x100000000
|
125
|
+
end
|
126
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
|
127
|
+
isPausedOrMaxed)
|
128
|
+
local score = getPriorityScore(priority, priorityCounterKey)
|
129
|
+
rcall("ZADD", prioritizedKey, score, jobId)
|
71
130
|
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
72
131
|
end
|
73
132
|
--[[
|
74
|
-
|
75
|
-
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
133
|
+
Function to get max events value or set by default 10000.
|
76
134
|
]]
|
77
|
-
local
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
135
|
+
local function getOrSetMaxEvents(metaKey)
|
136
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
137
|
+
if not maxEvents then
|
138
|
+
maxEvents = 10000
|
139
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
140
|
+
end
|
141
|
+
return maxEvents
|
82
142
|
end
|
83
143
|
--[[
|
84
144
|
Function to check for the meta.paused key to decide if we are paused or not
|
@@ -100,6 +160,58 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
100
160
|
end
|
101
161
|
return waitKey, false
|
102
162
|
end
|
163
|
+
--[[
|
164
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
165
|
+
(since an empty list and !EXISTS are not really the same).
|
166
|
+
]]
|
167
|
+
local function isQueuePaused(queueMetaKey)
|
168
|
+
return rcall("HEXISTS", queueMetaKey, "paused") == 1
|
169
|
+
end
|
170
|
+
--[[
|
171
|
+
Function to remove job.
|
172
|
+
]]
|
173
|
+
-- Includes
|
174
|
+
--[[
|
175
|
+
Function to remove deduplication key.
|
176
|
+
]]
|
177
|
+
local function removeDeduplicationKey(prefixKey, jobKey)
|
178
|
+
local deduplicationId = rcall("HGET", jobKey, "deid")
|
179
|
+
if deduplicationId then
|
180
|
+
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
181
|
+
rcall("DEL", deduplicationKey)
|
182
|
+
end
|
183
|
+
end
|
184
|
+
--[[
|
185
|
+
Function to remove job keys.
|
186
|
+
]]
|
187
|
+
local function removeJobKeys(jobKey)
|
188
|
+
return rcall("DEL", jobKey, jobKey .. ':logs',
|
189
|
+
jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
|
190
|
+
end
|
191
|
+
--[[
|
192
|
+
Check if this job has a parent. If so we will just remove it from
|
193
|
+
the parent child list, but if it is the last child we should move the parent to "wait/paused"
|
194
|
+
which requires code from "moveToFinished"
|
195
|
+
]]
|
196
|
+
-- Includes
|
197
|
+
--[[
|
198
|
+
Function to add job in target list and add marker if needed.
|
199
|
+
]]
|
200
|
+
-- Includes
|
201
|
+
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
202
|
+
rcall(pushCmd, targetKey, jobId)
|
203
|
+
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
204
|
+
end
|
205
|
+
--[[
|
206
|
+
Functions to destructure job key.
|
207
|
+
Just a bit of warning, these functions may be a bit slow and affect performance significantly.
|
208
|
+
]]
|
209
|
+
local getJobIdFromKey = function (jobKey)
|
210
|
+
return string.match(jobKey, ".*:(.*)")
|
211
|
+
end
|
212
|
+
local getJobKeyPrefix = function (jobKey, jobId)
|
213
|
+
return string.sub(jobKey, 0, #jobKey - #jobId)
|
214
|
+
end
|
103
215
|
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
104
216
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
105
217
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
@@ -180,6 +292,36 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
180
292
|
end
|
181
293
|
removeJobKeys(jobKey)
|
182
294
|
end
|
295
|
+
--[[
|
296
|
+
Function to store a job
|
297
|
+
]]
|
298
|
+
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
299
|
+
parentKey, parentData, repeatJobKey)
|
300
|
+
local jsonOpts = cjson.encode(opts)
|
301
|
+
local delay = opts['delay'] or 0
|
302
|
+
local priority = opts['priority'] or 0
|
303
|
+
local debounceId = opts['de'] and opts['de']['id']
|
304
|
+
local optionalValues = {}
|
305
|
+
if parentKey ~= nil then
|
306
|
+
table.insert(optionalValues, "parentKey")
|
307
|
+
table.insert(optionalValues, parentKey)
|
308
|
+
table.insert(optionalValues, "parent")
|
309
|
+
table.insert(optionalValues, parentData)
|
310
|
+
end
|
311
|
+
if repeatJobKey ~= nil then
|
312
|
+
table.insert(optionalValues, "rjk")
|
313
|
+
table.insert(optionalValues, repeatJobKey)
|
314
|
+
end
|
315
|
+
if debounceId then
|
316
|
+
table.insert(optionalValues, "deid")
|
317
|
+
table.insert(optionalValues, debounceId)
|
318
|
+
end
|
319
|
+
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
320
|
+
"timestamp", timestamp, "delay", delay, "priority", priority,
|
321
|
+
unpack(optionalValues))
|
322
|
+
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
323
|
+
return delay, priority
|
324
|
+
end
|
183
325
|
--[[
|
184
326
|
Function to store a job scheduler
|
185
327
|
]]
|
@@ -221,23 +363,73 @@ end
|
|
221
363
|
-- If we are overriding a repeatable job we must delete the delayed job for
|
222
364
|
-- the next iteration.
|
223
365
|
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
|
366
|
+
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
367
|
+
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
224
368
|
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
225
369
|
if prevMillis ~= false then
|
226
|
-
local
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
370
|
+
local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
371
|
+
if rcall("EXISTS", nextDelayedJobKey) ~= 1 or currentJobId == nextDelayedJobId then
|
372
|
+
if rcall("ZSCORE", delayedKey, currentJobId) ~= false then
|
373
|
+
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
374
|
+
rcall("ZREM", delayedKey, currentJobId)
|
375
|
+
elseif rcall("ZSCORE", prioritizedKey, currentJobId) ~= false then
|
376
|
+
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
377
|
+
rcall("ZREM", prioritizedKey, currentJobId)
|
378
|
+
else
|
379
|
+
if isQueuePaused(KEYS[5]) then
|
380
|
+
if rcall("LREM", KEYS[4], 1, currentJobId) > 0 then
|
381
|
+
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
382
|
+
end
|
383
|
+
else
|
384
|
+
if rcall("LREM", KEYS[3], 1, currentJobId) > 0 then
|
385
|
+
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
386
|
+
end
|
387
|
+
end
|
388
|
+
end
|
233
389
|
end
|
234
390
|
end
|
235
391
|
local schedulerOpts = cmsgpack.unpack(ARGV[2])
|
236
|
-
|
392
|
+
storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
|
393
|
+
if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
|
394
|
+
local eventsKey = KEYS[9]
|
395
|
+
local metaKey = KEYS[5]
|
396
|
+
local maxEvents = getOrSetMaxEvents(metaKey)
|
397
|
+
rcall("INCR", KEYS[8])
|
398
|
+
local delayedOpts = cmsgpack.unpack(ARGV[6])
|
399
|
+
local delay, priority = storeJob(eventsKey, nextDelayedJobKey, nextDelayedJobId, schedulerOpts['name'], ARGV[4],
|
400
|
+
delayedOpts, ARGV[7], nil, nil, jobSchedulerId)
|
401
|
+
if delay ~= 0 then
|
402
|
+
addDelayedJob(nextDelayedJobId, delayedKey, eventsKey,
|
403
|
+
ARGV[7], maxEvents, KEYS[7], delay)
|
404
|
+
else
|
405
|
+
local isPaused = isQueuePaused(KEYS[5])
|
406
|
+
-- Standard or priority add
|
407
|
+
if priority == 0 then
|
408
|
+
if isPaused then
|
409
|
+
-- LIFO or FIFO
|
410
|
+
local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
|
411
|
+
rcall(pushCmd, KEYS[4], nextDelayedJobId)
|
412
|
+
else
|
413
|
+
-- LIFO or FIFO
|
414
|
+
local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
|
415
|
+
rcall(pushCmd, KEYS[3], nextDelayedJobId)
|
416
|
+
end
|
417
|
+
else
|
418
|
+
-- Priority add
|
419
|
+
addJobWithPriority(KEYS[7], KEYS[6], priority, nextDelayedJobId, KEYS[10], isPaused)
|
420
|
+
end
|
421
|
+
-- Emit waiting event
|
422
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", nextDelayedJobId)
|
423
|
+
end
|
424
|
+
if ARGV[9] ~= "" then
|
425
|
+
rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
|
426
|
+
end
|
427
|
+
return nextDelayedJobId .. "" -- convert to string
|
428
|
+
end
|
237
429
|
`;
|
238
430
|
exports.addJobScheduler = {
|
239
431
|
name: 'addJobScheduler',
|
240
432
|
content,
|
241
|
-
keys:
|
433
|
+
keys: 10,
|
242
434
|
};
|
243
|
-
//# sourceMappingURL=addJobScheduler-
|
435
|
+
//# sourceMappingURL=addJobScheduler-10.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"addJobScheduler-10.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-10.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyaf,CAAC;AACW,QAAA,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|
@@ -71,12 +71,12 @@ end
|
|
71
71
|
Function to get max events value or set by default 10000.
|
72
72
|
]]
|
73
73
|
local function getOrSetMaxEvents(metaKey)
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
74
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
75
|
+
if not maxEvents then
|
76
|
+
maxEvents = 10000
|
77
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
78
|
+
end
|
79
|
+
return maxEvents
|
80
80
|
end
|
81
81
|
--[[
|
82
82
|
Function to handle the case when job is duplicated.
|
@@ -133,12 +133,12 @@ end
|
|
133
133
|
Function to get max events value or set by default 10000.
|
134
134
|
]]
|
135
135
|
local function getOrSetMaxEvents(metaKey)
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
136
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
137
|
+
if not maxEvents then
|
138
|
+
maxEvents = 10000
|
139
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
140
|
+
end
|
141
|
+
return maxEvents
|
142
142
|
end
|
143
143
|
--[[
|
144
144
|
Function to handle the case when job is duplicated.
|
@@ -97,12 +97,12 @@ end
|
|
97
97
|
Function to get max events value or set by default 10000.
|
98
98
|
]]
|
99
99
|
local function getOrSetMaxEvents(metaKey)
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
100
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
101
|
+
if not maxEvents then
|
102
|
+
maxEvents = 10000
|
103
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
104
|
+
end
|
105
|
+
return maxEvents
|
106
106
|
end
|
107
107
|
--[[
|
108
108
|
Function to check for the meta.paused key to decide if we are paused or not
|
@@ -73,12 +73,12 @@ end
|
|
73
73
|
Function to get max events value or set by default 10000.
|
74
74
|
]]
|
75
75
|
local function getOrSetMaxEvents(metaKey)
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
76
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
77
|
+
if not maxEvents then
|
78
|
+
maxEvents = 10000
|
79
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
80
|
+
end
|
81
|
+
return maxEvents
|
82
82
|
end
|
83
83
|
if rcall("EXISTS", ARGV[4]) == 1 then
|
84
84
|
local jobId = ARGV[3]
|
@@ -2,7 +2,7 @@
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
3
|
const tslib_1 = require("tslib");
|
4
4
|
tslib_1.__exportStar(require("./addDelayedJob-6"), exports);
|
5
|
-
tslib_1.__exportStar(require("./addJobScheduler-
|
5
|
+
tslib_1.__exportStar(require("./addJobScheduler-10"), exports);
|
6
6
|
tslib_1.__exportStar(require("./addLog-2"), exports);
|
7
7
|
tslib_1.__exportStar(require("./addParentJob-4"), exports);
|
8
8
|
tslib_1.__exportStar(require("./addPrioritizedJob-8"), exports);
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":";;;AAAA,4DAAkC;AAClC
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":";;;AAAA,4DAAkC;AAClC,+DAAqC;AACrC,qDAA2B;AAC3B,2DAAiC;AACjC,gEAAsC;AACtC,+DAAqC;AACrC,6DAAmC;AACnC,0DAAgC;AAChC,6DAAmC;AACnC,6DAAmC;AACnC,oDAA0B;AAC1B,yDAA+B;AAC/B,0DAAgC;AAChC,wDAA8B;AAC9B,mEAAyC;AACzC,8DAAoC;AACpC,wDAA8B;AAC9B,8DAAoC;AACpC,uDAA6B;AAC7B,yDAA+B;AAC/B,yDAA+B;AAC/B,0DAAgC;AAChC,sDAA4B;AAC5B,sEAA4C;AAC5C,6DAAmC;AACnC,oEAA0C;AAC1C,4DAAkC;AAClC,4DAAkC;AAClC,8DAAoC;AACpC,oEAA0C;AAC1C,yDAA+B;AAC/B,uDAA6B;AAC7B,oDAA0B;AAC1B,sDAA4B;AAC5B,0DAAgC;AAChC,oEAA0C;AAC1C,wDAA8B;AAC9B,iEAAuC;AACvC,+DAAqC;AACrC,2DAAiC;AACjC,wDAA8B;AAC9B,6DAAmC;AACnC,yDAA+B;AAC/B,iEAAuC;AACvC,6DAAmC;AACnC,wEAA8C"}
|
@@ -48,12 +48,12 @@ end
|
|
48
48
|
Function to get max events value or set by default 10000.
|
49
49
|
]]
|
50
50
|
local function getOrSetMaxEvents(metaKey)
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
51
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
52
|
+
if not maxEvents then
|
53
|
+
maxEvents = 10000
|
54
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
55
|
+
end
|
56
|
+
return maxEvents
|
57
57
|
end
|
58
58
|
--[[
|
59
59
|
Function to check for the meta.paused key to decide if we are paused or not
|
@@ -52,12 +52,12 @@ end
|
|
52
52
|
Function to get max events value or set by default 10000.
|
53
53
|
]]
|
54
54
|
local function getOrSetMaxEvents(metaKey)
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
55
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
56
|
+
if not maxEvents then
|
57
|
+
maxEvents = 10000
|
58
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
59
|
+
end
|
60
|
+
return maxEvents
|
61
61
|
end
|
62
62
|
--[[
|
63
63
|
Function to check for the meta.paused key to decide if we are paused or not
|
@@ -361,8 +361,11 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
|
|
361
361
|
end
|
362
362
|
end
|
363
363
|
end
|
364
|
-
local function moveParentFromWaitingChildrenToFailed(
|
365
|
-
|
364
|
+
local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
365
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
366
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
367
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) ~= false then
|
368
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
366
369
|
local parentQueuePrefix = parentQueueKey .. ":"
|
367
370
|
local parentFailedKey = parentQueueKey .. ":failed"
|
368
371
|
rcall("ZADD", parentFailedKey, timestamp, parentId)
|
@@ -398,6 +401,43 @@ local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey,
|
|
398
401
|
local parentRawOpts = jobAttributes[3]
|
399
402
|
local parentOpts = cjson.decode(parentRawOpts)
|
400
403
|
removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
|
404
|
+
elseif rcall("ZSCORE", parentDelayedKey, parentId) ~= false then
|
405
|
+
rcall("ZREM", parentDelayedKey, parentId)
|
406
|
+
local parentQueuePrefix = parentQueueKey .. ":"
|
407
|
+
local parentFailedKey = parentQueueKey .. ":failed"
|
408
|
+
rcall("ZADD", parentFailedKey, timestamp, parentId)
|
409
|
+
local failedReason = "child " .. jobIdKey .. " failed"
|
410
|
+
rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
|
411
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
|
412
|
+
failedReason, "prev", "delayed")
|
413
|
+
local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
|
414
|
+
removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
|
415
|
+
if jobAttributes[1] then
|
416
|
+
local parentData = cjson.decode(jobAttributes[1])
|
417
|
+
if parentData['fpof'] then
|
418
|
+
moveParentFromWaitingChildrenToFailed(
|
419
|
+
parentData['queueKey'],
|
420
|
+
parentData['queueKey'] .. ':' .. parentData['id'],
|
421
|
+
parentData['id'],
|
422
|
+
parentKey,
|
423
|
+
timestamp
|
424
|
+
)
|
425
|
+
elseif parentData['idof'] or parentData['rdof'] then
|
426
|
+
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
427
|
+
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
|
428
|
+
if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
|
429
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
|
430
|
+
grandParentKey, parentData['id'], timestamp)
|
431
|
+
if parentData['idof'] then
|
432
|
+
local grandParentFailedSet = grandParentKey .. ":failed"
|
433
|
+
rcall("HSET", grandParentFailedSet, parentKey, failedReason)
|
434
|
+
end
|
435
|
+
end
|
436
|
+
end
|
437
|
+
end
|
438
|
+
local parentRawOpts = jobAttributes[3]
|
439
|
+
local parentOpts = cjson.decode(parentRawOpts)
|
440
|
+
removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
|
401
441
|
end
|
402
442
|
end
|
403
443
|
--[[
|
@@ -408,12 +448,12 @@ end
|
|
408
448
|
Function to get max events value or set by default 10000.
|
409
449
|
]]
|
410
450
|
local function getOrSetMaxEvents(metaKey)
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
451
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
452
|
+
if not maxEvents then
|
453
|
+
maxEvents = 10000
|
454
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
455
|
+
end
|
456
|
+
return maxEvents
|
417
457
|
end
|
418
458
|
local function trimEvents(metaKey, eventStreamKey)
|
419
459
|
local maxEvents = getOrSetMaxEvents(metaKey)
|
@@ -476,7 +516,7 @@ if (#stalling > 0) then
|
|
476
516
|
rcall("XADD", eventStreamKey, "*", "event",
|
477
517
|
"failed", "jobId", jobId, 'prev', 'active',
|
478
518
|
'failedReason', failedReason)
|
479
|
-
if rawParentData
|
519
|
+
if rawParentData then
|
480
520
|
if opts['fpof'] then
|
481
521
|
local parentData = cjson.decode(rawParentData)
|
482
522
|
moveParentFromWaitingChildrenToFailed(
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAqjBS,CAAC;AACb,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -80,12 +80,12 @@ end
|
|
80
80
|
Function to get max events value or set by default 10000.
|
81
81
|
]]
|
82
82
|
local function getOrSetMaxEvents(metaKey)
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
83
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
84
|
+
if not maxEvents then
|
85
|
+
maxEvents = 10000
|
86
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
87
|
+
end
|
88
|
+
return maxEvents
|
89
89
|
end
|
90
90
|
local function removeLock(jobKey, stalledKey, token, jobId)
|
91
91
|
if token ~= "0" then
|