bullmq 5.58.7 → 5.58.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/job-scheduler.js +22 -24
- package/dist/cjs/classes/job-scheduler.js.map +1 -1
- package/dist/cjs/classes/job.js +6 -2
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-getters.js +6 -19
- package/dist/cjs/classes/queue-getters.js.map +1 -1
- package/dist/cjs/classes/queue.js +1 -1
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/scripts.js +47 -11
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +53 -18
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addJobScheduler-11.lua +81 -25
- package/dist/cjs/commands/addRepeatableJob-2.lua +1 -1
- package/dist/cjs/commands/getMetrics-2.lua +19 -0
- package/dist/cjs/commands/includes/addJobFromScheduler.lua +5 -3
- package/dist/cjs/commands/includes/getJobSchedulerEveryNextMillis.lua +28 -0
- package/dist/cjs/commands/includes/storeJobScheduler.lua +15 -1
- package/dist/cjs/commands/moveStalledJobsToWait-8.lua +14 -1
- package/dist/cjs/commands/updateJobScheduler-12.lua +50 -14
- package/dist/cjs/enums/error-code.js +2 -0
- package/dist/cjs/enums/error-code.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +108 -25
- package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +1 -1
- package/dist/cjs/scripts/getMetrics-2.js +25 -0
- package/dist/cjs/scripts/getMetrics-2.js.map +1 -0
- package/dist/cjs/scripts/index.js +1 -0
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js +11 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/cjs/scripts/updateJobScheduler-12.js +66 -17
- package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/job-scheduler.js +22 -24
- package/dist/esm/classes/job-scheduler.js.map +1 -1
- package/dist/esm/classes/job.js +6 -2
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-getters.js +6 -19
- package/dist/esm/classes/queue-getters.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +1 -1
- package/dist/esm/classes/queue.js +1 -1
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +2 -1
- package/dist/esm/classes/scripts.js +47 -11
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.js +53 -18
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addJobScheduler-11.lua +81 -25
- package/dist/esm/commands/addRepeatableJob-2.lua +1 -1
- package/dist/esm/commands/getMetrics-2.lua +19 -0
- package/dist/esm/commands/includes/addJobFromScheduler.lua +5 -3
- package/dist/esm/commands/includes/getJobSchedulerEveryNextMillis.lua +28 -0
- package/dist/esm/commands/includes/storeJobScheduler.lua +15 -1
- package/dist/esm/commands/moveStalledJobsToWait-8.lua +14 -1
- package/dist/esm/commands/updateJobScheduler-12.lua +50 -14
- package/dist/esm/enums/error-code.d.ts +3 -1
- package/dist/esm/enums/error-code.js +2 -0
- package/dist/esm/enums/error-code.js.map +1 -1
- package/dist/esm/interfaces/job-scheduler-json.d.ts +1 -0
- package/dist/esm/interfaces/repeatable-options.d.ts +1 -0
- package/dist/esm/scripts/addJobScheduler-11.js +108 -25
- package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +1 -1
- package/dist/esm/scripts/getMetrics-2.d.ts +5 -0
- package/dist/esm/scripts/getMetrics-2.js +22 -0
- package/dist/esm/scripts/getMetrics-2.js.map +1 -0
- package/dist/esm/scripts/index.d.ts +1 -0
- package/dist/esm/scripts/index.js +1 -0
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-8.js +11 -1
- package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/esm/scripts/updateJobScheduler-12.js +66 -17
- package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +5 -5
|
@@ -10,10 +10,12 @@
|
|
|
10
10
|
--- @include "getTargetQueueList"
|
|
11
11
|
--- @include "addJobInTargetList"
|
|
12
12
|
|
|
13
|
-
local function addJobFromScheduler(jobKey, jobId,
|
|
13
|
+
local function addJobFromScheduler(jobKey, jobId, opts, waitKey, pausedKey, activeKey, metaKey,
|
|
14
14
|
prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
|
|
15
|
-
data, jobSchedulerId)
|
|
16
|
-
|
|
15
|
+
data, jobSchedulerId, repeatDelay)
|
|
16
|
+
|
|
17
|
+
opts['delay'] = repeatDelay
|
|
18
|
+
opts['jobId'] = jobId
|
|
17
19
|
|
|
18
20
|
local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
|
|
19
21
|
opts, timestamp, nil, nil, jobSchedulerId)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
local function getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
|
|
4
|
+
local nextMillis
|
|
5
|
+
if not prevMillis then
|
|
6
|
+
if startDate then
|
|
7
|
+
-- Assuming startDate is passed as milliseconds from JavaScript
|
|
8
|
+
nextMillis = tonumber(startDate)
|
|
9
|
+
nextMillis = nextMillis > now and nextMillis or now
|
|
10
|
+
else
|
|
11
|
+
nextMillis = now
|
|
12
|
+
end
|
|
13
|
+
else
|
|
14
|
+
nextMillis = prevMillis + every
|
|
15
|
+
-- check if we may have missed some iterations
|
|
16
|
+
if nextMillis < now then
|
|
17
|
+
nextMillis = math.floor(now / every) * every + every + (offset or 0)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
if not offset or offset == 0 then
|
|
22
|
+
local timeSlot = math.floor(nextMillis / every) * every;
|
|
23
|
+
offset = nextMillis - timeSlot;
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
-- Return a tuple nextMillis, offset
|
|
27
|
+
return math.floor(nextMillis), math.floor(offset)
|
|
28
|
+
end
|
|
@@ -21,6 +21,11 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
|
|
|
21
21
|
table.insert(optionalValues, opts['pattern'])
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
+
if opts['startDate'] then
|
|
25
|
+
table.insert(optionalValues, "startDate")
|
|
26
|
+
table.insert(optionalValues, opts['startDate'])
|
|
27
|
+
end
|
|
28
|
+
|
|
24
29
|
if opts['endDate'] then
|
|
25
30
|
table.insert(optionalValues, "endDate")
|
|
26
31
|
table.insert(optionalValues, opts['endDate'])
|
|
@@ -34,6 +39,12 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
|
|
|
34
39
|
if opts['offset'] then
|
|
35
40
|
table.insert(optionalValues, "offset")
|
|
36
41
|
table.insert(optionalValues, opts['offset'])
|
|
42
|
+
else
|
|
43
|
+
local offset = rcall("HGET", schedulerKey, "offset")
|
|
44
|
+
if offset then
|
|
45
|
+
table.insert(optionalValues, "offset")
|
|
46
|
+
table.insert(optionalValues, tonumber(offset))
|
|
47
|
+
end
|
|
37
48
|
end
|
|
38
49
|
|
|
39
50
|
local jsonTemplateOpts = cjson.encode(templateOpts)
|
|
@@ -47,6 +58,9 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
|
|
|
47
58
|
table.insert(optionalValues, templateData)
|
|
48
59
|
end
|
|
49
60
|
|
|
61
|
+
table.insert(optionalValues, "ic")
|
|
62
|
+
table.insert(optionalValues, rcall("HGET", schedulerKey, "ic") or 1)
|
|
63
|
+
|
|
50
64
|
rcall("DEL", schedulerKey) -- remove all attributes and then re-insert new ones
|
|
51
|
-
rcall("HMSET", schedulerKey, "name", opts['name'],
|
|
65
|
+
rcall("HMSET", schedulerKey, "name", opts['name'], unpack(optionalValues))
|
|
52
66
|
end
|
|
@@ -73,10 +73,23 @@ if (#stalling > 0) then
|
|
|
73
73
|
if (removed > 0) then
|
|
74
74
|
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
|
75
75
|
local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
|
|
76
|
-
|
|
76
|
+
|
|
77
|
+
-- Check if this is a repeatable job by looking at job options
|
|
78
|
+
local jobOpts = rcall("HGET", jobKey, "opts")
|
|
79
|
+
local isRepeatableJob = false
|
|
80
|
+
if jobOpts then
|
|
81
|
+
local opts = cjson.decode(jobOpts)
|
|
82
|
+
if opts and opts["repeat"] then
|
|
83
|
+
isRepeatableJob = true
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
-- Only fail job if it exceeds stall limit AND is not a repeatable job
|
|
88
|
+
if stalledCount > maxStalledJobCount and not isRepeatableJob then
|
|
77
89
|
local failedReason = "job stalled more than allowable limit"
|
|
78
90
|
rcall("HSET", jobKey, "defa", failedReason)
|
|
79
91
|
end
|
|
92
|
+
|
|
80
93
|
moveJobToWait(metaKey, activeKey, waitKey, pausedKey, markerKey, eventStreamKey, jobId,
|
|
81
94
|
"RPUSH")
|
|
82
95
|
|
|
@@ -25,31 +25,60 @@
|
|
|
25
25
|
|
|
26
26
|
Output:
|
|
27
27
|
next delayed job id - OK
|
|
28
|
-
]]
|
|
29
|
-
local rcall = redis.call
|
|
28
|
+
]] local rcall = redis.call
|
|
30
29
|
local repeatKey = KEYS[1]
|
|
31
30
|
local delayedKey = KEYS[2]
|
|
32
31
|
local waitKey = KEYS[3]
|
|
33
32
|
local pausedKey = KEYS[4]
|
|
34
33
|
local metaKey = KEYS[5]
|
|
35
34
|
local prioritizedKey = KEYS[6]
|
|
36
|
-
local nextMillis = ARGV[1]
|
|
35
|
+
local nextMillis = tonumber(ARGV[1])
|
|
37
36
|
local jobSchedulerId = ARGV[2]
|
|
38
|
-
local timestamp = ARGV[5]
|
|
37
|
+
local timestamp = tonumber(ARGV[5])
|
|
39
38
|
local prefixKey = ARGV[6]
|
|
40
39
|
local producerId = ARGV[7]
|
|
40
|
+
local jobOpts = cmsgpack.unpack(ARGV[4])
|
|
41
41
|
|
|
42
42
|
-- Includes
|
|
43
43
|
--- @include "includes/addJobFromScheduler"
|
|
44
44
|
--- @include "includes/getOrSetMaxEvents"
|
|
45
|
+
--- @include "includes/getJobSchedulerEveryNextMillis"
|
|
45
46
|
|
|
46
|
-
local
|
|
47
|
-
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
|
48
|
-
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
|
47
|
+
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
|
49
48
|
|
|
50
49
|
-- Validate that scheduler exists.
|
|
51
|
-
|
|
50
|
+
-- If it does not exist we should not iterate anymore.
|
|
52
51
|
if prevMillis then
|
|
52
|
+
prevMillis = tonumber(prevMillis)
|
|
53
|
+
|
|
54
|
+
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
|
|
55
|
+
local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data", "every", "startDate", "offset")
|
|
56
|
+
|
|
57
|
+
local every = tonumber(schedulerAttributes[3])
|
|
58
|
+
local now = tonumber(timestamp)
|
|
59
|
+
|
|
60
|
+
-- If every is not found in scheduler attributes, try to get it from job options
|
|
61
|
+
if not every and jobOpts['repeat'] and jobOpts['repeat']['every'] then
|
|
62
|
+
every = tonumber(jobOpts['repeat']['every'])
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
if every then
|
|
66
|
+
local startDate = schedulerAttributes[4]
|
|
67
|
+
local jobOptsOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
|
|
68
|
+
local offset = schedulerAttributes[5] or jobOptsOffset or 0
|
|
69
|
+
local newOffset
|
|
70
|
+
|
|
71
|
+
nextMillis, newOffset = getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
|
|
72
|
+
|
|
73
|
+
if not offset then
|
|
74
|
+
rcall("HSET", schedulerKey, "offset", newOffset)
|
|
75
|
+
jobOpts['repeat']['offset'] = newOffset
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
|
80
|
+
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
|
81
|
+
|
|
53
82
|
local currentDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
|
54
83
|
|
|
55
84
|
if producerId == currentDelayedJobId then
|
|
@@ -57,7 +86,6 @@ if prevMillis then
|
|
|
57
86
|
local maxEvents = getOrSetMaxEvents(metaKey)
|
|
58
87
|
|
|
59
88
|
if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
|
|
60
|
-
local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data")
|
|
61
89
|
|
|
62
90
|
rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
|
|
63
91
|
rcall("HINCRBY", schedulerKey, "ic", 1)
|
|
@@ -72,9 +100,18 @@ if prevMillis then
|
|
|
72
100
|
rcall("HSET", schedulerKey, "data", templateData)
|
|
73
101
|
end
|
|
74
102
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
103
|
+
local delay = nextMillis - now
|
|
104
|
+
|
|
105
|
+
-- Fast Clamp delay to minimum of 0
|
|
106
|
+
if delay < 0 then
|
|
107
|
+
delay = 0
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
jobOpts["delay"] = delay
|
|
111
|
+
|
|
112
|
+
addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, jobOpts, waitKey, pausedKey, KEYS[12], metaKey,
|
|
113
|
+
prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey, schedulerAttributes[1], maxEvents, ARGV[5],
|
|
114
|
+
templateData or '{}', jobSchedulerId, delay)
|
|
78
115
|
|
|
79
116
|
-- TODO: remove this workaround in next breaking change
|
|
80
117
|
if KEYS[11] ~= "" then
|
|
@@ -83,8 +120,7 @@ if prevMillis then
|
|
|
83
120
|
|
|
84
121
|
return nextDelayedJobId .. "" -- convert to string
|
|
85
122
|
else
|
|
86
|
-
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
|
87
|
-
"duplicated", "jobId", nextDelayedJobId)
|
|
123
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "duplicated", "jobId", nextDelayedJobId)
|
|
88
124
|
end
|
|
89
125
|
end
|
|
90
126
|
end
|
|
@@ -9,5 +9,7 @@ export var ErrorCode;
|
|
|
9
9
|
ErrorCode[ErrorCode["ParentJobCannotBeReplaced"] = -7] = "ParentJobCannotBeReplaced";
|
|
10
10
|
ErrorCode[ErrorCode["JobBelongsToJobScheduler"] = -8] = "JobBelongsToJobScheduler";
|
|
11
11
|
ErrorCode[ErrorCode["JobHasFailedChildren"] = -9] = "JobHasFailedChildren";
|
|
12
|
+
ErrorCode[ErrorCode["SchedulerJobIdCollision"] = -10] = "SchedulerJobIdCollision";
|
|
13
|
+
ErrorCode[ErrorCode["SchedulerJobSlotsBusy"] = -11] = "SchedulerJobSlotsBusy";
|
|
12
14
|
})(ErrorCode || (ErrorCode = {}));
|
|
13
15
|
//# sourceMappingURL=error-code.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,
|
|
1
|
+
{"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,SAYX;AAZD,WAAY,SAAS;IACnB,wDAAgB,CAAA;IAChB,gEAAoB,CAAA;IACpB,4DAAkB,CAAA;IAClB,sEAAuB,CAAA;IACvB,oEAAsB,CAAA;IACtB,gEAAoB,CAAA;IACpB,oFAA8B,CAAA;IAC9B,kFAA6B,CAAA;IAC7B,0EAAyB,CAAA;IACzB,iFAA6B,CAAA;IAC7B,6EAA2B,CAAA;AAC7B,CAAC,EAZW,SAAS,KAAT,SAAS,QAYpB"}
|
|
@@ -16,7 +16,7 @@ const content = `--[[
|
|
|
16
16
|
ARGV[2] msgpacked options
|
|
17
17
|
[1] name
|
|
18
18
|
[2] tz?
|
|
19
|
-
[3]
|
|
19
|
+
[3] pattern?
|
|
20
20
|
[4] endDate?
|
|
21
21
|
[5] every?
|
|
22
22
|
ARGV[3] jobs scheduler id
|
|
@@ -40,7 +40,9 @@ local eventsKey = KEYS[9]
|
|
|
40
40
|
local nextMillis = ARGV[1]
|
|
41
41
|
local jobSchedulerId = ARGV[3]
|
|
42
42
|
local templateOpts = cmsgpack.unpack(ARGV[5])
|
|
43
|
+
local now = tonumber(ARGV[7])
|
|
43
44
|
local prefixKey = ARGV[8]
|
|
45
|
+
local jobOpts = cmsgpack.unpack(ARGV[6])
|
|
44
46
|
-- Includes
|
|
45
47
|
--[[
|
|
46
48
|
Add delay marker if needed.
|
|
@@ -200,10 +202,11 @@ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed
|
|
|
200
202
|
rcall(pushCmd, targetKey, jobId)
|
|
201
203
|
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
202
204
|
end
|
|
203
|
-
local function addJobFromScheduler(jobKey, jobId,
|
|
205
|
+
local function addJobFromScheduler(jobKey, jobId, opts, waitKey, pausedKey, activeKey, metaKey,
|
|
204
206
|
prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
|
|
205
|
-
data, jobSchedulerId)
|
|
206
|
-
|
|
207
|
+
data, jobSchedulerId, repeatDelay)
|
|
208
|
+
opts['delay'] = repeatDelay
|
|
209
|
+
opts['jobId'] = jobId
|
|
207
210
|
local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
|
|
208
211
|
opts, timestamp, nil, nil, jobSchedulerId)
|
|
209
212
|
if delay ~= 0 then
|
|
@@ -374,6 +377,10 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
|
|
|
374
377
|
table.insert(optionalValues, "pattern")
|
|
375
378
|
table.insert(optionalValues, opts['pattern'])
|
|
376
379
|
end
|
|
380
|
+
if opts['startDate'] then
|
|
381
|
+
table.insert(optionalValues, "startDate")
|
|
382
|
+
table.insert(optionalValues, opts['startDate'])
|
|
383
|
+
end
|
|
377
384
|
if opts['endDate'] then
|
|
378
385
|
table.insert(optionalValues, "endDate")
|
|
379
386
|
table.insert(optionalValues, opts['endDate'])
|
|
@@ -385,6 +392,12 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
|
|
|
385
392
|
if opts['offset'] then
|
|
386
393
|
table.insert(optionalValues, "offset")
|
|
387
394
|
table.insert(optionalValues, opts['offset'])
|
|
395
|
+
else
|
|
396
|
+
local offset = rcall("HGET", schedulerKey, "offset")
|
|
397
|
+
if offset then
|
|
398
|
+
table.insert(optionalValues, "offset")
|
|
399
|
+
table.insert(optionalValues, tonumber(offset))
|
|
400
|
+
end
|
|
388
401
|
end
|
|
389
402
|
local jsonTemplateOpts = cjson.encode(templateOpts)
|
|
390
403
|
if jsonTemplateOpts and jsonTemplateOpts ~= '{}' then
|
|
@@ -395,15 +408,55 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
|
|
|
395
408
|
table.insert(optionalValues, "data")
|
|
396
409
|
table.insert(optionalValues, templateData)
|
|
397
410
|
end
|
|
411
|
+
table.insert(optionalValues, "ic")
|
|
412
|
+
table.insert(optionalValues, rcall("HGET", schedulerKey, "ic") or 1)
|
|
398
413
|
rcall("DEL", schedulerKey) -- remove all attributes and then re-insert new ones
|
|
399
|
-
rcall("HMSET", schedulerKey, "name", opts['name'],
|
|
414
|
+
rcall("HMSET", schedulerKey, "name", opts['name'], unpack(optionalValues))
|
|
415
|
+
end
|
|
416
|
+
local function getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
|
|
417
|
+
local nextMillis
|
|
418
|
+
if not prevMillis then
|
|
419
|
+
if startDate then
|
|
420
|
+
-- Assuming startDate is passed as milliseconds from JavaScript
|
|
421
|
+
nextMillis = tonumber(startDate)
|
|
422
|
+
nextMillis = nextMillis > now and nextMillis or now
|
|
423
|
+
else
|
|
424
|
+
nextMillis = now
|
|
425
|
+
end
|
|
426
|
+
else
|
|
427
|
+
nextMillis = prevMillis + every
|
|
428
|
+
-- check if we may have missed some iterations
|
|
429
|
+
if nextMillis < now then
|
|
430
|
+
nextMillis = math.floor(now / every) * every + every + (offset or 0)
|
|
431
|
+
end
|
|
432
|
+
end
|
|
433
|
+
if not offset or offset == 0 then
|
|
434
|
+
local timeSlot = math.floor(nextMillis / every) * every;
|
|
435
|
+
offset = nextMillis - timeSlot;
|
|
436
|
+
end
|
|
437
|
+
-- Return a tuple nextMillis, offset
|
|
438
|
+
return math.floor(nextMillis), math.floor(offset)
|
|
400
439
|
end
|
|
401
440
|
-- If we are overriding a repeatable job we must delete the delayed job for
|
|
402
441
|
-- the next iteration.
|
|
403
442
|
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
|
|
404
|
-
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
|
405
|
-
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
|
406
443
|
local maxEvents = getOrSetMaxEvents(metaKey)
|
|
444
|
+
local templateData = ARGV[4]
|
|
445
|
+
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
|
446
|
+
if prevMillis then
|
|
447
|
+
prevMillis = tonumber(prevMillis)
|
|
448
|
+
end
|
|
449
|
+
local schedulerOpts = cmsgpack.unpack(ARGV[2])
|
|
450
|
+
local every = schedulerOpts['every']
|
|
451
|
+
-- For backwards compatibility we also check the offset from the job itself.
|
|
452
|
+
-- could be removed in future major versions.
|
|
453
|
+
local jobOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
|
|
454
|
+
local offset = schedulerOpts['offset'] or jobOffset or 0
|
|
455
|
+
local newOffset = offset
|
|
456
|
+
if every then
|
|
457
|
+
local startDate = schedulerOpts['startDate']
|
|
458
|
+
nextMillis, newOffset = getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
|
|
459
|
+
end
|
|
407
460
|
local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey, jobId,
|
|
408
461
|
metaKey, eventsKey)
|
|
409
462
|
if rcall("ZSCORE", delayedKey, jobId) then
|
|
@@ -426,33 +479,63 @@ local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, wai
|
|
|
426
479
|
end
|
|
427
480
|
return false
|
|
428
481
|
end
|
|
429
|
-
|
|
430
|
-
if not removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
|
|
431
|
-
nextDelayedJobId, metaKey, eventsKey) then
|
|
432
|
-
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
|
433
|
-
"duplicated", "jobId", nextDelayedJobId)
|
|
434
|
-
return nextDelayedJobId .. "" -- convert to string
|
|
435
|
-
end
|
|
436
|
-
end
|
|
437
|
-
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
|
482
|
+
local hadPrevJob = false
|
|
438
483
|
if prevMillis then
|
|
439
484
|
local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
|
440
|
-
local
|
|
441
|
-
|
|
442
|
-
|
|
485
|
+
local currentJobKey = schedulerKey .. ":" .. prevMillis
|
|
486
|
+
-- In theory it should always exist the currentJobKey if there is a prevMillis unless something has
|
|
487
|
+
-- gone really wrong.
|
|
488
|
+
if rcall("EXISTS", currentJobKey) == 1 then
|
|
489
|
+
hadPrevJob = removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
|
|
443
490
|
currentJobId, metaKey, eventsKey)
|
|
444
491
|
end
|
|
445
492
|
end
|
|
446
|
-
|
|
447
|
-
|
|
493
|
+
if hadPrevJob then
|
|
494
|
+
-- The jobs has been removed and we want to replace it, so lets use the same millis.
|
|
495
|
+
nextMillis = prevMillis
|
|
496
|
+
else
|
|
497
|
+
-- Special case where no job was removed, and we need to add the next iteration.
|
|
498
|
+
schedulerOpts['offset'] = newOffset
|
|
499
|
+
end
|
|
500
|
+
-- Check for job ID collision with existing jobs (in any state)
|
|
501
|
+
local jobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
|
502
|
+
local jobKey = prefixKey .. jobId
|
|
503
|
+
-- If there's already a job with this ID, handle the collision
|
|
504
|
+
if rcall("EXISTS", jobKey) == 1 then
|
|
505
|
+
if every then
|
|
506
|
+
-- For 'every' case: try next time slot to avoid collision
|
|
507
|
+
local nextSlotMillis = nextMillis + every
|
|
508
|
+
local nextSlotJobId = "repeat:" .. jobSchedulerId .. ":" .. nextSlotMillis
|
|
509
|
+
local nextSlotJobKey = prefixKey .. nextSlotJobId
|
|
510
|
+
if rcall("EXISTS", nextSlotJobKey) == 0 then
|
|
511
|
+
-- Next slot is free, use it
|
|
512
|
+
nextMillis = nextSlotMillis
|
|
513
|
+
jobId = nextSlotJobId
|
|
514
|
+
else
|
|
515
|
+
-- Next slot also has a job, return error code
|
|
516
|
+
return -11 -- SchedulerJobSlotsBusy
|
|
517
|
+
end
|
|
518
|
+
else
|
|
519
|
+
-- For 'pattern' case: return error code
|
|
520
|
+
return -10 -- SchedulerJobIdCollision
|
|
521
|
+
end
|
|
522
|
+
end
|
|
523
|
+
local delay = nextMillis - now
|
|
524
|
+
-- Fast Clamp delay to minimum of 0
|
|
525
|
+
if delay < 0 then
|
|
526
|
+
delay = 0
|
|
527
|
+
end
|
|
528
|
+
local nextJobKey = schedulerKey .. ":" .. nextMillis
|
|
529
|
+
-- jobId already calculated above during collision check
|
|
530
|
+
storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, templateData, templateOpts)
|
|
448
531
|
rcall("INCR", KEYS[8])
|
|
449
|
-
addJobFromScheduler(
|
|
532
|
+
addJobFromScheduler(nextJobKey, jobId, jobOpts, waitKey, pausedKey,
|
|
450
533
|
KEYS[11], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey,
|
|
451
|
-
schedulerOpts['name'], maxEvents,
|
|
534
|
+
schedulerOpts['name'], maxEvents, now, templateData, jobSchedulerId, delay)
|
|
452
535
|
if ARGV[9] ~= "" then
|
|
453
|
-
rcall("HSET", ARGV[9], "nrjid",
|
|
536
|
+
rcall("HSET", ARGV[9], "nrjid", jobId)
|
|
454
537
|
end
|
|
455
|
-
return
|
|
538
|
+
return {jobId .. "", delay}
|
|
456
539
|
`;
|
|
457
540
|
export const addJobScheduler = {
|
|
458
541
|
name: 'addJobScheduler',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
|
1
|
+
{"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0hBf,CAAC;AACF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
const content = `--[[
|
|
2
|
+
Get metrics
|
|
3
|
+
Input:
|
|
4
|
+
KEYS[1] 'metrics' key
|
|
5
|
+
KEYS[2] 'metrics data' key
|
|
6
|
+
ARGV[1] start index
|
|
7
|
+
ARGV[2] end index
|
|
8
|
+
]]
|
|
9
|
+
local rcall = redis.call;
|
|
10
|
+
local metricsKey = KEYS[1]
|
|
11
|
+
local dataKey = KEYS[2]
|
|
12
|
+
local metrics = rcall("HMGET", metricsKey, "count", "prevTS", "prevCount")
|
|
13
|
+
local data = rcall("LRANGE", dataKey, tonumber(ARGV[1]), tonumber(ARGV[2]))
|
|
14
|
+
local numPoints = rcall("LLEN", dataKey)
|
|
15
|
+
return {metrics, data, numPoints}
|
|
16
|
+
`;
|
|
17
|
+
export const getMetrics = {
|
|
18
|
+
name: 'getMetrics',
|
|
19
|
+
content,
|
|
20
|
+
keys: 2,
|
|
21
|
+
};
|
|
22
|
+
//# sourceMappingURL=getMetrics-2.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getMetrics-2.js","sourceRoot":"","sources":["../../../src/scripts/getMetrics-2.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;CAef,CAAC;AACF,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
|
@@ -15,6 +15,7 @@ export * from './getCounts-1';
|
|
|
15
15
|
export * from './getCountsPerPriority-4';
|
|
16
16
|
export * from './getDependencyCounts-4';
|
|
17
17
|
export * from './getJobScheduler-1';
|
|
18
|
+
export * from './getMetrics-2';
|
|
18
19
|
export * from './getRanges-1';
|
|
19
20
|
export * from './getRateLimitTtl-1';
|
|
20
21
|
export * from './getState-8';
|
|
@@ -15,6 +15,7 @@ export * from './getCounts-1';
|
|
|
15
15
|
export * from './getCountsPerPriority-4';
|
|
16
16
|
export * from './getDependencyCounts-4';
|
|
17
17
|
export * from './getJobScheduler-1';
|
|
18
|
+
export * from './getMetrics-2';
|
|
18
19
|
export * from './getRanges-1';
|
|
19
20
|
export * from './getRateLimitTtl-1';
|
|
20
21
|
export * from './getState-8';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,eAAe,CAAC;AAC9B,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,kBAAkB,CAAC;AACjC,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,eAAe,CAAC;AAC9B,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,kBAAkB,CAAC;AACjC,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC"}
|
|
@@ -141,7 +141,17 @@ if (#stalling > 0) then
|
|
|
141
141
|
if (removed > 0) then
|
|
142
142
|
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
|
143
143
|
local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
|
|
144
|
-
if
|
|
144
|
+
-- Check if this is a repeatable job by looking at job options
|
|
145
|
+
local jobOpts = rcall("HGET", jobKey, "opts")
|
|
146
|
+
local isRepeatableJob = false
|
|
147
|
+
if jobOpts then
|
|
148
|
+
local opts = cjson.decode(jobOpts)
|
|
149
|
+
if opts and opts["repeat"] then
|
|
150
|
+
isRepeatableJob = true
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
-- Only fail job if it exceeds stall limit AND is not a repeatable job
|
|
154
|
+
if stalledCount > maxStalledJobCount and not isRepeatableJob then
|
|
145
155
|
local failedReason = "job stalled more than allowable limit"
|
|
146
156
|
rcall("HSET", jobKey, "defa", failedReason)
|
|
147
157
|
end
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
|
1
|
+
{"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Kf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|