bullmq 5.58.6 → 5.58.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/job-scheduler.js +22 -24
- package/dist/cjs/classes/job-scheduler.js.map +1 -1
- package/dist/cjs/classes/job.js +19 -7
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue.js +1 -1
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/scripts.js +37 -11
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +53 -18
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addJobScheduler-11.lua +81 -25
- package/dist/cjs/commands/addRepeatableJob-2.lua +1 -1
- package/dist/cjs/commands/includes/addJobFromScheduler.lua +5 -3
- package/dist/cjs/commands/includes/getJobSchedulerEveryNextMillis.lua +28 -0
- package/dist/cjs/commands/includes/storeJobScheduler.lua +15 -1
- package/dist/cjs/commands/moveStalledJobsToWait-8.lua +14 -1
- package/dist/cjs/commands/promote-9.lua +0 -1
- package/dist/cjs/commands/updateJobScheduler-12.lua +50 -14
- package/dist/cjs/enums/error-code.js +2 -0
- package/dist/cjs/enums/error-code.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +108 -25
- package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js +11 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/cjs/scripts/promote-9.js +0 -1
- package/dist/cjs/scripts/promote-9.js.map +1 -1
- package/dist/cjs/scripts/updateJobScheduler-12.js +66 -17
- package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/job-scheduler.js +22 -24
- package/dist/esm/classes/job-scheduler.js.map +1 -1
- package/dist/esm/classes/job.js +19 -7
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +1 -1
- package/dist/esm/classes/queue.js +1 -1
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +1 -1
- package/dist/esm/classes/scripts.js +37 -11
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.js +53 -18
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addJobScheduler-11.lua +81 -25
- package/dist/esm/commands/addRepeatableJob-2.lua +1 -1
- package/dist/esm/commands/includes/addJobFromScheduler.lua +5 -3
- package/dist/esm/commands/includes/getJobSchedulerEveryNextMillis.lua +28 -0
- package/dist/esm/commands/includes/storeJobScheduler.lua +15 -1
- package/dist/esm/commands/moveStalledJobsToWait-8.lua +14 -1
- package/dist/esm/commands/promote-9.lua +0 -1
- package/dist/esm/commands/updateJobScheduler-12.lua +50 -14
- package/dist/esm/enums/error-code.d.ts +3 -1
- package/dist/esm/enums/error-code.js +2 -0
- package/dist/esm/enums/error-code.js.map +1 -1
- package/dist/esm/interfaces/job-scheduler-json.d.ts +1 -0
- package/dist/esm/interfaces/repeatable-options.d.ts +1 -0
- package/dist/esm/scripts/addJobScheduler-11.js +108 -25
- package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-8.js +11 -1
- package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/esm/scripts/promote-9.js +0 -1
- package/dist/esm/scripts/promote-9.js.map +1 -1
- package/dist/esm/scripts/updateJobScheduler-12.js +66 -17
- package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +5 -4
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
local function getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
|
|
4
|
+
local nextMillis
|
|
5
|
+
if not prevMillis then
|
|
6
|
+
if startDate then
|
|
7
|
+
-- Assuming startDate is passed as milliseconds from JavaScript
|
|
8
|
+
nextMillis = tonumber(startDate)
|
|
9
|
+
nextMillis = nextMillis > now and nextMillis or now
|
|
10
|
+
else
|
|
11
|
+
nextMillis = now
|
|
12
|
+
end
|
|
13
|
+
else
|
|
14
|
+
nextMillis = prevMillis + every
|
|
15
|
+
-- check if we may have missed some iterations
|
|
16
|
+
if nextMillis < now then
|
|
17
|
+
nextMillis = math.floor(now / every) * every + every + (offset or 0)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
if not offset or offset == 0 then
|
|
22
|
+
local timeSlot = math.floor(nextMillis / every) * every;
|
|
23
|
+
offset = nextMillis - timeSlot;
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
-- Return a tuple nextMillis, offset
|
|
27
|
+
return math.floor(nextMillis), math.floor(offset)
|
|
28
|
+
end
|
|
@@ -21,6 +21,11 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
|
|
|
21
21
|
table.insert(optionalValues, opts['pattern'])
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
+
if opts['startDate'] then
|
|
25
|
+
table.insert(optionalValues, "startDate")
|
|
26
|
+
table.insert(optionalValues, opts['startDate'])
|
|
27
|
+
end
|
|
28
|
+
|
|
24
29
|
if opts['endDate'] then
|
|
25
30
|
table.insert(optionalValues, "endDate")
|
|
26
31
|
table.insert(optionalValues, opts['endDate'])
|
|
@@ -34,6 +39,12 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
|
|
|
34
39
|
if opts['offset'] then
|
|
35
40
|
table.insert(optionalValues, "offset")
|
|
36
41
|
table.insert(optionalValues, opts['offset'])
|
|
42
|
+
else
|
|
43
|
+
local offset = rcall("HGET", schedulerKey, "offset")
|
|
44
|
+
if offset then
|
|
45
|
+
table.insert(optionalValues, "offset")
|
|
46
|
+
table.insert(optionalValues, tonumber(offset))
|
|
47
|
+
end
|
|
37
48
|
end
|
|
38
49
|
|
|
39
50
|
local jsonTemplateOpts = cjson.encode(templateOpts)
|
|
@@ -47,6 +58,9 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
|
|
|
47
58
|
table.insert(optionalValues, templateData)
|
|
48
59
|
end
|
|
49
60
|
|
|
61
|
+
table.insert(optionalValues, "ic")
|
|
62
|
+
table.insert(optionalValues, rcall("HGET", schedulerKey, "ic") or 1)
|
|
63
|
+
|
|
50
64
|
rcall("DEL", schedulerKey) -- remove all attributes and then re-insert new ones
|
|
51
|
-
rcall("HMSET", schedulerKey, "name", opts['name'],
|
|
65
|
+
rcall("HMSET", schedulerKey, "name", opts['name'], unpack(optionalValues))
|
|
52
66
|
end
|
|
@@ -73,10 +73,23 @@ if (#stalling > 0) then
|
|
|
73
73
|
if (removed > 0) then
|
|
74
74
|
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
|
75
75
|
local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
|
|
76
|
-
|
|
76
|
+
|
|
77
|
+
-- Check if this is a repeatable job by looking at job options
|
|
78
|
+
local jobOpts = rcall("HGET", jobKey, "opts")
|
|
79
|
+
local isRepeatableJob = false
|
|
80
|
+
if jobOpts then
|
|
81
|
+
local opts = cjson.decode(jobOpts)
|
|
82
|
+
if opts and opts["repeat"] then
|
|
83
|
+
isRepeatableJob = true
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
-- Only fail job if it exceeds stall limit AND is not a repeatable job
|
|
88
|
+
if stalledCount > maxStalledJobCount and not isRepeatableJob then
|
|
77
89
|
local failedReason = "job stalled more than allowable limit"
|
|
78
90
|
rcall("HSET", jobKey, "defa", failedReason)
|
|
79
91
|
end
|
|
92
|
+
|
|
80
93
|
moveJobToWait(metaKey, activeKey, waitKey, pausedKey, markerKey, eventStreamKey, jobId,
|
|
81
94
|
"RPUSH")
|
|
82
95
|
|
|
@@ -50,7 +50,6 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then
|
|
|
50
50
|
addJobWithPriority(markerKey, KEYS[5], priority, jobId, KEYS[7], isPausedOrMaxed)
|
|
51
51
|
end
|
|
52
52
|
|
|
53
|
-
-- Emit waiting event (wait..ing@token)
|
|
54
53
|
rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId, "prev",
|
|
55
54
|
"delayed");
|
|
56
55
|
|
|
@@ -25,31 +25,60 @@
|
|
|
25
25
|
|
|
26
26
|
Output:
|
|
27
27
|
next delayed job id - OK
|
|
28
|
-
]]
|
|
29
|
-
local rcall = redis.call
|
|
28
|
+
]] local rcall = redis.call
|
|
30
29
|
local repeatKey = KEYS[1]
|
|
31
30
|
local delayedKey = KEYS[2]
|
|
32
31
|
local waitKey = KEYS[3]
|
|
33
32
|
local pausedKey = KEYS[4]
|
|
34
33
|
local metaKey = KEYS[5]
|
|
35
34
|
local prioritizedKey = KEYS[6]
|
|
36
|
-
local nextMillis = ARGV[1]
|
|
35
|
+
local nextMillis = tonumber(ARGV[1])
|
|
37
36
|
local jobSchedulerId = ARGV[2]
|
|
38
|
-
local timestamp = ARGV[5]
|
|
37
|
+
local timestamp = tonumber(ARGV[5])
|
|
39
38
|
local prefixKey = ARGV[6]
|
|
40
39
|
local producerId = ARGV[7]
|
|
40
|
+
local jobOpts = cmsgpack.unpack(ARGV[4])
|
|
41
41
|
|
|
42
42
|
-- Includes
|
|
43
43
|
--- @include "includes/addJobFromScheduler"
|
|
44
44
|
--- @include "includes/getOrSetMaxEvents"
|
|
45
|
+
--- @include "includes/getJobSchedulerEveryNextMillis"
|
|
45
46
|
|
|
46
|
-
local
|
|
47
|
-
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
|
48
|
-
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
|
47
|
+
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
|
49
48
|
|
|
50
49
|
-- Validate that scheduler exists.
|
|
51
|
-
|
|
50
|
+
-- If it does not exist we should not iterate anymore.
|
|
52
51
|
if prevMillis then
|
|
52
|
+
prevMillis = tonumber(prevMillis)
|
|
53
|
+
|
|
54
|
+
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
|
|
55
|
+
local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data", "every", "startDate", "offset")
|
|
56
|
+
|
|
57
|
+
local every = tonumber(schedulerAttributes[3])
|
|
58
|
+
local now = tonumber(timestamp)
|
|
59
|
+
|
|
60
|
+
-- If every is not found in scheduler attributes, try to get it from job options
|
|
61
|
+
if not every and jobOpts['repeat'] and jobOpts['repeat']['every'] then
|
|
62
|
+
every = tonumber(jobOpts['repeat']['every'])
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
if every then
|
|
66
|
+
local startDate = schedulerAttributes[4]
|
|
67
|
+
local jobOptsOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
|
|
68
|
+
local offset = schedulerAttributes[5] or jobOptsOffset or 0
|
|
69
|
+
local newOffset
|
|
70
|
+
|
|
71
|
+
nextMillis, newOffset = getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
|
|
72
|
+
|
|
73
|
+
if not offset then
|
|
74
|
+
rcall("HSET", schedulerKey, "offset", newOffset)
|
|
75
|
+
jobOpts['repeat']['offset'] = newOffset
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
|
80
|
+
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
|
81
|
+
|
|
53
82
|
local currentDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
|
54
83
|
|
|
55
84
|
if producerId == currentDelayedJobId then
|
|
@@ -57,7 +86,6 @@ if prevMillis then
|
|
|
57
86
|
local maxEvents = getOrSetMaxEvents(metaKey)
|
|
58
87
|
|
|
59
88
|
if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
|
|
60
|
-
local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data")
|
|
61
89
|
|
|
62
90
|
rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
|
|
63
91
|
rcall("HINCRBY", schedulerKey, "ic", 1)
|
|
@@ -72,9 +100,18 @@ if prevMillis then
|
|
|
72
100
|
rcall("HSET", schedulerKey, "data", templateData)
|
|
73
101
|
end
|
|
74
102
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
103
|
+
local delay = nextMillis - now
|
|
104
|
+
|
|
105
|
+
-- Fast Clamp delay to minimum of 0
|
|
106
|
+
if delay < 0 then
|
|
107
|
+
delay = 0
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
jobOpts["delay"] = delay
|
|
111
|
+
|
|
112
|
+
addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, jobOpts, waitKey, pausedKey, KEYS[12], metaKey,
|
|
113
|
+
prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey, schedulerAttributes[1], maxEvents, ARGV[5],
|
|
114
|
+
templateData or '{}', jobSchedulerId, delay)
|
|
78
115
|
|
|
79
116
|
-- TODO: remove this workaround in next breaking change
|
|
80
117
|
if KEYS[11] ~= "" then
|
|
@@ -83,8 +120,7 @@ if prevMillis then
|
|
|
83
120
|
|
|
84
121
|
return nextDelayedJobId .. "" -- convert to string
|
|
85
122
|
else
|
|
86
|
-
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
|
87
|
-
"duplicated", "jobId", nextDelayedJobId)
|
|
123
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "duplicated", "jobId", nextDelayedJobId)
|
|
88
124
|
end
|
|
89
125
|
end
|
|
90
126
|
end
|
|
@@ -12,5 +12,7 @@ var ErrorCode;
|
|
|
12
12
|
ErrorCode[ErrorCode["ParentJobCannotBeReplaced"] = -7] = "ParentJobCannotBeReplaced";
|
|
13
13
|
ErrorCode[ErrorCode["JobBelongsToJobScheduler"] = -8] = "JobBelongsToJobScheduler";
|
|
14
14
|
ErrorCode[ErrorCode["JobHasFailedChildren"] = -9] = "JobHasFailedChildren";
|
|
15
|
+
ErrorCode[ErrorCode["SchedulerJobIdCollision"] = -10] = "SchedulerJobIdCollision";
|
|
16
|
+
ErrorCode[ErrorCode["SchedulerJobSlotsBusy"] = -11] = "SchedulerJobSlotsBusy";
|
|
15
17
|
})(ErrorCode || (exports.ErrorCode = ErrorCode = {}));
|
|
16
18
|
//# sourceMappingURL=error-code.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":";;;AAAA,IAAY,
|
|
1
|
+
{"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":";;;AAAA,IAAY,SAYX;AAZD,WAAY,SAAS;IACnB,wDAAgB,CAAA;IAChB,gEAAoB,CAAA;IACpB,4DAAkB,CAAA;IAClB,sEAAuB,CAAA;IACvB,oEAAsB,CAAA;IACtB,gEAAoB,CAAA;IACpB,oFAA8B,CAAA;IAC9B,kFAA6B,CAAA;IAC7B,0EAAyB,CAAA;IACzB,iFAA6B,CAAA;IAC7B,6EAA2B,CAAA;AAC7B,CAAC,EAZW,SAAS,yBAAT,SAAS,QAYpB"}
|
|
@@ -19,7 +19,7 @@ const content = `--[[
|
|
|
19
19
|
ARGV[2] msgpacked options
|
|
20
20
|
[1] name
|
|
21
21
|
[2] tz?
|
|
22
|
-
[3]
|
|
22
|
+
[3] pattern?
|
|
23
23
|
[4] endDate?
|
|
24
24
|
[5] every?
|
|
25
25
|
ARGV[3] jobs scheduler id
|
|
@@ -43,7 +43,9 @@ local eventsKey = KEYS[9]
|
|
|
43
43
|
local nextMillis = ARGV[1]
|
|
44
44
|
local jobSchedulerId = ARGV[3]
|
|
45
45
|
local templateOpts = cmsgpack.unpack(ARGV[5])
|
|
46
|
+
local now = tonumber(ARGV[7])
|
|
46
47
|
local prefixKey = ARGV[8]
|
|
48
|
+
local jobOpts = cmsgpack.unpack(ARGV[6])
|
|
47
49
|
-- Includes
|
|
48
50
|
--[[
|
|
49
51
|
Add delay marker if needed.
|
|
@@ -203,10 +205,11 @@ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed
|
|
|
203
205
|
rcall(pushCmd, targetKey, jobId)
|
|
204
206
|
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
205
207
|
end
|
|
206
|
-
local function addJobFromScheduler(jobKey, jobId,
|
|
208
|
+
local function addJobFromScheduler(jobKey, jobId, opts, waitKey, pausedKey, activeKey, metaKey,
|
|
207
209
|
prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
|
|
208
|
-
data, jobSchedulerId)
|
|
209
|
-
|
|
210
|
+
data, jobSchedulerId, repeatDelay)
|
|
211
|
+
opts['delay'] = repeatDelay
|
|
212
|
+
opts['jobId'] = jobId
|
|
210
213
|
local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
|
|
211
214
|
opts, timestamp, nil, nil, jobSchedulerId)
|
|
212
215
|
if delay ~= 0 then
|
|
@@ -377,6 +380,10 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
|
|
|
377
380
|
table.insert(optionalValues, "pattern")
|
|
378
381
|
table.insert(optionalValues, opts['pattern'])
|
|
379
382
|
end
|
|
383
|
+
if opts['startDate'] then
|
|
384
|
+
table.insert(optionalValues, "startDate")
|
|
385
|
+
table.insert(optionalValues, opts['startDate'])
|
|
386
|
+
end
|
|
380
387
|
if opts['endDate'] then
|
|
381
388
|
table.insert(optionalValues, "endDate")
|
|
382
389
|
table.insert(optionalValues, opts['endDate'])
|
|
@@ -388,6 +395,12 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
|
|
|
388
395
|
if opts['offset'] then
|
|
389
396
|
table.insert(optionalValues, "offset")
|
|
390
397
|
table.insert(optionalValues, opts['offset'])
|
|
398
|
+
else
|
|
399
|
+
local offset = rcall("HGET", schedulerKey, "offset")
|
|
400
|
+
if offset then
|
|
401
|
+
table.insert(optionalValues, "offset")
|
|
402
|
+
table.insert(optionalValues, tonumber(offset))
|
|
403
|
+
end
|
|
391
404
|
end
|
|
392
405
|
local jsonTemplateOpts = cjson.encode(templateOpts)
|
|
393
406
|
if jsonTemplateOpts and jsonTemplateOpts ~= '{}' then
|
|
@@ -398,15 +411,55 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
|
|
|
398
411
|
table.insert(optionalValues, "data")
|
|
399
412
|
table.insert(optionalValues, templateData)
|
|
400
413
|
end
|
|
414
|
+
table.insert(optionalValues, "ic")
|
|
415
|
+
table.insert(optionalValues, rcall("HGET", schedulerKey, "ic") or 1)
|
|
401
416
|
rcall("DEL", schedulerKey) -- remove all attributes and then re-insert new ones
|
|
402
|
-
rcall("HMSET", schedulerKey, "name", opts['name'],
|
|
417
|
+
rcall("HMSET", schedulerKey, "name", opts['name'], unpack(optionalValues))
|
|
418
|
+
end
|
|
419
|
+
local function getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
|
|
420
|
+
local nextMillis
|
|
421
|
+
if not prevMillis then
|
|
422
|
+
if startDate then
|
|
423
|
+
-- Assuming startDate is passed as milliseconds from JavaScript
|
|
424
|
+
nextMillis = tonumber(startDate)
|
|
425
|
+
nextMillis = nextMillis > now and nextMillis or now
|
|
426
|
+
else
|
|
427
|
+
nextMillis = now
|
|
428
|
+
end
|
|
429
|
+
else
|
|
430
|
+
nextMillis = prevMillis + every
|
|
431
|
+
-- check if we may have missed some iterations
|
|
432
|
+
if nextMillis < now then
|
|
433
|
+
nextMillis = math.floor(now / every) * every + every + (offset or 0)
|
|
434
|
+
end
|
|
435
|
+
end
|
|
436
|
+
if not offset or offset == 0 then
|
|
437
|
+
local timeSlot = math.floor(nextMillis / every) * every;
|
|
438
|
+
offset = nextMillis - timeSlot;
|
|
439
|
+
end
|
|
440
|
+
-- Return a tuple nextMillis, offset
|
|
441
|
+
return math.floor(nextMillis), math.floor(offset)
|
|
403
442
|
end
|
|
404
443
|
-- If we are overriding a repeatable job we must delete the delayed job for
|
|
405
444
|
-- the next iteration.
|
|
406
445
|
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
|
|
407
|
-
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
|
408
|
-
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
|
409
446
|
local maxEvents = getOrSetMaxEvents(metaKey)
|
|
447
|
+
local templateData = ARGV[4]
|
|
448
|
+
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
|
449
|
+
if prevMillis then
|
|
450
|
+
prevMillis = tonumber(prevMillis)
|
|
451
|
+
end
|
|
452
|
+
local schedulerOpts = cmsgpack.unpack(ARGV[2])
|
|
453
|
+
local every = schedulerOpts['every']
|
|
454
|
+
-- For backwards compatibility we also check the offset from the job itself.
|
|
455
|
+
-- could be removed in future major versions.
|
|
456
|
+
local jobOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
|
|
457
|
+
local offset = schedulerOpts['offset'] or jobOffset or 0
|
|
458
|
+
local newOffset = offset
|
|
459
|
+
if every then
|
|
460
|
+
local startDate = schedulerOpts['startDate']
|
|
461
|
+
nextMillis, newOffset = getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
|
|
462
|
+
end
|
|
410
463
|
local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey, jobId,
|
|
411
464
|
metaKey, eventsKey)
|
|
412
465
|
if rcall("ZSCORE", delayedKey, jobId) then
|
|
@@ -429,33 +482,63 @@ local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, wai
|
|
|
429
482
|
end
|
|
430
483
|
return false
|
|
431
484
|
end
|
|
432
|
-
|
|
433
|
-
if not removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
|
|
434
|
-
nextDelayedJobId, metaKey, eventsKey) then
|
|
435
|
-
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
|
436
|
-
"duplicated", "jobId", nextDelayedJobId)
|
|
437
|
-
return nextDelayedJobId .. "" -- convert to string
|
|
438
|
-
end
|
|
439
|
-
end
|
|
440
|
-
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
|
485
|
+
local hadPrevJob = false
|
|
441
486
|
if prevMillis then
|
|
442
487
|
local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
|
443
|
-
local
|
|
444
|
-
|
|
445
|
-
|
|
488
|
+
local currentJobKey = schedulerKey .. ":" .. prevMillis
|
|
489
|
+
-- In theory it should always exist the currentJobKey if there is a prevMillis unless something has
|
|
490
|
+
-- gone really wrong.
|
|
491
|
+
if rcall("EXISTS", currentJobKey) == 1 then
|
|
492
|
+
hadPrevJob = removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
|
|
446
493
|
currentJobId, metaKey, eventsKey)
|
|
447
494
|
end
|
|
448
495
|
end
|
|
449
|
-
|
|
450
|
-
|
|
496
|
+
if hadPrevJob then
|
|
497
|
+
-- The jobs has been removed and we want to replace it, so lets use the same millis.
|
|
498
|
+
nextMillis = prevMillis
|
|
499
|
+
else
|
|
500
|
+
-- Special case where no job was removed, and we need to add the next iteration.
|
|
501
|
+
schedulerOpts['offset'] = newOffset
|
|
502
|
+
end
|
|
503
|
+
-- Check for job ID collision with existing jobs (in any state)
|
|
504
|
+
local jobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
|
505
|
+
local jobKey = prefixKey .. jobId
|
|
506
|
+
-- If there's already a job with this ID, handle the collision
|
|
507
|
+
if rcall("EXISTS", jobKey) == 1 then
|
|
508
|
+
if every then
|
|
509
|
+
-- For 'every' case: try next time slot to avoid collision
|
|
510
|
+
local nextSlotMillis = nextMillis + every
|
|
511
|
+
local nextSlotJobId = "repeat:" .. jobSchedulerId .. ":" .. nextSlotMillis
|
|
512
|
+
local nextSlotJobKey = prefixKey .. nextSlotJobId
|
|
513
|
+
if rcall("EXISTS", nextSlotJobKey) == 0 then
|
|
514
|
+
-- Next slot is free, use it
|
|
515
|
+
nextMillis = nextSlotMillis
|
|
516
|
+
jobId = nextSlotJobId
|
|
517
|
+
else
|
|
518
|
+
-- Next slot also has a job, return error code
|
|
519
|
+
return -11 -- SchedulerJobSlotsBusy
|
|
520
|
+
end
|
|
521
|
+
else
|
|
522
|
+
-- For 'pattern' case: return error code
|
|
523
|
+
return -10 -- SchedulerJobIdCollision
|
|
524
|
+
end
|
|
525
|
+
end
|
|
526
|
+
local delay = nextMillis - now
|
|
527
|
+
-- Fast Clamp delay to minimum of 0
|
|
528
|
+
if delay < 0 then
|
|
529
|
+
delay = 0
|
|
530
|
+
end
|
|
531
|
+
local nextJobKey = schedulerKey .. ":" .. nextMillis
|
|
532
|
+
-- jobId already calculated above during collision check
|
|
533
|
+
storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, templateData, templateOpts)
|
|
451
534
|
rcall("INCR", KEYS[8])
|
|
452
|
-
addJobFromScheduler(
|
|
535
|
+
addJobFromScheduler(nextJobKey, jobId, jobOpts, waitKey, pausedKey,
|
|
453
536
|
KEYS[11], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey,
|
|
454
|
-
schedulerOpts['name'], maxEvents,
|
|
537
|
+
schedulerOpts['name'], maxEvents, now, templateData, jobSchedulerId, delay)
|
|
455
538
|
if ARGV[9] ~= "" then
|
|
456
|
-
rcall("HSET", ARGV[9], "nrjid",
|
|
539
|
+
rcall("HSET", ARGV[9], "nrjid", jobId)
|
|
457
540
|
end
|
|
458
|
-
return
|
|
541
|
+
return {jobId .. "", delay}
|
|
459
542
|
`;
|
|
460
543
|
exports.addJobScheduler = {
|
|
461
544
|
name: 'addJobScheduler',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
|
1
|
+
{"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0hBf,CAAC;AACW,QAAA,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|
|
@@ -144,7 +144,17 @@ if (#stalling > 0) then
|
|
|
144
144
|
if (removed > 0) then
|
|
145
145
|
-- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
|
|
146
146
|
local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
|
|
147
|
-
if
|
|
147
|
+
-- Check if this is a repeatable job by looking at job options
|
|
148
|
+
local jobOpts = rcall("HGET", jobKey, "opts")
|
|
149
|
+
local isRepeatableJob = false
|
|
150
|
+
if jobOpts then
|
|
151
|
+
local opts = cjson.decode(jobOpts)
|
|
152
|
+
if opts and opts["repeat"] then
|
|
153
|
+
isRepeatableJob = true
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
-- Only fail job if it exceeds stall limit AND is not a repeatable job
|
|
157
|
+
if stalledCount > maxStalledJobCount and not isRepeatableJob then
|
|
148
158
|
local failedReason = "job stalled more than allowable limit"
|
|
149
159
|
rcall("HSET", jobKey, "defa", failedReason)
|
|
150
160
|
end
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
|
1
|
+
{"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Kf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
|
@@ -94,7 +94,6 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then
|
|
|
94
94
|
else
|
|
95
95
|
addJobWithPriority(markerKey, KEYS[5], priority, jobId, KEYS[7], isPausedOrMaxed)
|
|
96
96
|
end
|
|
97
|
-
-- Emit waiting event (wait..ing@token)
|
|
98
97
|
rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId, "prev",
|
|
99
98
|
"delayed");
|
|
100
99
|
rcall("HSET", jobKey, "delay", 0)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"promote-9.js","sourceRoot":"","sources":["../../../src/scripts/promote-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
|
1
|
+
{"version":3,"file":"promote-9.js","sourceRoot":"","sources":["../../../src/scripts/promote-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoGf,CAAC;AACW,QAAA,OAAO,GAAG;IACrB,IAAI,EAAE,SAAS;IACf,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
|
@@ -25,19 +25,19 @@ const content = `--[[
|
|
|
25
25
|
ARGV[7] producer id
|
|
26
26
|
Output:
|
|
27
27
|
next delayed job id - OK
|
|
28
|
-
]]
|
|
29
|
-
local rcall = redis.call
|
|
28
|
+
]] local rcall = redis.call
|
|
30
29
|
local repeatKey = KEYS[1]
|
|
31
30
|
local delayedKey = KEYS[2]
|
|
32
31
|
local waitKey = KEYS[3]
|
|
33
32
|
local pausedKey = KEYS[4]
|
|
34
33
|
local metaKey = KEYS[5]
|
|
35
34
|
local prioritizedKey = KEYS[6]
|
|
36
|
-
local nextMillis = ARGV[1]
|
|
35
|
+
local nextMillis = tonumber(ARGV[1])
|
|
37
36
|
local jobSchedulerId = ARGV[2]
|
|
38
|
-
local timestamp = ARGV[5]
|
|
37
|
+
local timestamp = tonumber(ARGV[5])
|
|
39
38
|
local prefixKey = ARGV[6]
|
|
40
39
|
local producerId = ARGV[7]
|
|
40
|
+
local jobOpts = cmsgpack.unpack(ARGV[4])
|
|
41
41
|
-- Includes
|
|
42
42
|
--[[
|
|
43
43
|
Add delay marker if needed.
|
|
@@ -197,10 +197,11 @@ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed
|
|
|
197
197
|
rcall(pushCmd, targetKey, jobId)
|
|
198
198
|
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
|
199
199
|
end
|
|
200
|
-
local function addJobFromScheduler(jobKey, jobId,
|
|
200
|
+
local function addJobFromScheduler(jobKey, jobId, opts, waitKey, pausedKey, activeKey, metaKey,
|
|
201
201
|
prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
|
|
202
|
-
data, jobSchedulerId)
|
|
203
|
-
|
|
202
|
+
data, jobSchedulerId, repeatDelay)
|
|
203
|
+
opts['delay'] = repeatDelay
|
|
204
|
+
opts['jobId'] = jobId
|
|
204
205
|
local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
|
|
205
206
|
opts, timestamp, nil, nil, jobSchedulerId)
|
|
206
207
|
if delay ~= 0 then
|
|
@@ -230,18 +231,61 @@ local function getOrSetMaxEvents(metaKey)
|
|
|
230
231
|
end
|
|
231
232
|
return maxEvents
|
|
232
233
|
end
|
|
233
|
-
local
|
|
234
|
-
local
|
|
235
|
-
|
|
236
|
-
|
|
234
|
+
local function getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
|
|
235
|
+
local nextMillis
|
|
236
|
+
if not prevMillis then
|
|
237
|
+
if startDate then
|
|
238
|
+
-- Assuming startDate is passed as milliseconds from JavaScript
|
|
239
|
+
nextMillis = tonumber(startDate)
|
|
240
|
+
nextMillis = nextMillis > now and nextMillis or now
|
|
241
|
+
else
|
|
242
|
+
nextMillis = now
|
|
243
|
+
end
|
|
244
|
+
else
|
|
245
|
+
nextMillis = prevMillis + every
|
|
246
|
+
-- check if we may have missed some iterations
|
|
247
|
+
if nextMillis < now then
|
|
248
|
+
nextMillis = math.floor(now / every) * every + every + (offset or 0)
|
|
249
|
+
end
|
|
250
|
+
end
|
|
251
|
+
if not offset or offset == 0 then
|
|
252
|
+
local timeSlot = math.floor(nextMillis / every) * every;
|
|
253
|
+
offset = nextMillis - timeSlot;
|
|
254
|
+
end
|
|
255
|
+
-- Return a tuple nextMillis, offset
|
|
256
|
+
return math.floor(nextMillis), math.floor(offset)
|
|
257
|
+
end
|
|
237
258
|
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
|
259
|
+
-- Validate that scheduler exists.
|
|
260
|
+
-- If it does not exist we should not iterate anymore.
|
|
238
261
|
if prevMillis then
|
|
262
|
+
prevMillis = tonumber(prevMillis)
|
|
263
|
+
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
|
|
264
|
+
local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data", "every", "startDate", "offset")
|
|
265
|
+
local every = tonumber(schedulerAttributes[3])
|
|
266
|
+
local now = tonumber(timestamp)
|
|
267
|
+
-- If every is not found in scheduler attributes, try to get it from job options
|
|
268
|
+
if not every and jobOpts['repeat'] and jobOpts['repeat']['every'] then
|
|
269
|
+
every = tonumber(jobOpts['repeat']['every'])
|
|
270
|
+
end
|
|
271
|
+
if every then
|
|
272
|
+
local startDate = schedulerAttributes[4]
|
|
273
|
+
local jobOptsOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
|
|
274
|
+
local offset = schedulerAttributes[5] or jobOptsOffset or 0
|
|
275
|
+
local newOffset
|
|
276
|
+
nextMillis, newOffset = getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
|
|
277
|
+
if not offset then
|
|
278
|
+
rcall("HSET", schedulerKey, "offset", newOffset)
|
|
279
|
+
jobOpts['repeat']['offset'] = newOffset
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
|
283
|
+
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
|
239
284
|
local currentDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
|
240
285
|
if producerId == currentDelayedJobId then
|
|
241
286
|
local eventsKey = KEYS[9]
|
|
242
287
|
local maxEvents = getOrSetMaxEvents(metaKey)
|
|
243
288
|
if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
|
|
244
|
-
local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data")
|
|
245
289
|
rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
|
|
246
290
|
rcall("HINCRBY", schedulerKey, "ic", 1)
|
|
247
291
|
rcall("INCR", KEYS[8])
|
|
@@ -251,17 +295,22 @@ if prevMillis then
|
|
|
251
295
|
if templateData and templateData ~= '{}' then
|
|
252
296
|
rcall("HSET", schedulerKey, "data", templateData)
|
|
253
297
|
end
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
298
|
+
local delay = nextMillis - now
|
|
299
|
+
-- Fast Clamp delay to minimum of 0
|
|
300
|
+
if delay < 0 then
|
|
301
|
+
delay = 0
|
|
302
|
+
end
|
|
303
|
+
jobOpts["delay"] = delay
|
|
304
|
+
addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, jobOpts, waitKey, pausedKey, KEYS[12], metaKey,
|
|
305
|
+
prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey, schedulerAttributes[1], maxEvents, ARGV[5],
|
|
306
|
+
templateData or '{}', jobSchedulerId, delay)
|
|
257
307
|
-- TODO: remove this workaround in next breaking change
|
|
258
308
|
if KEYS[11] ~= "" then
|
|
259
309
|
rcall("HSET", KEYS[11], "nrjid", nextDelayedJobId)
|
|
260
310
|
end
|
|
261
311
|
return nextDelayedJobId .. "" -- convert to string
|
|
262
312
|
else
|
|
263
|
-
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
|
264
|
-
"duplicated", "jobId", nextDelayedJobId)
|
|
313
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "duplicated", "jobId", nextDelayedJobId)
|
|
265
314
|
end
|
|
266
315
|
end
|
|
267
316
|
end
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"updateJobScheduler-12.js","sourceRoot":"","sources":["../../../src/scripts/updateJobScheduler-12.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
|
1
|
+
{"version":3,"file":"updateJobScheduler-12.js","sourceRoot":"","sources":["../../../src/scripts/updateJobScheduler-12.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyTf,CAAC;AACW,QAAA,kBAAkB,GAAG;IAChC,IAAI,EAAE,oBAAoB;IAC1B,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|