bullmq 5.41.5 → 5.41.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/job-scheduler.js +40 -15
- package/dist/cjs/classes/job-scheduler.js.map +1 -1
- package/dist/cjs/classes/job.js +9 -12
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/scripts.js +11 -1
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/commands/addDelayedJob-6.lua +5 -2
- package/dist/cjs/commands/addJobScheduler-10.lua +130 -0
- package/dist/cjs/commands/includes/addDelayedJob.lua +2 -7
- package/dist/cjs/commands/includes/getOrSetMaxEvents.lua +6 -6
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +1 -1
- package/dist/cjs/commands/moveToFinished-14.lua +1 -1
- package/dist/cjs/scripts/addDelayedJob-6.js +41 -43
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/{addJobScheduler-6.js → addJobScheduler-10.js} +208 -42
- package/dist/cjs/scripts/addJobScheduler-10.js.map +1 -0
- package/dist/cjs/scripts/addParentJob-4.js +6 -6
- package/dist/cjs/scripts/addPrioritizedJob-8.js +6 -6
- package/dist/cjs/scripts/addStandardJob-8.js +6 -6
- package/dist/cjs/scripts/changeDelay-4.js +6 -6
- package/dist/cjs/scripts/index.js +1 -1
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +6 -6
- package/dist/cjs/scripts/moveJobsToWait-8.js +6 -6
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +49 -9
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToDelayed-8.js +6 -6
- package/dist/cjs/scripts/moveToFinished-14.js +49 -9
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/removeJob-3.js +6 -6
- package/dist/cjs/scripts/reprocessJob-8.js +6 -6
- package/dist/cjs/scripts/retryJob-11.js +6 -6
- package/dist/cjs/scripts/updateJobScheduler-7.js +8 -41
- package/dist/cjs/scripts/updateJobScheduler-7.js.map +1 -1
- package/dist/cjs/scripts/updateProgress-3.js +6 -6
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/job-scheduler.d.ts +1 -0
- package/dist/esm/classes/job-scheduler.js +40 -15
- package/dist/esm/classes/job-scheduler.js.map +1 -1
- package/dist/esm/classes/job.js +9 -12
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +2 -2
- package/dist/esm/classes/scripts.js +11 -1
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/commands/addDelayedJob-6.lua +5 -2
- package/dist/esm/commands/addJobScheduler-10.lua +130 -0
- package/dist/esm/commands/includes/addDelayedJob.lua +2 -7
- package/dist/esm/commands/includes/getOrSetMaxEvents.lua +6 -6
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +1 -1
- package/dist/esm/commands/moveToFinished-14.lua +1 -1
- package/dist/esm/scripts/addDelayedJob-6.js +41 -43
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/{addJobScheduler-6.js → addJobScheduler-10.js} +208 -42
- package/dist/esm/scripts/addJobScheduler-10.js.map +1 -0
- package/dist/esm/scripts/addParentJob-4.js +6 -6
- package/dist/esm/scripts/addPrioritizedJob-8.js +6 -6
- package/dist/esm/scripts/addStandardJob-8.js +6 -6
- package/dist/esm/scripts/changeDelay-4.js +6 -6
- package/dist/esm/scripts/index.d.ts +1 -1
- package/dist/esm/scripts/index.js +1 -1
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveJobFromActiveToWait-9.js +6 -6
- package/dist/esm/scripts/moveJobsToWait-8.js +6 -6
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +49 -9
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToDelayed-8.js +6 -6
- package/dist/esm/scripts/moveToFinished-14.js +49 -9
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/removeJob-3.js +6 -6
- package/dist/esm/scripts/reprocessJob-8.js +6 -6
- package/dist/esm/scripts/retryJob-11.js +6 -6
- package/dist/esm/scripts/updateJobScheduler-7.js +8 -41
- package/dist/esm/scripts/updateJobScheduler-7.js.map +1 -1
- package/dist/esm/scripts/updateProgress-3.js +6 -6
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/addJobScheduler-6.lua +0 -72
- package/dist/cjs/scripts/addJobScheduler-6.js.map +0 -1
- package/dist/esm/commands/addJobScheduler-6.lua +0 -72
- package/dist/esm/scripts/addJobScheduler-6.js.map +0 -1
- /package/dist/esm/scripts/{addJobScheduler-6.d.ts → addJobScheduler-10.d.ts} +0 -0
@@ -0,0 +1,130 @@
|
|
1
|
+
--[[
|
2
|
+
Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] 'repeat' key
|
6
|
+
KEYS[2] 'delayed' key
|
7
|
+
KEYS[3] 'wait' key
|
8
|
+
KEYS[4] 'paused' key
|
9
|
+
KEYS[5] 'meta' key
|
10
|
+
KEYS[6] 'prioritized' key
|
11
|
+
KEYS[7] 'marker' key
|
12
|
+
KEYS[8] 'id' key
|
13
|
+
KEYS[9] 'events' key
|
14
|
+
KEYS[10] 'pc' priority counter
|
15
|
+
|
16
|
+
ARGV[1] next milliseconds
|
17
|
+
ARGV[2] msgpacked options
|
18
|
+
[1] name
|
19
|
+
[2] tz?
|
20
|
+
[3] patten?
|
21
|
+
[4] endDate?
|
22
|
+
[5] every?
|
23
|
+
ARGV[3] jobs scheduler id
|
24
|
+
ARGV[4] Json stringified template data
|
25
|
+
ARGV[5] mspacked template opts
|
26
|
+
ARGV[6] msgpacked delayed opts
|
27
|
+
ARGV[7] timestamp
|
28
|
+
ARGV[8] prefix key
|
29
|
+
ARGV[9] producer key
|
30
|
+
|
31
|
+
Output:
|
32
|
+
repeatableKey - OK
|
33
|
+
]]
|
34
|
+
local rcall = redis.call
|
35
|
+
local repeatKey = KEYS[1]
|
36
|
+
local delayedKey = KEYS[2]
|
37
|
+
local prioritizedKey = KEYS[6]
|
38
|
+
|
39
|
+
local nextMillis = ARGV[1]
|
40
|
+
local jobSchedulerId = ARGV[3]
|
41
|
+
local templateOpts = cmsgpack.unpack(ARGV[5])
|
42
|
+
local prefixKey = ARGV[8]
|
43
|
+
|
44
|
+
-- Includes
|
45
|
+
--- @include "includes/addDelayedJob"
|
46
|
+
--- @include "includes/addJobWithPriority"
|
47
|
+
--- @include "includes/getOrSetMaxEvents"
|
48
|
+
--- @include "includes/getTargetQueueList"
|
49
|
+
--- @include "includes/isQueuePaused"
|
50
|
+
--- @include "includes/removeJob"
|
51
|
+
--- @include "includes/storeJob"
|
52
|
+
--- @include "includes/storeJobScheduler"
|
53
|
+
|
54
|
+
-- If we are overriding a repeatable job we must delete the delayed job for
|
55
|
+
-- the next iteration.
|
56
|
+
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
|
57
|
+
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
58
|
+
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
59
|
+
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
60
|
+
|
61
|
+
if prevMillis ~= false then
|
62
|
+
local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
63
|
+
|
64
|
+
if rcall("EXISTS", nextDelayedJobKey) ~= 1 or currentJobId == nextDelayedJobId then
|
65
|
+
if rcall("ZSCORE", delayedKey, currentJobId) ~= false then
|
66
|
+
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
67
|
+
rcall("ZREM", delayedKey, currentJobId)
|
68
|
+
elseif rcall("ZSCORE", prioritizedKey, currentJobId) ~= false then
|
69
|
+
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
70
|
+
rcall("ZREM", prioritizedKey, currentJobId)
|
71
|
+
else
|
72
|
+
if isQueuePaused(KEYS[5]) then
|
73
|
+
if rcall("LREM", KEYS[4], 1, currentJobId) > 0 then
|
74
|
+
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
75
|
+
end
|
76
|
+
else
|
77
|
+
if rcall("LREM", KEYS[3], 1, currentJobId) > 0 then
|
78
|
+
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
local schedulerOpts = cmsgpack.unpack(ARGV[2])
|
86
|
+
storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
|
87
|
+
|
88
|
+
if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
|
89
|
+
local eventsKey = KEYS[9]
|
90
|
+
local metaKey = KEYS[5]
|
91
|
+
local maxEvents = getOrSetMaxEvents(metaKey)
|
92
|
+
|
93
|
+
rcall("INCR", KEYS[8])
|
94
|
+
|
95
|
+
local delayedOpts = cmsgpack.unpack(ARGV[6])
|
96
|
+
|
97
|
+
local delay, priority = storeJob(eventsKey, nextDelayedJobKey, nextDelayedJobId, schedulerOpts['name'], ARGV[4],
|
98
|
+
delayedOpts, ARGV[7], nil, nil, jobSchedulerId)
|
99
|
+
|
100
|
+
if delay ~= 0 then
|
101
|
+
addDelayedJob(nextDelayedJobId, delayedKey, eventsKey,
|
102
|
+
ARGV[7], maxEvents, KEYS[7], delay)
|
103
|
+
else
|
104
|
+
local isPaused = isQueuePaused(KEYS[5])
|
105
|
+
|
106
|
+
-- Standard or priority add
|
107
|
+
if priority == 0 then
|
108
|
+
if isPaused then
|
109
|
+
-- LIFO or FIFO
|
110
|
+
local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
|
111
|
+
rcall(pushCmd, KEYS[4], nextDelayedJobId)
|
112
|
+
else
|
113
|
+
-- LIFO or FIFO
|
114
|
+
local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
|
115
|
+
rcall(pushCmd, KEYS[3], nextDelayedJobId)
|
116
|
+
end
|
117
|
+
else
|
118
|
+
-- Priority add
|
119
|
+
addJobWithPriority(KEYS[7], KEYS[6], priority, nextDelayedJobId, KEYS[10], isPaused)
|
120
|
+
end
|
121
|
+
-- Emit waiting event
|
122
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", nextDelayedJobId)
|
123
|
+
end
|
124
|
+
|
125
|
+
if ARGV[9] ~= "" then
|
126
|
+
rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
|
127
|
+
end
|
128
|
+
|
129
|
+
return nextDelayedJobId .. "" -- convert to string
|
130
|
+
end
|
@@ -8,13 +8,9 @@
|
|
8
8
|
-- Includes
|
9
9
|
--- @include "addDelayMarkerIfNeeded"
|
10
10
|
--- @include "getDelayedScore"
|
11
|
-
--- @include "storeJob"
|
12
11
|
|
13
|
-
local function addDelayedJob(
|
14
|
-
maxEvents, markerKey,
|
15
|
-
-- Store the job.
|
16
|
-
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
|
17
|
-
opts, timestamp, parentKey, parentData, repeatJobKey)
|
12
|
+
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
13
|
+
maxEvents, markerKey, delay)
|
18
14
|
|
19
15
|
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
|
20
16
|
|
@@ -25,4 +21,3 @@ local function addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, name, data,
|
|
25
21
|
-- mark that a delayed job is available
|
26
22
|
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
27
23
|
end
|
28
|
-
|
@@ -2,10 +2,10 @@
|
|
2
2
|
Function to get max events value or set by default 10000.
|
3
3
|
]]
|
4
4
|
local function getOrSetMaxEvents(metaKey)
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
5
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
6
|
+
if not maxEvents then
|
7
|
+
maxEvents = 10000
|
8
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
9
|
+
end
|
10
|
+
return maxEvents
|
11
11
|
end
|
@@ -7,8 +7,11 @@
|
|
7
7
|
--- @include "removeDeduplicationKeyIfNeeded"
|
8
8
|
--- @include "removeJobsOnFail"
|
9
9
|
|
10
|
-
local function moveParentFromWaitingChildrenToFailed(
|
11
|
-
|
10
|
+
local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
11
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
12
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
13
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) ~= false then
|
14
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
12
15
|
local parentQueuePrefix = parentQueueKey .. ":"
|
13
16
|
local parentFailedKey = parentQueueKey .. ":failed"
|
14
17
|
rcall("ZADD", parentFailedKey, timestamp, parentId)
|
@@ -48,6 +51,48 @@ local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey,
|
|
48
51
|
local parentRawOpts = jobAttributes[3]
|
49
52
|
local parentOpts = cjson.decode(parentRawOpts)
|
50
53
|
|
54
|
+
removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
|
55
|
+
elseif rcall("ZSCORE", parentDelayedKey, parentId) ~= false then
|
56
|
+
rcall("ZREM", parentDelayedKey, parentId)
|
57
|
+
local parentQueuePrefix = parentQueueKey .. ":"
|
58
|
+
local parentFailedKey = parentQueueKey .. ":failed"
|
59
|
+
rcall("ZADD", parentFailedKey, timestamp, parentId)
|
60
|
+
local failedReason = "child " .. jobIdKey .. " failed"
|
61
|
+
rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
|
62
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
|
63
|
+
failedReason, "prev", "delayed")
|
64
|
+
|
65
|
+
local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
|
66
|
+
|
67
|
+
removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
|
68
|
+
|
69
|
+
if jobAttributes[1] then
|
70
|
+
local parentData = cjson.decode(jobAttributes[1])
|
71
|
+
if parentData['fpof'] then
|
72
|
+
moveParentFromWaitingChildrenToFailed(
|
73
|
+
parentData['queueKey'],
|
74
|
+
parentData['queueKey'] .. ':' .. parentData['id'],
|
75
|
+
parentData['id'],
|
76
|
+
parentKey,
|
77
|
+
timestamp
|
78
|
+
)
|
79
|
+
elseif parentData['idof'] or parentData['rdof'] then
|
80
|
+
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
81
|
+
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
|
82
|
+
if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
|
83
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
|
84
|
+
grandParentKey, parentData['id'], timestamp)
|
85
|
+
if parentData['idof'] then
|
86
|
+
local grandParentFailedSet = grandParentKey .. ":failed"
|
87
|
+
rcall("HSET", grandParentFailedSet, parentKey, failedReason)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
local parentRawOpts = jobAttributes[3]
|
94
|
+
local parentOpts = cjson.decode(parentRawOpts)
|
95
|
+
|
51
96
|
removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
|
52
97
|
end
|
53
98
|
end
|
@@ -95,7 +95,7 @@ if (#stalling > 0) then
|
|
95
95
|
"failed", "jobId", jobId, 'prev', 'active',
|
96
96
|
'failedReason', failedReason)
|
97
97
|
|
98
|
-
if rawParentData
|
98
|
+
if rawParentData then
|
99
99
|
if opts['fpof'] then
|
100
100
|
local parentData = cjson.decode(rawParentData)
|
101
101
|
moveParentFromWaitingChildrenToFailed(
|
@@ -103,7 +103,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
103
103
|
local parentKey = jobAttributes[1] or ""
|
104
104
|
local parentId = ""
|
105
105
|
local parentQueueKey = ""
|
106
|
-
if jobAttributes[2]
|
106
|
+
if jobAttributes[2] then
|
107
107
|
local jsonDecodedParent = cjson.decode(jobAttributes[2])
|
108
108
|
parentId = jsonDecodedParent['id']
|
109
109
|
parentQueueKey = jsonDecodedParent['queueKey']
|
@@ -103,41 +103,8 @@ local function getDelayedScore(delayedKey, timestamp, delay)
|
|
103
103
|
end
|
104
104
|
return minScore, delayedTimestamp
|
105
105
|
end
|
106
|
-
|
107
|
-
|
108
|
-
]]
|
109
|
-
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
110
|
-
parentKey, parentData, repeatJobKey)
|
111
|
-
local jsonOpts = cjson.encode(opts)
|
112
|
-
local delay = opts['delay'] or 0
|
113
|
-
local priority = opts['priority'] or 0
|
114
|
-
local debounceId = opts['de'] and opts['de']['id']
|
115
|
-
local optionalValues = {}
|
116
|
-
if parentKey ~= nil then
|
117
|
-
table.insert(optionalValues, "parentKey")
|
118
|
-
table.insert(optionalValues, parentKey)
|
119
|
-
table.insert(optionalValues, "parent")
|
120
|
-
table.insert(optionalValues, parentData)
|
121
|
-
end
|
122
|
-
if repeatJobKey ~= nil then
|
123
|
-
table.insert(optionalValues, "rjk")
|
124
|
-
table.insert(optionalValues, repeatJobKey)
|
125
|
-
end
|
126
|
-
if debounceId then
|
127
|
-
table.insert(optionalValues, "deid")
|
128
|
-
table.insert(optionalValues, debounceId)
|
129
|
-
end
|
130
|
-
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
131
|
-
"timestamp", timestamp, "delay", delay, "priority", priority,
|
132
|
-
unpack(optionalValues))
|
133
|
-
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
134
|
-
return delay, priority
|
135
|
-
end
|
136
|
-
local function addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, name, data, opts, timestamp, repeatJobKey,
|
137
|
-
maxEvents, markerKey, parentKey, parentData)
|
138
|
-
-- Store the job.
|
139
|
-
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
|
140
|
-
opts, timestamp, parentKey, parentData, repeatJobKey)
|
106
|
+
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
107
|
+
maxEvents, markerKey, delay)
|
141
108
|
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
|
142
109
|
rcall("ZADD", delayedKey, score, jobId)
|
143
110
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
@@ -172,12 +139,12 @@ end
|
|
172
139
|
Function to get max events value or set by default 10000.
|
173
140
|
]]
|
174
141
|
local function getOrSetMaxEvents(metaKey)
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
142
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
143
|
+
if not maxEvents then
|
144
|
+
maxEvents = 10000
|
145
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
146
|
+
end
|
147
|
+
return maxEvents
|
181
148
|
end
|
182
149
|
--[[
|
183
150
|
Function to handle the case when job is duplicated.
|
@@ -347,6 +314,36 @@ local function handleDuplicatedJob(jobKey, jobId, currentParentKey, currentParen
|
|
347
314
|
"duplicated", "jobId", jobId)
|
348
315
|
return jobId .. "" -- convert to string
|
349
316
|
end
|
317
|
+
--[[
|
318
|
+
Function to store a job
|
319
|
+
]]
|
320
|
+
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
321
|
+
parentKey, parentData, repeatJobKey)
|
322
|
+
local jsonOpts = cjson.encode(opts)
|
323
|
+
local delay = opts['delay'] or 0
|
324
|
+
local priority = opts['priority'] or 0
|
325
|
+
local debounceId = opts['de'] and opts['de']['id']
|
326
|
+
local optionalValues = {}
|
327
|
+
if parentKey ~= nil then
|
328
|
+
table.insert(optionalValues, "parentKey")
|
329
|
+
table.insert(optionalValues, parentKey)
|
330
|
+
table.insert(optionalValues, "parent")
|
331
|
+
table.insert(optionalValues, parentData)
|
332
|
+
end
|
333
|
+
if repeatJobKey ~= nil then
|
334
|
+
table.insert(optionalValues, "rjk")
|
335
|
+
table.insert(optionalValues, repeatJobKey)
|
336
|
+
end
|
337
|
+
if debounceId then
|
338
|
+
table.insert(optionalValues, "deid")
|
339
|
+
table.insert(optionalValues, debounceId)
|
340
|
+
end
|
341
|
+
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
342
|
+
"timestamp", timestamp, "delay", delay, "priority", priority,
|
343
|
+
unpack(optionalValues))
|
344
|
+
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
345
|
+
return delay, priority
|
346
|
+
end
|
350
347
|
if parentKey ~= nil then
|
351
348
|
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
352
349
|
parentData = cjson.encode(parent)
|
@@ -373,8 +370,9 @@ local deduplicationJobId = deduplicateJob(opts['de'], jobId, deduplicationKey,
|
|
373
370
|
if deduplicationJobId then
|
374
371
|
return deduplicationJobId
|
375
372
|
end
|
376
|
-
|
377
|
-
|
373
|
+
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
374
|
+
opts, timestamp, parentKey, parentData, repeatJobKey)
|
375
|
+
addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, KEYS[1], delay)
|
378
376
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
379
377
|
if parentDependenciesKey ~= nil then
|
380
378
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyXf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -4,12 +4,16 @@ exports.addJobScheduler = void 0;
|
|
4
4
|
const content = `--[[
|
5
5
|
Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
|
6
6
|
Input:
|
7
|
-
KEYS[1]
|
8
|
-
KEYS[2]
|
9
|
-
KEYS[3]
|
10
|
-
KEYS[4]
|
11
|
-
KEYS[5]
|
12
|
-
KEYS[6]
|
7
|
+
KEYS[1] 'repeat' key
|
8
|
+
KEYS[2] 'delayed' key
|
9
|
+
KEYS[3] 'wait' key
|
10
|
+
KEYS[4] 'paused' key
|
11
|
+
KEYS[5] 'meta' key
|
12
|
+
KEYS[6] 'prioritized' key
|
13
|
+
KEYS[7] 'marker' key
|
14
|
+
KEYS[8] 'id' key
|
15
|
+
KEYS[9] 'events' key
|
16
|
+
KEYS[10] 'pc' priority counter
|
13
17
|
ARGV[1] next milliseconds
|
14
18
|
ARGV[2] msgpacked options
|
15
19
|
[1] name
|
@@ -20,18 +24,142 @@ const content = `--[[
|
|
20
24
|
ARGV[3] jobs scheduler id
|
21
25
|
ARGV[4] Json stringified template data
|
22
26
|
ARGV[5] mspacked template opts
|
23
|
-
ARGV[6]
|
27
|
+
ARGV[6] msgpacked delayed opts
|
28
|
+
ARGV[7] timestamp
|
29
|
+
ARGV[8] prefix key
|
30
|
+
ARGV[9] producer key
|
24
31
|
Output:
|
25
32
|
repeatableKey - OK
|
26
|
-
]]
|
33
|
+
]]
|
34
|
+
local rcall = redis.call
|
27
35
|
local repeatKey = KEYS[1]
|
28
36
|
local delayedKey = KEYS[2]
|
29
37
|
local prioritizedKey = KEYS[6]
|
30
38
|
local nextMillis = ARGV[1]
|
31
39
|
local jobSchedulerId = ARGV[3]
|
32
40
|
local templateOpts = cmsgpack.unpack(ARGV[5])
|
33
|
-
local prefixKey = ARGV[
|
41
|
+
local prefixKey = ARGV[8]
|
42
|
+
-- Includes
|
43
|
+
--[[
|
44
|
+
Adds a delayed job to the queue by doing the following:
|
45
|
+
- Creates a new job key with the job data.
|
46
|
+
- adds to delayed zset.
|
47
|
+
- Emits a global event 'delayed' if the job is delayed.
|
48
|
+
]]
|
49
|
+
-- Includes
|
50
|
+
--[[
|
51
|
+
Add delay marker if needed.
|
52
|
+
]]
|
53
|
+
-- Includes
|
54
|
+
--[[
|
55
|
+
Function to return the next delayed job timestamp.
|
56
|
+
]]
|
57
|
+
local function getNextDelayedTimestamp(delayedKey)
|
58
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
59
|
+
if #result then
|
60
|
+
local nextTimestamp = tonumber(result[2])
|
61
|
+
if nextTimestamp ~= nil then
|
62
|
+
return nextTimestamp / 0x1000
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
67
|
+
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
68
|
+
if nextTimestamp ~= nil then
|
69
|
+
-- Replace the score of the marker with the newest known
|
70
|
+
-- next timestamp.
|
71
|
+
rcall("ZADD", markerKey, nextTimestamp, "1")
|
72
|
+
end
|
73
|
+
end
|
74
|
+
--[[
|
75
|
+
Bake in the job id first 12 bits into the timestamp
|
76
|
+
to guarantee correct execution order of delayed jobs
|
77
|
+
(up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
|
78
|
+
WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
|
79
|
+
]]
|
80
|
+
local function getDelayedScore(delayedKey, timestamp, delay)
|
81
|
+
local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
|
82
|
+
local minScore = delayedTimestamp * 0x1000
|
83
|
+
local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
|
84
|
+
local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
|
85
|
+
minScore, "WITHSCORES","LIMIT", 0, 1)
|
86
|
+
if #result then
|
87
|
+
local currentMaxScore = tonumber(result[2])
|
88
|
+
if currentMaxScore ~= nil then
|
89
|
+
if currentMaxScore >= maxScore then
|
90
|
+
return maxScore, delayedTimestamp
|
91
|
+
else
|
92
|
+
return currentMaxScore + 1, delayedTimestamp
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
return minScore, delayedTimestamp
|
97
|
+
end
|
98
|
+
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
99
|
+
maxEvents, markerKey, delay)
|
100
|
+
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
|
101
|
+
rcall("ZADD", delayedKey, score, jobId)
|
102
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
103
|
+
"jobId", jobId, "delay", delayedTimestamp)
|
104
|
+
-- mark that a delayed job is available
|
105
|
+
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
106
|
+
end
|
107
|
+
--[[
|
108
|
+
Function to add job considering priority.
|
109
|
+
]]
|
34
110
|
-- Includes
|
111
|
+
--[[
|
112
|
+
Add marker if needed when a job is available.
|
113
|
+
]]
|
114
|
+
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
115
|
+
if not isPausedOrMaxed then
|
116
|
+
rcall("ZADD", markerKey, 0, "0")
|
117
|
+
end
|
118
|
+
end
|
119
|
+
--[[
|
120
|
+
Function to get priority score.
|
121
|
+
]]
|
122
|
+
local function getPriorityScore(priority, priorityCounterKey)
|
123
|
+
local prioCounter = rcall("INCR", priorityCounterKey)
|
124
|
+
return priority * 0x100000000 + prioCounter % 0x100000000
|
125
|
+
end
|
126
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
|
127
|
+
isPausedOrMaxed)
|
128
|
+
local score = getPriorityScore(priority, priorityCounterKey)
|
129
|
+
rcall("ZADD", prioritizedKey, score, jobId)
|
130
|
+
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
131
|
+
end
|
132
|
+
--[[
|
133
|
+
Function to get max events value or set by default 10000.
|
134
|
+
]]
|
135
|
+
local function getOrSetMaxEvents(metaKey)
|
136
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
137
|
+
if not maxEvents then
|
138
|
+
maxEvents = 10000
|
139
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
140
|
+
end
|
141
|
+
return maxEvents
|
142
|
+
end
|
143
|
+
--[[
|
144
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
145
|
+
(since an empty list and !EXISTS are not really the same).
|
146
|
+
]]
|
147
|
+
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
148
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
149
|
+
if queueAttributes[1] then
|
150
|
+
return pausedKey, true
|
151
|
+
else
|
152
|
+
if queueAttributes[2] then
|
153
|
+
local activeCount = rcall("LLEN", activeKey)
|
154
|
+
if activeCount >= tonumber(queueAttributes[2]) then
|
155
|
+
return waitKey, true
|
156
|
+
else
|
157
|
+
return waitKey, false
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
return waitKey, false
|
162
|
+
end
|
35
163
|
--[[
|
36
164
|
Function to check for the meta.paused key to decide if we are paused or not
|
37
165
|
(since an empty list and !EXISTS are not really the same).
|
@@ -70,14 +198,6 @@ end
|
|
70
198
|
Function to add job in target list and add marker if needed.
|
71
199
|
]]
|
72
200
|
-- Includes
|
73
|
-
--[[
|
74
|
-
Add marker if needed when a job is available.
|
75
|
-
]]
|
76
|
-
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
77
|
-
if not isPausedOrMaxed then
|
78
|
-
rcall("ZADD", markerKey, 0, "0")
|
79
|
-
end
|
80
|
-
end
|
81
201
|
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
82
202
|
rcall(pushCmd, targetKey, jobId)
|
83
203
|
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
@@ -92,26 +212,6 @@ end
|
|
92
212
|
local getJobKeyPrefix = function (jobKey, jobId)
|
93
213
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
94
214
|
end
|
95
|
-
--[[
|
96
|
-
Function to check for the meta.paused key to decide if we are paused or not
|
97
|
-
(since an empty list and !EXISTS are not really the same).
|
98
|
-
]]
|
99
|
-
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
100
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
101
|
-
if queueAttributes[1] then
|
102
|
-
return pausedKey, true
|
103
|
-
else
|
104
|
-
if queueAttributes[2] then
|
105
|
-
local activeCount = rcall("LLEN", activeKey)
|
106
|
-
if activeCount >= tonumber(queueAttributes[2]) then
|
107
|
-
return waitKey, true
|
108
|
-
else
|
109
|
-
return waitKey, false
|
110
|
-
end
|
111
|
-
end
|
112
|
-
end
|
113
|
-
return waitKey, false
|
114
|
-
end
|
115
215
|
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
116
216
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
117
217
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
@@ -192,6 +292,36 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
192
292
|
end
|
193
293
|
removeJobKeys(jobKey)
|
194
294
|
end
|
295
|
+
--[[
|
296
|
+
Function to store a job
|
297
|
+
]]
|
298
|
+
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
299
|
+
parentKey, parentData, repeatJobKey)
|
300
|
+
local jsonOpts = cjson.encode(opts)
|
301
|
+
local delay = opts['delay'] or 0
|
302
|
+
local priority = opts['priority'] or 0
|
303
|
+
local debounceId = opts['de'] and opts['de']['id']
|
304
|
+
local optionalValues = {}
|
305
|
+
if parentKey ~= nil then
|
306
|
+
table.insert(optionalValues, "parentKey")
|
307
|
+
table.insert(optionalValues, parentKey)
|
308
|
+
table.insert(optionalValues, "parent")
|
309
|
+
table.insert(optionalValues, parentData)
|
310
|
+
end
|
311
|
+
if repeatJobKey ~= nil then
|
312
|
+
table.insert(optionalValues, "rjk")
|
313
|
+
table.insert(optionalValues, repeatJobKey)
|
314
|
+
end
|
315
|
+
if debounceId then
|
316
|
+
table.insert(optionalValues, "deid")
|
317
|
+
table.insert(optionalValues, debounceId)
|
318
|
+
end
|
319
|
+
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
320
|
+
"timestamp", timestamp, "delay", delay, "priority", priority,
|
321
|
+
unpack(optionalValues))
|
322
|
+
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
323
|
+
return delay, priority
|
324
|
+
end
|
195
325
|
--[[
|
196
326
|
Function to store a job scheduler
|
197
327
|
]]
|
@@ -233,11 +363,11 @@ end
|
|
233
363
|
-- If we are overriding a repeatable job we must delete the delayed job for
|
234
364
|
-- the next iteration.
|
235
365
|
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
|
366
|
+
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
367
|
+
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
236
368
|
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
237
369
|
if prevMillis ~= false then
|
238
370
|
local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
239
|
-
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
240
|
-
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
241
371
|
if rcall("EXISTS", nextDelayedJobKey) ~= 1 or currentJobId == nextDelayedJobId then
|
242
372
|
if rcall("ZSCORE", delayedKey, currentJobId) ~= false then
|
243
373
|
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
@@ -259,11 +389,47 @@ if prevMillis ~= false then
|
|
259
389
|
end
|
260
390
|
end
|
261
391
|
local schedulerOpts = cmsgpack.unpack(ARGV[2])
|
262
|
-
|
392
|
+
storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
|
393
|
+
if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
|
394
|
+
local eventsKey = KEYS[9]
|
395
|
+
local metaKey = KEYS[5]
|
396
|
+
local maxEvents = getOrSetMaxEvents(metaKey)
|
397
|
+
rcall("INCR", KEYS[8])
|
398
|
+
local delayedOpts = cmsgpack.unpack(ARGV[6])
|
399
|
+
local delay, priority = storeJob(eventsKey, nextDelayedJobKey, nextDelayedJobId, schedulerOpts['name'], ARGV[4],
|
400
|
+
delayedOpts, ARGV[7], nil, nil, jobSchedulerId)
|
401
|
+
if delay ~= 0 then
|
402
|
+
addDelayedJob(nextDelayedJobId, delayedKey, eventsKey,
|
403
|
+
ARGV[7], maxEvents, KEYS[7], delay)
|
404
|
+
else
|
405
|
+
local isPaused = isQueuePaused(KEYS[5])
|
406
|
+
-- Standard or priority add
|
407
|
+
if priority == 0 then
|
408
|
+
if isPaused then
|
409
|
+
-- LIFO or FIFO
|
410
|
+
local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
|
411
|
+
rcall(pushCmd, KEYS[4], nextDelayedJobId)
|
412
|
+
else
|
413
|
+
-- LIFO or FIFO
|
414
|
+
local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
|
415
|
+
rcall(pushCmd, KEYS[3], nextDelayedJobId)
|
416
|
+
end
|
417
|
+
else
|
418
|
+
-- Priority add
|
419
|
+
addJobWithPriority(KEYS[7], KEYS[6], priority, nextDelayedJobId, KEYS[10], isPaused)
|
420
|
+
end
|
421
|
+
-- Emit waiting event
|
422
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", nextDelayedJobId)
|
423
|
+
end
|
424
|
+
if ARGV[9] ~= "" then
|
425
|
+
rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
|
426
|
+
end
|
427
|
+
return nextDelayedJobId .. "" -- convert to string
|
428
|
+
end
|
263
429
|
`;
|
264
430
|
exports.addJobScheduler = {
|
265
431
|
name: 'addJobScheduler',
|
266
432
|
content,
|
267
|
-
keys:
|
433
|
+
keys: 10,
|
268
434
|
};
|
269
|
-
//# sourceMappingURL=addJobScheduler-
|
435
|
+
//# sourceMappingURL=addJobScheduler-10.js.map
|