bullmq 3.2.3 → 3.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/commands/addJob-8.lua +42 -41
- package/dist/cjs/commands/includes/getNextDelayedTimestamp.lua +7 -7
- package/dist/cjs/commands/includes/getRateLimitTTL.lua +10 -10
- package/dist/cjs/commands/includes/getTimestamp.lua +1 -1
- package/dist/cjs/commands/includes/isLocked.lua +12 -12
- package/dist/cjs/scripts/addJob-8.js +47 -47
- package/dist/cjs/scripts/moveToActive-9.js +17 -17
- package/dist/cjs/scripts/moveToDelayed-8.js +7 -7
- package/dist/cjs/scripts/moveToFinished-12.js +17 -17
- package/dist/cjs/scripts/removeJob-1.js +12 -12
- package/dist/esm/commands/addJob-8.lua +42 -41
- package/dist/esm/commands/includes/getNextDelayedTimestamp.lua +7 -7
- package/dist/esm/commands/includes/getRateLimitTTL.lua +10 -10
- package/dist/esm/commands/includes/getTimestamp.lua +1 -1
- package/dist/esm/commands/includes/isLocked.lua +12 -12
- package/dist/esm/scripts/addJob-8.js +47 -47
- package/dist/esm/scripts/moveToActive-9.js +17 -17
- package/dist/esm/scripts/moveToDelayed-8.js +7 -7
- package/dist/esm/scripts/moveToFinished-12.js +17 -17
- package/dist/esm/scripts/removeJob-1.js +12 -12
- package/package.json +1 -1
@@ -86,21 +86,22 @@ if args[2] == "" then
|
|
86
86
|
else
|
87
87
|
jobId = args[2]
|
88
88
|
jobIdKey = args[1] .. jobId
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
89
|
+
end
|
90
|
+
|
91
|
+
if rcall("EXISTS", jobIdKey) == 1 then
|
92
|
+
if parentKey ~= nil then
|
93
|
+
if rcall("ZSCORE", KEYS[7], jobId) ~= false then
|
94
|
+
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
|
95
|
+
updateParentDepsIfNeeded(parentKey, parent['queueKey'], parentDependenciesKey,
|
96
|
+
parent['id'], jobIdKey, returnvalue, timestamp)
|
97
|
+
else
|
98
|
+
if parentDependenciesKey ~= nil then
|
99
|
+
rcall("SADD", parentDependenciesKey, jobIdKey)
|
99
100
|
end
|
100
|
-
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
101
101
|
end
|
102
|
-
|
102
|
+
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
103
103
|
end
|
104
|
+
return jobId .. "" -- convert to string
|
104
105
|
end
|
105
106
|
|
106
107
|
-- Store the job.
|
@@ -132,44 +133,44 @@ local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
|
|
132
133
|
-- Check if job is a parent, if so add to the parents set
|
133
134
|
local waitChildrenKey = args[6]
|
134
135
|
if waitChildrenKey ~= nil then
|
135
|
-
|
136
|
-
|
136
|
+
rcall("ZADD", waitChildrenKey, timestamp, jobId)
|
137
|
+
rcall("XADD", KEYS[8], "*", "event", "waiting-children", "jobId", jobId)
|
137
138
|
elseif (delayedTimestamp ~= 0) then
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
end
|
139
|
+
local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
|
140
|
+
rcall("ZADD", KEYS[5], score, jobId)
|
141
|
+
rcall("XADD", KEYS[8], "*", "event", "delayed", "jobId", jobId, "delay",
|
142
|
+
delayedTimestamp)
|
143
|
+
-- If wait list is empty, and this delayed job is the next one to be processed,
|
144
|
+
-- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
|
145
|
+
local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
|
146
|
+
if rcall("LLEN", target) == 0 then
|
147
|
+
local nextTimestamp = getNextDelayedTimestamp(KEYS[5])
|
148
|
+
if not nextTimestamp or (delayedTimestamp < nextTimestamp) then
|
149
|
+
local delay = delayedTimestamp - tonumber(timestamp)
|
150
|
+
rcall("LPUSH", target, "0:" .. delay)
|
151
151
|
end
|
152
|
+
end
|
152
153
|
else
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
154
|
+
local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
|
155
|
+
|
156
|
+
-- Standard or priority add
|
157
|
+
if priority == 0 then
|
158
|
+
-- LIFO or FIFO
|
159
|
+
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH';
|
160
|
+
rcall(pushCmd, target, jobId)
|
161
|
+
else
|
162
|
+
-- Priority add
|
163
|
+
addJobWithPriority(KEYS[6], priority, target, jobId)
|
164
|
+
end
|
165
|
+
-- Emit waiting event
|
166
|
+
rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId)
|
166
167
|
end
|
167
168
|
|
168
169
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
169
170
|
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
170
171
|
-- fail in this case.
|
171
172
|
if parentDependenciesKey ~= nil then
|
172
|
-
|
173
|
+
rcall("SADD", parentDependenciesKey, jobIdKey)
|
173
174
|
end
|
174
175
|
|
175
176
|
return jobId .. "" -- convert to string
|
@@ -2,12 +2,12 @@
|
|
2
2
|
Function to return the next delayed job timestamp.
|
3
3
|
]]
|
4
4
|
local function getNextDelayedTimestamp(delayedKey)
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
end
|
11
|
-
return nextTimestamp
|
5
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
6
|
+
if #result then
|
7
|
+
local nextTimestamp = tonumber(result[2])
|
8
|
+
if (nextTimestamp ~= nil) then
|
9
|
+
nextTimestamp = nextTimestamp / 0x1000
|
12
10
|
end
|
11
|
+
return nextTimestamp
|
12
|
+
end
|
13
13
|
end
|
@@ -1,13 +1,13 @@
|
|
1
1
|
local function getRateLimitTTL(opts, limiterKey)
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
end
|
2
|
+
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
3
|
+
if maxJobs then
|
4
|
+
local jobCounter = tonumber(rcall("GET", limiterKey))
|
5
|
+
if jobCounter ~= nil and jobCounter >= maxJobs then
|
6
|
+
local pttl = rcall("PTTL", KEYS[6])
|
7
|
+
if pttl > 0 then
|
8
|
+
return pttl
|
9
|
+
end
|
11
10
|
end
|
12
|
-
|
11
|
+
end
|
12
|
+
return 0
|
13
13
|
end
|
@@ -13,19 +13,19 @@ local function isLocked( prefix, jobId)
|
|
13
13
|
local lockKey = jobKey .. ':lock'
|
14
14
|
local lock = rcall("GET", lockKey)
|
15
15
|
if not lock then
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
end
|
16
|
+
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
17
|
+
if (#dependencies > 0) then
|
18
|
+
for i, childJobKey in ipairs(dependencies) do
|
19
|
+
-- We need to get the jobId for this job.
|
20
|
+
local childJobId = getJobIdFromKey(childJobKey)
|
21
|
+
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
22
|
+
local result = isLocked( childJobPrefix, childJobId )
|
23
|
+
if result then
|
24
|
+
return true
|
25
|
+
end
|
27
26
|
end
|
28
|
-
|
27
|
+
end
|
28
|
+
return false
|
29
29
|
end
|
30
30
|
return true
|
31
31
|
end
|
@@ -91,14 +91,14 @@ end
|
|
91
91
|
Function to return the next delayed job timestamp.
|
92
92
|
]]
|
93
93
|
local function getNextDelayedTimestamp(delayedKey)
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
end
|
100
|
-
return nextTimestamp
|
94
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
95
|
+
if #result then
|
96
|
+
local nextTimestamp = tonumber(result[2])
|
97
|
+
if (nextTimestamp ~= nil) then
|
98
|
+
nextTimestamp = nextTimestamp / 0x1000
|
101
99
|
end
|
100
|
+
return nextTimestamp
|
101
|
+
end
|
102
102
|
end
|
103
103
|
if parentKey ~= nil then
|
104
104
|
if rcall("EXISTS", parentKey) ~= 1 then
|
@@ -154,21 +154,21 @@ if args[2] == "" then
|
|
154
154
|
else
|
155
155
|
jobId = args[2]
|
156
156
|
jobIdKey = args[1] .. jobId
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
157
|
+
end
|
158
|
+
if rcall("EXISTS", jobIdKey) == 1 then
|
159
|
+
if parentKey ~= nil then
|
160
|
+
if rcall("ZSCORE", KEYS[7], jobId) ~= false then
|
161
|
+
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
|
162
|
+
updateParentDepsIfNeeded(parentKey, parent['queueKey'], parentDependenciesKey,
|
163
|
+
parent['id'], jobIdKey, returnvalue, timestamp)
|
164
|
+
else
|
165
|
+
if parentDependenciesKey ~= nil then
|
166
|
+
rcall("SADD", parentDependenciesKey, jobIdKey)
|
167
167
|
end
|
168
|
-
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
169
168
|
end
|
170
|
-
|
169
|
+
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
171
170
|
end
|
171
|
+
return jobId .. "" -- convert to string
|
172
172
|
end
|
173
173
|
-- Store the job.
|
174
174
|
local jsonOpts = cjson.encode(opts)
|
@@ -193,42 +193,42 @@ local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
|
|
193
193
|
-- Check if job is a parent, if so add to the parents set
|
194
194
|
local waitChildrenKey = args[6]
|
195
195
|
if waitChildrenKey ~= nil then
|
196
|
-
|
197
|
-
|
196
|
+
rcall("ZADD", waitChildrenKey, timestamp, jobId)
|
197
|
+
rcall("XADD", KEYS[8], "*", "event", "waiting-children", "jobId", jobId)
|
198
198
|
elseif (delayedTimestamp ~= 0) then
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
end
|
199
|
+
local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
|
200
|
+
rcall("ZADD", KEYS[5], score, jobId)
|
201
|
+
rcall("XADD", KEYS[8], "*", "event", "delayed", "jobId", jobId, "delay",
|
202
|
+
delayedTimestamp)
|
203
|
+
-- If wait list is empty, and this delayed job is the next one to be processed,
|
204
|
+
-- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
|
205
|
+
local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
|
206
|
+
if rcall("LLEN", target) == 0 then
|
207
|
+
local nextTimestamp = getNextDelayedTimestamp(KEYS[5])
|
208
|
+
if not nextTimestamp or (delayedTimestamp < nextTimestamp) then
|
209
|
+
local delay = delayedTimestamp - tonumber(timestamp)
|
210
|
+
rcall("LPUSH", target, "0:" .. delay)
|
212
211
|
end
|
212
|
+
end
|
213
213
|
else
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
214
|
+
local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
|
215
|
+
-- Standard or priority add
|
216
|
+
if priority == 0 then
|
217
|
+
-- LIFO or FIFO
|
218
|
+
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH';
|
219
|
+
rcall(pushCmd, target, jobId)
|
220
|
+
else
|
221
|
+
-- Priority add
|
222
|
+
addJobWithPriority(KEYS[6], priority, target, jobId)
|
223
|
+
end
|
224
|
+
-- Emit waiting event
|
225
|
+
rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId)
|
226
226
|
end
|
227
227
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
228
228
|
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
229
229
|
-- fail in this case.
|
230
230
|
if parentDependenciesKey ~= nil then
|
231
|
-
|
231
|
+
rcall("SADD", parentDependenciesKey, jobIdKey)
|
232
232
|
end
|
233
233
|
return jobId .. "" -- convert to string
|
234
234
|
`;
|
@@ -89,14 +89,14 @@ end
|
|
89
89
|
Function to return the next delayed job timestamp.
|
90
90
|
]]
|
91
91
|
local function getNextDelayedTimestamp(delayedKey)
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
end
|
98
|
-
return nextTimestamp
|
92
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
93
|
+
if #result then
|
94
|
+
local nextTimestamp = tonumber(result[2])
|
95
|
+
if (nextTimestamp ~= nil) then
|
96
|
+
nextTimestamp = nextTimestamp / 0x1000
|
99
97
|
end
|
98
|
+
return nextTimestamp
|
99
|
+
end
|
100
100
|
end
|
101
101
|
--[[
|
102
102
|
Updates the delay set, by moving delayed jobs that should
|
@@ -156,17 +156,17 @@ local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
|
|
156
156
|
end
|
157
157
|
end
|
158
158
|
local function getRateLimitTTL(opts, limiterKey)
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
end
|
159
|
+
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
160
|
+
if maxJobs then
|
161
|
+
local jobCounter = tonumber(rcall("GET", limiterKey))
|
162
|
+
if jobCounter ~= nil and jobCounter >= maxJobs then
|
163
|
+
local pttl = rcall("PTTL", KEYS[6])
|
164
|
+
if pttl > 0 then
|
165
|
+
return pttl
|
166
|
+
end
|
168
167
|
end
|
169
|
-
|
168
|
+
end
|
169
|
+
return 0
|
170
170
|
end
|
171
171
|
-- Check if there are delayed jobs that we can move to wait.
|
172
172
|
promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[9], KEYS[4], ARGV[1], ARGV[2])
|
@@ -87,14 +87,14 @@ end
|
|
87
87
|
Function to return the next delayed job timestamp.
|
88
88
|
]]
|
89
89
|
local function getNextDelayedTimestamp(delayedKey)
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
end
|
96
|
-
return nextTimestamp
|
90
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
91
|
+
if #result then
|
92
|
+
local nextTimestamp = tonumber(result[2])
|
93
|
+
if (nextTimestamp ~= nil) then
|
94
|
+
nextTimestamp = nextTimestamp / 0x1000
|
97
95
|
end
|
96
|
+
return nextTimestamp
|
97
|
+
end
|
98
98
|
end
|
99
99
|
local jobKey = KEYS[5]
|
100
100
|
if rcall("EXISTS", jobKey) == 1 then
|
@@ -108,14 +108,14 @@ end
|
|
108
108
|
Function to return the next delayed job timestamp.
|
109
109
|
]]
|
110
110
|
local function getNextDelayedTimestamp(delayedKey)
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
end
|
117
|
-
return nextTimestamp
|
111
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
112
|
+
if #result then
|
113
|
+
local nextTimestamp = tonumber(result[2])
|
114
|
+
if (nextTimestamp ~= nil) then
|
115
|
+
nextTimestamp = nextTimestamp / 0x1000
|
118
116
|
end
|
117
|
+
return nextTimestamp
|
118
|
+
end
|
119
119
|
end
|
120
120
|
--[[
|
121
121
|
Function to move job from wait state to active.
|
@@ -416,17 +416,17 @@ local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDepende
|
|
416
416
|
end
|
417
417
|
end
|
418
418
|
local function getRateLimitTTL(opts, limiterKey)
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
end
|
419
|
+
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
420
|
+
if maxJobs then
|
421
|
+
local jobCounter = tonumber(rcall("GET", limiterKey))
|
422
|
+
if jobCounter ~= nil and jobCounter >= maxJobs then
|
423
|
+
local pttl = rcall("PTTL", KEYS[6])
|
424
|
+
if pttl > 0 then
|
425
|
+
return pttl
|
426
|
+
end
|
428
427
|
end
|
429
|
-
|
428
|
+
end
|
429
|
+
return 0
|
430
430
|
end
|
431
431
|
local jobIdKey = KEYS[10]
|
432
432
|
if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
@@ -34,19 +34,19 @@ local function isLocked( prefix, jobId)
|
|
34
34
|
local lockKey = jobKey .. ':lock'
|
35
35
|
local lock = rcall("GET", lockKey)
|
36
36
|
if not lock then
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
end
|
37
|
+
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
38
|
+
if (#dependencies > 0) then
|
39
|
+
for i, childJobKey in ipairs(dependencies) do
|
40
|
+
-- We need to get the jobId for this job.
|
41
|
+
local childJobId = getJobIdFromKey(childJobKey)
|
42
|
+
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
43
|
+
local result = isLocked( childJobPrefix, childJobId )
|
44
|
+
if result then
|
45
|
+
return true
|
46
|
+
end
|
48
47
|
end
|
49
|
-
|
48
|
+
end
|
49
|
+
return false
|
50
50
|
end
|
51
51
|
return true
|
52
52
|
end
|
@@ -86,21 +86,22 @@ if args[2] == "" then
|
|
86
86
|
else
|
87
87
|
jobId = args[2]
|
88
88
|
jobIdKey = args[1] .. jobId
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
89
|
+
end
|
90
|
+
|
91
|
+
if rcall("EXISTS", jobIdKey) == 1 then
|
92
|
+
if parentKey ~= nil then
|
93
|
+
if rcall("ZSCORE", KEYS[7], jobId) ~= false then
|
94
|
+
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
|
95
|
+
updateParentDepsIfNeeded(parentKey, parent['queueKey'], parentDependenciesKey,
|
96
|
+
parent['id'], jobIdKey, returnvalue, timestamp)
|
97
|
+
else
|
98
|
+
if parentDependenciesKey ~= nil then
|
99
|
+
rcall("SADD", parentDependenciesKey, jobIdKey)
|
99
100
|
end
|
100
|
-
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
101
101
|
end
|
102
|
-
|
102
|
+
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
103
103
|
end
|
104
|
+
return jobId .. "" -- convert to string
|
104
105
|
end
|
105
106
|
|
106
107
|
-- Store the job.
|
@@ -132,44 +133,44 @@ local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
|
|
132
133
|
-- Check if job is a parent, if so add to the parents set
|
133
134
|
local waitChildrenKey = args[6]
|
134
135
|
if waitChildrenKey ~= nil then
|
135
|
-
|
136
|
-
|
136
|
+
rcall("ZADD", waitChildrenKey, timestamp, jobId)
|
137
|
+
rcall("XADD", KEYS[8], "*", "event", "waiting-children", "jobId", jobId)
|
137
138
|
elseif (delayedTimestamp ~= 0) then
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
end
|
139
|
+
local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
|
140
|
+
rcall("ZADD", KEYS[5], score, jobId)
|
141
|
+
rcall("XADD", KEYS[8], "*", "event", "delayed", "jobId", jobId, "delay",
|
142
|
+
delayedTimestamp)
|
143
|
+
-- If wait list is empty, and this delayed job is the next one to be processed,
|
144
|
+
-- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
|
145
|
+
local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
|
146
|
+
if rcall("LLEN", target) == 0 then
|
147
|
+
local nextTimestamp = getNextDelayedTimestamp(KEYS[5])
|
148
|
+
if not nextTimestamp or (delayedTimestamp < nextTimestamp) then
|
149
|
+
local delay = delayedTimestamp - tonumber(timestamp)
|
150
|
+
rcall("LPUSH", target, "0:" .. delay)
|
151
151
|
end
|
152
|
+
end
|
152
153
|
else
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
154
|
+
local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
|
155
|
+
|
156
|
+
-- Standard or priority add
|
157
|
+
if priority == 0 then
|
158
|
+
-- LIFO or FIFO
|
159
|
+
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH';
|
160
|
+
rcall(pushCmd, target, jobId)
|
161
|
+
else
|
162
|
+
-- Priority add
|
163
|
+
addJobWithPriority(KEYS[6], priority, target, jobId)
|
164
|
+
end
|
165
|
+
-- Emit waiting event
|
166
|
+
rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId)
|
166
167
|
end
|
167
168
|
|
168
169
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
169
170
|
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
170
171
|
-- fail in this case.
|
171
172
|
if parentDependenciesKey ~= nil then
|
172
|
-
|
173
|
+
rcall("SADD", parentDependenciesKey, jobIdKey)
|
173
174
|
end
|
174
175
|
|
175
176
|
return jobId .. "" -- convert to string
|
@@ -2,12 +2,12 @@
|
|
2
2
|
Function to return the next delayed job timestamp.
|
3
3
|
]]
|
4
4
|
local function getNextDelayedTimestamp(delayedKey)
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
end
|
11
|
-
return nextTimestamp
|
5
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
6
|
+
if #result then
|
7
|
+
local nextTimestamp = tonumber(result[2])
|
8
|
+
if (nextTimestamp ~= nil) then
|
9
|
+
nextTimestamp = nextTimestamp / 0x1000
|
12
10
|
end
|
11
|
+
return nextTimestamp
|
12
|
+
end
|
13
13
|
end
|
@@ -1,13 +1,13 @@
|
|
1
1
|
local function getRateLimitTTL(opts, limiterKey)
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
end
|
2
|
+
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
3
|
+
if maxJobs then
|
4
|
+
local jobCounter = tonumber(rcall("GET", limiterKey))
|
5
|
+
if jobCounter ~= nil and jobCounter >= maxJobs then
|
6
|
+
local pttl = rcall("PTTL", KEYS[6])
|
7
|
+
if pttl > 0 then
|
8
|
+
return pttl
|
9
|
+
end
|
11
10
|
end
|
12
|
-
|
11
|
+
end
|
12
|
+
return 0
|
13
13
|
end
|
@@ -13,19 +13,19 @@ local function isLocked( prefix, jobId)
|
|
13
13
|
local lockKey = jobKey .. ':lock'
|
14
14
|
local lock = rcall("GET", lockKey)
|
15
15
|
if not lock then
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
end
|
16
|
+
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
17
|
+
if (#dependencies > 0) then
|
18
|
+
for i, childJobKey in ipairs(dependencies) do
|
19
|
+
-- We need to get the jobId for this job.
|
20
|
+
local childJobId = getJobIdFromKey(childJobKey)
|
21
|
+
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
22
|
+
local result = isLocked( childJobPrefix, childJobId )
|
23
|
+
if result then
|
24
|
+
return true
|
25
|
+
end
|
27
26
|
end
|
28
|
-
|
27
|
+
end
|
28
|
+
return false
|
29
29
|
end
|
30
30
|
return true
|
31
31
|
end
|
@@ -88,14 +88,14 @@ end
|
|
88
88
|
Function to return the next delayed job timestamp.
|
89
89
|
]]
|
90
90
|
local function getNextDelayedTimestamp(delayedKey)
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
end
|
97
|
-
return nextTimestamp
|
91
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
92
|
+
if #result then
|
93
|
+
local nextTimestamp = tonumber(result[2])
|
94
|
+
if (nextTimestamp ~= nil) then
|
95
|
+
nextTimestamp = nextTimestamp / 0x1000
|
98
96
|
end
|
97
|
+
return nextTimestamp
|
98
|
+
end
|
99
99
|
end
|
100
100
|
if parentKey ~= nil then
|
101
101
|
if rcall("EXISTS", parentKey) ~= 1 then
|
@@ -151,21 +151,21 @@ if args[2] == "" then
|
|
151
151
|
else
|
152
152
|
jobId = args[2]
|
153
153
|
jobIdKey = args[1] .. jobId
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
154
|
+
end
|
155
|
+
if rcall("EXISTS", jobIdKey) == 1 then
|
156
|
+
if parentKey ~= nil then
|
157
|
+
if rcall("ZSCORE", KEYS[7], jobId) ~= false then
|
158
|
+
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
|
159
|
+
updateParentDepsIfNeeded(parentKey, parent['queueKey'], parentDependenciesKey,
|
160
|
+
parent['id'], jobIdKey, returnvalue, timestamp)
|
161
|
+
else
|
162
|
+
if parentDependenciesKey ~= nil then
|
163
|
+
rcall("SADD", parentDependenciesKey, jobIdKey)
|
164
164
|
end
|
165
|
-
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
166
165
|
end
|
167
|
-
|
166
|
+
rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
|
168
167
|
end
|
168
|
+
return jobId .. "" -- convert to string
|
169
169
|
end
|
170
170
|
-- Store the job.
|
171
171
|
local jsonOpts = cjson.encode(opts)
|
@@ -190,42 +190,42 @@ local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
|
|
190
190
|
-- Check if job is a parent, if so add to the parents set
|
191
191
|
local waitChildrenKey = args[6]
|
192
192
|
if waitChildrenKey ~= nil then
|
193
|
-
|
194
|
-
|
193
|
+
rcall("ZADD", waitChildrenKey, timestamp, jobId)
|
194
|
+
rcall("XADD", KEYS[8], "*", "event", "waiting-children", "jobId", jobId)
|
195
195
|
elseif (delayedTimestamp ~= 0) then
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
end
|
196
|
+
local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
|
197
|
+
rcall("ZADD", KEYS[5], score, jobId)
|
198
|
+
rcall("XADD", KEYS[8], "*", "event", "delayed", "jobId", jobId, "delay",
|
199
|
+
delayedTimestamp)
|
200
|
+
-- If wait list is empty, and this delayed job is the next one to be processed,
|
201
|
+
-- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
|
202
|
+
local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
|
203
|
+
if rcall("LLEN", target) == 0 then
|
204
|
+
local nextTimestamp = getNextDelayedTimestamp(KEYS[5])
|
205
|
+
if not nextTimestamp or (delayedTimestamp < nextTimestamp) then
|
206
|
+
local delay = delayedTimestamp - tonumber(timestamp)
|
207
|
+
rcall("LPUSH", target, "0:" .. delay)
|
209
208
|
end
|
209
|
+
end
|
210
210
|
else
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
211
|
+
local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
|
212
|
+
-- Standard or priority add
|
213
|
+
if priority == 0 then
|
214
|
+
-- LIFO or FIFO
|
215
|
+
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH';
|
216
|
+
rcall(pushCmd, target, jobId)
|
217
|
+
else
|
218
|
+
-- Priority add
|
219
|
+
addJobWithPriority(KEYS[6], priority, target, jobId)
|
220
|
+
end
|
221
|
+
-- Emit waiting event
|
222
|
+
rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId)
|
223
223
|
end
|
224
224
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
225
225
|
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
226
226
|
-- fail in this case.
|
227
227
|
if parentDependenciesKey ~= nil then
|
228
|
-
|
228
|
+
rcall("SADD", parentDependenciesKey, jobIdKey)
|
229
229
|
end
|
230
230
|
return jobId .. "" -- convert to string
|
231
231
|
`;
|
@@ -86,14 +86,14 @@ end
|
|
86
86
|
Function to return the next delayed job timestamp.
|
87
87
|
]]
|
88
88
|
local function getNextDelayedTimestamp(delayedKey)
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
end
|
95
|
-
return nextTimestamp
|
89
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
90
|
+
if #result then
|
91
|
+
local nextTimestamp = tonumber(result[2])
|
92
|
+
if (nextTimestamp ~= nil) then
|
93
|
+
nextTimestamp = nextTimestamp / 0x1000
|
96
94
|
end
|
95
|
+
return nextTimestamp
|
96
|
+
end
|
97
97
|
end
|
98
98
|
--[[
|
99
99
|
Updates the delay set, by moving delayed jobs that should
|
@@ -153,17 +153,17 @@ local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
|
|
153
153
|
end
|
154
154
|
end
|
155
155
|
local function getRateLimitTTL(opts, limiterKey)
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
end
|
156
|
+
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
157
|
+
if maxJobs then
|
158
|
+
local jobCounter = tonumber(rcall("GET", limiterKey))
|
159
|
+
if jobCounter ~= nil and jobCounter >= maxJobs then
|
160
|
+
local pttl = rcall("PTTL", KEYS[6])
|
161
|
+
if pttl > 0 then
|
162
|
+
return pttl
|
163
|
+
end
|
165
164
|
end
|
166
|
-
|
165
|
+
end
|
166
|
+
return 0
|
167
167
|
end
|
168
168
|
-- Check if there are delayed jobs that we can move to wait.
|
169
169
|
promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[9], KEYS[4], ARGV[1], ARGV[2])
|
@@ -84,14 +84,14 @@ end
|
|
84
84
|
Function to return the next delayed job timestamp.
|
85
85
|
]]
|
86
86
|
local function getNextDelayedTimestamp(delayedKey)
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
end
|
93
|
-
return nextTimestamp
|
87
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
88
|
+
if #result then
|
89
|
+
local nextTimestamp = tonumber(result[2])
|
90
|
+
if (nextTimestamp ~= nil) then
|
91
|
+
nextTimestamp = nextTimestamp / 0x1000
|
94
92
|
end
|
93
|
+
return nextTimestamp
|
94
|
+
end
|
95
95
|
end
|
96
96
|
local jobKey = KEYS[5]
|
97
97
|
if rcall("EXISTS", jobKey) == 1 then
|
@@ -105,14 +105,14 @@ end
|
|
105
105
|
Function to return the next delayed job timestamp.
|
106
106
|
]]
|
107
107
|
local function getNextDelayedTimestamp(delayedKey)
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
end
|
114
|
-
return nextTimestamp
|
108
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
109
|
+
if #result then
|
110
|
+
local nextTimestamp = tonumber(result[2])
|
111
|
+
if (nextTimestamp ~= nil) then
|
112
|
+
nextTimestamp = nextTimestamp / 0x1000
|
115
113
|
end
|
114
|
+
return nextTimestamp
|
115
|
+
end
|
116
116
|
end
|
117
117
|
--[[
|
118
118
|
Function to move job from wait state to active.
|
@@ -413,17 +413,17 @@ local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDepende
|
|
413
413
|
end
|
414
414
|
end
|
415
415
|
local function getRateLimitTTL(opts, limiterKey)
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
end
|
416
|
+
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
417
|
+
if maxJobs then
|
418
|
+
local jobCounter = tonumber(rcall("GET", limiterKey))
|
419
|
+
if jobCounter ~= nil and jobCounter >= maxJobs then
|
420
|
+
local pttl = rcall("PTTL", KEYS[6])
|
421
|
+
if pttl > 0 then
|
422
|
+
return pttl
|
423
|
+
end
|
425
424
|
end
|
426
|
-
|
425
|
+
end
|
426
|
+
return 0
|
427
427
|
end
|
428
428
|
local jobIdKey = KEYS[10]
|
429
429
|
if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
@@ -31,19 +31,19 @@ local function isLocked( prefix, jobId)
|
|
31
31
|
local lockKey = jobKey .. ':lock'
|
32
32
|
local lock = rcall("GET", lockKey)
|
33
33
|
if not lock then
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
end
|
34
|
+
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
35
|
+
if (#dependencies > 0) then
|
36
|
+
for i, childJobKey in ipairs(dependencies) do
|
37
|
+
-- We need to get the jobId for this job.
|
38
|
+
local childJobId = getJobIdFromKey(childJobKey)
|
39
|
+
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
40
|
+
local result = isLocked( childJobPrefix, childJobId )
|
41
|
+
if result then
|
42
|
+
return true
|
43
|
+
end
|
45
44
|
end
|
46
|
-
|
45
|
+
end
|
46
|
+
return false
|
47
47
|
end
|
48
48
|
return true
|
49
49
|
end
|