bullmq 5.41.5 → 5.41.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/job-scheduler.js +40 -15
- package/dist/cjs/classes/job-scheduler.js.map +1 -1
- package/dist/cjs/classes/job.js +9 -12
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/scripts.js +11 -1
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/commands/addDelayedJob-6.lua +5 -2
- package/dist/cjs/commands/addJobScheduler-10.lua +130 -0
- package/dist/cjs/commands/includes/addDelayedJob.lua +2 -7
- package/dist/cjs/commands/includes/getOrSetMaxEvents.lua +6 -6
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +1 -1
- package/dist/cjs/commands/moveToFinished-14.lua +1 -1
- package/dist/cjs/scripts/addDelayedJob-6.js +41 -43
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/{addJobScheduler-6.js → addJobScheduler-10.js} +208 -42
- package/dist/cjs/scripts/addJobScheduler-10.js.map +1 -0
- package/dist/cjs/scripts/addParentJob-4.js +6 -6
- package/dist/cjs/scripts/addPrioritizedJob-8.js +6 -6
- package/dist/cjs/scripts/addStandardJob-8.js +6 -6
- package/dist/cjs/scripts/changeDelay-4.js +6 -6
- package/dist/cjs/scripts/index.js +1 -1
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +6 -6
- package/dist/cjs/scripts/moveJobsToWait-8.js +6 -6
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +49 -9
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToDelayed-8.js +6 -6
- package/dist/cjs/scripts/moveToFinished-14.js +49 -9
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/removeJob-3.js +6 -6
- package/dist/cjs/scripts/reprocessJob-8.js +6 -6
- package/dist/cjs/scripts/retryJob-11.js +6 -6
- package/dist/cjs/scripts/updateJobScheduler-7.js +8 -41
- package/dist/cjs/scripts/updateJobScheduler-7.js.map +1 -1
- package/dist/cjs/scripts/updateProgress-3.js +6 -6
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/job-scheduler.d.ts +1 -0
- package/dist/esm/classes/job-scheduler.js +40 -15
- package/dist/esm/classes/job-scheduler.js.map +1 -1
- package/dist/esm/classes/job.js +9 -12
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +2 -2
- package/dist/esm/classes/scripts.js +11 -1
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/commands/addDelayedJob-6.lua +5 -2
- package/dist/esm/commands/addJobScheduler-10.lua +130 -0
- package/dist/esm/commands/includes/addDelayedJob.lua +2 -7
- package/dist/esm/commands/includes/getOrSetMaxEvents.lua +6 -6
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +1 -1
- package/dist/esm/commands/moveToFinished-14.lua +1 -1
- package/dist/esm/scripts/addDelayedJob-6.js +41 -43
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/{addJobScheduler-6.js → addJobScheduler-10.js} +208 -42
- package/dist/esm/scripts/addJobScheduler-10.js.map +1 -0
- package/dist/esm/scripts/addParentJob-4.js +6 -6
- package/dist/esm/scripts/addPrioritizedJob-8.js +6 -6
- package/dist/esm/scripts/addStandardJob-8.js +6 -6
- package/dist/esm/scripts/changeDelay-4.js +6 -6
- package/dist/esm/scripts/index.d.ts +1 -1
- package/dist/esm/scripts/index.js +1 -1
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveJobFromActiveToWait-9.js +6 -6
- package/dist/esm/scripts/moveJobsToWait-8.js +6 -6
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +49 -9
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToDelayed-8.js +6 -6
- package/dist/esm/scripts/moveToFinished-14.js +49 -9
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/removeJob-3.js +6 -6
- package/dist/esm/scripts/reprocessJob-8.js +6 -6
- package/dist/esm/scripts/retryJob-11.js +6 -6
- package/dist/esm/scripts/updateJobScheduler-7.js +8 -41
- package/dist/esm/scripts/updateJobScheduler-7.js.map +1 -1
- package/dist/esm/scripts/updateProgress-3.js +6 -6
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/addJobScheduler-6.lua +0 -72
- package/dist/cjs/scripts/addJobScheduler-6.js.map +0 -1
- package/dist/esm/commands/addJobScheduler-6.lua +0 -72
- package/dist/esm/scripts/addJobScheduler-6.js.map +0 -1
- /package/dist/esm/scripts/{addJobScheduler-6.d.ts → addJobScheduler-10.d.ts} +0 -0
@@ -100,41 +100,8 @@ local function getDelayedScore(delayedKey, timestamp, delay)
|
|
100
100
|
end
|
101
101
|
return minScore, delayedTimestamp
|
102
102
|
end
|
103
|
-
|
104
|
-
|
105
|
-
]]
|
106
|
-
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
107
|
-
parentKey, parentData, repeatJobKey)
|
108
|
-
local jsonOpts = cjson.encode(opts)
|
109
|
-
local delay = opts['delay'] or 0
|
110
|
-
local priority = opts['priority'] or 0
|
111
|
-
local debounceId = opts['de'] and opts['de']['id']
|
112
|
-
local optionalValues = {}
|
113
|
-
if parentKey ~= nil then
|
114
|
-
table.insert(optionalValues, "parentKey")
|
115
|
-
table.insert(optionalValues, parentKey)
|
116
|
-
table.insert(optionalValues, "parent")
|
117
|
-
table.insert(optionalValues, parentData)
|
118
|
-
end
|
119
|
-
if repeatJobKey ~= nil then
|
120
|
-
table.insert(optionalValues, "rjk")
|
121
|
-
table.insert(optionalValues, repeatJobKey)
|
122
|
-
end
|
123
|
-
if debounceId then
|
124
|
-
table.insert(optionalValues, "deid")
|
125
|
-
table.insert(optionalValues, debounceId)
|
126
|
-
end
|
127
|
-
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
128
|
-
"timestamp", timestamp, "delay", delay, "priority", priority,
|
129
|
-
unpack(optionalValues))
|
130
|
-
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
131
|
-
return delay, priority
|
132
|
-
end
|
133
|
-
local function addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, name, data, opts, timestamp, repeatJobKey,
|
134
|
-
maxEvents, markerKey, parentKey, parentData)
|
135
|
-
-- Store the job.
|
136
|
-
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
|
137
|
-
opts, timestamp, parentKey, parentData, repeatJobKey)
|
103
|
+
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
104
|
+
maxEvents, markerKey, delay)
|
138
105
|
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
|
139
106
|
rcall("ZADD", delayedKey, score, jobId)
|
140
107
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
@@ -169,12 +136,12 @@ end
|
|
169
136
|
Function to get max events value or set by default 10000.
|
170
137
|
]]
|
171
138
|
local function getOrSetMaxEvents(metaKey)
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
139
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
140
|
+
if not maxEvents then
|
141
|
+
maxEvents = 10000
|
142
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
143
|
+
end
|
144
|
+
return maxEvents
|
178
145
|
end
|
179
146
|
--[[
|
180
147
|
Function to handle the case when job is duplicated.
|
@@ -344,6 +311,36 @@ local function handleDuplicatedJob(jobKey, jobId, currentParentKey, currentParen
|
|
344
311
|
"duplicated", "jobId", jobId)
|
345
312
|
return jobId .. "" -- convert to string
|
346
313
|
end
|
314
|
+
--[[
|
315
|
+
Function to store a job
|
316
|
+
]]
|
317
|
+
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
318
|
+
parentKey, parentData, repeatJobKey)
|
319
|
+
local jsonOpts = cjson.encode(opts)
|
320
|
+
local delay = opts['delay'] or 0
|
321
|
+
local priority = opts['priority'] or 0
|
322
|
+
local debounceId = opts['de'] and opts['de']['id']
|
323
|
+
local optionalValues = {}
|
324
|
+
if parentKey ~= nil then
|
325
|
+
table.insert(optionalValues, "parentKey")
|
326
|
+
table.insert(optionalValues, parentKey)
|
327
|
+
table.insert(optionalValues, "parent")
|
328
|
+
table.insert(optionalValues, parentData)
|
329
|
+
end
|
330
|
+
if repeatJobKey ~= nil then
|
331
|
+
table.insert(optionalValues, "rjk")
|
332
|
+
table.insert(optionalValues, repeatJobKey)
|
333
|
+
end
|
334
|
+
if debounceId then
|
335
|
+
table.insert(optionalValues, "deid")
|
336
|
+
table.insert(optionalValues, debounceId)
|
337
|
+
end
|
338
|
+
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
339
|
+
"timestamp", timestamp, "delay", delay, "priority", priority,
|
340
|
+
unpack(optionalValues))
|
341
|
+
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
342
|
+
return delay, priority
|
343
|
+
end
|
347
344
|
if parentKey ~= nil then
|
348
345
|
if rcall("EXISTS", parentKey) ~= 1 then return -5 end
|
349
346
|
parentData = cjson.encode(parent)
|
@@ -370,8 +367,9 @@ local deduplicationJobId = deduplicateJob(opts['de'], jobId, deduplicationKey,
|
|
370
367
|
if deduplicationJobId then
|
371
368
|
return deduplicationJobId
|
372
369
|
end
|
373
|
-
|
374
|
-
|
370
|
+
local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
|
371
|
+
opts, timestamp, parentKey, parentData, repeatJobKey)
|
372
|
+
addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, KEYS[1], delay)
|
375
373
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
376
374
|
if parentDependenciesKey ~= nil then
|
377
375
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyXf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -1,12 +1,16 @@
|
|
1
1
|
const content = `--[[
|
2
2
|
Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
|
3
3
|
Input:
|
4
|
-
KEYS[1]
|
5
|
-
KEYS[2]
|
6
|
-
KEYS[3]
|
7
|
-
KEYS[4]
|
8
|
-
KEYS[5]
|
9
|
-
KEYS[6]
|
4
|
+
KEYS[1] 'repeat' key
|
5
|
+
KEYS[2] 'delayed' key
|
6
|
+
KEYS[3] 'wait' key
|
7
|
+
KEYS[4] 'paused' key
|
8
|
+
KEYS[5] 'meta' key
|
9
|
+
KEYS[6] 'prioritized' key
|
10
|
+
KEYS[7] 'marker' key
|
11
|
+
KEYS[8] 'id' key
|
12
|
+
KEYS[9] 'events' key
|
13
|
+
KEYS[10] 'pc' priority counter
|
10
14
|
ARGV[1] next milliseconds
|
11
15
|
ARGV[2] msgpacked options
|
12
16
|
[1] name
|
@@ -17,18 +21,142 @@ const content = `--[[
|
|
17
21
|
ARGV[3] jobs scheduler id
|
18
22
|
ARGV[4] Json stringified template data
|
19
23
|
ARGV[5] mspacked template opts
|
20
|
-
ARGV[6]
|
24
|
+
ARGV[6] msgpacked delayed opts
|
25
|
+
ARGV[7] timestamp
|
26
|
+
ARGV[8] prefix key
|
27
|
+
ARGV[9] producer key
|
21
28
|
Output:
|
22
29
|
repeatableKey - OK
|
23
|
-
]]
|
30
|
+
]]
|
31
|
+
local rcall = redis.call
|
24
32
|
local repeatKey = KEYS[1]
|
25
33
|
local delayedKey = KEYS[2]
|
26
34
|
local prioritizedKey = KEYS[6]
|
27
35
|
local nextMillis = ARGV[1]
|
28
36
|
local jobSchedulerId = ARGV[3]
|
29
37
|
local templateOpts = cmsgpack.unpack(ARGV[5])
|
30
|
-
local prefixKey = ARGV[
|
38
|
+
local prefixKey = ARGV[8]
|
39
|
+
-- Includes
|
40
|
+
--[[
|
41
|
+
Adds a delayed job to the queue by doing the following:
|
42
|
+
- Creates a new job key with the job data.
|
43
|
+
- adds to delayed zset.
|
44
|
+
- Emits a global event 'delayed' if the job is delayed.
|
45
|
+
]]
|
46
|
+
-- Includes
|
47
|
+
--[[
|
48
|
+
Add delay marker if needed.
|
49
|
+
]]
|
50
|
+
-- Includes
|
51
|
+
--[[
|
52
|
+
Function to return the next delayed job timestamp.
|
53
|
+
]]
|
54
|
+
local function getNextDelayedTimestamp(delayedKey)
|
55
|
+
local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
|
56
|
+
if #result then
|
57
|
+
local nextTimestamp = tonumber(result[2])
|
58
|
+
if nextTimestamp ~= nil then
|
59
|
+
return nextTimestamp / 0x1000
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
64
|
+
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
65
|
+
if nextTimestamp ~= nil then
|
66
|
+
-- Replace the score of the marker with the newest known
|
67
|
+
-- next timestamp.
|
68
|
+
rcall("ZADD", markerKey, nextTimestamp, "1")
|
69
|
+
end
|
70
|
+
end
|
71
|
+
--[[
|
72
|
+
Bake in the job id first 12 bits into the timestamp
|
73
|
+
to guarantee correct execution order of delayed jobs
|
74
|
+
(up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
|
75
|
+
WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
|
76
|
+
]]
|
77
|
+
local function getDelayedScore(delayedKey, timestamp, delay)
|
78
|
+
local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
|
79
|
+
local minScore = delayedTimestamp * 0x1000
|
80
|
+
local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
|
81
|
+
local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
|
82
|
+
minScore, "WITHSCORES","LIMIT", 0, 1)
|
83
|
+
if #result then
|
84
|
+
local currentMaxScore = tonumber(result[2])
|
85
|
+
if currentMaxScore ~= nil then
|
86
|
+
if currentMaxScore >= maxScore then
|
87
|
+
return maxScore, delayedTimestamp
|
88
|
+
else
|
89
|
+
return currentMaxScore + 1, delayedTimestamp
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
return minScore, delayedTimestamp
|
94
|
+
end
|
95
|
+
local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
96
|
+
maxEvents, markerKey, delay)
|
97
|
+
local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
|
98
|
+
rcall("ZADD", delayedKey, score, jobId)
|
99
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
100
|
+
"jobId", jobId, "delay", delayedTimestamp)
|
101
|
+
-- mark that a delayed job is available
|
102
|
+
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
103
|
+
end
|
104
|
+
--[[
|
105
|
+
Function to add job considering priority.
|
106
|
+
]]
|
31
107
|
-- Includes
|
108
|
+
--[[
|
109
|
+
Add marker if needed when a job is available.
|
110
|
+
]]
|
111
|
+
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
112
|
+
if not isPausedOrMaxed then
|
113
|
+
rcall("ZADD", markerKey, 0, "0")
|
114
|
+
end
|
115
|
+
end
|
116
|
+
--[[
|
117
|
+
Function to get priority score.
|
118
|
+
]]
|
119
|
+
local function getPriorityScore(priority, priorityCounterKey)
|
120
|
+
local prioCounter = rcall("INCR", priorityCounterKey)
|
121
|
+
return priority * 0x100000000 + prioCounter % 0x100000000
|
122
|
+
end
|
123
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
|
124
|
+
isPausedOrMaxed)
|
125
|
+
local score = getPriorityScore(priority, priorityCounterKey)
|
126
|
+
rcall("ZADD", prioritizedKey, score, jobId)
|
127
|
+
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
128
|
+
end
|
129
|
+
--[[
|
130
|
+
Function to get max events value or set by default 10000.
|
131
|
+
]]
|
132
|
+
local function getOrSetMaxEvents(metaKey)
|
133
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
134
|
+
if not maxEvents then
|
135
|
+
maxEvents = 10000
|
136
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
137
|
+
end
|
138
|
+
return maxEvents
|
139
|
+
end
|
140
|
+
--[[
|
141
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
142
|
+
(since an empty list and !EXISTS are not really the same).
|
143
|
+
]]
|
144
|
+
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
145
|
+
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
146
|
+
if queueAttributes[1] then
|
147
|
+
return pausedKey, true
|
148
|
+
else
|
149
|
+
if queueAttributes[2] then
|
150
|
+
local activeCount = rcall("LLEN", activeKey)
|
151
|
+
if activeCount >= tonumber(queueAttributes[2]) then
|
152
|
+
return waitKey, true
|
153
|
+
else
|
154
|
+
return waitKey, false
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
return waitKey, false
|
159
|
+
end
|
32
160
|
--[[
|
33
161
|
Function to check for the meta.paused key to decide if we are paused or not
|
34
162
|
(since an empty list and !EXISTS are not really the same).
|
@@ -67,14 +195,6 @@ end
|
|
67
195
|
Function to add job in target list and add marker if needed.
|
68
196
|
]]
|
69
197
|
-- Includes
|
70
|
-
--[[
|
71
|
-
Add marker if needed when a job is available.
|
72
|
-
]]
|
73
|
-
local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
74
|
-
if not isPausedOrMaxed then
|
75
|
-
rcall("ZADD", markerKey, 0, "0")
|
76
|
-
end
|
77
|
-
end
|
78
198
|
local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
|
79
199
|
rcall(pushCmd, targetKey, jobId)
|
80
200
|
addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
|
@@ -89,26 +209,6 @@ end
|
|
89
209
|
local getJobKeyPrefix = function (jobKey, jobId)
|
90
210
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
91
211
|
end
|
92
|
-
--[[
|
93
|
-
Function to check for the meta.paused key to decide if we are paused or not
|
94
|
-
(since an empty list and !EXISTS are not really the same).
|
95
|
-
]]
|
96
|
-
local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
97
|
-
local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
|
98
|
-
if queueAttributes[1] then
|
99
|
-
return pausedKey, true
|
100
|
-
else
|
101
|
-
if queueAttributes[2] then
|
102
|
-
local activeCount = rcall("LLEN", activeKey)
|
103
|
-
if activeCount >= tonumber(queueAttributes[2]) then
|
104
|
-
return waitKey, true
|
105
|
-
else
|
106
|
-
return waitKey, false
|
107
|
-
end
|
108
|
-
end
|
109
|
-
end
|
110
|
-
return waitKey, false
|
111
|
-
end
|
112
212
|
local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
113
213
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
114
214
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
@@ -189,6 +289,36 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
189
289
|
end
|
190
290
|
removeJobKeys(jobKey)
|
191
291
|
end
|
292
|
+
--[[
|
293
|
+
Function to store a job
|
294
|
+
]]
|
295
|
+
local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
296
|
+
parentKey, parentData, repeatJobKey)
|
297
|
+
local jsonOpts = cjson.encode(opts)
|
298
|
+
local delay = opts['delay'] or 0
|
299
|
+
local priority = opts['priority'] or 0
|
300
|
+
local debounceId = opts['de'] and opts['de']['id']
|
301
|
+
local optionalValues = {}
|
302
|
+
if parentKey ~= nil then
|
303
|
+
table.insert(optionalValues, "parentKey")
|
304
|
+
table.insert(optionalValues, parentKey)
|
305
|
+
table.insert(optionalValues, "parent")
|
306
|
+
table.insert(optionalValues, parentData)
|
307
|
+
end
|
308
|
+
if repeatJobKey ~= nil then
|
309
|
+
table.insert(optionalValues, "rjk")
|
310
|
+
table.insert(optionalValues, repeatJobKey)
|
311
|
+
end
|
312
|
+
if debounceId then
|
313
|
+
table.insert(optionalValues, "deid")
|
314
|
+
table.insert(optionalValues, debounceId)
|
315
|
+
end
|
316
|
+
rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
|
317
|
+
"timestamp", timestamp, "delay", delay, "priority", priority,
|
318
|
+
unpack(optionalValues))
|
319
|
+
rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
|
320
|
+
return delay, priority
|
321
|
+
end
|
192
322
|
--[[
|
193
323
|
Function to store a job scheduler
|
194
324
|
]]
|
@@ -230,11 +360,11 @@ end
|
|
230
360
|
-- If we are overriding a repeatable job we must delete the delayed job for
|
231
361
|
-- the next iteration.
|
232
362
|
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
|
363
|
+
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
364
|
+
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
233
365
|
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
234
366
|
if prevMillis ~= false then
|
235
367
|
local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
236
|
-
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
237
|
-
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
238
368
|
if rcall("EXISTS", nextDelayedJobKey) ~= 1 or currentJobId == nextDelayedJobId then
|
239
369
|
if rcall("ZSCORE", delayedKey, currentJobId) ~= false then
|
240
370
|
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
@@ -256,11 +386,47 @@ if prevMillis ~= false then
|
|
256
386
|
end
|
257
387
|
end
|
258
388
|
local schedulerOpts = cmsgpack.unpack(ARGV[2])
|
259
|
-
|
389
|
+
storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
|
390
|
+
if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
|
391
|
+
local eventsKey = KEYS[9]
|
392
|
+
local metaKey = KEYS[5]
|
393
|
+
local maxEvents = getOrSetMaxEvents(metaKey)
|
394
|
+
rcall("INCR", KEYS[8])
|
395
|
+
local delayedOpts = cmsgpack.unpack(ARGV[6])
|
396
|
+
local delay, priority = storeJob(eventsKey, nextDelayedJobKey, nextDelayedJobId, schedulerOpts['name'], ARGV[4],
|
397
|
+
delayedOpts, ARGV[7], nil, nil, jobSchedulerId)
|
398
|
+
if delay ~= 0 then
|
399
|
+
addDelayedJob(nextDelayedJobId, delayedKey, eventsKey,
|
400
|
+
ARGV[7], maxEvents, KEYS[7], delay)
|
401
|
+
else
|
402
|
+
local isPaused = isQueuePaused(KEYS[5])
|
403
|
+
-- Standard or priority add
|
404
|
+
if priority == 0 then
|
405
|
+
if isPaused then
|
406
|
+
-- LIFO or FIFO
|
407
|
+
local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
|
408
|
+
rcall(pushCmd, KEYS[4], nextDelayedJobId)
|
409
|
+
else
|
410
|
+
-- LIFO or FIFO
|
411
|
+
local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
|
412
|
+
rcall(pushCmd, KEYS[3], nextDelayedJobId)
|
413
|
+
end
|
414
|
+
else
|
415
|
+
-- Priority add
|
416
|
+
addJobWithPriority(KEYS[7], KEYS[6], priority, nextDelayedJobId, KEYS[10], isPaused)
|
417
|
+
end
|
418
|
+
-- Emit waiting event
|
419
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", nextDelayedJobId)
|
420
|
+
end
|
421
|
+
if ARGV[9] ~= "" then
|
422
|
+
rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
|
423
|
+
end
|
424
|
+
return nextDelayedJobId .. "" -- convert to string
|
425
|
+
end
|
260
426
|
`;
|
261
427
|
export const addJobScheduler = {
|
262
428
|
name: 'addJobScheduler',
|
263
429
|
content,
|
264
|
-
keys:
|
430
|
+
keys: 10,
|
265
431
|
};
|
266
|
-
//# sourceMappingURL=addJobScheduler-
|
432
|
+
//# sourceMappingURL=addJobScheduler-10.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"addJobScheduler-10.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-10.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyaf,CAAC;AACF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|
@@ -68,12 +68,12 @@ end
|
|
68
68
|
Function to get max events value or set by default 10000.
|
69
69
|
]]
|
70
70
|
local function getOrSetMaxEvents(metaKey)
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
71
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
72
|
+
if not maxEvents then
|
73
|
+
maxEvents = 10000
|
74
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
75
|
+
end
|
76
|
+
return maxEvents
|
77
77
|
end
|
78
78
|
--[[
|
79
79
|
Function to handle the case when job is duplicated.
|
@@ -130,12 +130,12 @@ end
|
|
130
130
|
Function to get max events value or set by default 10000.
|
131
131
|
]]
|
132
132
|
local function getOrSetMaxEvents(metaKey)
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
133
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
134
|
+
if not maxEvents then
|
135
|
+
maxEvents = 10000
|
136
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
137
|
+
end
|
138
|
+
return maxEvents
|
139
139
|
end
|
140
140
|
--[[
|
141
141
|
Function to handle the case when job is duplicated.
|
@@ -94,12 +94,12 @@ end
|
|
94
94
|
Function to get max events value or set by default 10000.
|
95
95
|
]]
|
96
96
|
local function getOrSetMaxEvents(metaKey)
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
97
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
98
|
+
if not maxEvents then
|
99
|
+
maxEvents = 10000
|
100
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
101
|
+
end
|
102
|
+
return maxEvents
|
103
103
|
end
|
104
104
|
--[[
|
105
105
|
Function to check for the meta.paused key to decide if we are paused or not
|
@@ -70,12 +70,12 @@ end
|
|
70
70
|
Function to get max events value or set by default 10000.
|
71
71
|
]]
|
72
72
|
local function getOrSetMaxEvents(metaKey)
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
73
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
74
|
+
if not maxEvents then
|
75
|
+
maxEvents = 10000
|
76
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
77
|
+
end
|
78
|
+
return maxEvents
|
79
79
|
end
|
80
80
|
if rcall("EXISTS", ARGV[4]) == 1 then
|
81
81
|
local jobId = ARGV[3]
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,eAAe,CAAC;AAC9B,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC"}
|
@@ -45,12 +45,12 @@ end
|
|
45
45
|
Function to get max events value or set by default 10000.
|
46
46
|
]]
|
47
47
|
local function getOrSetMaxEvents(metaKey)
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
48
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
49
|
+
if not maxEvents then
|
50
|
+
maxEvents = 10000
|
51
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
52
|
+
end
|
53
|
+
return maxEvents
|
54
54
|
end
|
55
55
|
--[[
|
56
56
|
Function to check for the meta.paused key to decide if we are paused or not
|
@@ -49,12 +49,12 @@ end
|
|
49
49
|
Function to get max events value or set by default 10000.
|
50
50
|
]]
|
51
51
|
local function getOrSetMaxEvents(metaKey)
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
52
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
53
|
+
if not maxEvents then
|
54
|
+
maxEvents = 10000
|
55
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
56
|
+
end
|
57
|
+
return maxEvents
|
58
58
|
end
|
59
59
|
--[[
|
60
60
|
Function to check for the meta.paused key to decide if we are paused or not
|
@@ -358,8 +358,11 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
|
|
358
358
|
end
|
359
359
|
end
|
360
360
|
end
|
361
|
-
local function moveParentFromWaitingChildrenToFailed(
|
362
|
-
|
361
|
+
local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
362
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
363
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
364
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) ~= false then
|
365
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
363
366
|
local parentQueuePrefix = parentQueueKey .. ":"
|
364
367
|
local parentFailedKey = parentQueueKey .. ":failed"
|
365
368
|
rcall("ZADD", parentFailedKey, timestamp, parentId)
|
@@ -395,6 +398,43 @@ local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey,
|
|
395
398
|
local parentRawOpts = jobAttributes[3]
|
396
399
|
local parentOpts = cjson.decode(parentRawOpts)
|
397
400
|
removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
|
401
|
+
elseif rcall("ZSCORE", parentDelayedKey, parentId) ~= false then
|
402
|
+
rcall("ZREM", parentDelayedKey, parentId)
|
403
|
+
local parentQueuePrefix = parentQueueKey .. ":"
|
404
|
+
local parentFailedKey = parentQueueKey .. ":failed"
|
405
|
+
rcall("ZADD", parentFailedKey, timestamp, parentId)
|
406
|
+
local failedReason = "child " .. jobIdKey .. " failed"
|
407
|
+
rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
|
408
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
|
409
|
+
failedReason, "prev", "delayed")
|
410
|
+
local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
|
411
|
+
removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
|
412
|
+
if jobAttributes[1] then
|
413
|
+
local parentData = cjson.decode(jobAttributes[1])
|
414
|
+
if parentData['fpof'] then
|
415
|
+
moveParentFromWaitingChildrenToFailed(
|
416
|
+
parentData['queueKey'],
|
417
|
+
parentData['queueKey'] .. ':' .. parentData['id'],
|
418
|
+
parentData['id'],
|
419
|
+
parentKey,
|
420
|
+
timestamp
|
421
|
+
)
|
422
|
+
elseif parentData['idof'] or parentData['rdof'] then
|
423
|
+
local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
|
424
|
+
local grandParentDependenciesSet = grandParentKey .. ":dependencies"
|
425
|
+
if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
|
426
|
+
moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
|
427
|
+
grandParentKey, parentData['id'], timestamp)
|
428
|
+
if parentData['idof'] then
|
429
|
+
local grandParentFailedSet = grandParentKey .. ":failed"
|
430
|
+
rcall("HSET", grandParentFailedSet, parentKey, failedReason)
|
431
|
+
end
|
432
|
+
end
|
433
|
+
end
|
434
|
+
end
|
435
|
+
local parentRawOpts = jobAttributes[3]
|
436
|
+
local parentOpts = cjson.decode(parentRawOpts)
|
437
|
+
removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
|
398
438
|
end
|
399
439
|
end
|
400
440
|
--[[
|
@@ -405,12 +445,12 @@ end
|
|
405
445
|
Function to get max events value or set by default 10000.
|
406
446
|
]]
|
407
447
|
local function getOrSetMaxEvents(metaKey)
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
448
|
+
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
449
|
+
if not maxEvents then
|
450
|
+
maxEvents = 10000
|
451
|
+
rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
|
452
|
+
end
|
453
|
+
return maxEvents
|
414
454
|
end
|
415
455
|
local function trimEvents(metaKey, eventStreamKey)
|
416
456
|
local maxEvents = getOrSetMaxEvents(metaKey)
|
@@ -473,7 +513,7 @@ if (#stalling > 0) then
|
|
473
513
|
rcall("XADD", eventStreamKey, "*", "event",
|
474
514
|
"failed", "jobId", jobId, 'prev', 'active',
|
475
515
|
'failedReason', failedReason)
|
476
|
-
if rawParentData
|
516
|
+
if rawParentData then
|
477
517
|
if opts['fpof'] then
|
478
518
|
local parentData = cjson.decode(rawParentData)
|
479
519
|
moveParentFromWaitingChildrenToFailed(
|