bullmq 4.17.0 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/flow-producer.js +3 -10
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job.js +21 -11
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-base.js +2 -9
- package/dist/cjs/classes/queue-base.js.map +1 -1
- package/dist/cjs/classes/queue-events.js +3 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js +1 -0
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/queue.js +1 -1
- package/dist/cjs/classes/redis-connection.js +9 -7
- package/dist/cjs/classes/redis-connection.js.map +1 -1
- package/dist/cjs/classes/repeat.js +22 -5
- package/dist/cjs/classes/repeat.js.map +1 -1
- package/dist/cjs/classes/scripts.js +34 -34
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +27 -32
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
- package/dist/cjs/commands/addParentJob-4.lua +0 -2
- package/dist/{esm/commands/addPrioritizedJob-8.lua → cjs/commands/addPrioritizedJob-7.lua} +18 -24
- package/dist/cjs/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
- package/dist/cjs/commands/changePriority-6.lua +57 -0
- package/dist/cjs/commands/getCounts-1.lua +1 -0
- package/dist/cjs/commands/getRanges-1.lua +1 -0
- package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
- package/dist/cjs/commands/includes/addJobWithPriority.lua +3 -7
- package/dist/cjs/commands/includes/checkStalledJobs.lua +1 -1
- package/dist/cjs/commands/includes/isQueuePaused.lua +7 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
- package/dist/cjs/commands/includes/prepareJobForProcessing.lua +1 -1
- package/dist/cjs/commands/includes/promoteDelayedJobs.lua +7 -4
- package/dist/{esm/commands/moveToActive-10.lua → cjs/commands/moveToActive-11.lua} +18 -29
- package/dist/cjs/commands/moveToDelayed-7.lua +77 -0
- package/dist/cjs/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
- package/dist/cjs/commands/moveToWaitingChildren-4.lua +7 -4
- package/dist/cjs/commands/pause-7.lua +41 -0
- package/dist/cjs/commands/promote-8.lua +60 -0
- package/dist/cjs/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
- package/dist/cjs/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
- package/dist/cjs/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +51 -61
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
- package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -0
- package/dist/cjs/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
- package/dist/cjs/scripts/addStandardJob-7.js.map +1 -0
- package/dist/cjs/scripts/changePriority-6.js +85 -0
- package/dist/cjs/scripts/changePriority-6.js.map +1 -0
- package/dist/cjs/scripts/getCounts-1.js +1 -0
- package/dist/cjs/scripts/getCounts-1.js.map +1 -1
- package/dist/cjs/scripts/getRanges-1.js +1 -0
- package/dist/cjs/scripts/getRanges-1.js.map +1 -1
- package/dist/cjs/scripts/index.js +10 -10
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js +1 -0
- package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/cjs/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
- package/dist/cjs/scripts/moveToActive-11.js.map +1 -0
- package/dist/cjs/scripts/moveToDelayed-7.js +103 -0
- package/dist/cjs/scripts/moveToDelayed-7.js.map +1 -0
- package/dist/cjs/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
- package/dist/cjs/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-4.js +6 -3
- package/dist/cjs/scripts/moveToWaitingChildren-4.js.map +1 -1
- package/dist/cjs/scripts/pause-7.js +67 -0
- package/dist/cjs/scripts/pause-7.js.map +1 -0
- package/dist/cjs/scripts/promote-8.js +79 -0
- package/dist/cjs/scripts/promote-8.js.map +1 -0
- package/dist/cjs/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
- package/dist/cjs/scripts/retryJob-10.js.map +1 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/esm/classes/flow-producer.js +3 -10
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job.d.ts +5 -0
- package/dist/esm/classes/job.js +21 -11
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-base.js +2 -9
- package/dist/esm/classes/queue-base.js.map +1 -1
- package/dist/esm/classes/queue-events.js +3 -1
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-keys.js +1 -0
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +1 -1
- package/dist/esm/classes/queue.js +1 -1
- package/dist/esm/classes/redis-connection.js +9 -7
- package/dist/esm/classes/redis-connection.js.map +1 -1
- package/dist/esm/classes/repeat.js +22 -5
- package/dist/esm/classes/repeat.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +4 -4
- package/dist/esm/classes/scripts.js +34 -34
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +1 -1
- package/dist/esm/classes/worker.js +27 -32
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/{addDelayedJob-7.lua → addDelayedJob-6.lua} +18 -22
- package/dist/esm/commands/addParentJob-4.lua +0 -2
- package/dist/{cjs/commands/addPrioritizedJob-8.lua → esm/commands/addPrioritizedJob-7.lua} +18 -24
- package/dist/esm/commands/{addStandardJob-6.lua → addStandardJob-7.lua} +6 -2
- package/dist/esm/commands/changePriority-6.lua +57 -0
- package/dist/esm/commands/getCounts-1.lua +1 -0
- package/dist/esm/commands/getRanges-1.lua +1 -0
- package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +5 -19
- package/dist/esm/commands/includes/addJobWithPriority.lua +3 -7
- package/dist/esm/commands/includes/checkStalledJobs.lua +1 -1
- package/dist/esm/commands/includes/isQueuePaused.lua +7 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +39 -30
- package/dist/esm/commands/includes/prepareJobForProcessing.lua +1 -1
- package/dist/esm/commands/includes/promoteDelayedJobs.lua +7 -4
- package/dist/{cjs/commands/moveToActive-10.lua → esm/commands/moveToActive-11.lua} +18 -29
- package/dist/esm/commands/moveToDelayed-7.lua +77 -0
- package/dist/esm/commands/{moveToFinished-13.lua → moveToFinished-14.lua} +29 -22
- package/dist/esm/commands/moveToWaitingChildren-4.lua +7 -4
- package/dist/esm/commands/pause-7.lua +41 -0
- package/dist/esm/commands/promote-8.lua +60 -0
- package/dist/esm/commands/{retryJob-9.lua → retryJob-10.lua} +16 -11
- package/dist/esm/interfaces/job-json.d.ts +4 -1
- package/dist/esm/interfaces/minimal-job.d.ts +3 -0
- package/dist/esm/interfaces/queue-options.d.ts +1 -1
- package/dist/esm/interfaces/repeat-options.d.ts +7 -0
- package/dist/esm/scripts/{addDelayedJob-7.js → addDelayedJob-6.js} +77 -88
- package/dist/esm/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map} +1 -1
- package/dist/esm/scripts/addParentJob-4.js +51 -61
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-7.js} +80 -93
- package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -0
- package/dist/esm/scripts/{addStandardJob-6.js → addStandardJob-7.js} +58 -63
- package/dist/esm/scripts/addStandardJob-7.js.map +1 -0
- package/dist/esm/scripts/changePriority-6.js +82 -0
- package/dist/esm/scripts/changePriority-6.js.map +1 -0
- package/dist/esm/scripts/getCounts-1.js +1 -0
- package/dist/esm/scripts/getCounts-1.js.map +1 -1
- package/dist/esm/scripts/getRanges-1.js +1 -0
- package/dist/esm/scripts/getRanges-1.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +10 -10
- package/dist/esm/scripts/index.js +10 -10
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-8.js +1 -0
- package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
- package/dist/esm/scripts/{moveToActive-10.js → moveToActive-11.js} +30 -48
- package/dist/esm/scripts/moveToActive-11.js.map +1 -0
- package/dist/esm/scripts/moveToDelayed-7.js +100 -0
- package/dist/esm/scripts/moveToDelayed-7.js.map +1 -0
- package/dist/esm/scripts/{moveToFinished-13.js → moveToFinished-14.js} +89 -88
- package/dist/esm/scripts/{moveToFinished-13.js.map → moveToFinished-14.js.map} +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-4.js +6 -3
- package/dist/esm/scripts/moveToWaitingChildren-4.js.map +1 -1
- package/dist/esm/scripts/pause-7.js +64 -0
- package/dist/esm/scripts/pause-7.js.map +1 -0
- package/dist/esm/scripts/promote-8.js +76 -0
- package/dist/esm/scripts/promote-8.js.map +1 -0
- package/dist/esm/scripts/{retryJob-9.js → retryJob-10.js} +26 -32
- package/dist/esm/scripts/retryJob-10.js.map +1 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -1
- package/dist/cjs/commands/changePriority-5.lua +0 -52
- package/dist/cjs/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
- package/dist/cjs/commands/moveToDelayed-8.lua +0 -72
- package/dist/cjs/commands/pause-5.lua +0 -36
- package/dist/cjs/commands/promote-7.lua +0 -57
- package/dist/cjs/scripts/addDelayedJob-7.js.map +0 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +0 -1
- package/dist/cjs/scripts/addStandardJob-6.js.map +0 -1
- package/dist/cjs/scripts/changePriority-5.js +0 -87
- package/dist/cjs/scripts/changePriority-5.js.map +0 -1
- package/dist/cjs/scripts/moveToDelayed-8.js +0 -168
- package/dist/cjs/scripts/moveToDelayed-8.js.map +0 -1
- package/dist/cjs/scripts/pause-5.js +0 -48
- package/dist/cjs/scripts/pause-5.js.map +0 -1
- package/dist/cjs/scripts/promote-7.js +0 -87
- package/dist/cjs/scripts/promote-7.js.map +0 -1
- package/dist/cjs/scripts/retryJob-9.js.map +0 -1
- package/dist/esm/commands/changePriority-5.lua +0 -52
- package/dist/esm/commands/includes/addPriorityMarkerIfNeeded.lua +0 -12
- package/dist/esm/commands/moveToDelayed-8.lua +0 -72
- package/dist/esm/commands/pause-5.lua +0 -36
- package/dist/esm/commands/promote-7.lua +0 -57
- package/dist/esm/scripts/addDelayedJob-7.js.map +0 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +0 -1
- package/dist/esm/scripts/addStandardJob-6.js.map +0 -1
- package/dist/esm/scripts/changePriority-5.js +0 -84
- package/dist/esm/scripts/changePriority-5.js.map +0 -1
- package/dist/esm/scripts/moveToDelayed-8.js +0 -165
- package/dist/esm/scripts/moveToDelayed-8.js.map +0 -1
- package/dist/esm/scripts/pause-5.js +0 -45
- package/dist/esm/scripts/pause-5.js.map +0 -1
- package/dist/esm/scripts/promote-7.js +0 -84
- package/dist/esm/scripts/promote-7.js.map +0 -1
- package/dist/esm/scripts/retryJob-9.js.map +0 -1
- /package/dist/esm/scripts/{addDelayedJob-7.d.ts → addDelayedJob-6.d.ts} +0 -0
- /package/dist/esm/scripts/{addPrioritizedJob-8.d.ts → addPrioritizedJob-7.d.ts} +0 -0
- /package/dist/esm/scripts/{addStandardJob-6.d.ts → addStandardJob-7.d.ts} +0 -0
- /package/dist/esm/scripts/{changePriority-5.d.ts → changePriority-6.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToActive-10.d.ts → moveToActive-11.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToDelayed-8.d.ts → moveToDelayed-7.d.ts} +0 -0
- /package/dist/esm/scripts/{moveToFinished-13.d.ts → moveToFinished-14.d.ts} +0 -0
- /package/dist/esm/scripts/{pause-5.d.ts → pause-7.d.ts} +0 -0
- /package/dist/esm/scripts/{promote-7.d.ts → promote-8.d.ts} +0 -0
- /package/dist/esm/scripts/{retryJob-9.d.ts → retryJob-10.d.ts} +0 -0
@@ -2,15 +2,16 @@
|
|
2
2
|
Retries a failed job by moving it back to the wait queue.
|
3
3
|
|
4
4
|
Input:
|
5
|
-
KEYS[1]
|
6
|
-
KEYS[2]
|
7
|
-
KEYS[3]
|
8
|
-
KEYS[4]
|
9
|
-
KEYS[5]
|
10
|
-
KEYS[6]
|
11
|
-
KEYS[7]
|
12
|
-
KEYS[8]
|
13
|
-
KEYS[9]
|
5
|
+
KEYS[1] 'active',
|
6
|
+
KEYS[2] 'wait'
|
7
|
+
KEYS[3] 'paused'
|
8
|
+
KEYS[4] job key
|
9
|
+
KEYS[5] 'meta'
|
10
|
+
KEYS[6] events stream
|
11
|
+
KEYS[7] delayed key
|
12
|
+
KEYS[8] prioritized key
|
13
|
+
KEYS[9] 'pc' priority counter
|
14
|
+
KEYS[10] 'marker'
|
14
15
|
|
15
16
|
ARGV[1] key prefix
|
16
17
|
ARGV[2] timestamp
|
@@ -34,9 +35,11 @@ local rcall = redis.call
|
|
34
35
|
--- @include "includes/promoteDelayedJobs"
|
35
36
|
|
36
37
|
local target, paused = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
|
38
|
+
local markerKey = KEYS[10]
|
39
|
+
|
37
40
|
-- Check if there are delayed jobs that we can move to wait.
|
38
41
|
-- test example: when there are delayed jobs between retries
|
39
|
-
promoteDelayedJobs(KEYS[7],
|
42
|
+
promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], paused)
|
40
43
|
|
41
44
|
if rcall("EXISTS", KEYS[4]) == 1 then
|
42
45
|
|
@@ -57,9 +60,11 @@ if rcall("EXISTS", KEYS[4]) == 1 then
|
|
57
60
|
if priority == 0 then
|
58
61
|
rcall(ARGV[3], target, ARGV[4])
|
59
62
|
else
|
60
|
-
addJobWithPriority(
|
63
|
+
addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], paused)
|
61
64
|
end
|
62
65
|
|
66
|
+
rcall("HINCRBY", KEYS[4], "atm", 1)
|
67
|
+
|
63
68
|
local maxEvents = rcall("HGET", KEYS[5], "opts.maxLenEvents") or 10000
|
64
69
|
|
65
70
|
-- Emit waiting event
|
@@ -7,6 +7,7 @@ export interface JobJson {
|
|
7
7
|
opts: RedisJobOptions;
|
8
8
|
progress: number | object;
|
9
9
|
attemptsMade: number;
|
10
|
+
attemptsStarted: number;
|
10
11
|
finishedOn?: number;
|
11
12
|
processedOn?: number;
|
12
13
|
timestamp: number;
|
@@ -24,7 +25,7 @@ export interface JobJsonRaw {
|
|
24
25
|
delay: string;
|
25
26
|
opts: string;
|
26
27
|
progress: string;
|
27
|
-
attemptsMade
|
28
|
+
attemptsMade?: string;
|
28
29
|
finishedOn?: string;
|
29
30
|
processedOn?: string;
|
30
31
|
timestamp: string;
|
@@ -34,4 +35,6 @@ export interface JobJsonRaw {
|
|
34
35
|
parentKey?: string;
|
35
36
|
parent?: string;
|
36
37
|
rjk?: string;
|
38
|
+
atm?: string;
|
39
|
+
ats?: string;
|
37
40
|
}
|
@@ -2,6 +2,9 @@ import { JobsOptions, JobJsonSandbox } from '../types';
|
|
2
2
|
import { JobJson } from './job-json';
|
3
3
|
import { ParentKeys } from './parent';
|
4
4
|
export type BulkJobOptions = Omit<JobsOptions, 'repeat'>;
|
5
|
+
export interface MoveToDelayedOpts {
|
6
|
+
skipAttempt?: boolean;
|
7
|
+
}
|
5
8
|
export interface MoveToWaitingChildrenOpts {
|
6
9
|
child?: {
|
7
10
|
id: string;
|
@@ -9,6 +9,13 @@ export interface RepeatOptions extends Omit<ParserOptions, 'iterator'> {
|
|
9
9
|
* A repeat pattern
|
10
10
|
*/
|
11
11
|
pattern?: string;
|
12
|
+
/**
|
13
|
+
* Custom repeatable key. This is the key that holds the "metadata"
|
14
|
+
* of a given repeatable job. This key is normally auto-generated but
|
15
|
+
* it is sometimes useful to specify a custom key for easier retrieval
|
16
|
+
* of repeatable jobs.
|
17
|
+
*/
|
18
|
+
key?: string;
|
12
19
|
/**
|
13
20
|
* Number of times the job should repeat at max.
|
14
21
|
*/
|
@@ -6,13 +6,12 @@ const content = `--[[
|
|
6
6
|
- adds to delayed zset.
|
7
7
|
- Emits a global event 'delayed' if the job is delayed.
|
8
8
|
Input:
|
9
|
-
KEYS[1] '
|
10
|
-
KEYS[2] '
|
11
|
-
KEYS[3] '
|
12
|
-
KEYS[4] '
|
13
|
-
KEYS[5] '
|
14
|
-
KEYS[6]
|
15
|
-
KEYS[7] events stream key
|
9
|
+
KEYS[1] 'marker',
|
10
|
+
KEYS[2] 'meta'
|
11
|
+
KEYS[3] 'id'
|
12
|
+
KEYS[4] 'delayed'
|
13
|
+
KEYS[5] 'completed'
|
14
|
+
KEYS[6] events stream key
|
16
15
|
ARGV[1] msgpacked arguments array
|
17
16
|
[1] key prefix,
|
18
17
|
[2] custom id (use custom instead of one generated automatically)
|
@@ -29,13 +28,11 @@ const content = `--[[
|
|
29
28
|
jobId - OK
|
30
29
|
-5 - Missing parent key
|
31
30
|
]]
|
32
|
-
local
|
33
|
-
local
|
34
|
-
local
|
35
|
-
local
|
36
|
-
local
|
37
|
-
local completedKey = KEYS[6]
|
38
|
-
local eventsKey = KEYS[7]
|
31
|
+
local metaKey = KEYS[2]
|
32
|
+
local idKey = KEYS[3]
|
33
|
+
local delayedKey = KEYS[4]
|
34
|
+
local completedKey = KEYS[5]
|
35
|
+
local eventsKey = KEYS[6]
|
39
36
|
local jobId
|
40
37
|
local jobIdKey
|
41
38
|
local rcall = redis.call
|
@@ -74,7 +71,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
74
71
|
end
|
75
72
|
--[[
|
76
73
|
Add delay marker if needed.
|
77
|
-
]]
|
74
|
+
]]
|
78
75
|
-- Includes
|
79
76
|
--[[
|
80
77
|
Function to return the next delayed job timestamp.
|
@@ -89,36 +86,20 @@ local function getNextDelayedTimestamp(delayedKey)
|
|
89
86
|
return nextTimestamp
|
90
87
|
end
|
91
88
|
end
|
92
|
-
local function addDelayMarkerIfNeeded(
|
93
|
-
local waitLen = rcall("LLEN", targetKey)
|
94
|
-
if waitLen <= 1 then
|
89
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
95
90
|
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
96
91
|
if nextTimestamp ~= nil then
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
local marker = rcall("LINDEX", targetKey, 0)
|
101
|
-
local oldTimestamp = tonumber(marker:sub(3))
|
102
|
-
if oldTimestamp and oldTimestamp > nextTimestamp then
|
103
|
-
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
104
|
-
end
|
105
|
-
else
|
106
|
-
-- if there is no marker, then we need to add one
|
107
|
-
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
108
|
-
end
|
92
|
+
-- Replace the score of the marker with the newest known
|
93
|
+
-- next timestamp.
|
94
|
+
rcall("ZADD", markerKey, nextTimestamp, "0")
|
109
95
|
end
|
110
|
-
end
|
111
96
|
end
|
112
97
|
--[[
|
113
98
|
Function to check for the meta.paused key to decide if we are paused or not
|
114
99
|
(since an empty list and !EXISTS are not really the same).
|
115
100
|
]]
|
116
|
-
local function
|
117
|
-
|
118
|
-
return waitKey, false
|
119
|
-
else
|
120
|
-
return pausedKey, true
|
121
|
-
end
|
101
|
+
local function isQueuePaused(queueMetaKey)
|
102
|
+
return rcall("HEXISTS", queueMetaKey, "paused") == 1
|
122
103
|
end
|
123
104
|
--[[
|
124
105
|
Validate and move or add dependencies to parent.
|
@@ -131,56 +112,64 @@ end
|
|
131
112
|
--[[
|
132
113
|
Function to add job considering priority.
|
133
114
|
]]
|
134
|
-
|
135
|
-
--[[
|
136
|
-
Function priority marker to wait if needed
|
137
|
-
in order to wake up our workers and to respect priority
|
138
|
-
order as much as possible
|
139
|
-
]]
|
140
|
-
local function addPriorityMarkerIfNeeded(waitKey)
|
141
|
-
local waitLen = rcall("LLEN", waitKey)
|
142
|
-
if waitLen == 0 then
|
143
|
-
rcall("LPUSH", waitKey, "0:0")
|
144
|
-
end
|
145
|
-
end
|
146
|
-
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
115
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
147
116
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
148
117
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
149
118
|
rcall("ZADD", prioritizedKey, score, jobId)
|
150
|
-
if not
|
151
|
-
|
119
|
+
if not isPaused then
|
120
|
+
rcall("ZADD", markerKey, 0, "0")
|
152
121
|
end
|
153
122
|
end
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
local delay = tonumber(jobAttributes[2]) or 0
|
164
|
-
if delay > 0 then
|
165
|
-
local delayedTimestamp = tonumber(timestamp) + delay
|
166
|
-
local score = delayedTimestamp * 0x1000
|
167
|
-
local parentDelayedKey = parentQueueKey .. ":delayed"
|
168
|
-
rcall("ZADD", parentDelayedKey, score, parentId)
|
169
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
|
170
|
-
"delay", delayedTimestamp)
|
171
|
-
addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
|
172
|
-
else
|
173
|
-
if priority == 0 then
|
174
|
-
rcall("RPUSH", parentTarget, parentId)
|
175
|
-
else
|
176
|
-
addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
|
177
|
-
parentId, parentQueueKey .. ":pc")
|
178
|
-
end
|
179
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
|
180
|
-
"prev", "waiting-children")
|
181
|
-
end
|
123
|
+
--[[
|
124
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
125
|
+
(since an empty list and !EXISTS are not really the same).
|
126
|
+
]]
|
127
|
+
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
128
|
+
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
129
|
+
return waitKey, false
|
130
|
+
else
|
131
|
+
return pausedKey, true
|
182
132
|
end
|
183
133
|
end
|
134
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
135
|
+
parentKey, parentId, timestamp)
|
136
|
+
local isParentActive = rcall("ZSCORE",
|
137
|
+
parentQueueKey .. ":waiting-children", parentId)
|
138
|
+
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
139
|
+
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
140
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
141
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
142
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
143
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
144
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
145
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
146
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
147
|
+
if delay > 0 then
|
148
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
149
|
+
local score = delayedTimestamp * 0x1000
|
150
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
151
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
152
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
153
|
+
"jobId", parentId, "delay", delayedTimestamp)
|
154
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
155
|
+
else
|
156
|
+
if priority == 0 then
|
157
|
+
local parentTarget, _paused =
|
158
|
+
getTargetQueueList(parentMetaKey, parentWaitKey,
|
159
|
+
parentPausedKey)
|
160
|
+
rcall("RPUSH", parentTarget, parentId)
|
161
|
+
rcall("ZADD", parentMarkerKey, 0, "0")
|
162
|
+
else
|
163
|
+
local isPaused = isQueuePaused(parentMetaKey)
|
164
|
+
addJobWithPriority(parentMarkerKey,
|
165
|
+
parentQueueKey .. ":prioritized", priority,
|
166
|
+
parentId, parentQueueKey .. ":pc", isPaused)
|
167
|
+
end
|
168
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
169
|
+
"jobId", parentId, "prev", "waiting-children")
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
184
173
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
185
174
|
parentId, jobIdKey, returnvalue, timestamp )
|
186
175
|
local processedSet = parentKey .. ":processed"
|
@@ -252,13 +241,13 @@ local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
|
|
252
241
|
rcall("ZADD", delayedKey, score, jobId)
|
253
242
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
|
254
243
|
"jobId", jobId, "delay", delayedTimestamp)
|
255
|
-
--
|
256
|
-
|
257
|
-
|
258
|
-
|
244
|
+
-- mark that a delayed job is available
|
245
|
+
local isPaused = isQueuePaused(metaKey)
|
246
|
+
if not isPaused then
|
247
|
+
local markerKey = KEYS[1]
|
248
|
+
addDelayMarkerIfNeeded(markerKey, delayedKey)
|
249
|
+
end
|
259
250
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
260
|
-
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
261
|
-
-- fail in this case.
|
262
251
|
if parentDependenciesKey ~= nil then
|
263
252
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
264
253
|
end
|
@@ -267,6 +256,6 @@ return jobId .. "" -- convert to string
|
|
267
256
|
export const addDelayedJob = {
|
268
257
|
name: 'addDelayedJob',
|
269
258
|
content,
|
270
|
-
keys:
|
259
|
+
keys: 6,
|
271
260
|
};
|
272
|
-
//# sourceMappingURL=addDelayedJob-
|
261
|
+
//# sourceMappingURL=addDelayedJob-6.js.map
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"
|
1
|
+
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Pf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -74,7 +74,7 @@ end
|
|
74
74
|
-- Includes
|
75
75
|
--[[
|
76
76
|
Add delay marker if needed.
|
77
|
-
]]
|
77
|
+
]]
|
78
78
|
-- Includes
|
79
79
|
--[[
|
80
80
|
Function to return the next delayed job timestamp.
|
@@ -89,47 +89,30 @@ local function getNextDelayedTimestamp(delayedKey)
|
|
89
89
|
return nextTimestamp
|
90
90
|
end
|
91
91
|
end
|
92
|
-
local function addDelayMarkerIfNeeded(
|
93
|
-
local waitLen = rcall("LLEN", targetKey)
|
94
|
-
if waitLen <= 1 then
|
92
|
+
local function addDelayMarkerIfNeeded(markerKey, delayedKey)
|
95
93
|
local nextTimestamp = getNextDelayedTimestamp(delayedKey)
|
96
94
|
if nextTimestamp ~= nil then
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
local marker = rcall("LINDEX", targetKey, 0)
|
101
|
-
local oldTimestamp = tonumber(marker:sub(3))
|
102
|
-
if oldTimestamp and oldTimestamp > nextTimestamp then
|
103
|
-
rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
|
104
|
-
end
|
105
|
-
else
|
106
|
-
-- if there is no marker, then we need to add one
|
107
|
-
rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
|
108
|
-
end
|
95
|
+
-- Replace the score of the marker with the newest known
|
96
|
+
-- next timestamp.
|
97
|
+
rcall("ZADD", markerKey, nextTimestamp, "0")
|
109
98
|
end
|
110
|
-
end
|
111
99
|
end
|
112
100
|
--[[
|
113
|
-
Function to
|
101
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
102
|
+
(since an empty list and !EXISTS are not really the same).
|
114
103
|
]]
|
115
|
-
|
104
|
+
local function isQueuePaused(queueMetaKey)
|
105
|
+
return rcall("HEXISTS", queueMetaKey, "paused") == 1
|
106
|
+
end
|
116
107
|
--[[
|
117
|
-
Function
|
118
|
-
in order to wake up our workers and to respect priority
|
119
|
-
order as much as possible
|
108
|
+
Function to add job considering priority.
|
120
109
|
]]
|
121
|
-
local function
|
122
|
-
local waitLen = rcall("LLEN", waitKey)
|
123
|
-
if waitLen == 0 then
|
124
|
-
rcall("LPUSH", waitKey, "0:0")
|
125
|
-
end
|
126
|
-
end
|
127
|
-
local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
|
110
|
+
local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
|
128
111
|
local prioCounter = rcall("INCR", priorityCounterKey)
|
129
112
|
local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
|
130
113
|
rcall("ZADD", prioritizedKey, score, jobId)
|
131
|
-
if not
|
132
|
-
|
114
|
+
if not isPaused then
|
115
|
+
rcall("ZADD", markerKey, 0, "0")
|
133
116
|
end
|
134
117
|
end
|
135
118
|
--[[
|
@@ -143,35 +126,44 @@ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
|
143
126
|
return pausedKey, true
|
144
127
|
end
|
145
128
|
end
|
146
|
-
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
129
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
|
130
|
+
parentKey, parentId, timestamp)
|
131
|
+
local isParentActive = rcall("ZSCORE",
|
132
|
+
parentQueueKey .. ":waiting-children", parentId)
|
133
|
+
if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
|
134
|
+
rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
|
135
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
136
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
137
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
138
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
139
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
140
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
141
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
142
|
+
if delay > 0 then
|
143
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
144
|
+
local score = delayedTimestamp * 0x1000
|
145
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
146
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
147
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
|
148
|
+
"jobId", parentId, "delay", delayedTimestamp)
|
149
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
150
|
+
else
|
151
|
+
if priority == 0 then
|
152
|
+
local parentTarget, _paused =
|
153
|
+
getTargetQueueList(parentMetaKey, parentWaitKey,
|
154
|
+
parentPausedKey)
|
155
|
+
rcall("RPUSH", parentTarget, parentId)
|
156
|
+
rcall("ZADD", parentMarkerKey, 0, "0")
|
157
|
+
else
|
158
|
+
local isPaused = isQueuePaused(parentMetaKey)
|
159
|
+
addJobWithPriority(parentMarkerKey,
|
160
|
+
parentQueueKey .. ":prioritized", priority,
|
161
|
+
parentId, parentQueueKey .. ":pc", isPaused)
|
162
|
+
end
|
163
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
164
|
+
"jobId", parentId, "prev", "waiting-children")
|
165
|
+
end
|
173
166
|
end
|
174
|
-
end
|
175
167
|
end
|
176
168
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
177
169
|
parentId, jobIdKey, returnvalue, timestamp )
|
@@ -241,8 +233,6 @@ rcall("ZADD", waitChildrenKey, timestamp, jobId)
|
|
241
233
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
242
234
|
"waiting-children", "jobId", jobId)
|
243
235
|
-- Check if this job is a child of another job, if so add it to the parents dependencies
|
244
|
-
-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
|
245
|
-
-- fail in this case.
|
246
236
|
if parentDependenciesKey ~= nil then
|
247
237
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
248
238
|
end
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Of,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|