bullmq 2.4.0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cjs/classes/backoffs.d.ts +3 -6
- package/dist/cjs/classes/backoffs.js +8 -8
- package/dist/cjs/classes/backoffs.js.map +1 -1
- package/dist/cjs/classes/flow-producer.js +2 -1
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job.d.ts +3 -3
- package/dist/cjs/classes/job.js +1 -1
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-base.js +1 -1
- package/dist/cjs/classes/queue-base.js.map +1 -1
- package/dist/cjs/classes/queue.d.ts +8 -6
- package/dist/cjs/classes/queue.js +10 -9
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/repeat.d.ts +0 -1
- package/dist/cjs/classes/repeat.js +4 -5
- package/dist/cjs/classes/repeat.js.map +1 -1
- package/dist/cjs/classes/scripts.d.ts +6 -6
- package/dist/cjs/classes/scripts.js +43 -22
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.d.ts +12 -3
- package/dist/cjs/classes/worker.js +50 -34
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/includes/getRateLimitTTL.lua +13 -0
- package/dist/cjs/commands/includes/moveJobFromWaitToActive.lua +15 -34
- package/dist/cjs/commands/includes/promoteDelayedJobs.lua +0 -6
- package/dist/cjs/commands/moveToActive-9.lua +11 -2
- package/dist/cjs/commands/moveToDelayed-8.lua +63 -0
- package/dist/cjs/commands/moveToFinished-12.lua +8 -1
- package/dist/{esm/commands/retryJob-6.lua → cjs/commands/retryJob-8.lua} +15 -8
- package/dist/cjs/interfaces/advanced-options.d.ts +4 -4
- package/dist/cjs/interfaces/base-job-options.d.ts +0 -6
- package/dist/cjs/interfaces/queue-options.d.ts +0 -10
- package/dist/cjs/interfaces/rate-limiter-options.d.ts +0 -18
- package/dist/cjs/interfaces/repeat-options.d.ts +1 -6
- package/dist/cjs/scripts/index.d.ts +2 -2
- package/dist/cjs/scripts/index.js +2 -2
- package/dist/cjs/scripts/moveToActive-9.js +35 -36
- package/dist/cjs/scripts/moveToActive-9.js.map +1 -1
- package/dist/cjs/scripts/{moveToDelayed-5.d.ts → moveToDelayed-8.d.ts} +0 -0
- package/dist/cjs/scripts/moveToDelayed-8.js +116 -0
- package/dist/cjs/scripts/moveToDelayed-8.js.map +1 -0
- package/dist/cjs/scripts/moveToFinished-12.js +33 -36
- package/dist/cjs/scripts/moveToFinished-12.js.map +1 -1
- package/dist/cjs/scripts/{retryJob-6.d.ts → retryJob-8.d.ts} +0 -0
- package/dist/cjs/scripts/retryJob-8.js +112 -0
- package/dist/cjs/scripts/retryJob-8.js.map +1 -0
- package/dist/cjs/types/backoff-strategy.d.ts +2 -0
- package/dist/cjs/types/backoff-strategy.js +3 -0
- package/dist/cjs/types/backoff-strategy.js.map +1 -0
- package/dist/cjs/types/index.d.ts +2 -1
- package/dist/cjs/types/index.js +2 -1
- package/dist/cjs/types/index.js.map +1 -1
- package/dist/cjs/utils.d.ts +1 -3
- package/dist/cjs/utils.js +1 -13
- package/dist/cjs/utils.js.map +1 -1
- package/dist/esm/classes/backoffs.d.ts +3 -6
- package/dist/esm/classes/backoffs.js +8 -8
- package/dist/esm/classes/backoffs.js.map +1 -1
- package/dist/esm/classes/flow-producer.js +3 -2
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job.d.ts +3 -3
- package/dist/esm/classes/job.js +1 -1
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-base.js +1 -1
- package/dist/esm/classes/queue-base.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +8 -6
- package/dist/esm/classes/queue.js +11 -10
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/repeat.d.ts +0 -1
- package/dist/esm/classes/repeat.js +4 -5
- package/dist/esm/classes/repeat.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +6 -6
- package/dist/esm/classes/scripts.js +41 -20
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +12 -3
- package/dist/esm/classes/worker.js +50 -34
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/includes/getRateLimitTTL.lua +13 -0
- package/dist/esm/commands/includes/moveJobFromWaitToActive.lua +15 -34
- package/dist/esm/commands/includes/promoteDelayedJobs.lua +0 -6
- package/dist/esm/commands/moveToActive-9.lua +11 -2
- package/dist/esm/commands/moveToDelayed-8.lua +63 -0
- package/dist/esm/commands/moveToFinished-12.lua +8 -1
- package/dist/{cjs/commands/retryJob-6.lua → esm/commands/retryJob-8.lua} +15 -8
- package/dist/esm/interfaces/advanced-options.d.ts +4 -4
- package/dist/esm/interfaces/base-job-options.d.ts +0 -6
- package/dist/esm/interfaces/queue-options.d.ts +0 -10
- package/dist/esm/interfaces/rate-limiter-options.d.ts +0 -18
- package/dist/esm/interfaces/repeat-options.d.ts +1 -6
- package/dist/esm/scripts/index.d.ts +2 -2
- package/dist/esm/scripts/index.js +2 -2
- package/dist/esm/scripts/moveToActive-9.js +35 -36
- package/dist/esm/scripts/moveToActive-9.js.map +1 -1
- package/dist/esm/scripts/{moveToDelayed-5.d.ts → moveToDelayed-8.d.ts} +0 -0
- package/dist/esm/scripts/moveToDelayed-8.js +113 -0
- package/dist/esm/scripts/moveToDelayed-8.js.map +1 -0
- package/dist/esm/scripts/moveToFinished-12.js +33 -36
- package/dist/esm/scripts/moveToFinished-12.js.map +1 -1
- package/dist/esm/scripts/{retryJob-6.d.ts → retryJob-8.d.ts} +0 -0
- package/dist/esm/scripts/retryJob-8.js +109 -0
- package/dist/esm/scripts/retryJob-8.js.map +1 -0
- package/dist/esm/types/backoff-strategy.d.ts +2 -0
- package/dist/esm/types/backoff-strategy.js +2 -0
- package/dist/esm/types/backoff-strategy.js.map +1 -0
- package/dist/esm/types/index.d.ts +2 -1
- package/dist/esm/types/index.js +2 -1
- package/dist/esm/types/index.js.map +1 -1
- package/dist/esm/utils.d.ts +1 -3
- package/dist/esm/utils.js +0 -11
- package/dist/esm/utils.js.map +1 -1
- package/package.json +1 -1
- package/dist/cjs/commands/moveToDelayed-5.lua +0 -54
- package/dist/cjs/scripts/moveToDelayed-5.js +0 -52
- package/dist/cjs/scripts/moveToDelayed-5.js.map +0 -1
- package/dist/cjs/scripts/retryJob-6.js +0 -60
- package/dist/cjs/scripts/retryJob-6.js.map +0 -1
- package/dist/esm/commands/moveToDelayed-5.lua +0 -54
- package/dist/esm/scripts/moveToDelayed-5.js +0 -49
- package/dist/esm/scripts/moveToDelayed-5.js.map +0 -1
- package/dist/esm/scripts/retryJob-6.js +0 -57
- package/dist/esm/scripts/retryJob-6.js.map +0 -1
@@ -0,0 +1,63 @@
|
|
1
|
+
--[[
|
2
|
+
Moves job from active to delayed set.
|
3
|
+
|
4
|
+
Input:
|
5
|
+
KEYS[1] wait key
|
6
|
+
KEYS[2] active key
|
7
|
+
KEYS[3] priority key
|
8
|
+
KEYS[4] delayed key
|
9
|
+
KEYS[5] job key
|
10
|
+
KEYS[6] events stream
|
11
|
+
KEYS[7] paused key
|
12
|
+
KEYS[8] meta key
|
13
|
+
|
14
|
+
ARGV[1] key prefix
|
15
|
+
ARGV[2] timestamp
|
16
|
+
ARGV[3] delayedTimestamp
|
17
|
+
ARGV[4] the id of the job
|
18
|
+
ARGV[5] queue token
|
19
|
+
|
20
|
+
Output:
|
21
|
+
0 - OK
|
22
|
+
-1 - Missing job.
|
23
|
+
-3 - Job not in active set.
|
24
|
+
|
25
|
+
Events:
|
26
|
+
- delayed key.
|
27
|
+
]]
|
28
|
+
local rcall = redis.call
|
29
|
+
|
30
|
+
-- Includes
|
31
|
+
--- @include "includes/promoteDelayedJobs"
|
32
|
+
|
33
|
+
promoteDelayedJobs(KEYS[4], KEYS[1], KEYS[3], KEYS[7], KEYS[8], KEYS[6], ARGV[1], ARGV[2])
|
34
|
+
|
35
|
+
if rcall("EXISTS", KEYS[5]) == 1 then
|
36
|
+
|
37
|
+
if ARGV[5] ~= "0" then
|
38
|
+
local lockKey = KEYS[5] .. ':lock'
|
39
|
+
if rcall("GET", lockKey) == ARGV[5] then
|
40
|
+
rcall("DEL", lockKey)
|
41
|
+
else
|
42
|
+
return -2
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
local jobId = ARGV[4]
|
47
|
+
local score = tonumber(ARGV[3])
|
48
|
+
local delayedTimestamp = (score / 0x1000)
|
49
|
+
|
50
|
+
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
51
|
+
|
52
|
+
if(numRemovedElements < 1) then
|
53
|
+
return -3
|
54
|
+
end
|
55
|
+
|
56
|
+
rcall("ZADD", KEYS[4], score, jobId)
|
57
|
+
|
58
|
+
rcall("XADD", KEYS[6], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
|
59
|
+
|
60
|
+
return 0
|
61
|
+
else
|
62
|
+
return -1
|
63
|
+
end
|
@@ -64,6 +64,7 @@ local rcall = redis.call
|
|
64
64
|
--- @include "includes/removeJobsByMaxCount"
|
65
65
|
--- @include "includes/trimEvents"
|
66
66
|
--- @include "includes/updateParentDepsIfNeeded"
|
67
|
+
--- @include "includes/getRateLimitTTL"
|
67
68
|
|
68
69
|
local jobIdKey = KEYS[10]
|
69
70
|
if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
@@ -169,6 +170,12 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
169
170
|
-- Check if there are delayed jobs that can be promoted
|
170
171
|
promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[11], KEYS[4], ARGV[8], timestamp)
|
171
172
|
|
173
|
+
-- Check if we are rate limited first.
|
174
|
+
local pttl = getRateLimitTTL(opts, KEYS[6])
|
175
|
+
if pttl > 0 then
|
176
|
+
return { 0, 0, pttl }
|
177
|
+
end
|
178
|
+
|
172
179
|
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
173
180
|
|
174
181
|
if jobId == "0" then
|
@@ -182,7 +189,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
182
189
|
if (nextTimestamp ~= nil) then
|
183
190
|
-- The result is guaranteed to be positive, since the
|
184
191
|
-- ZRANGEBYSCORE command would have return a job otherwise.
|
185
|
-
return nextTimestamp - timestamp
|
192
|
+
return {0, 0, 0, nextTimestamp - timestamp}
|
186
193
|
end
|
187
194
|
end
|
188
195
|
|
@@ -8,10 +8,14 @@
|
|
8
8
|
KEYS[4] job key
|
9
9
|
KEYS[5] 'meta'
|
10
10
|
KEYS[6] events stream
|
11
|
+
KEYS[7] delayed key
|
12
|
+
KEYS[8] priority key
|
11
13
|
|
12
|
-
ARGV[1]
|
13
|
-
ARGV[2]
|
14
|
-
ARGV[3]
|
14
|
+
ARGV[1] key prefix
|
15
|
+
ARGV[2] timestamp
|
16
|
+
ARGV[3] pushCmd
|
17
|
+
ARGV[4] jobId
|
18
|
+
ARGV[5] token
|
15
19
|
|
16
20
|
Events:
|
17
21
|
'waiting'
|
@@ -25,12 +29,15 @@ local rcall = redis.call
|
|
25
29
|
|
26
30
|
-- Includes
|
27
31
|
--- @include "includes/getTargetQueueList"
|
32
|
+
--- @include "includes/promoteDelayedJobs"
|
33
|
+
|
34
|
+
promoteDelayedJobs(KEYS[7], KEYS[2], KEYS[8], KEYS[3], KEYS[5], KEYS[6], ARGV[1], ARGV[2])
|
28
35
|
|
29
36
|
if rcall("EXISTS", KEYS[4]) == 1 then
|
30
37
|
|
31
|
-
if ARGV[
|
38
|
+
if ARGV[5] ~= "0" then
|
32
39
|
local lockKey = KEYS[4] .. ':lock'
|
33
|
-
if rcall("GET", lockKey) == ARGV[
|
40
|
+
if rcall("GET", lockKey) == ARGV[5] then
|
34
41
|
rcall("DEL", lockKey)
|
35
42
|
else
|
36
43
|
return -2
|
@@ -39,11 +46,11 @@ if rcall("EXISTS", KEYS[4]) == 1 then
|
|
39
46
|
|
40
47
|
local target = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
|
41
48
|
|
42
|
-
rcall("LREM", KEYS[1], 0, ARGV[
|
43
|
-
rcall(ARGV[
|
49
|
+
rcall("LREM", KEYS[1], 0, ARGV[4])
|
50
|
+
rcall(ARGV[3], target, ARGV[4])
|
44
51
|
|
45
52
|
-- Emit waiting event
|
46
|
-
rcall("XADD", KEYS[6], "*", "event", "waiting", "jobId", ARGV[
|
53
|
+
rcall("XADD", KEYS[6], "*", "event", "waiting", "jobId", ARGV[4], "prev", "failed");
|
47
54
|
|
48
55
|
return 0
|
49
56
|
else
|
@@ -1,13 +1,13 @@
|
|
1
|
-
import { RepeatStrategy } from '../types';
|
1
|
+
import { BackoffStrategy, RepeatStrategy } from '../types';
|
2
2
|
export interface AdvancedRepeatOptions {
|
3
3
|
/**
|
4
|
-
* A
|
4
|
+
* A custom cron strategy.
|
5
5
|
*/
|
6
6
|
repeatStrategy?: RepeatStrategy;
|
7
7
|
}
|
8
8
|
export interface AdvancedOptions extends AdvancedRepeatOptions {
|
9
9
|
/**
|
10
|
-
* A
|
10
|
+
* A custom backoff strategy.
|
11
11
|
*/
|
12
|
-
|
12
|
+
backoffStrategy?: BackoffStrategy;
|
13
13
|
}
|
@@ -23,12 +23,6 @@ export interface DefaultJobOptions {
|
|
23
23
|
* @defaultValue 0
|
24
24
|
*/
|
25
25
|
attempts?: number;
|
26
|
-
/**
|
27
|
-
* Rate limiter key to use if rate limiter enabled.
|
28
|
-
*
|
29
|
-
* @see {@link https://docs.bullmq.io/guide/rate-limiting}
|
30
|
-
*/
|
31
|
-
rateLimiterKey?: string;
|
32
26
|
/**
|
33
27
|
* Backoff setting for automatic retries if the job fails
|
34
28
|
*/
|
@@ -31,16 +31,6 @@ export interface QueueBaseOptions {
|
|
31
31
|
*/
|
32
32
|
export interface QueueOptions extends QueueBaseOptions {
|
33
33
|
defaultJobOptions?: DefaultJobOptions;
|
34
|
-
/**
|
35
|
-
* Options for the rate limiter.
|
36
|
-
*/
|
37
|
-
limiter?: {
|
38
|
-
/**
|
39
|
-
* Group key to be used by the limiter when
|
40
|
-
* limiting by group keys.
|
41
|
-
*/
|
42
|
-
groupKey: string;
|
43
|
-
};
|
44
34
|
/**
|
45
35
|
* Options for the streams used internally in BullMQ.
|
46
36
|
*/
|
@@ -9,22 +9,4 @@ export interface RateLimiterOptions {
|
|
9
9
|
* of `max` jobs will be processed.
|
10
10
|
*/
|
11
11
|
duration: number;
|
12
|
-
/**
|
13
|
-
* It is possible to define a rate limiter based on group keys,
|
14
|
-
* for example you may want to have a rate limiter per customer
|
15
|
-
* instead of a global rate limiter for all customers
|
16
|
-
*
|
17
|
-
* @see {@link https://docs.bullmq.io/guide/rate-limiting}
|
18
|
-
*/
|
19
|
-
groupKey?: string;
|
20
|
-
/**
|
21
|
-
* This option enables a heuristic so that when a queue is heavily
|
22
|
-
* rete limited, it delays the workers so that they do not try
|
23
|
-
* to pick jobs when there is no point in doing so.
|
24
|
-
* Note: It is not recommended to use this option when using
|
25
|
-
* groupKeys unless you have a big amount of workers since
|
26
|
-
* you may be delaying workers that could pick jobs in groups that
|
27
|
-
* have not been rate limited.
|
28
|
-
*/
|
29
|
-
workerDelay?: boolean;
|
30
12
|
}
|
@@ -5,11 +5,6 @@ import { ParserOptions } from 'cron-parser';
|
|
5
5
|
* @see {@link https://docs.bullmq.io/guide/jobs/repeatable}
|
6
6
|
*/
|
7
7
|
export interface RepeatOptions extends Omit<ParserOptions, 'iterator'> {
|
8
|
-
/**
|
9
|
-
* @deprecated Use pattern option instead.
|
10
|
-
* A cron pattern
|
11
|
-
*/
|
12
|
-
cron?: string;
|
13
8
|
/**
|
14
9
|
* A repeat pattern
|
15
10
|
*/
|
@@ -20,7 +15,7 @@ export interface RepeatOptions extends Omit<ParserOptions, 'iterator'> {
|
|
20
15
|
limit?: number;
|
21
16
|
/**
|
22
17
|
* Repeat after this amount of milliseconds
|
23
|
-
* (`
|
18
|
+
* (`pattern` setting cannot be used together with this setting.)
|
24
19
|
*/
|
25
20
|
every?: number;
|
26
21
|
/**
|
@@ -9,7 +9,7 @@ export * from './isFinished-3';
|
|
9
9
|
export * from './isJobInList-1';
|
10
10
|
export * from './moveStalledJobsToWait-8';
|
11
11
|
export * from './moveToActive-9';
|
12
|
-
export * from './moveToDelayed-
|
12
|
+
export * from './moveToDelayed-8';
|
13
13
|
export * from './moveToFinished-12';
|
14
14
|
export * from './moveToWaitingChildren-4';
|
15
15
|
export * from './obliterate-2';
|
@@ -19,7 +19,7 @@ export * from './releaseLock-1';
|
|
19
19
|
export * from './removeJob-1';
|
20
20
|
export * from './removeRepeatable-2';
|
21
21
|
export * from './reprocessJob-4';
|
22
|
-
export * from './retryJob-
|
22
|
+
export * from './retryJob-8';
|
23
23
|
export * from './retryJobs-6';
|
24
24
|
export * from './takeLock-1';
|
25
25
|
export * from './updateData-1';
|
@@ -12,7 +12,7 @@ tslib_1.__exportStar(require("./isFinished-3"), exports);
|
|
12
12
|
tslib_1.__exportStar(require("./isJobInList-1"), exports);
|
13
13
|
tslib_1.__exportStar(require("./moveStalledJobsToWait-8"), exports);
|
14
14
|
tslib_1.__exportStar(require("./moveToActive-9"), exports);
|
15
|
-
tslib_1.__exportStar(require("./moveToDelayed-
|
15
|
+
tslib_1.__exportStar(require("./moveToDelayed-8"), exports);
|
16
16
|
tslib_1.__exportStar(require("./moveToFinished-12"), exports);
|
17
17
|
tslib_1.__exportStar(require("./moveToWaitingChildren-4"), exports);
|
18
18
|
tslib_1.__exportStar(require("./obliterate-2"), exports);
|
@@ -22,7 +22,7 @@ tslib_1.__exportStar(require("./releaseLock-1"), exports);
|
|
22
22
|
tslib_1.__exportStar(require("./removeJob-1"), exports);
|
23
23
|
tslib_1.__exportStar(require("./removeRepeatable-2"), exports);
|
24
24
|
tslib_1.__exportStar(require("./reprocessJob-4"), exports);
|
25
|
-
tslib_1.__exportStar(require("./retryJob-
|
25
|
+
tslib_1.__exportStar(require("./retryJob-8"), exports);
|
26
26
|
tslib_1.__exportStar(require("./retryJobs-6"), exports);
|
27
27
|
tslib_1.__exportStar(require("./takeLock-1"), exports);
|
28
28
|
tslib_1.__exportStar(require("./updateData-1"), exports);
|
@@ -49,41 +49,25 @@ local rcall = redis.call
|
|
49
49
|
local function moveJobFromWaitToActive(keys, keyPrefix, jobId, processedOn, opts)
|
50
50
|
-- Check if we need to perform rate limiting.
|
51
51
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
52
|
+
local expireTime
|
52
53
|
if(maxJobs) then
|
53
54
|
local rateLimiterKey = keys[6];
|
54
|
-
local
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
if groupKey ~= jobId then
|
59
|
-
rateLimiterKey = rateLimiterKey .. ":" .. groupKey
|
60
|
-
end
|
55
|
+
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
56
|
+
if jobCounter == 1 then
|
57
|
+
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
|
58
|
+
rcall("PEXPIRE", rateLimiterKey, limiterDuration)
|
61
59
|
end
|
62
|
-
|
63
|
-
if
|
64
|
-
|
65
|
-
|
66
|
-
end
|
67
|
-
else
|
68
|
-
jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
69
|
-
end
|
70
|
-
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
|
71
|
-
-- check if rate limit hit
|
72
|
-
if jobCounter ~= nil and jobCounter > maxJobs then
|
73
|
-
local exceedingJobs = jobCounter - maxJobs
|
74
|
-
local expireTime = tonumber(rcall("PTTL", rateLimiterKey))
|
75
|
-
local delay = expireTime + ((exceedingJobs - 1) * limiterDuration) / maxJobs;
|
76
|
-
local timestamp = delay + tonumber(processedOn)
|
77
|
-
-- put job into delayed queue
|
78
|
-
rcall("ZADD", keys[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId);
|
79
|
-
rcall("XADD", keys[4], "*", "event", "delayed", "jobId", jobId, "delay", timestamp);
|
80
|
-
-- remove from active queue
|
60
|
+
-- check if we passed rate limit, we need to remove the job and return expireTime
|
61
|
+
if jobCounter > maxJobs then
|
62
|
+
expireTime = rcall("PTTL", rateLimiterKey)
|
63
|
+
-- remove from active queue and add back to the wait list
|
81
64
|
rcall("LREM", keys[2], 1, jobId)
|
65
|
+
rcall("RPUSH", keys[1], jobId)
|
82
66
|
-- Return when we can process more jobs
|
83
|
-
return expireTime
|
67
|
+
return {0, 0, expireTime}
|
84
68
|
else
|
85
|
-
if jobCounter ==
|
86
|
-
rcall("
|
69
|
+
if jobCounter == maxJobs then
|
70
|
+
expireTime = rcall("PTTL", rateLimiterKey)
|
87
71
|
end
|
88
72
|
end
|
89
73
|
end
|
@@ -97,7 +81,7 @@ local function moveJobFromWaitToActive(keys, keyPrefix, jobId, processedOn, opts
|
|
97
81
|
rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
|
98
82
|
rcall("HSET", jobKey, "processedOn", processedOn)
|
99
83
|
rcall("HINCRBY", jobKey, "attemptsMade", 1)
|
100
|
-
return {rcall("HGETALL", jobKey), jobId} -- get job data
|
84
|
+
return {rcall("HGETALL", jobKey), jobId, expireTime} -- get job data
|
101
85
|
end
|
102
86
|
--[[
|
103
87
|
Function to return the next delayed job timestamp.
|
@@ -169,19 +153,34 @@ local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
|
|
169
153
|
rcall("HSET", prefix .. jobId, "delay", 0)
|
170
154
|
end
|
171
155
|
end
|
172
|
-
|
173
|
-
|
174
|
-
|
156
|
+
end
|
157
|
+
local function getRateLimitTTL(opts, limiterKey)
|
158
|
+
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
159
|
+
if maxJobs then
|
160
|
+
local jobCounter = tonumber(rcall("GET", limiterKey))
|
161
|
+
if jobCounter ~= nil and jobCounter >= maxJobs then
|
162
|
+
local pttl = rcall("PTTL", KEYS[6])
|
163
|
+
if pttl > 0 then
|
164
|
+
return pttl
|
165
|
+
end
|
166
|
+
end
|
175
167
|
end
|
176
|
-
return
|
168
|
+
return 0
|
177
169
|
end
|
178
170
|
-- Check if there are delayed jobs that we can move to wait.
|
179
171
|
promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[9], KEYS[4], ARGV[1], ARGV[2])
|
172
|
+
local opts
|
180
173
|
if (ARGV[3] ~= "") then
|
181
174
|
jobId = ARGV[3]
|
182
175
|
-- clean stalled key
|
183
176
|
rcall("SREM", KEYS[5], jobId)
|
184
177
|
else
|
178
|
+
-- Check if we are rate limited first.
|
179
|
+
opts = cmsgpack.unpack(ARGV[4])
|
180
|
+
local pttl = getRateLimitTTL(opts, KEYS[6])
|
181
|
+
if pttl > 0 then
|
182
|
+
return { 0, 0, pttl }
|
183
|
+
end
|
185
184
|
-- no job ID, try non-blocking move from wait to active
|
186
185
|
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
187
186
|
end
|
@@ -189,14 +188,14 @@ end
|
|
189
188
|
if jobId == "0" then
|
190
189
|
rcall("LREM", KEYS[2], 1, 0)
|
191
190
|
elseif jobId then
|
192
|
-
|
191
|
+
opts = opts or cmsgpack.unpack(ARGV[4])
|
193
192
|
-- this script is not really moving, it is preparing the job for processing
|
194
193
|
return moveJobFromWaitToActive(KEYS, ARGV[1], jobId, ARGV[2], opts)
|
195
194
|
end
|
196
195
|
-- Return the timestamp for the next delayed job if any.
|
197
196
|
local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
|
198
197
|
if (nextTimestamp ~= nil) then
|
199
|
-
return nextTimestamp - tonumber(ARGV[2])
|
198
|
+
return { 0, 0, 0, nextTimestamp - tonumber(ARGV[2])}
|
200
199
|
end
|
201
200
|
`;
|
202
201
|
exports.moveToActive = {
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveToActive-9.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveToActive-9.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoMf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
File without changes
|
@@ -0,0 +1,116 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.moveToDelayed = void 0;
|
4
|
+
const content = `--[[
|
5
|
+
Moves job from active to delayed set.
|
6
|
+
Input:
|
7
|
+
KEYS[1] wait key
|
8
|
+
KEYS[2] active key
|
9
|
+
KEYS[3] priority key
|
10
|
+
KEYS[4] delayed key
|
11
|
+
KEYS[5] job key
|
12
|
+
KEYS[6] events stream
|
13
|
+
KEYS[7] paused key
|
14
|
+
KEYS[8] meta key
|
15
|
+
ARGV[1] key prefix
|
16
|
+
ARGV[2] timestamp
|
17
|
+
ARGV[3] delayedTimestamp
|
18
|
+
ARGV[4] the id of the job
|
19
|
+
ARGV[5] queue token
|
20
|
+
Output:
|
21
|
+
0 - OK
|
22
|
+
-1 - Missing job.
|
23
|
+
-3 - Job not in active set.
|
24
|
+
Events:
|
25
|
+
- delayed key.
|
26
|
+
]]
|
27
|
+
local rcall = redis.call
|
28
|
+
-- Includes
|
29
|
+
--[[
|
30
|
+
Updates the delay set, by moving delayed jobs that should
|
31
|
+
be processed now to "wait".
|
32
|
+
Events:
|
33
|
+
'waiting'
|
34
|
+
]]
|
35
|
+
local rcall = redis.call
|
36
|
+
-- Includes
|
37
|
+
--[[
|
38
|
+
Function to add job considering priority.
|
39
|
+
]]
|
40
|
+
local function addJobWithPriority(priorityKey, priority, targetKey, jobId)
|
41
|
+
rcall("ZADD", priorityKey, priority, jobId)
|
42
|
+
local count = rcall("ZCOUNT", priorityKey, 0, priority)
|
43
|
+
local len = rcall("LLEN", targetKey)
|
44
|
+
local id = rcall("LINDEX", targetKey, len - (count - 1))
|
45
|
+
if id then
|
46
|
+
rcall("LINSERT", targetKey, "BEFORE", id, jobId)
|
47
|
+
else
|
48
|
+
rcall("RPUSH", targetKey, jobId)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
--[[
|
52
|
+
Function to check for the meta.paused key to decide if we are paused or not
|
53
|
+
(since an empty list and !EXISTS are not really the same).
|
54
|
+
]]
|
55
|
+
local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
|
56
|
+
if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
|
57
|
+
return waitKey
|
58
|
+
else
|
59
|
+
return pausedKey
|
60
|
+
end
|
61
|
+
end
|
62
|
+
-- Try to get as much as 1000 jobs at once, and returns the nextTimestamp if
|
63
|
+
-- there are more delayed jobs to process.
|
64
|
+
local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
|
65
|
+
metaKey, eventStreamKey, prefix, timestamp)
|
66
|
+
local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
|
67
|
+
if (#jobs > 0) then
|
68
|
+
rcall("ZREM", delayedKey, unpack(jobs))
|
69
|
+
-- check if we need to use push in paused instead of waiting
|
70
|
+
local target = getTargetQueueList(metaKey, waitKey, pausedKey)
|
71
|
+
for _, jobId in ipairs(jobs) do
|
72
|
+
local priority =
|
73
|
+
tonumber(rcall("HGET", prefix .. jobId, "priority")) or 0
|
74
|
+
if priority == 0 then
|
75
|
+
-- LIFO or FIFO
|
76
|
+
rcall("LPUSH", target, jobId)
|
77
|
+
else
|
78
|
+
addJobWithPriority(priorityKey, priority, target, jobId)
|
79
|
+
end
|
80
|
+
-- Emit waiting event
|
81
|
+
rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
|
82
|
+
jobId, "prev", "delayed")
|
83
|
+
rcall("HSET", prefix .. jobId, "delay", 0)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
promoteDelayedJobs(KEYS[4], KEYS[1], KEYS[3], KEYS[7], KEYS[8], KEYS[6], ARGV[1], ARGV[2])
|
88
|
+
if rcall("EXISTS", KEYS[5]) == 1 then
|
89
|
+
if ARGV[5] ~= "0" then
|
90
|
+
local lockKey = KEYS[5] .. ':lock'
|
91
|
+
if rcall("GET", lockKey) == ARGV[5] then
|
92
|
+
rcall("DEL", lockKey)
|
93
|
+
else
|
94
|
+
return -2
|
95
|
+
end
|
96
|
+
end
|
97
|
+
local jobId = ARGV[4]
|
98
|
+
local score = tonumber(ARGV[3])
|
99
|
+
local delayedTimestamp = (score / 0x1000)
|
100
|
+
local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
|
101
|
+
if(numRemovedElements < 1) then
|
102
|
+
return -3
|
103
|
+
end
|
104
|
+
rcall("ZADD", KEYS[4], score, jobId)
|
105
|
+
rcall("XADD", KEYS[6], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
|
106
|
+
return 0
|
107
|
+
else
|
108
|
+
return -1
|
109
|
+
end
|
110
|
+
`;
|
111
|
+
exports.moveToDelayed = {
|
112
|
+
name: 'moveToDelayed',
|
113
|
+
content,
|
114
|
+
keys: 8,
|
115
|
+
};
|
116
|
+
//# sourceMappingURL=moveToDelayed-8.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"moveToDelayed-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToDelayed-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Gf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -134,41 +134,25 @@ end
|
|
134
134
|
local function moveJobFromWaitToActive(keys, keyPrefix, jobId, processedOn, opts)
|
135
135
|
-- Check if we need to perform rate limiting.
|
136
136
|
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
137
|
+
local expireTime
|
137
138
|
if(maxJobs) then
|
138
139
|
local rateLimiterKey = keys[6];
|
139
|
-
local
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
if groupKey ~= jobId then
|
144
|
-
rateLimiterKey = rateLimiterKey .. ":" .. groupKey
|
145
|
-
end
|
146
|
-
end
|
147
|
-
local jobCounter
|
148
|
-
if groupKey ~= nil then
|
149
|
-
if rateLimiterKey ~= keys[6] then
|
150
|
-
jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
151
|
-
end
|
152
|
-
else
|
153
|
-
jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
140
|
+
local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
|
141
|
+
if jobCounter == 1 then
|
142
|
+
local limiterDuration = opts['limiter'] and opts['limiter']['duration']
|
143
|
+
rcall("PEXPIRE", rateLimiterKey, limiterDuration)
|
154
144
|
end
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
local expireTime = tonumber(rcall("PTTL", rateLimiterKey))
|
160
|
-
local delay = expireTime + ((exceedingJobs - 1) * limiterDuration) / maxJobs;
|
161
|
-
local timestamp = delay + tonumber(processedOn)
|
162
|
-
-- put job into delayed queue
|
163
|
-
rcall("ZADD", keys[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId);
|
164
|
-
rcall("XADD", keys[4], "*", "event", "delayed", "jobId", jobId, "delay", timestamp);
|
165
|
-
-- remove from active queue
|
145
|
+
-- check if we passed rate limit, we need to remove the job and return expireTime
|
146
|
+
if jobCounter > maxJobs then
|
147
|
+
expireTime = rcall("PTTL", rateLimiterKey)
|
148
|
+
-- remove from active queue and add back to the wait list
|
166
149
|
rcall("LREM", keys[2], 1, jobId)
|
150
|
+
rcall("RPUSH", keys[1], jobId)
|
167
151
|
-- Return when we can process more jobs
|
168
|
-
return expireTime
|
152
|
+
return {0, 0, expireTime}
|
169
153
|
else
|
170
|
-
if jobCounter ==
|
171
|
-
rcall("
|
154
|
+
if jobCounter == maxJobs then
|
155
|
+
expireTime = rcall("PTTL", rateLimiterKey)
|
172
156
|
end
|
173
157
|
end
|
174
158
|
end
|
@@ -182,7 +166,7 @@ local function moveJobFromWaitToActive(keys, keyPrefix, jobId, processedOn, opts
|
|
182
166
|
rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
|
183
167
|
rcall("HSET", jobKey, "processedOn", processedOn)
|
184
168
|
rcall("HINCRBY", jobKey, "attemptsMade", 1)
|
185
|
-
return {rcall("HGETALL", jobKey), jobId} -- get job data
|
169
|
+
return {rcall("HGETALL", jobKey), jobId, expireTime} -- get job data
|
186
170
|
end
|
187
171
|
--[[
|
188
172
|
Function to recursively move from waitingChildren to failed.
|
@@ -266,11 +250,6 @@ local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
|
|
266
250
|
rcall("HSET", prefix .. jobId, "delay", 0)
|
267
251
|
end
|
268
252
|
end
|
269
|
-
local nextTimestamp = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")[2]
|
270
|
-
if (nextTimestamp ~= nil) then
|
271
|
-
nextTimestamp = nextTimestamp / 0x1000
|
272
|
-
end
|
273
|
-
return nextTimestamp
|
274
253
|
end
|
275
254
|
--[[
|
276
255
|
Functions to remove jobs by max age.
|
@@ -421,6 +400,19 @@ local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDepende
|
|
421
400
|
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
|
422
401
|
end
|
423
402
|
end
|
403
|
+
local function getRateLimitTTL(opts, limiterKey)
|
404
|
+
local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
|
405
|
+
if maxJobs then
|
406
|
+
local jobCounter = tonumber(rcall("GET", limiterKey))
|
407
|
+
if jobCounter ~= nil and jobCounter >= maxJobs then
|
408
|
+
local pttl = rcall("PTTL", KEYS[6])
|
409
|
+
if pttl > 0 then
|
410
|
+
return pttl
|
411
|
+
end
|
412
|
+
end
|
413
|
+
end
|
414
|
+
return 0
|
415
|
+
end
|
424
416
|
local jobIdKey = KEYS[10]
|
425
417
|
if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
426
418
|
local opts = cmsgpack.unpack(ARGV[9])
|
@@ -507,6 +499,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
507
499
|
if (ARGV[7] == "1") then
|
508
500
|
-- Check if there are delayed jobs that can be promoted
|
509
501
|
promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[11], KEYS[4], ARGV[8], timestamp)
|
502
|
+
-- Check if we are rate limited first.
|
503
|
+
local pttl = getRateLimitTTL(opts, KEYS[6])
|
504
|
+
if pttl > 0 then
|
505
|
+
return { 0, 0, pttl }
|
506
|
+
end
|
510
507
|
jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
|
511
508
|
if jobId == "0" then
|
512
509
|
rcall("LREM", KEYS[2], 1, 0)
|
@@ -518,7 +515,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
518
515
|
if (nextTimestamp ~= nil) then
|
519
516
|
-- The result is guaranteed to be positive, since the
|
520
517
|
-- ZRANGEBYSCORE command would have return a job otherwise.
|
521
|
-
return nextTimestamp - timestamp
|
518
|
+
return {0, 0, 0, nextTimestamp - timestamp}
|
522
519
|
end
|
523
520
|
end
|
524
521
|
local waitLen = rcall("LLEN", KEYS[1])
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveToFinished-12.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-12.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveToFinished-12.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-12.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAghBf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|
File without changes
|