bullmq 2.0.1 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/job.d.ts +4 -2
- package/dist/cjs/classes/job.js +38 -3
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue.d.ts +2 -2
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/repeat.d.ts +2 -1
- package/dist/cjs/classes/scripts.d.ts +3 -8
- package/dist/cjs/classes/scripts.js +5 -20
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +25 -0
- package/dist/cjs/commands/includes/promoteDelayedJobs.lua +3 -11
- package/dist/cjs/commands/moveToFinished-12.lua +16 -13
- package/dist/cjs/interfaces/{jobs-options.d.ts → base-job-options.d.ts} +2 -2
- package/dist/cjs/interfaces/base-job-options.js +3 -0
- package/dist/cjs/interfaces/base-job-options.js.map +1 -0
- package/dist/cjs/interfaces/flow-job.d.ts +1 -1
- package/dist/cjs/interfaces/index.d.ts +1 -1
- package/dist/cjs/interfaces/index.js +1 -1
- package/dist/cjs/interfaces/index.js.map +1 -1
- package/dist/cjs/interfaces/job-json.d.ts +2 -2
- package/dist/cjs/interfaces/parent.d.ts +1 -1
- package/dist/cjs/interfaces/queue-options.d.ts +2 -2
- package/dist/cjs/interfaces/sandboxed-job.d.ts +1 -1
- package/dist/cjs/types/index.d.ts +1 -0
- package/dist/cjs/types/index.js +1 -0
- package/dist/cjs/types/index.js.map +1 -1
- package/dist/cjs/types/job-options.d.ts +16 -0
- package/dist/cjs/{interfaces/jobs-options.js → types/job-options.js} +1 -1
- package/dist/cjs/types/job-options.js.map +1 -0
- package/dist/cjs/utils.d.ts +2 -1
- package/dist/esm/classes/job.d.ts +4 -2
- package/dist/esm/classes/job.js +39 -4
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +2 -2
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/repeat.d.ts +2 -1
- package/dist/esm/classes/scripts.d.ts +3 -8
- package/dist/esm/classes/scripts.js +5 -20
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +25 -0
- package/dist/esm/commands/includes/promoteDelayedJobs.lua +3 -11
- package/dist/esm/commands/moveToFinished-12.lua +16 -13
- package/dist/esm/interfaces/{jobs-options.d.ts → base-job-options.d.ts} +2 -2
- package/dist/esm/interfaces/base-job-options.js +2 -0
- package/dist/esm/interfaces/base-job-options.js.map +1 -0
- package/dist/esm/interfaces/flow-job.d.ts +1 -1
- package/dist/esm/interfaces/index.d.ts +1 -1
- package/dist/esm/interfaces/index.js +1 -1
- package/dist/esm/interfaces/index.js.map +1 -1
- package/dist/esm/interfaces/job-json.d.ts +2 -2
- package/dist/esm/interfaces/parent.d.ts +1 -1
- package/dist/esm/interfaces/queue-options.d.ts +2 -2
- package/dist/esm/interfaces/sandboxed-job.d.ts +1 -1
- package/dist/esm/types/index.d.ts +1 -0
- package/dist/esm/types/index.js +1 -0
- package/dist/esm/types/index.js.map +1 -1
- package/dist/esm/types/job-options.d.ts +16 -0
- package/dist/esm/types/job-options.js +2 -0
- package/dist/esm/types/job-options.js.map +1 -0
- package/dist/esm/utils.d.ts +2 -1
- package/package.json +1 -1
- package/dist/cjs/commands/updateDelaySet-7.lua +0 -71
- package/dist/cjs/interfaces/jobs-options.js.map +0 -1
- package/dist/esm/commands/updateDelaySet-7.lua +0 -71
- package/dist/esm/interfaces/jobs-options.js +0 -2
- package/dist/esm/interfaces/jobs-options.js.map +0 -1
@@ -24,13 +24,13 @@
|
|
24
24
|
|
25
25
|
ARGV[1] jobId
|
26
26
|
ARGV[2] timestamp
|
27
|
-
ARGV[3] msg property
|
27
|
+
ARGV[3] msg property returnvalue / failedReason
|
28
28
|
ARGV[4] return value / failed reason
|
29
29
|
ARGV[5] target (completed/failed)
|
30
30
|
ARGV[6] event data (? maybe just send jobid).
|
31
31
|
ARGV[7] fetch next?
|
32
32
|
ARGV[8] keys prefix
|
33
|
-
ARGV[9]
|
33
|
+
ARGV[9] opts
|
34
34
|
|
35
35
|
opts - token - lock token
|
36
36
|
opts - keepJobs
|
@@ -40,6 +40,7 @@
|
|
40
40
|
opts - attempts max attempts
|
41
41
|
opts - attemptsMade
|
42
42
|
opts - maxMetricsSize
|
43
|
+
opts - fpof - fail parent on fail
|
43
44
|
|
44
45
|
Output:
|
45
46
|
0 OK
|
@@ -54,15 +55,15 @@
|
|
54
55
|
local rcall = redis.call
|
55
56
|
|
56
57
|
--- Includes
|
57
|
-
--- @include "includes/
|
58
|
+
--- @include "includes/collectMetrics"
|
59
|
+
--- @include "includes/getNextDelayedTimestamp"
|
58
60
|
--- @include "includes/moveJobFromWaitToActive"
|
61
|
+
--- @include "includes/moveParentFromWaitingChildrenToFailed"
|
62
|
+
--- @include "includes/promoteDelayedJobs"
|
59
63
|
--- @include "includes/removeJobsByMaxAge"
|
60
64
|
--- @include "includes/removeJobsByMaxCount"
|
61
65
|
--- @include "includes/trimEvents"
|
62
66
|
--- @include "includes/updateParentDepsIfNeeded"
|
63
|
-
--- @include "includes/collectMetrics"
|
64
|
-
--- @include "includes/getNextDelayedTimestamp"
|
65
|
-
--- @include "includes/promoteDelayedJobs"
|
66
67
|
|
67
68
|
local jobIdKey = KEYS[10]
|
68
69
|
if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
@@ -112,13 +113,15 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
|
|
112
113
|
parentId = getJobIdFromKey(parentKey)
|
113
114
|
parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
|
114
115
|
end
|
115
|
-
if parentId ~= ""
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
116
|
+
if parentId ~= "" then
|
117
|
+
if ARGV[5] == "completed" then
|
118
|
+
local dependenciesSet = parentKey .. ":dependencies"
|
119
|
+
if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
|
120
|
+
updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet,
|
121
|
+
parentId, jobIdKey, ARGV[4])
|
122
|
+
end
|
123
|
+
elseif opts['fpof'] then
|
124
|
+
moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
|
122
125
|
end
|
123
126
|
end
|
124
127
|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
import { RepeatOptions, KeepJobs, BackoffOptions } from './';
|
2
|
-
export interface
|
2
|
+
export interface DefaultJobOptions {
|
3
3
|
/**
|
4
4
|
* Timestamp when the job was created.
|
5
5
|
* @defaultValue Date.now()
|
@@ -63,7 +63,7 @@ export interface BaseJobOptions {
|
|
63
63
|
*/
|
64
64
|
sizeLimit?: number;
|
65
65
|
}
|
66
|
-
export interface
|
66
|
+
export interface BaseJobOptions extends DefaultJobOptions {
|
67
67
|
/**
|
68
68
|
* Repeat this job, for example based on a `cron` schedule.
|
69
69
|
*/
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"base-job-options.js","sourceRoot":"","sources":["../../../src/interfaces/base-job-options.ts"],"names":[],"mappings":""}
|
@@ -1,12 +1,12 @@
|
|
1
1
|
export * from './advanced-options';
|
2
2
|
export * from './backoff-options';
|
3
|
+
export * from './base-job-options';
|
3
4
|
export * from './child-command';
|
4
5
|
export * from './child-message';
|
5
6
|
export * from './connection';
|
6
7
|
export * from './flow-job';
|
7
8
|
export * from './ioredis-events';
|
8
9
|
export * from './job-json';
|
9
|
-
export * from './jobs-options';
|
10
10
|
export * from './keep-jobs';
|
11
11
|
export * from './metrics-options';
|
12
12
|
export * from './metrics';
|
@@ -1,12 +1,12 @@
|
|
1
1
|
export * from './advanced-options';
|
2
2
|
export * from './backoff-options';
|
3
|
+
export * from './base-job-options';
|
3
4
|
export * from './child-command';
|
4
5
|
export * from './child-message';
|
5
6
|
export * from './connection';
|
6
7
|
export * from './flow-job';
|
7
8
|
export * from './ioredis-events';
|
8
9
|
export * from './job-json';
|
9
|
-
export * from './jobs-options';
|
10
10
|
export * from './keep-jobs';
|
11
11
|
export * from './metrics-options';
|
12
12
|
export * from './metrics';
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,YAAY,CAAC;AAC3B,cAAc,
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,WAAW,CAAC;AAC1B,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC"}
|
@@ -1,10 +1,10 @@
|
|
1
|
-
import {
|
1
|
+
import { RedisJobOptions } from '../types';
|
2
2
|
import { ParentKeys } from './parent';
|
3
3
|
export interface JobJson {
|
4
4
|
id: string;
|
5
5
|
name: string;
|
6
6
|
data: string;
|
7
|
-
opts:
|
7
|
+
opts: RedisJobOptions;
|
8
8
|
progress: number | object;
|
9
9
|
attemptsMade: number;
|
10
10
|
finishedOn?: number;
|
@@ -1,5 +1,5 @@
|
|
1
1
|
import { AdvancedRepeatOptions } from './advanced-options';
|
2
|
-
import {
|
2
|
+
import { DefaultJobOptions } from './base-job-options';
|
3
3
|
import { ConnectionOptions } from './redis-options';
|
4
4
|
export declare enum ClientType {
|
5
5
|
blocking = "blocking",
|
@@ -30,7 +30,7 @@ export interface QueueBaseOptions {
|
|
30
30
|
* Options for the Queue class.
|
31
31
|
*/
|
32
32
|
export interface QueueOptions extends QueueBaseOptions {
|
33
|
-
defaultJobOptions?:
|
33
|
+
defaultJobOptions?: DefaultJobOptions;
|
34
34
|
/**
|
35
35
|
* Options for the rate limiter.
|
36
36
|
*/
|
package/dist/esm/types/index.js
CHANGED
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/types/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC"}
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/types/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC"}
|
@@ -0,0 +1,16 @@
|
|
1
|
+
import { BaseJobOptions } from '../interfaces';
|
2
|
+
export declare type JobsOptions = BaseJobOptions & {
|
3
|
+
/**
|
4
|
+
* If true, moves parent to failed.
|
5
|
+
*/
|
6
|
+
failParentOnFailure?: boolean;
|
7
|
+
};
|
8
|
+
/**
|
9
|
+
* These fields are the ones stored in Redis with smaller keys for compactness.
|
10
|
+
*/
|
11
|
+
export declare type RedisJobOptions = BaseJobOptions & {
|
12
|
+
/**
|
13
|
+
* If true, moves parent to failed.
|
14
|
+
*/
|
15
|
+
fpof?: boolean;
|
16
|
+
};
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"job-options.js","sourceRoot":"","sources":["../../../src/types/job-options.ts"],"names":[],"mappings":""}
|
package/dist/esm/utils.d.ts
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
/// <reference types="node" />
|
2
2
|
/// <reference types="node" />
|
3
3
|
import { Cluster, Redis } from 'ioredis';
|
4
|
-
import { ChildMessage,
|
4
|
+
import { ChildMessage, ParentMessage, QueueOptions, RedisClient } from './interfaces';
|
5
|
+
import { JobsOptions } from './types';
|
5
6
|
import { ChildProcess } from 'child_process';
|
6
7
|
export declare const errorObject: {
|
7
8
|
[index: string]: any;
|
package/package.json
CHANGED
@@ -1,71 +0,0 @@
|
|
1
|
-
--[[
|
2
|
-
Updates the delay set, by picking a delayed job that should
|
3
|
-
be processed now.
|
4
|
-
|
5
|
-
Input:
|
6
|
-
KEYS[1] 'delayed'
|
7
|
-
KEYS[2] 'wait'
|
8
|
-
KEYS[3] 'priority'
|
9
|
-
KEYS[4] 'paused'
|
10
|
-
KEYS[5] 'meta'
|
11
|
-
|
12
|
-
KEYS[6] event's stream
|
13
|
-
KEYS[7] delayed stream
|
14
|
-
|
15
|
-
ARGV[1] queue.toKey('')
|
16
|
-
ARGV[2] delayed timestamp
|
17
|
-
|
18
|
-
Events:
|
19
|
-
'waiting'
|
20
|
-
]]
|
21
|
-
local rcall = redis.call
|
22
|
-
|
23
|
-
-- Includes
|
24
|
-
--- @include "includes/getTargetQueueList"
|
25
|
-
|
26
|
-
-- Try to get as much as 1000 jobs at once
|
27
|
-
local jobs = rcall("ZRANGEBYSCORE", KEYS[1], 0, tonumber(ARGV[2]) * 0x1000,
|
28
|
-
"LIMIT", 0, 1000)
|
29
|
-
|
30
|
-
if (#jobs > 0) then
|
31
|
-
rcall("ZREM", KEYS[1], unpack(jobs))
|
32
|
-
|
33
|
-
-- check if we need to use push in paused instead of waiting
|
34
|
-
local target = getTargetQueueList(KEYS[5], KEYS[2], KEYS[4])
|
35
|
-
|
36
|
-
for _, jobId in ipairs(jobs) do
|
37
|
-
local priority =
|
38
|
-
tonumber(rcall("HGET", ARGV[1] .. jobId, "priority")) or 0
|
39
|
-
|
40
|
-
if priority == 0 then
|
41
|
-
-- LIFO or FIFO
|
42
|
-
rcall("LPUSH", target, jobId)
|
43
|
-
else
|
44
|
-
-- Priority add
|
45
|
-
rcall("ZADD", KEYS[3], priority, jobId)
|
46
|
-
local count = rcall("ZCOUNT", KEYS[3], 0, priority)
|
47
|
-
|
48
|
-
local len = rcall("LLEN", target)
|
49
|
-
local id = rcall("LINDEX", target, len - (count - 1))
|
50
|
-
if id then
|
51
|
-
rcall("LINSERT", target, "BEFORE", id, jobId)
|
52
|
-
else
|
53
|
-
rcall("RPUSH", target, jobId)
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
-- Emit waiting event
|
58
|
-
rcall("XADD", KEYS[6], "*", "event", "waiting", "jobId", jobId, "prev",
|
59
|
-
"delayed")
|
60
|
-
rcall("HSET", ARGV[1] .. jobId, "delay", 0)
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
local nextTimestamp = rcall("ZRANGE", KEYS[1], 0, 0, "WITHSCORES")[2]
|
65
|
-
local id
|
66
|
-
if (nextTimestamp ~= nil) then
|
67
|
-
nextTimestamp = nextTimestamp / 0x1000
|
68
|
-
id = rcall("XADD", KEYS[7], "*", "nextTimestamp", nextTimestamp)
|
69
|
-
end
|
70
|
-
|
71
|
-
return {nextTimestamp, id}
|
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"jobs-options.js","sourceRoot":"","sources":["../../../src/interfaces/jobs-options.ts"],"names":[],"mappings":""}
|
@@ -1,71 +0,0 @@
|
|
1
|
-
--[[
|
2
|
-
Updates the delay set, by picking a delayed job that should
|
3
|
-
be processed now.
|
4
|
-
|
5
|
-
Input:
|
6
|
-
KEYS[1] 'delayed'
|
7
|
-
KEYS[2] 'wait'
|
8
|
-
KEYS[3] 'priority'
|
9
|
-
KEYS[4] 'paused'
|
10
|
-
KEYS[5] 'meta'
|
11
|
-
|
12
|
-
KEYS[6] event's stream
|
13
|
-
KEYS[7] delayed stream
|
14
|
-
|
15
|
-
ARGV[1] queue.toKey('')
|
16
|
-
ARGV[2] delayed timestamp
|
17
|
-
|
18
|
-
Events:
|
19
|
-
'waiting'
|
20
|
-
]]
|
21
|
-
local rcall = redis.call
|
22
|
-
|
23
|
-
-- Includes
|
24
|
-
--- @include "includes/getTargetQueueList"
|
25
|
-
|
26
|
-
-- Try to get as much as 1000 jobs at once
|
27
|
-
local jobs = rcall("ZRANGEBYSCORE", KEYS[1], 0, tonumber(ARGV[2]) * 0x1000,
|
28
|
-
"LIMIT", 0, 1000)
|
29
|
-
|
30
|
-
if (#jobs > 0) then
|
31
|
-
rcall("ZREM", KEYS[1], unpack(jobs))
|
32
|
-
|
33
|
-
-- check if we need to use push in paused instead of waiting
|
34
|
-
local target = getTargetQueueList(KEYS[5], KEYS[2], KEYS[4])
|
35
|
-
|
36
|
-
for _, jobId in ipairs(jobs) do
|
37
|
-
local priority =
|
38
|
-
tonumber(rcall("HGET", ARGV[1] .. jobId, "priority")) or 0
|
39
|
-
|
40
|
-
if priority == 0 then
|
41
|
-
-- LIFO or FIFO
|
42
|
-
rcall("LPUSH", target, jobId)
|
43
|
-
else
|
44
|
-
-- Priority add
|
45
|
-
rcall("ZADD", KEYS[3], priority, jobId)
|
46
|
-
local count = rcall("ZCOUNT", KEYS[3], 0, priority)
|
47
|
-
|
48
|
-
local len = rcall("LLEN", target)
|
49
|
-
local id = rcall("LINDEX", target, len - (count - 1))
|
50
|
-
if id then
|
51
|
-
rcall("LINSERT", target, "BEFORE", id, jobId)
|
52
|
-
else
|
53
|
-
rcall("RPUSH", target, jobId)
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
-- Emit waiting event
|
58
|
-
rcall("XADD", KEYS[6], "*", "event", "waiting", "jobId", jobId, "prev",
|
59
|
-
"delayed")
|
60
|
-
rcall("HSET", ARGV[1] .. jobId, "delay", 0)
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
local nextTimestamp = rcall("ZRANGE", KEYS[1], 0, 0, "WITHSCORES")[2]
|
65
|
-
local id
|
66
|
-
if (nextTimestamp ~= nil) then
|
67
|
-
nextTimestamp = nextTimestamp / 0x1000
|
68
|
-
id = rcall("XADD", KEYS[7], "*", "nextTimestamp", nextTimestamp)
|
69
|
-
end
|
70
|
-
|
71
|
-
return {nextTimestamp, id}
|
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"jobs-options.js","sourceRoot":"","sources":["../../../src/interfaces/jobs-options.ts"],"names":[],"mappings":""}
|