bullmq 1.76.6 → 1.77.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bullmq.d.ts +1 -1
- package/dist/cjs/classes/queue-scheduler.d.ts +2 -1
- package/dist/cjs/classes/queue-scheduler.js +2 -2
- package/dist/cjs/classes/queue-scheduler.js.map +1 -1
- package/dist/cjs/commands/addJob-9.lua +3 -2
- package/dist/cjs/commands/cleanJobsInSet-2.lua +14 -4
- package/dist/cjs/commands/includes/removeJob.lua +1 -1
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +34 -5
- package/dist/cjs/commands/includes/trimEvents.lua +1 -1
- package/dist/cjs/commands/moveStalledJobsToWait-8.lua +3 -2
- package/dist/cjs/commands/removeJob-1.lua +23 -14
- package/dist/esm/classes/queue-scheduler.d.ts +2 -1
- package/dist/esm/classes/queue-scheduler.js +2 -2
- package/dist/esm/classes/queue-scheduler.js.map +1 -1
- package/dist/esm/commands/addJob-9.lua +3 -2
- package/dist/esm/commands/cleanJobsInSet-2.lua +14 -4
- package/dist/esm/commands/includes/removeJob.lua +1 -1
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +34 -5
- package/dist/esm/commands/includes/trimEvents.lua +1 -1
- package/dist/esm/commands/moveStalledJobsToWait-8.lua +3 -2
- package/dist/esm/commands/removeJob-1.lua +23 -14
- package/package.json +1 -1
package/dist/bullmq.d.ts
CHANGED
|
@@ -1811,7 +1811,7 @@ export declare class QueueScheduler extends QueueBase {
|
|
|
1811
1811
|
private nextTimestamp;
|
|
1812
1812
|
private isBlocked;
|
|
1813
1813
|
private running;
|
|
1814
|
-
constructor(name: string, { connection, autorun, ...opts }?: QueueSchedulerOptions);
|
|
1814
|
+
constructor(name: string, { connection, autorun, ...opts }?: QueueSchedulerOptions, Connection?: typeof RedisConnection);
|
|
1815
1815
|
emit<U extends keyof QueueSchedulerListener>(event: U, ...args: Parameters<QueueSchedulerListener[U]>): boolean;
|
|
1816
1816
|
off<U extends keyof QueueSchedulerListener>(eventName: U, listener: QueueSchedulerListener[U]): this;
|
|
1817
1817
|
on<U extends keyof QueueSchedulerListener>(event: U, listener: QueueSchedulerListener[U]): this;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { QueueSchedulerOptions } from '../interfaces';
|
|
2
2
|
import { QueueBase } from './queue-base';
|
|
3
|
+
import { RedisConnection } from './redis-connection';
|
|
3
4
|
export interface QueueSchedulerListener {
|
|
4
5
|
/**
|
|
5
6
|
* Listen to 'stalled' event.
|
|
@@ -34,7 +35,7 @@ export declare class QueueScheduler extends QueueBase {
|
|
|
34
35
|
private nextTimestamp;
|
|
35
36
|
private isBlocked;
|
|
36
37
|
private running;
|
|
37
|
-
constructor(name: string, { connection, autorun, ...opts }?: QueueSchedulerOptions);
|
|
38
|
+
constructor(name: string, { connection, autorun, ...opts }?: QueueSchedulerOptions, Connection?: typeof RedisConnection);
|
|
38
39
|
emit<U extends keyof QueueSchedulerListener>(event: U, ...args: Parameters<QueueSchedulerListener[U]>): boolean;
|
|
39
40
|
off<U extends keyof QueueSchedulerListener>(eventName: U, listener: QueueSchedulerListener[U]): this;
|
|
40
41
|
on<U extends keyof QueueSchedulerListener>(event: U, listener: QueueSchedulerListener[U]): this;
|
|
@@ -22,11 +22,11 @@ const scripts_1 = require("./scripts");
|
|
|
22
22
|
*
|
|
23
23
|
*/
|
|
24
24
|
class QueueScheduler extends queue_base_1.QueueBase {
|
|
25
|
-
constructor(name, _a = {}) {
|
|
25
|
+
constructor(name, _a = {}, Connection) {
|
|
26
26
|
var { connection, autorun = true } = _a, opts = tslib_1.__rest(_a, ["connection", "autorun"]);
|
|
27
27
|
super(name, Object.assign(Object.assign({ maxStalledCount: 1, stalledInterval: 30000 }, opts), { connection: utils_1.isRedisInstance(connection)
|
|
28
28
|
? connection.duplicate()
|
|
29
|
-
: connection, sharedConnection: false }));
|
|
29
|
+
: connection, sharedConnection: false }), Connection);
|
|
30
30
|
this.nextTimestamp = Number.MAX_VALUE;
|
|
31
31
|
this.isBlocked = false;
|
|
32
32
|
this.running = false;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue-scheduler.js","sourceRoot":"","sources":["../../../src/classes/queue-scheduler.ts"],"names":[],"mappings":";;;;AAKA,oCAA+E;AAC/E,6CAAyC;AACzC,uCAAoC;
|
|
1
|
+
{"version":3,"file":"queue-scheduler.js","sourceRoot":"","sources":["../../../src/classes/queue-scheduler.ts"],"names":[],"mappings":";;;;AAKA,oCAA+E;AAC/E,6CAAyC;AACzC,uCAAoC;AAmBpC;;;;;;;;;;;;;;;GAeG;AACH,MAAa,cAAe,SAAQ,sBAAS;IAK3C,YACE,IAAY,EACZ,KAAiE,EAAE,EACnE,UAAmC;YADnC,EAAE,UAAU,EAAE,OAAO,GAAG,IAAI,OAAuC,EAAlC,IAAI,sBAArC,yBAAuC,CAAF;QAGrC,KAAK,CACH,IAAI,gCAEF,eAAe,EAAE,CAAC,EAClB,eAAe,EAAE,KAAK,IACnB,IAAI,KACP,UAAU,EAAE,uBAAe,CAAC,UAAU,CAAC;gBACrC,CAAC,CAAe,UAAW,CAAC,SAAS,EAAE;gBACvC,CAAC,CAAC,UAAU,EACd,gBAAgB,EAAE,KAAK,KAEzB,UAAU,CACX,CAAC;QArBI,kBAAa,GAAG,MAAM,CAAC,SAAS,CAAC;QACjC,cAAS,GAAG,KAAK,CAAC;QAClB,YAAO,GAAG,KAAK,CAAC;QAqBtB,IAAI,CAAE,IAAI,CAAC,IAA8B,CAAC,eAAe,EAAE;YACzD,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;SACjE;QAED,IAAI,OAAO,EAAE;YACX,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACvB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC,CAAC,CAAC;SACJ;IACH,CAAC;IAED,IAAI,CACF,KAAQ,EACR,GAAG,IAA2C;QAE9C,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CACD,SAAY,EACZ,QAAmC;QAEnC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,EAAE,CACA,KAAQ,EACR,QAAmC;QAEnC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CACF,KAAQ,EACR,QAAmC;QAEnC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI;gBACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;gBAE3C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;gBAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,IAA6B,CAAC;gBAEhD,IAAI;oBACF,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;iBACnD;gBAAC,OAAO,GAAG,EAAE;oBACZ,IAAI,CAAC,+BAAuB,CAAC,IAAI,CAAS,GAAI,CAAC,OAAO,CAAC,EAAE;wBACvD,MAAM,GAAG,CAAC;qBACX;iBACF;gBAED,MAAM,CAAC,aAAa,EAAE,QAAQ,GAAG,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CACjE,IAAI,CAAC,GAAG,EAAE,CACX,CAAC;gBACF,IAAI,YAAY,GAAG,QAAQ,CAAC;gBAE5B,IAAI,aAAa,EAAE;oBACjB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;iBACpC;gBAED,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE;oBACpB,2DAA2D;oBAC3D,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;oBAEnC,iEAAiE;oBACjE,2CAA2C;oBAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAElD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CACvD,CAAC;oBAEF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CACrC,MAAM,EACN,GAAG,EACH,YAAY,EACZ,SAAS,CACV,CAAC;oBAEF,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE;wBACnB,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;wBACvB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;wBAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;4BACtC,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;4BAC5B,MAAM,IAAI,GAAG,iBAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;4BACrC,MAAM,aAAa,GAAW,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;4BAE3D,IAAI,aAAa,GAAG,IAAI,CAAC,aAAa,EAAE;gCACtC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;6BACpC;yBACF;wBAED,EAAE;wBACF,gEAAgE;wBAChE,6BAA6B;wBAC7B,EAAE;wBACF,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;4BACjB,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;yBAC7C;qBACF;oBAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBACvB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC;oBAEvC,IAAI,KAAK,IAAI,CAAC,EAAE;wBACd,MAAM,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;wBAC3D,IAAI,aAAa,EAAE;4BACjB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;4BACnC,YAAY,GAAG,EAAE,CAAC;yBACnB;6BAAM;4BACL,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC;yBACvC;qBACF;iBACF;gBACD,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;aACtB;YAAC,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAC;aACb;SACF;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;SACxD;IACH,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,MAAmB,EACnB,GAAW,EACX,YAAoB,EACpB,SAAiB;QAEjB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI,IAAI,CAAC;YACT,IAAI,SAAS,EAAE;gBACb,IAAI;oBACF,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;oBACtB,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CACvB,OAAO,EACP,SAAS,EACT,SAAS,EACT,GAAG,EACH,YAAY,CACb,CAAC;iBACH;gBAAC,OAAO,GAAG,EAAE;oBACZ,yCAAyC;oBACzC,IAAY,GAAI,CAAC,OAAO,KAAK,uBAAuB,EAAE;wBACpD,MAAM,GAAG,CAAC;qBACX;iBACF;wBAAS;oBACR,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;iBACxB;aACF;iBAAM;gBACL,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,YAAY,CAAC,CAAC;aACzD;YAED,mGAAmG;YACnG,OAAO,IAAW,CAAC;SACpB;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,SAAiB;QAC5C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,OAAO,iBAAO,CAAC,cAAc,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;SAChD;QACD,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,qBAAqB;QACjC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,iBAAO,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;YAEpE,MAAM,CAAC,OAAO,CAAC,CAAC,KAAa,EAAE,EAAE,CAC/B,IAAI,CAAC,IAAI,CACP,QAAQ,EACR,KAAK,EACL,IAAI,KAAK,CAAC,uCAAuC,CAAC,EAClD,QAAQ,CACT,CACF,CAAC;YACF,OAAO,CAAC,OAAO,CAAC,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;SAC3E;IACH,CAAC;IAES,UAAU;QAClB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1C,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,eAAe,KAAK,CAAC;IACrD,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,OAAO,IAAI,CAAC,OAAO,CAAC;SACrB;QACD,IAAI,IAAI,CAAC,SAAS,EAAE;YAClB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;SAClC;aAAM;YACL,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;SAC9B;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF;AA3OD,wCA2OC"}
|
|
@@ -77,6 +77,9 @@ local jobCounter = rcall("INCR", KEYS[4])
|
|
|
77
77
|
-- Includes
|
|
78
78
|
--- @include "includes/updateParentDepsIfNeeded"
|
|
79
79
|
|
|
80
|
+
-- Trim events before emiting them to avoid trimming events emitted in this script
|
|
81
|
+
trimEvents(KEYS[3], KEYS[8])
|
|
82
|
+
|
|
80
83
|
local parentDependenciesKey = args[7]
|
|
81
84
|
if args[2] == "" then
|
|
82
85
|
jobId = jobCounter
|
|
@@ -174,6 +177,4 @@ if parentDependenciesKey ~= nil then
|
|
|
174
177
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
|
175
178
|
end
|
|
176
179
|
|
|
177
|
-
trimEvents(KEYS[3], KEYS[8])
|
|
178
|
-
|
|
179
180
|
return jobId .. "" -- convert to string
|
|
@@ -42,8 +42,10 @@ local jobs = rcall(command, KEYS[1], rangeStart, rangeEnd)
|
|
|
42
42
|
local deleted = {}
|
|
43
43
|
local deletedCount = 0
|
|
44
44
|
local jobTS
|
|
45
|
+
local deletionMarker = ''
|
|
46
|
+
local jobIdsLen = #jobs
|
|
45
47
|
if ARGV[4] == "active" then
|
|
46
|
-
for
|
|
48
|
+
for i, job in ipairs(jobs) do
|
|
47
49
|
if limit > 0 and deletedCount >= limit then
|
|
48
50
|
break
|
|
49
51
|
end
|
|
@@ -52,7 +54,9 @@ if ARGV[4] == "active" then
|
|
|
52
54
|
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
|
|
53
55
|
jobTS = rcall("HGET", jobKey, "timestamp")
|
|
54
56
|
if (not jobTS or jobTS < ARGV[2]) then
|
|
55
|
-
|
|
57
|
+
-- replace the entry with a deletion marker; the actual deletion will
|
|
58
|
+
-- occur at the end of the script
|
|
59
|
+
rcall("LSET", KEYS[1], rangeEnd - jobIdsLen + i, deletionMarker)
|
|
56
60
|
removeJob(job, true, ARGV[1])
|
|
57
61
|
deletedCount = deletedCount + 1
|
|
58
62
|
table.insert(deleted, job)
|
|
@@ -60,7 +64,7 @@ if ARGV[4] == "active" then
|
|
|
60
64
|
end
|
|
61
65
|
end
|
|
62
66
|
else
|
|
63
|
-
for
|
|
67
|
+
for i, job in ipairs(jobs) do
|
|
64
68
|
if limit > 0 and deletedCount >= limit then
|
|
65
69
|
break
|
|
66
70
|
end
|
|
@@ -69,7 +73,9 @@ else
|
|
|
69
73
|
jobTS = rcall("HGET", jobKey, "timestamp")
|
|
70
74
|
if (not jobTS or jobTS < ARGV[2]) then
|
|
71
75
|
if isList then
|
|
72
|
-
|
|
76
|
+
-- replace the entry with a deletion marker; the actual deletion will
|
|
77
|
+
-- occur at the end of the script
|
|
78
|
+
rcall("LSET", KEYS[1], rangeEnd - jobIdsLen + i, deletionMarker)
|
|
73
79
|
end
|
|
74
80
|
removeJob(job, true, ARGV[1])
|
|
75
81
|
deletedCount = deletedCount + 1
|
|
@@ -86,6 +92,10 @@ else
|
|
|
86
92
|
end
|
|
87
93
|
end
|
|
88
94
|
|
|
95
|
+
if isList then
|
|
96
|
+
rcall("LREM", KEYS[1], 0, deletionMarker)
|
|
97
|
+
end
|
|
98
|
+
|
|
89
99
|
rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", deletedCount)
|
|
90
100
|
|
|
91
101
|
return deleted
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
local function removeJob(key, hard, baseKey)
|
|
9
9
|
local jobKey = baseKey .. key
|
|
10
|
-
removeParentDependencyKey(jobKey, hard, baseKey)
|
|
10
|
+
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
|
11
11
|
rcall("DEL", jobKey, jobKey .. ':logs',
|
|
12
12
|
jobKey .. ':dependencies', jobKey .. ':processed')
|
|
13
13
|
end
|
|
@@ -19,9 +19,10 @@ local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
|
|
19
19
|
end
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
local function removeParentDependencyKey(jobKey, hard, baseKey)
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
|
23
|
+
if parentKey then
|
|
24
|
+
local parentProcessedKey = parentKey .. ":processed"
|
|
25
|
+
rcall("HDEL", parentProcessedKey, jobKey)
|
|
25
26
|
local parentDependenciesKey = parentKey .. ":dependencies"
|
|
26
27
|
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
27
28
|
if result > 0 then
|
|
@@ -34,7 +35,7 @@ local function removeParentDependencyKey(jobKey, hard, baseKey)
|
|
|
34
35
|
|
|
35
36
|
if hard then
|
|
36
37
|
if parentPrefix == baseKey then
|
|
37
|
-
removeParentDependencyKey(parentKey, hard, baseKey)
|
|
38
|
+
removeParentDependencyKey(parentKey, hard, nil, baseKey)
|
|
38
39
|
rcall("DEL", parentKey, parentKey .. ':logs',
|
|
39
40
|
parentKey .. ':dependencies', parentKey .. ':processed')
|
|
40
41
|
else
|
|
@@ -45,6 +46,34 @@ local function removeParentDependencyKey(jobKey, hard, baseKey)
|
|
|
45
46
|
end
|
|
46
47
|
end
|
|
47
48
|
end
|
|
49
|
+
else
|
|
50
|
+
local missedParentKey = rcall("HGET", jobKey, "parentKey")
|
|
51
|
+
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
|
|
52
|
+
local parentProcessedKey = missedParentKey .. ":processed"
|
|
53
|
+
rcall("HDEL", parentProcessedKey, jobKey)
|
|
54
|
+
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
|
55
|
+
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
56
|
+
if result > 0 then
|
|
57
|
+
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
|
58
|
+
if pendingDependencies == 0 then
|
|
59
|
+
local parentId = getJobIdFromKey(missedParentKey)
|
|
60
|
+
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
|
61
|
+
|
|
62
|
+
rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
|
63
|
+
|
|
64
|
+
if hard then
|
|
65
|
+
if parentPrefix == baseKey then
|
|
66
|
+
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
|
|
67
|
+
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
|
|
68
|
+
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
|
|
69
|
+
else
|
|
70
|
+
moveParentToWait(parentPrefix, parentId)
|
|
71
|
+
end
|
|
72
|
+
else
|
|
73
|
+
moveParentToWait(parentPrefix, parentId, true)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
48
78
|
end
|
|
49
79
|
end
|
|
50
|
-
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
local function trimEvents(metaKey, eventStreamKey)
|
|
6
6
|
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
|
7
|
-
if
|
|
7
|
+
if maxEvents ~= false then
|
|
8
8
|
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
|
|
9
9
|
else
|
|
10
10
|
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
|
|
@@ -32,6 +32,9 @@ if rcall("EXISTS", KEYS[5]) == 1 then return {{}, {}} end
|
|
|
32
32
|
|
|
33
33
|
rcall("SET", KEYS[5], ARGV[3], "PX", ARGV[4])
|
|
34
34
|
|
|
35
|
+
-- Trim events before emiting them to avoid trimming events emitted in this script
|
|
36
|
+
trimEvents(KEYS[6], KEYS[8])
|
|
37
|
+
|
|
35
38
|
-- Move all stalled jobs to wait
|
|
36
39
|
local stalling = rcall('SMEMBERS', KEYS[1])
|
|
37
40
|
local stalled = {}
|
|
@@ -97,6 +100,4 @@ if (#active > 0) then
|
|
|
97
100
|
end
|
|
98
101
|
end
|
|
99
102
|
|
|
100
|
-
trimEvents(KEYS[6], KEYS[8])
|
|
101
|
-
|
|
102
103
|
return {failed, stalled}
|
|
@@ -42,35 +42,44 @@ local function isLocked( prefix, jobId)
|
|
|
42
42
|
return true
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
-
local function removeJob( prefix, jobId)
|
|
45
|
+
local function removeJob( prefix, jobId, parentKey)
|
|
46
46
|
local jobKey = prefix .. jobId;
|
|
47
47
|
|
|
48
|
-
removeParentDependencyKey(jobKey)
|
|
49
|
-
|
|
50
|
-
rcall("LREM", prefix .. "active", 0, jobId)
|
|
51
|
-
rcall("LREM", prefix .. "wait", 0, jobId)
|
|
52
|
-
rcall("ZREM", prefix .. "delayed", jobId)
|
|
53
|
-
rcall("LREM", prefix .. "paused", 0, jobId)
|
|
54
|
-
rcall("ZREM", prefix .. "completed", jobId)
|
|
55
|
-
rcall("ZREM", prefix .. "failed", jobId)
|
|
56
|
-
rcall("ZREM", prefix .. "priority", jobId)
|
|
57
|
-
rcall("ZREM", prefix .. "waiting-children", jobId)
|
|
58
|
-
rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":processed")
|
|
48
|
+
removeParentDependencyKey(jobKey, false, parentKey)
|
|
59
49
|
|
|
60
50
|
-- Check if this job has children
|
|
61
51
|
-- If so, we are going to try to remove the children recursively in deep first way because
|
|
62
52
|
-- if some job is locked we must exit with and error.
|
|
53
|
+
--local countProcessed = rcall("HLEN", jobKey .. ":processed")
|
|
54
|
+
local processed = rcall("HGETALL", jobKey .. ":processed")
|
|
55
|
+
|
|
56
|
+
if (#processed > 0) then
|
|
57
|
+
for i = 1, #processed, 2 do
|
|
58
|
+
local childJobId = getJobIdFromKey(processed[i])
|
|
59
|
+
local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
|
|
60
|
+
removeJob( childJobPrefix, childJobId, jobKey )
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
63
64
|
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
|
64
65
|
if (#dependencies > 0) then
|
|
65
66
|
for i, childJobKey in ipairs(dependencies) do
|
|
66
67
|
-- We need to get the jobId for this job.
|
|
67
68
|
local childJobId = getJobIdFromKey(childJobKey)
|
|
68
69
|
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
|
69
|
-
removeJob( childJobPrefix, childJobId )
|
|
70
|
+
removeJob( childJobPrefix, childJobId, jobKey )
|
|
70
71
|
end
|
|
71
72
|
end
|
|
72
73
|
|
|
73
|
-
rcall("
|
|
74
|
+
rcall("LREM", prefix .. "active", 0, jobId)
|
|
75
|
+
rcall("LREM", prefix .. "wait", 0, jobId)
|
|
76
|
+
rcall("ZREM", prefix .. "delayed", jobId)
|
|
77
|
+
rcall("LREM", prefix .. "paused", 0, jobId)
|
|
78
|
+
rcall("ZREM", prefix .. "completed", jobId)
|
|
79
|
+
rcall("ZREM", prefix .. "failed", jobId)
|
|
80
|
+
rcall("ZREM", prefix .. "priority", jobId)
|
|
81
|
+
rcall("ZREM", prefix .. "waiting-children", jobId)
|
|
82
|
+
rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed")
|
|
74
83
|
|
|
75
84
|
-- -- delete keys related to rate limiter
|
|
76
85
|
-- local limiterIndexTable = KEYS[10] .. ":index"
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { QueueSchedulerOptions } from '../interfaces';
|
|
2
2
|
import { QueueBase } from './queue-base';
|
|
3
|
+
import { RedisConnection } from './redis-connection';
|
|
3
4
|
export interface QueueSchedulerListener {
|
|
4
5
|
/**
|
|
5
6
|
* Listen to 'stalled' event.
|
|
@@ -34,7 +35,7 @@ export declare class QueueScheduler extends QueueBase {
|
|
|
34
35
|
private nextTimestamp;
|
|
35
36
|
private isBlocked;
|
|
36
37
|
private running;
|
|
37
|
-
constructor(name: string, { connection, autorun, ...opts }?: QueueSchedulerOptions);
|
|
38
|
+
constructor(name: string, { connection, autorun, ...opts }?: QueueSchedulerOptions, Connection?: typeof RedisConnection);
|
|
38
39
|
emit<U extends keyof QueueSchedulerListener>(event: U, ...args: Parameters<QueueSchedulerListener[U]>): boolean;
|
|
39
40
|
off<U extends keyof QueueSchedulerListener>(eventName: U, listener: QueueSchedulerListener[U]): this;
|
|
40
41
|
on<U extends keyof QueueSchedulerListener>(event: U, listener: QueueSchedulerListener[U]): this;
|
|
@@ -19,11 +19,11 @@ import { Scripts } from './scripts';
|
|
|
19
19
|
*
|
|
20
20
|
*/
|
|
21
21
|
export class QueueScheduler extends QueueBase {
|
|
22
|
-
constructor(name, _a = {}) {
|
|
22
|
+
constructor(name, _a = {}, Connection) {
|
|
23
23
|
var { connection, autorun = true } = _a, opts = __rest(_a, ["connection", "autorun"]);
|
|
24
24
|
super(name, Object.assign(Object.assign({ maxStalledCount: 1, stalledInterval: 30000 }, opts), { connection: isRedisInstance(connection)
|
|
25
25
|
? connection.duplicate()
|
|
26
|
-
: connection, sharedConnection: false }));
|
|
26
|
+
: connection, sharedConnection: false }), Connection);
|
|
27
27
|
this.nextTimestamp = Number.MAX_VALUE;
|
|
28
28
|
this.isBlocked = false;
|
|
29
29
|
this.running = false;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"queue-scheduler.js","sourceRoot":"","sources":["../../../src/classes/queue-scheduler.ts"],"names":[],"mappings":";AAKA,OAAO,EAAE,SAAS,EAAE,uBAAuB,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"queue-scheduler.js","sourceRoot":"","sources":["../../../src/classes/queue-scheduler.ts"],"names":[],"mappings":";AAKA,OAAO,EAAE,SAAS,EAAE,uBAAuB,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAmBpC;;;;;;;;;;;;;;;GAeG;AACH,MAAM,OAAO,cAAe,SAAQ,SAAS;IAK3C,YACE,IAAY,EACZ,KAAiE,EAAE,EACnE,UAAmC;YADnC,EAAE,UAAU,EAAE,OAAO,GAAG,IAAI,OAAuC,EAAlC,IAAI,cAArC,yBAAuC,CAAF;QAGrC,KAAK,CACH,IAAI,gCAEF,eAAe,EAAE,CAAC,EAClB,eAAe,EAAE,KAAK,IACnB,IAAI,KACP,UAAU,EAAE,eAAe,CAAC,UAAU,CAAC;gBACrC,CAAC,CAAe,UAAW,CAAC,SAAS,EAAE;gBACvC,CAAC,CAAC,UAAU,EACd,gBAAgB,EAAE,KAAK,KAEzB,UAAU,CACX,CAAC;QArBI,kBAAa,GAAG,MAAM,CAAC,SAAS,CAAC;QACjC,cAAS,GAAG,KAAK,CAAC;QAClB,YAAO,GAAG,KAAK,CAAC;QAqBtB,IAAI,CAAE,IAAI,CAAC,IAA8B,CAAC,eAAe,EAAE;YACzD,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;SACjE;QAED,IAAI,OAAO,EAAE;YACX,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACvB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACvB,CAAC,CAAC,CAAC;SACJ;IACH,CAAC;IAED,IAAI,CACF,KAAQ,EACR,GAAG,IAA2C;QAE9C,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,GAAG,CACD,SAAY,EACZ,QAAmC;QAEnC,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,EAAE,CACA,KAAQ,EACR,QAAmC;QAEnC,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CACF,KAAQ,EACR,QAAmC;QAEnC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,GAAG;QACP,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI;gBACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;gBACpB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;gBAE3C,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;gBAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,IAA6B,CAAC;gBAEhD,IAAI;oBACF,MAAM,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;iBACnD;gBAAC,OAAO,GAAG,EAAE;oBACZ,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAS,GAAI,CAAC,OAAO,CAAC,EAAE;wBACvD,MAAM,GAAG,CAAC;qBACX;iBACF;gBAED,MAAM,CAAC,aAAa,EAAE,QAAQ,GAAG,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CACjE,IAAI,CAAC,GAAG,EAAE,CACX,CAAC;gBACF,IAAI,YAAY,GAAG,QAAQ,CAAC;gBAE5B,IAAI,aAAa,EAAE;oBACjB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;iBACpC;gBAED,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE;oBACpB,2DAA2D;oBAC3D,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;oBAEnC,iEAAiE;oBACjE,2CAA2C;oBAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAElD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAC1B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CACvD,CAAC;oBAEF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CACrC,MAAM,EACN,GAAG,EACH,YAAY,EACZ,SAAS,CACV,CAAC;oBAEF,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE;wBACnB,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;wBACvB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;wBAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;4BACtC,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;4BAC5B,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;4BACrC,MAAM,aAAa,GAAW,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;4BAE3D,IAAI,aAAa,GAAG,IAAI,CAAC,aAAa,EAAE;gCACtC,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;6BACpC;yBACF;wBAED,EAAE;wBACF,gEAAgE;wBAChE,6BAA6B;wBAC7B,EAAE;wBACF,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;4BACjB,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;yBAC7C;qBACF;oBAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBACvB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC;oBAEvC,IAAI,KAAK,IAAI,CAAC,EAAE;wBACd,MAAM,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;wBAC3D,IAAI,aAAa,EAAE;4BACjB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;4BACnC,YAAY,GAAG,EAAE,CAAC;yBACnB;6BAAM;4BACL,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC;yBACvC;qBACF;iBACF;gBACD,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;aACtB;YAAC,OAAO,KAAK,EAAE;gBACd,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;gBACrB,MAAM,KAAK,CAAC;aACb;SACF;aAAM;YACL,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;SACxD;IACH,CAAC;IAED,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,MAAmB,EACnB,GAAW,EACX,YAAoB,EACpB,SAAiB;QAEjB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,IAAI,IAAI,CAAC;YACT,IAAI,SAAS,EAAE;gBACb,IAAI;oBACF,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;oBACtB,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CACvB,OAAO,EACP,SAAS,EACT,SAAS,EACT,GAAG,EACH,YAAY,CACb,CAAC;iBACH;gBAAC,OAAO,GAAG,EAAE;oBACZ,yCAAyC;oBACzC,IAAY,GAAI,CAAC,OAAO,KAAK,uBAAuB,EAAE;wBACpD,MAAM,GAAG,CAAC;qBACX;iBACF;wBAAS;oBACR,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;iBACxB;aACF;iBAAM;gBACL,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,EAAE,YAAY,CAAC,CAAC;aACzD;YAED,mGAAmG;YACnG,OAAO,IAAW,CAAC;SACpB;IACH,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,SAAiB;QAC5C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,OAAO,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;SAChD;QACD,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,qBAAqB;QACjC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;YACjB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;YAEpE,MAAM,CAAC,OAAO,CAAC,CAAC,KAAa,EAAE,EAAE,CAC/B,IAAI,CAAC,IAAI,CACP,QAAQ,EACR,KAAK,EACL,IAAI,KAAK,CAAC,uCAAuC,CAAC,EAClD,QAAQ,CACT,CACF,CAAC;YACF,OAAO,CAAC,OAAO,CAAC,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;SAC3E;IACH,CAAC;IAES,UAAU;QAClB,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1C,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,eAAe,KAAK,CAAC;IACrD,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,OAAO,EAAE;YAChB,OAAO,IAAI,CAAC,OAAO,CAAC;SACrB;QACD,IAAI,IAAI,CAAC,SAAS,EAAE;YAClB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;SAClC;aAAM;YACL,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;SAC9B;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;CACF"}
|
|
@@ -77,6 +77,9 @@ local jobCounter = rcall("INCR", KEYS[4])
|
|
|
77
77
|
-- Includes
|
|
78
78
|
--- @include "includes/updateParentDepsIfNeeded"
|
|
79
79
|
|
|
80
|
+
-- Trim events before emiting them to avoid trimming events emitted in this script
|
|
81
|
+
trimEvents(KEYS[3], KEYS[8])
|
|
82
|
+
|
|
80
83
|
local parentDependenciesKey = args[7]
|
|
81
84
|
if args[2] == "" then
|
|
82
85
|
jobId = jobCounter
|
|
@@ -174,6 +177,4 @@ if parentDependenciesKey ~= nil then
|
|
|
174
177
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
|
175
178
|
end
|
|
176
179
|
|
|
177
|
-
trimEvents(KEYS[3], KEYS[8])
|
|
178
|
-
|
|
179
180
|
return jobId .. "" -- convert to string
|
|
@@ -42,8 +42,10 @@ local jobs = rcall(command, KEYS[1], rangeStart, rangeEnd)
|
|
|
42
42
|
local deleted = {}
|
|
43
43
|
local deletedCount = 0
|
|
44
44
|
local jobTS
|
|
45
|
+
local deletionMarker = ''
|
|
46
|
+
local jobIdsLen = #jobs
|
|
45
47
|
if ARGV[4] == "active" then
|
|
46
|
-
for
|
|
48
|
+
for i, job in ipairs(jobs) do
|
|
47
49
|
if limit > 0 and deletedCount >= limit then
|
|
48
50
|
break
|
|
49
51
|
end
|
|
@@ -52,7 +54,9 @@ if ARGV[4] == "active" then
|
|
|
52
54
|
if (rcall("EXISTS", jobKey .. ":lock") == 0) then
|
|
53
55
|
jobTS = rcall("HGET", jobKey, "timestamp")
|
|
54
56
|
if (not jobTS or jobTS < ARGV[2]) then
|
|
55
|
-
|
|
57
|
+
-- replace the entry with a deletion marker; the actual deletion will
|
|
58
|
+
-- occur at the end of the script
|
|
59
|
+
rcall("LSET", KEYS[1], rangeEnd - jobIdsLen + i, deletionMarker)
|
|
56
60
|
removeJob(job, true, ARGV[1])
|
|
57
61
|
deletedCount = deletedCount + 1
|
|
58
62
|
table.insert(deleted, job)
|
|
@@ -60,7 +64,7 @@ if ARGV[4] == "active" then
|
|
|
60
64
|
end
|
|
61
65
|
end
|
|
62
66
|
else
|
|
63
|
-
for
|
|
67
|
+
for i, job in ipairs(jobs) do
|
|
64
68
|
if limit > 0 and deletedCount >= limit then
|
|
65
69
|
break
|
|
66
70
|
end
|
|
@@ -69,7 +73,9 @@ else
|
|
|
69
73
|
jobTS = rcall("HGET", jobKey, "timestamp")
|
|
70
74
|
if (not jobTS or jobTS < ARGV[2]) then
|
|
71
75
|
if isList then
|
|
72
|
-
|
|
76
|
+
-- replace the entry with a deletion marker; the actual deletion will
|
|
77
|
+
-- occur at the end of the script
|
|
78
|
+
rcall("LSET", KEYS[1], rangeEnd - jobIdsLen + i, deletionMarker)
|
|
73
79
|
end
|
|
74
80
|
removeJob(job, true, ARGV[1])
|
|
75
81
|
deletedCount = deletedCount + 1
|
|
@@ -86,6 +92,10 @@ else
|
|
|
86
92
|
end
|
|
87
93
|
end
|
|
88
94
|
|
|
95
|
+
if isList then
|
|
96
|
+
rcall("LREM", KEYS[1], 0, deletionMarker)
|
|
97
|
+
end
|
|
98
|
+
|
|
89
99
|
rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", deletedCount)
|
|
90
100
|
|
|
91
101
|
return deleted
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
local function removeJob(key, hard, baseKey)
|
|
9
9
|
local jobKey = baseKey .. key
|
|
10
|
-
removeParentDependencyKey(jobKey, hard, baseKey)
|
|
10
|
+
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
|
11
11
|
rcall("DEL", jobKey, jobKey .. ':logs',
|
|
12
12
|
jobKey .. ':dependencies', jobKey .. ':processed')
|
|
13
13
|
end
|
|
@@ -19,9 +19,10 @@ local function moveParentToWait(parentPrefix, parentId, emitEvent)
|
|
|
19
19
|
end
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
local function removeParentDependencyKey(jobKey, hard, baseKey)
|
|
23
|
-
|
|
24
|
-
|
|
22
|
+
local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
|
|
23
|
+
if parentKey then
|
|
24
|
+
local parentProcessedKey = parentKey .. ":processed"
|
|
25
|
+
rcall("HDEL", parentProcessedKey, jobKey)
|
|
25
26
|
local parentDependenciesKey = parentKey .. ":dependencies"
|
|
26
27
|
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
27
28
|
if result > 0 then
|
|
@@ -34,7 +35,7 @@ local function removeParentDependencyKey(jobKey, hard, baseKey)
|
|
|
34
35
|
|
|
35
36
|
if hard then
|
|
36
37
|
if parentPrefix == baseKey then
|
|
37
|
-
removeParentDependencyKey(parentKey, hard, baseKey)
|
|
38
|
+
removeParentDependencyKey(parentKey, hard, nil, baseKey)
|
|
38
39
|
rcall("DEL", parentKey, parentKey .. ':logs',
|
|
39
40
|
parentKey .. ':dependencies', parentKey .. ':processed')
|
|
40
41
|
else
|
|
@@ -45,6 +46,34 @@ local function removeParentDependencyKey(jobKey, hard, baseKey)
|
|
|
45
46
|
end
|
|
46
47
|
end
|
|
47
48
|
end
|
|
49
|
+
else
|
|
50
|
+
local missedParentKey = rcall("HGET", jobKey, "parentKey")
|
|
51
|
+
if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
|
|
52
|
+
local parentProcessedKey = missedParentKey .. ":processed"
|
|
53
|
+
rcall("HDEL", parentProcessedKey, jobKey)
|
|
54
|
+
local parentDependenciesKey = missedParentKey .. ":dependencies"
|
|
55
|
+
local result = rcall("SREM", parentDependenciesKey, jobKey)
|
|
56
|
+
if result > 0 then
|
|
57
|
+
local pendingDependencies = rcall("SCARD", parentDependenciesKey)
|
|
58
|
+
if pendingDependencies == 0 then
|
|
59
|
+
local parentId = getJobIdFromKey(missedParentKey)
|
|
60
|
+
local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
|
|
61
|
+
|
|
62
|
+
rcall("ZREM", parentPrefix .. "waiting-children", parentId)
|
|
63
|
+
|
|
64
|
+
if hard then
|
|
65
|
+
if parentPrefix == baseKey then
|
|
66
|
+
removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
|
|
67
|
+
rcall("DEL", missedParentKey, missedParentKey .. ':logs',
|
|
68
|
+
missedParentKey .. ':dependencies', missedParentKey .. ':processed')
|
|
69
|
+
else
|
|
70
|
+
moveParentToWait(parentPrefix, parentId)
|
|
71
|
+
end
|
|
72
|
+
else
|
|
73
|
+
moveParentToWait(parentPrefix, parentId, true)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
48
78
|
end
|
|
49
79
|
end
|
|
50
|
-
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
local function trimEvents(metaKey, eventStreamKey)
|
|
6
6
|
local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
|
|
7
|
-
if
|
|
7
|
+
if maxEvents ~= false then
|
|
8
8
|
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
|
|
9
9
|
else
|
|
10
10
|
rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
|
|
@@ -32,6 +32,9 @@ if rcall("EXISTS", KEYS[5]) == 1 then return {{}, {}} end
|
|
|
32
32
|
|
|
33
33
|
rcall("SET", KEYS[5], ARGV[3], "PX", ARGV[4])
|
|
34
34
|
|
|
35
|
+
-- Trim events before emiting them to avoid trimming events emitted in this script
|
|
36
|
+
trimEvents(KEYS[6], KEYS[8])
|
|
37
|
+
|
|
35
38
|
-- Move all stalled jobs to wait
|
|
36
39
|
local stalling = rcall('SMEMBERS', KEYS[1])
|
|
37
40
|
local stalled = {}
|
|
@@ -97,6 +100,4 @@ if (#active > 0) then
|
|
|
97
100
|
end
|
|
98
101
|
end
|
|
99
102
|
|
|
100
|
-
trimEvents(KEYS[6], KEYS[8])
|
|
101
|
-
|
|
102
103
|
return {failed, stalled}
|
|
@@ -42,35 +42,44 @@ local function isLocked( prefix, jobId)
|
|
|
42
42
|
return true
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
-
local function removeJob( prefix, jobId)
|
|
45
|
+
local function removeJob( prefix, jobId, parentKey)
|
|
46
46
|
local jobKey = prefix .. jobId;
|
|
47
47
|
|
|
48
|
-
removeParentDependencyKey(jobKey)
|
|
49
|
-
|
|
50
|
-
rcall("LREM", prefix .. "active", 0, jobId)
|
|
51
|
-
rcall("LREM", prefix .. "wait", 0, jobId)
|
|
52
|
-
rcall("ZREM", prefix .. "delayed", jobId)
|
|
53
|
-
rcall("LREM", prefix .. "paused", 0, jobId)
|
|
54
|
-
rcall("ZREM", prefix .. "completed", jobId)
|
|
55
|
-
rcall("ZREM", prefix .. "failed", jobId)
|
|
56
|
-
rcall("ZREM", prefix .. "priority", jobId)
|
|
57
|
-
rcall("ZREM", prefix .. "waiting-children", jobId)
|
|
58
|
-
rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":processed")
|
|
48
|
+
removeParentDependencyKey(jobKey, false, parentKey)
|
|
59
49
|
|
|
60
50
|
-- Check if this job has children
|
|
61
51
|
-- If so, we are going to try to remove the children recursively in deep first way because
|
|
62
52
|
-- if some job is locked we must exit with and error.
|
|
53
|
+
--local countProcessed = rcall("HLEN", jobKey .. ":processed")
|
|
54
|
+
local processed = rcall("HGETALL", jobKey .. ":processed")
|
|
55
|
+
|
|
56
|
+
if (#processed > 0) then
|
|
57
|
+
for i = 1, #processed, 2 do
|
|
58
|
+
local childJobId = getJobIdFromKey(processed[i])
|
|
59
|
+
local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
|
|
60
|
+
removeJob( childJobPrefix, childJobId, jobKey )
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
63
64
|
local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
|
|
64
65
|
if (#dependencies > 0) then
|
|
65
66
|
for i, childJobKey in ipairs(dependencies) do
|
|
66
67
|
-- We need to get the jobId for this job.
|
|
67
68
|
local childJobId = getJobIdFromKey(childJobKey)
|
|
68
69
|
local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
|
|
69
|
-
removeJob( childJobPrefix, childJobId )
|
|
70
|
+
removeJob( childJobPrefix, childJobId, jobKey )
|
|
70
71
|
end
|
|
71
72
|
end
|
|
72
73
|
|
|
73
|
-
rcall("
|
|
74
|
+
rcall("LREM", prefix .. "active", 0, jobId)
|
|
75
|
+
rcall("LREM", prefix .. "wait", 0, jobId)
|
|
76
|
+
rcall("ZREM", prefix .. "delayed", jobId)
|
|
77
|
+
rcall("LREM", prefix .. "paused", 0, jobId)
|
|
78
|
+
rcall("ZREM", prefix .. "completed", jobId)
|
|
79
|
+
rcall("ZREM", prefix .. "failed", jobId)
|
|
80
|
+
rcall("ZREM", prefix .. "priority", jobId)
|
|
81
|
+
rcall("ZREM", prefix .. "waiting-children", jobId)
|
|
82
|
+
rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed")
|
|
74
83
|
|
|
75
84
|
-- -- delete keys related to rate limiter
|
|
76
85
|
-- local limiterIndexTable = KEYS[10] .. ":index"
|