bullmq 1.85.3 → 1.85.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/child-processor.js +1 -1
- package/dist/cjs/classes/child-processor.js.map +1 -1
- package/dist/cjs/commands/cleanJobsInSet-2.lua +2 -83
- package/dist/cjs/commands/includes/cleanList.lua +44 -0
- package/dist/cjs/commands/includes/cleanSet.lua +38 -0
- package/dist/cjs/commands/includes/getTimestamp.lua +19 -0
- package/dist/cjs/commands/includes/getZSetItems.lua +0 -1
- package/dist/esm/classes/child-processor.js +1 -1
- package/dist/esm/classes/child-processor.js.map +1 -1
- package/dist/esm/commands/cleanJobsInSet-2.lua +2 -83
- package/dist/esm/commands/includes/cleanList.lua +44 -0
- package/dist/esm/commands/includes/cleanSet.lua +38 -0
- package/dist/esm/commands/includes/getTimestamp.lua +19 -0
- package/dist/esm/commands/includes/getZSetItems.lua +0 -1
- package/package.json +1 -1
|
@@ -99,7 +99,7 @@ exports.ChildProcessor = ChildProcessor;
|
|
|
99
99
|
// https://stackoverflow.com/questions/18391212/is-it-not-possible-to-stringify-an-error-using-json-stringify
|
|
100
100
|
if (!('toJSON' in Error.prototype)) {
|
|
101
101
|
Object.defineProperty(Error.prototype, 'toJSON', {
|
|
102
|
-
value: function () {
|
|
102
|
+
value: function toJSONByBull() {
|
|
103
103
|
const alt = {};
|
|
104
104
|
const _this = this;
|
|
105
105
|
Object.getOwnPropertyNames(_this).forEach(function (key) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"child-processor.js","sourceRoot":"","sources":["../../../src/classes/child-processor.ts"],"names":[],"mappings":";;;AAAA,+BAAiC;AACjC,8CAAqE;AACrE,oCAAqC;AAErC,IAAK,WAKJ;AALD,WAAK,WAAW;IACd,6CAAI,CAAA;IACJ,mDAAO,CAAA;IACP,2DAAW,CAAA;IACX,mDAAO,CAAA;AACT,CAAC,EALI,WAAW,KAAX,WAAW,QAKf;AAED;;;;;;GAMG;AACH,MAAa,cAAc;IAKlB,KAAK,CAAC,IAAI,CAAC,aAAqB;QACrC,IAAI,SAAS,CAAC;QACd,IAAI;YACF,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;SACpC;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;YAClC,OAAO,IAAA,iBAAS,EAAC,OAAO,EAAE;gBACxB,GAAG,EAAE,0BAAa,CAAC,UAAU;gBAC7B,GAAG,EAAS,GAAG;aAChB,CAAC,CAAC;SACJ;QAED,IAAI,SAAS,CAAC,OAAO,EAAE;YACrB,yBAAyB;YACzB,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC;SAC/B;QACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;YACxB,SAAS,GAAG,IAAA,gBAAS,EAAC,SAAS,CAAC,CAAC;SAClC;aAAM;YACL,MAAM,aAAa,GAAG,SAAS,CAAC;YAChC,SAAS,GAAG,UAAU,GAAG,IAAW;gBAClC,IAAI;oBACF,OAAO,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;iBAChD;gBAAC,OAAO,GAAG,EAAE;oBACZ,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;iBAC5B;YACH,CAAC,CAAC;SACH;QACD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;QAC/B,MAAM,IAAA,iBAAS,EAAC,OAAO,EAAE;YACvB,GAAG,EAAE,0BAAa,CAAC,aAAa;SACjC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,OAAgB;QACjC,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,IAAI,EAAE;YACpC,OAAO,IAAA,iBAAS,EAAC,OAAO,EAAE;gBACxB,GAAG,EAAE,0BAAa,CAAC,KAAK;gBACxB,GAAG,EAAE,IAAI,KAAK,CAAC,yCAAyC,CAAC;aAC1D,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,iBAAiB,GAAG,CAAC,KAAK,IAAI,EAAE;YACnC,IAAI;gBACF,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC7B,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBACjD,MAAM,IAAA,iBAAS,EAAC,OAAO,EAAE;oBACvB,GAAG,EAAE,0BAAa,CAAC,SAAS;oBAC5B,KAAK,EAAE,MAAM;iBACd,CAAC,CAAC;aACJ;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,IAAA,iBAAS,EAAC,OAAO,EAAE;oBACvB,GAAG,EAAE,0BAAa,CAAC,MAAM;oBACzB,KAAK,EAAE,CAAS,GAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG;iBACzD,CAAC,CAAC;aACJ;oBAAS;gBACR,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC/B,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;aACpC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,IAAI,KAAmB,CAAC;IAErC,KAAK,CAAC,wBAAwB;QAC5B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC;QACtC,IAAI;YACF,MAAM,IAAI,CAAC,iBAAiB,CAAC;SAC9B;gBAAS;YACR,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;CACF;AA9ED,wCA8EC;AAED,6GAA6G;AAC7G,IAAI,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,SAAS,CAAC,EAAE;IAClC,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE;QAC/C,KAAK,EAAE;
|
|
1
|
+
{"version":3,"file":"child-processor.js","sourceRoot":"","sources":["../../../src/classes/child-processor.ts"],"names":[],"mappings":";;;AAAA,+BAAiC;AACjC,8CAAqE;AACrE,oCAAqC;AAErC,IAAK,WAKJ;AALD,WAAK,WAAW;IACd,6CAAI,CAAA;IACJ,mDAAO,CAAA;IACP,2DAAW,CAAA;IACX,mDAAO,CAAA;AACT,CAAC,EALI,WAAW,KAAX,WAAW,QAKf;AAED;;;;;;GAMG;AACH,MAAa,cAAc;IAKlB,KAAK,CAAC,IAAI,CAAC,aAAqB;QACrC,IAAI,SAAS,CAAC;QACd,IAAI;YACF,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;SACpC;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;YAClC,OAAO,IAAA,iBAAS,EAAC,OAAO,EAAE;gBACxB,GAAG,EAAE,0BAAa,CAAC,UAAU;gBAC7B,GAAG,EAAS,GAAG;aAChB,CAAC,CAAC;SACJ;QAED,IAAI,SAAS,CAAC,OAAO,EAAE;YACrB,yBAAyB;YACzB,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC;SAC/B;QACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;YACxB,SAAS,GAAG,IAAA,gBAAS,EAAC,SAAS,CAAC,CAAC;SAClC;aAAM;YACL,MAAM,aAAa,GAAG,SAAS,CAAC;YAChC,SAAS,GAAG,UAAU,GAAG,IAAW;gBAClC,IAAI;oBACF,OAAO,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;iBAChD;gBAAC,OAAO,GAAG,EAAE;oBACZ,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;iBAC5B;YACH,CAAC,CAAC;SACH;QACD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;QAC/B,MAAM,IAAA,iBAAS,EAAC,OAAO,EAAE;YACvB,GAAG,EAAE,0BAAa,CAAC,aAAa;SACjC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,OAAgB;QACjC,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,IAAI,EAAE;YACpC,OAAO,IAAA,iBAAS,EAAC,OAAO,EAAE;gBACxB,GAAG,EAAE,0BAAa,CAAC,KAAK;gBACxB,GAAG,EAAE,IAAI,KAAK,CAAC,yCAAyC,CAAC;aAC1D,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,iBAAiB,GAAG,CAAC,KAAK,IAAI,EAAE;YACnC,IAAI;gBACF,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC7B,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBACjD,MAAM,IAAA,iBAAS,EAAC,OAAO,EAAE;oBACvB,GAAG,EAAE,0BAAa,CAAC,SAAS;oBAC5B,KAAK,EAAE,MAAM;iBACd,CAAC,CAAC;aACJ;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,IAAA,iBAAS,EAAC,OAAO,EAAE;oBACvB,GAAG,EAAE,0BAAa,CAAC,MAAM;oBACzB,KAAK,EAAE,CAAS,GAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG;iBACzD,CAAC,CAAC;aACJ;oBAAS;gBACR,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC/B,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;aACpC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,IAAI,KAAmB,CAAC;IAErC,KAAK,CAAC,wBAAwB;QAC5B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC;QACtC,IAAI;YACF,MAAM,IAAI,CAAC,iBAAiB,CAAC;SAC9B;gBAAS;YACR,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;CACF;AA9ED,wCA8EC;AAED,6GAA6G;AAC7G,IAAI,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,SAAS,CAAC,EAAE;IAClC,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE;QAC/C,KAAK,EAAE,SAAS,YAAY;YAC1B,MAAM,GAAG,GAAQ,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,IAAI,CAAC;YAEnB,MAAM,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,UAAU,GAAG;gBACrD,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,OAAO,GAAG,CAAC;QACb,CAAC;QACD,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;CACJ;AAED;;;;;;;;GAQG;AACH,SAAS,OAAO,CAAC,GAAY;IAC3B,IAAI,aAAa,GAAG,GAAG,CAAC,QAAQ,CAAC;IAEjC,MAAM,cAAc,GAAG,KAAK,EAAE,QAAyB,EAAE,EAAE;QACzD,gDAAgD;QAChD,4DAA4D;QAC5D,aAAa,GAAG,QAAQ,CAAC;QACzB,uCAAuC;QACvC,MAAM,IAAA,iBAAS,EAAC,OAAO,EAAE;YACvB,GAAG,EAAE,0BAAa,CAAC,QAAQ;YAC3B,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,CAAC,QAA0B,EAAE,EAAE;QAC9C,OAAO,CAAC,IAAI,CACV;YACE,gGAAgG;YAChG,qFAAqF;SACtF,CAAC,IAAI,CAAC,GAAG,CAAC,CACZ,CAAC;QAEF,IAAI,QAAQ,EAAE;YACZ,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;SACjC;aAAM;YACL,wCAAwC;YACxC,OAAO,aAAa,CAAC;SACtB;IACH,CAAC,CAAC;IAEF,uCACK,GAAG,KACN,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,EAClC,IAAI,EAAE,GAAG,CAAC,IAAI,EACd,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC;QAChD;;;;;WAKG;QACH,QAAQ;QACR;;WAEG;QACH,cAAc;QACd;;WAEG;QACH,GAAG,EAAE,KAAK,EAAE,GAAQ,EAAE,EAAE;YACtB,IAAA,iBAAS,EAAC,OAAO,EAAE;gBACjB,GAAG,EAAE,0BAAa,CAAC,GAAG;gBACtB,KAAK,EAAE,GAAG;aACX,CAAC,CAAC;QACL,CAAC,IACD;AACJ,CAAC"}
|
|
@@ -27,89 +27,8 @@ if limit > 0 then
|
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
-- Includes
|
|
30
|
-
--- @include "includes/
|
|
31
|
-
--- @include "includes/
|
|
32
|
-
|
|
33
|
-
local function getTimestamp(jobKey, attributes)
|
|
34
|
-
if #attributes == 1 then
|
|
35
|
-
return rcall("HGET", jobKey, attributes[1])
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
local jobTs
|
|
39
|
-
for _, ts in ipairs(rcall("HMGET", jobKey, unpack(attributes))) do
|
|
40
|
-
if (ts) then
|
|
41
|
-
jobTs = ts
|
|
42
|
-
break
|
|
43
|
-
end
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
return jobTs
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd, timestamp, skipCheckLock)
|
|
50
|
-
local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
|
|
51
|
-
local deleted = {}
|
|
52
|
-
local deletedCount = 0
|
|
53
|
-
local jobTS
|
|
54
|
-
local deletionMarker = ''
|
|
55
|
-
local jobIdsLen = #jobs
|
|
56
|
-
for i, job in ipairs(jobs) do
|
|
57
|
-
if limit > 0 and deletedCount >= limit then
|
|
58
|
-
break
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
local jobKey = jobKeyPrefix .. job
|
|
62
|
-
if (skipCheckLock or rcall("EXISTS", jobKey .. ":lock") == 0) then
|
|
63
|
-
-- Find the right timestamp of the job to compare to maxTimestamp:
|
|
64
|
-
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
|
65
|
-
-- * processedOn represents when the job was last attempted, but it doesn't get populated until the job is first tried
|
|
66
|
-
-- * timestamp is the original job submission time
|
|
67
|
-
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs that have been active within the grace period:
|
|
68
|
-
jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
|
|
69
|
-
if (not jobTS or jobTS < timestamp) then
|
|
70
|
-
-- replace the entry with a deletion marker; the actual deletion will
|
|
71
|
-
-- occur at the end of the script
|
|
72
|
-
rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
|
|
73
|
-
removeJob(job, true, jobKeyPrefix)
|
|
74
|
-
deletedCount = deletedCount + 1
|
|
75
|
-
table.insert(deleted, job)
|
|
76
|
-
end
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
rcall("LREM", listKey, 0, deletionMarker)
|
|
81
|
-
|
|
82
|
-
return {deleted, deletedCount}
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
local function cleanSet(setKey, jobKeyPrefix, rangeStart, rangeEnd, timestamp, attributes)
|
|
86
|
-
local jobs = rcall("ZRANGE", setKey, rangeStart, rangeEnd)
|
|
87
|
-
local deleted = {}
|
|
88
|
-
local deletedCount = 0
|
|
89
|
-
local jobTS
|
|
90
|
-
for i, job in ipairs(jobs) do
|
|
91
|
-
if limit > 0 and deletedCount >= limit then
|
|
92
|
-
break
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
local jobKey = jobKeyPrefix .. job
|
|
96
|
-
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
|
97
|
-
jobTS = getTimestamp(jobKey, attributes)
|
|
98
|
-
if (not jobTS or jobTS < timestamp) then
|
|
99
|
-
removeJob(job, true, jobKeyPrefix)
|
|
100
|
-
deletedCount = deletedCount + 1
|
|
101
|
-
table.insert(deleted, job)
|
|
102
|
-
end
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
if(#deleted > 0) then
|
|
106
|
-
for from, to in batches(#deleted, 7000) do
|
|
107
|
-
rcall("ZREM", setKey, unpack(deleted, from, to))
|
|
108
|
-
end
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
return {deleted, deletedCount}
|
|
112
|
-
end
|
|
30
|
+
--- @include "includes/cleanList"
|
|
31
|
+
--- @include "includes/cleanSet"
|
|
113
32
|
|
|
114
33
|
local result
|
|
115
34
|
if ARGV[4] == "active" then
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
Function to clean job list.
|
|
3
|
+
Returns jobIds and deleted count number.
|
|
4
|
+
]]
|
|
5
|
+
|
|
6
|
+
-- Includes
|
|
7
|
+
--- @include "getTimestamp"
|
|
8
|
+
--- @include "removeJob"
|
|
9
|
+
|
|
10
|
+
local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd, timestamp, skipCheckLock)
|
|
11
|
+
local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
|
|
12
|
+
local deleted = {}
|
|
13
|
+
local deletedCount = 0
|
|
14
|
+
local jobTS
|
|
15
|
+
local deletionMarker = ''
|
|
16
|
+
local jobIdsLen = #jobs
|
|
17
|
+
for i, job in ipairs(jobs) do
|
|
18
|
+
if limit > 0 and deletedCount >= limit then
|
|
19
|
+
break
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
local jobKey = jobKeyPrefix .. job
|
|
23
|
+
if (skipCheckLock or rcall("EXISTS", jobKey .. ":lock") == 0) then
|
|
24
|
+
-- Find the right timestamp of the job to compare to maxTimestamp:
|
|
25
|
+
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
|
26
|
+
-- * processedOn represents when the job was last attempted, but it doesn't get populated until the job is first tried
|
|
27
|
+
-- * timestamp is the original job submission time
|
|
28
|
+
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs that have been active within the grace period:
|
|
29
|
+
jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
|
|
30
|
+
if (not jobTS or jobTS < timestamp) then
|
|
31
|
+
-- replace the entry with a deletion marker; the actual deletion will
|
|
32
|
+
-- occur at the end of the script
|
|
33
|
+
rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
|
|
34
|
+
removeJob(job, true, jobKeyPrefix)
|
|
35
|
+
deletedCount = deletedCount + 1
|
|
36
|
+
table.insert(deleted, job)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
rcall("LREM", listKey, 0, deletionMarker)
|
|
42
|
+
|
|
43
|
+
return {deleted, deletedCount}
|
|
44
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
Function to clean job set.
|
|
3
|
+
Returns jobIds and deleted count number.
|
|
4
|
+
]]
|
|
5
|
+
|
|
6
|
+
-- Includes
|
|
7
|
+
--- @include "batches"
|
|
8
|
+
--- @include "getTimestamp"
|
|
9
|
+
--- @include "removeJob"
|
|
10
|
+
|
|
11
|
+
local function cleanSet(setKey, jobKeyPrefix, rangeStart, rangeEnd, timestamp, attributes)
|
|
12
|
+
local jobs = rcall("ZRANGE", setKey, rangeStart, rangeEnd)
|
|
13
|
+
local deleted = {}
|
|
14
|
+
local deletedCount = 0
|
|
15
|
+
local jobTS
|
|
16
|
+
for i, job in ipairs(jobs) do
|
|
17
|
+
if limit > 0 and deletedCount >= limit then
|
|
18
|
+
break
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
local jobKey = jobKeyPrefix .. job
|
|
22
|
+
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
|
23
|
+
jobTS = getTimestamp(jobKey, attributes)
|
|
24
|
+
if (not jobTS or jobTS < timestamp) then
|
|
25
|
+
removeJob(job, true, jobKeyPrefix)
|
|
26
|
+
deletedCount = deletedCount + 1
|
|
27
|
+
table.insert(deleted, job)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
if(#deleted > 0) then
|
|
32
|
+
for from, to in batches(#deleted, 7000) do
|
|
33
|
+
rcall("ZREM", setKey, unpack(deleted, from, to))
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
return {deleted, deletedCount}
|
|
38
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
Function to get the latest saved timestamp.
|
|
3
|
+
]]
|
|
4
|
+
|
|
5
|
+
local function getTimestamp(jobKey, attributes)
|
|
6
|
+
if #attributes == 1 then
|
|
7
|
+
return rcall("HGET", jobKey, attributes[1])
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
local jobTs
|
|
11
|
+
for _, ts in ipairs(rcall("HMGET", jobKey, unpack(attributes))) do
|
|
12
|
+
if (ts) then
|
|
13
|
+
jobTs = ts
|
|
14
|
+
break
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
return jobTs
|
|
19
|
+
end
|
|
@@ -95,7 +95,7 @@ export class ChildProcessor {
|
|
|
95
95
|
// https://stackoverflow.com/questions/18391212/is-it-not-possible-to-stringify-an-error-using-json-stringify
|
|
96
96
|
if (!('toJSON' in Error.prototype)) {
|
|
97
97
|
Object.defineProperty(Error.prototype, 'toJSON', {
|
|
98
|
-
value: function () {
|
|
98
|
+
value: function toJSONByBull() {
|
|
99
99
|
const alt = {};
|
|
100
100
|
const _this = this;
|
|
101
101
|
Object.getOwnPropertyNames(_this).forEach(function (key) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"child-processor.js","sourceRoot":"","sources":["../../../src/classes/child-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,EAAW,aAAa,EAAgB,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,IAAK,WAKJ;AALD,WAAK,WAAW;IACd,6CAAI,CAAA;IACJ,mDAAO,CAAA;IACP,2DAAW,CAAA;IACX,mDAAO,CAAA;AACT,CAAC,EALI,WAAW,KAAX,WAAW,QAKf;AAED;;;;;;GAMG;AACH,MAAM,OAAO,cAAc;IAKlB,KAAK,CAAC,IAAI,CAAC,aAAqB;QACrC,IAAI,SAAS,CAAC;QACd,IAAI;YACF,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;SACpC;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;YAClC,OAAO,SAAS,CAAC,OAAO,EAAE;gBACxB,GAAG,EAAE,aAAa,CAAC,UAAU;gBAC7B,GAAG,EAAS,GAAG;aAChB,CAAC,CAAC;SACJ;QAED,IAAI,SAAS,CAAC,OAAO,EAAE;YACrB,yBAAyB;YACzB,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC;SAC/B;QACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;YACxB,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;SAClC;aAAM;YACL,MAAM,aAAa,GAAG,SAAS,CAAC;YAChC,SAAS,GAAG,UAAU,GAAG,IAAW;gBAClC,IAAI;oBACF,OAAO,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;iBAChD;gBAAC,OAAO,GAAG,EAAE;oBACZ,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;iBAC5B;YACH,CAAC,CAAC;SACH;QACD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;QAC/B,MAAM,SAAS,CAAC,OAAO,EAAE;YACvB,GAAG,EAAE,aAAa,CAAC,aAAa;SACjC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,OAAgB;QACjC,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,IAAI,EAAE;YACpC,OAAO,SAAS,CAAC,OAAO,EAAE;gBACxB,GAAG,EAAE,aAAa,CAAC,KAAK;gBACxB,GAAG,EAAE,IAAI,KAAK,CAAC,yCAAyC,CAAC;aAC1D,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,iBAAiB,GAAG,CAAC,KAAK,IAAI,EAAE;YACnC,IAAI;gBACF,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC7B,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBACjD,MAAM,SAAS,CAAC,OAAO,EAAE;oBACvB,GAAG,EAAE,aAAa,CAAC,SAAS;oBAC5B,KAAK,EAAE,MAAM;iBACd,CAAC,CAAC;aACJ;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,SAAS,CAAC,OAAO,EAAE;oBACvB,GAAG,EAAE,aAAa,CAAC,MAAM;oBACzB,KAAK,EAAE,CAAS,GAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG;iBACzD,CAAC,CAAC;aACJ;oBAAS;gBACR,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC/B,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;aACpC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,IAAI,KAAmB,CAAC;IAErC,KAAK,CAAC,wBAAwB;QAC5B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC;QACtC,IAAI;YACF,MAAM,IAAI,CAAC,iBAAiB,CAAC;SAC9B;gBAAS;YACR,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;CACF;AAED,6GAA6G;AAC7G,IAAI,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,SAAS,CAAC,EAAE;IAClC,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE;QAC/C,KAAK,EAAE;
|
|
1
|
+
{"version":3,"file":"child-processor.js","sourceRoot":"","sources":["../../../src/classes/child-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AACjC,OAAO,EAAW,aAAa,EAAgB,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,IAAK,WAKJ;AALD,WAAK,WAAW;IACd,6CAAI,CAAA;IACJ,mDAAO,CAAA;IACP,2DAAW,CAAA;IACX,mDAAO,CAAA;AACT,CAAC,EALI,WAAW,KAAX,WAAW,QAKf;AAED;;;;;;GAMG;AACH,MAAM,OAAO,cAAc;IAKlB,KAAK,CAAC,IAAI,CAAC,aAAqB;QACrC,IAAI,SAAS,CAAC;QACd,IAAI;YACF,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;SACpC;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;YAClC,OAAO,SAAS,CAAC,OAAO,EAAE;gBACxB,GAAG,EAAE,aAAa,CAAC,UAAU;gBAC7B,GAAG,EAAS,GAAG;aAChB,CAAC,CAAC;SACJ;QAED,IAAI,SAAS,CAAC,OAAO,EAAE;YACrB,yBAAyB;YACzB,SAAS,GAAG,SAAS,CAAC,OAAO,CAAC;SAC/B;QACD,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;YACxB,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;SAClC;aAAM;YACL,MAAM,aAAa,GAAG,SAAS,CAAC;YAChC,SAAS,GAAG,UAAU,GAAG,IAAW;gBAClC,IAAI;oBACF,OAAO,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;iBAChD;gBAAC,OAAO,GAAG,EAAE;oBACZ,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;iBAC5B;YACH,CAAC,CAAC;SACH;QACD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;QAC/B,MAAM,SAAS,CAAC,OAAO,EAAE;YACvB,GAAG,EAAE,aAAa,CAAC,aAAa;SACjC,CAAC,CAAC;IACL,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,OAAgB;QACjC,IAAI,IAAI,CAAC,MAAM,KAAK,WAAW,CAAC,IAAI,EAAE;YACpC,OAAO,SAAS,CAAC,OAAO,EAAE;gBACxB,GAAG,EAAE,aAAa,CAAC,KAAK;gBACxB,GAAG,EAAE,IAAI,KAAK,CAAC,yCAAyC,CAAC;aAC1D,CAAC,CAAC;SACJ;QACD,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,iBAAiB,GAAG,CAAC,KAAK,IAAI,EAAE;YACnC,IAAI;gBACF,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC7B,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBACjD,MAAM,SAAS,CAAC,OAAO,EAAE;oBACvB,GAAG,EAAE,aAAa,CAAC,SAAS;oBAC5B,KAAK,EAAE,MAAM;iBACd,CAAC,CAAC;aACJ;YAAC,OAAO,GAAG,EAAE;gBACZ,MAAM,SAAS,CAAC,OAAO,EAAE;oBACvB,GAAG,EAAE,aAAa,CAAC,MAAM;oBACzB,KAAK,EAAE,CAAS,GAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG;iBACzD,CAAC,CAAC;aACJ;oBAAS;gBACR,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC/B,IAAI,CAAC,iBAAiB,GAAG,SAAS,CAAC;aACpC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAEM,KAAK,CAAC,IAAI,KAAmB,CAAC;IAErC,KAAK,CAAC,wBAAwB;QAC5B,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC;QACtC,IAAI;YACF,MAAM,IAAI,CAAC,iBAAiB,CAAC;SAC9B;gBAAS;YACR,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;SACrC;IACH,CAAC;CACF;AAED,6GAA6G;AAC7G,IAAI,CAAC,CAAC,QAAQ,IAAI,KAAK,CAAC,SAAS,CAAC,EAAE;IAClC,MAAM,CAAC,cAAc,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE;QAC/C,KAAK,EAAE,SAAS,YAAY;YAC1B,MAAM,GAAG,GAAQ,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,IAAI,CAAC;YAEnB,MAAM,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,UAAU,GAAG;gBACrD,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC,EAAE,IAAI,CAAC,CAAC;YAET,OAAO,GAAG,CAAC;QACb,CAAC;QACD,YAAY,EAAE,IAAI;QAClB,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;CACJ;AAED;;;;;;;;GAQG;AACH,SAAS,OAAO,CAAC,GAAY;IAC3B,IAAI,aAAa,GAAG,GAAG,CAAC,QAAQ,CAAC;IAEjC,MAAM,cAAc,GAAG,KAAK,EAAE,QAAyB,EAAE,EAAE;QACzD,gDAAgD;QAChD,4DAA4D;QAC5D,aAAa,GAAG,QAAQ,CAAC;QACzB,uCAAuC;QACvC,MAAM,SAAS,CAAC,OAAO,EAAE;YACvB,GAAG,EAAE,aAAa,CAAC,QAAQ;YAC3B,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,CAAC,QAA0B,EAAE,EAAE;QAC9C,OAAO,CAAC,IAAI,CACV;YACE,gGAAgG;YAChG,qFAAqF;SACtF,CAAC,IAAI,CAAC,GAAG,CAAC,CACZ,CAAC;QAEF,IAAI,QAAQ,EAAE;YACZ,OAAO,cAAc,CAAC,QAAQ,CAAC,CAAC;SACjC;aAAM;YACL,wCAAwC;YACxC,OAAO,aAAa,CAAC;SACtB;IACH,CAAC,CAAC;IAEF,uCACK,GAAG,KACN,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,EAClC,IAAI,EAAE,GAAG,CAAC,IAAI,EACd,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,IAAI,IAAI,CAAC;QAChD;;;;;WAKG;QACH,QAAQ;QACR;;WAEG;QACH,cAAc;QACd;;WAEG;QACH,GAAG,EAAE,KAAK,EAAE,GAAQ,EAAE,EAAE;YACtB,SAAS,CAAC,OAAO,EAAE;gBACjB,GAAG,EAAE,aAAa,CAAC,GAAG;gBACtB,KAAK,EAAE,GAAG;aACX,CAAC,CAAC;QACL,CAAC,IACD;AACJ,CAAC"}
|
|
@@ -27,89 +27,8 @@ if limit > 0 then
|
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
-- Includes
|
|
30
|
-
--- @include "includes/
|
|
31
|
-
--- @include "includes/
|
|
32
|
-
|
|
33
|
-
local function getTimestamp(jobKey, attributes)
|
|
34
|
-
if #attributes == 1 then
|
|
35
|
-
return rcall("HGET", jobKey, attributes[1])
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
local jobTs
|
|
39
|
-
for _, ts in ipairs(rcall("HMGET", jobKey, unpack(attributes))) do
|
|
40
|
-
if (ts) then
|
|
41
|
-
jobTs = ts
|
|
42
|
-
break
|
|
43
|
-
end
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
return jobTs
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd, timestamp, skipCheckLock)
|
|
50
|
-
local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
|
|
51
|
-
local deleted = {}
|
|
52
|
-
local deletedCount = 0
|
|
53
|
-
local jobTS
|
|
54
|
-
local deletionMarker = ''
|
|
55
|
-
local jobIdsLen = #jobs
|
|
56
|
-
for i, job in ipairs(jobs) do
|
|
57
|
-
if limit > 0 and deletedCount >= limit then
|
|
58
|
-
break
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
local jobKey = jobKeyPrefix .. job
|
|
62
|
-
if (skipCheckLock or rcall("EXISTS", jobKey .. ":lock") == 0) then
|
|
63
|
-
-- Find the right timestamp of the job to compare to maxTimestamp:
|
|
64
|
-
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
|
65
|
-
-- * processedOn represents when the job was last attempted, but it doesn't get populated until the job is first tried
|
|
66
|
-
-- * timestamp is the original job submission time
|
|
67
|
-
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs that have been active within the grace period:
|
|
68
|
-
jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
|
|
69
|
-
if (not jobTS or jobTS < timestamp) then
|
|
70
|
-
-- replace the entry with a deletion marker; the actual deletion will
|
|
71
|
-
-- occur at the end of the script
|
|
72
|
-
rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
|
|
73
|
-
removeJob(job, true, jobKeyPrefix)
|
|
74
|
-
deletedCount = deletedCount + 1
|
|
75
|
-
table.insert(deleted, job)
|
|
76
|
-
end
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
rcall("LREM", listKey, 0, deletionMarker)
|
|
81
|
-
|
|
82
|
-
return {deleted, deletedCount}
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
local function cleanSet(setKey, jobKeyPrefix, rangeStart, rangeEnd, timestamp, attributes)
|
|
86
|
-
local jobs = rcall("ZRANGE", setKey, rangeStart, rangeEnd)
|
|
87
|
-
local deleted = {}
|
|
88
|
-
local deletedCount = 0
|
|
89
|
-
local jobTS
|
|
90
|
-
for i, job in ipairs(jobs) do
|
|
91
|
-
if limit > 0 and deletedCount >= limit then
|
|
92
|
-
break
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
local jobKey = jobKeyPrefix .. job
|
|
96
|
-
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
|
97
|
-
jobTS = getTimestamp(jobKey, attributes)
|
|
98
|
-
if (not jobTS or jobTS < timestamp) then
|
|
99
|
-
removeJob(job, true, jobKeyPrefix)
|
|
100
|
-
deletedCount = deletedCount + 1
|
|
101
|
-
table.insert(deleted, job)
|
|
102
|
-
end
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
if(#deleted > 0) then
|
|
106
|
-
for from, to in batches(#deleted, 7000) do
|
|
107
|
-
rcall("ZREM", setKey, unpack(deleted, from, to))
|
|
108
|
-
end
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
return {deleted, deletedCount}
|
|
112
|
-
end
|
|
30
|
+
--- @include "includes/cleanList"
|
|
31
|
+
--- @include "includes/cleanSet"
|
|
113
32
|
|
|
114
33
|
local result
|
|
115
34
|
if ARGV[4] == "active" then
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
Function to clean job list.
|
|
3
|
+
Returns jobIds and deleted count number.
|
|
4
|
+
]]
|
|
5
|
+
|
|
6
|
+
-- Includes
|
|
7
|
+
--- @include "getTimestamp"
|
|
8
|
+
--- @include "removeJob"
|
|
9
|
+
|
|
10
|
+
local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd, timestamp, skipCheckLock)
|
|
11
|
+
local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
|
|
12
|
+
local deleted = {}
|
|
13
|
+
local deletedCount = 0
|
|
14
|
+
local jobTS
|
|
15
|
+
local deletionMarker = ''
|
|
16
|
+
local jobIdsLen = #jobs
|
|
17
|
+
for i, job in ipairs(jobs) do
|
|
18
|
+
if limit > 0 and deletedCount >= limit then
|
|
19
|
+
break
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
local jobKey = jobKeyPrefix .. job
|
|
23
|
+
if (skipCheckLock or rcall("EXISTS", jobKey .. ":lock") == 0) then
|
|
24
|
+
-- Find the right timestamp of the job to compare to maxTimestamp:
|
|
25
|
+
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
|
26
|
+
-- * processedOn represents when the job was last attempted, but it doesn't get populated until the job is first tried
|
|
27
|
+
-- * timestamp is the original job submission time
|
|
28
|
+
-- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs that have been active within the grace period:
|
|
29
|
+
jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
|
|
30
|
+
if (not jobTS or jobTS < timestamp) then
|
|
31
|
+
-- replace the entry with a deletion marker; the actual deletion will
|
|
32
|
+
-- occur at the end of the script
|
|
33
|
+
rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
|
|
34
|
+
removeJob(job, true, jobKeyPrefix)
|
|
35
|
+
deletedCount = deletedCount + 1
|
|
36
|
+
table.insert(deleted, job)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
rcall("LREM", listKey, 0, deletionMarker)
|
|
42
|
+
|
|
43
|
+
return {deleted, deletedCount}
|
|
44
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
Function to clean job set.
|
|
3
|
+
Returns jobIds and deleted count number.
|
|
4
|
+
]]
|
|
5
|
+
|
|
6
|
+
-- Includes
|
|
7
|
+
--- @include "batches"
|
|
8
|
+
--- @include "getTimestamp"
|
|
9
|
+
--- @include "removeJob"
|
|
10
|
+
|
|
11
|
+
local function cleanSet(setKey, jobKeyPrefix, rangeStart, rangeEnd, timestamp, attributes)
|
|
12
|
+
local jobs = rcall("ZRANGE", setKey, rangeStart, rangeEnd)
|
|
13
|
+
local deleted = {}
|
|
14
|
+
local deletedCount = 0
|
|
15
|
+
local jobTS
|
|
16
|
+
for i, job in ipairs(jobs) do
|
|
17
|
+
if limit > 0 and deletedCount >= limit then
|
|
18
|
+
break
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
local jobKey = jobKeyPrefix .. job
|
|
22
|
+
-- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
|
|
23
|
+
jobTS = getTimestamp(jobKey, attributes)
|
|
24
|
+
if (not jobTS or jobTS < timestamp) then
|
|
25
|
+
removeJob(job, true, jobKeyPrefix)
|
|
26
|
+
deletedCount = deletedCount + 1
|
|
27
|
+
table.insert(deleted, job)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
if(#deleted > 0) then
|
|
32
|
+
for from, to in batches(#deleted, 7000) do
|
|
33
|
+
rcall("ZREM", setKey, unpack(deleted, from, to))
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
return {deleted, deletedCount}
|
|
38
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
Function to get the latest saved timestamp.
|
|
3
|
+
]]
|
|
4
|
+
|
|
5
|
+
local function getTimestamp(jobKey, attributes)
|
|
6
|
+
if #attributes == 1 then
|
|
7
|
+
return rcall("HGET", jobKey, attributes[1])
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
local jobTs
|
|
11
|
+
for _, ts in ipairs(rcall("HMGET", jobKey, unpack(attributes))) do
|
|
12
|
+
if (ts) then
|
|
13
|
+
jobTs = ts
|
|
14
|
+
break
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
return jobTs
|
|
19
|
+
end
|