bullmq 5.55.0 → 5.56.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/scripts.js +3 -0
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +4 -3
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addDelayedJob-6.lua +2 -2
- package/dist/cjs/commands/{addParentJob-4.lua → addParentJob-5.lua} +7 -6
- package/dist/cjs/commands/{addPrioritizedJob-8.lua → addPrioritizedJob-9.lua} +11 -10
- package/dist/{esm/commands/addStandardJob-8.lua → cjs/commands/addStandardJob-9.lua} +9 -8
- package/dist/cjs/commands/includes/collectMetrics.lua +1 -1
- package/dist/cjs/commands/includes/deduplicateJob.lua +66 -20
- package/dist/cjs/commands/includes/prepareJobForProcessing.lua +2 -1
- package/dist/cjs/commands/includes/storeJob.lua +1 -1
- package/dist/cjs/interfaces/index.js +0 -1
- package/dist/cjs/interfaces/index.js.map +1 -1
- package/dist/cjs/scripts/addDelayedJob-6.js +71 -22
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/{addParentJob-4.js → addParentJob-5.js} +78 -28
- package/dist/cjs/scripts/addParentJob-5.js.map +1 -0
- package/dist/cjs/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-9.js} +82 -32
- package/dist/cjs/scripts/{addPrioritizedJob-8.js.map → addPrioritizedJob-9.js.map} +1 -1
- package/dist/cjs/scripts/{addStandardJob-8.js → addStandardJob-9.js} +80 -30
- package/dist/cjs/scripts/{addStandardJob-8.js.map → addStandardJob-9.js.map} +1 -1
- package/dist/cjs/scripts/index.js +3 -3
- package/dist/cjs/scripts/moveToActive-11.js +2 -0
- package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +3 -1
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/{interfaces/debounce-options.js → types/deduplication-options.js} +1 -1
- package/dist/cjs/types/deduplication-options.js.map +1 -0
- package/dist/cjs/types/index.js +1 -0
- package/dist/cjs/types/index.js.map +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/scripts.js +3 -0
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.js +4 -3
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addDelayedJob-6.lua +2 -2
- package/dist/esm/commands/{addParentJob-4.lua → addParentJob-5.lua} +7 -6
- package/dist/esm/commands/{addPrioritizedJob-8.lua → addPrioritizedJob-9.lua} +11 -10
- package/dist/{cjs/commands/addStandardJob-8.lua → esm/commands/addStandardJob-9.lua} +9 -8
- package/dist/esm/commands/includes/collectMetrics.lua +1 -1
- package/dist/esm/commands/includes/deduplicateJob.lua +66 -20
- package/dist/esm/commands/includes/prepareJobForProcessing.lua +2 -1
- package/dist/esm/commands/includes/storeJob.lua +1 -1
- package/dist/esm/interfaces/index.d.ts +0 -1
- package/dist/esm/interfaces/index.js +0 -1
- package/dist/esm/interfaces/index.js.map +1 -1
- package/dist/esm/scripts/addDelayedJob-6.js +71 -22
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/{addParentJob-4.js → addParentJob-5.js} +78 -28
- package/dist/esm/scripts/addParentJob-5.js.map +1 -0
- package/dist/esm/scripts/{addPrioritizedJob-8.js → addPrioritizedJob-9.js} +82 -32
- package/dist/esm/scripts/{addPrioritizedJob-8.js.map → addPrioritizedJob-9.js.map} +1 -1
- package/dist/esm/scripts/{addStandardJob-8.js → addStandardJob-9.js} +80 -30
- package/dist/esm/scripts/{addStandardJob-8.js.map → addStandardJob-9.js.map} +1 -1
- package/dist/esm/scripts/index.d.ts +3 -3
- package/dist/esm/scripts/index.js +3 -3
- package/dist/esm/scripts/moveToActive-11.js +2 -0
- package/dist/esm/scripts/moveToActive-11.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +3 -1
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/deduplication-options.d.ts +22 -0
- package/dist/esm/types/deduplication-options.js +2 -0
- package/dist/esm/types/deduplication-options.js.map +1 -0
- package/dist/esm/types/index.d.ts +1 -0
- package/dist/esm/types/index.js +1 -0
- package/dist/esm/types/index.js.map +1 -1
- package/dist/esm/types/job-options.d.ts +6 -6
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +2 -5
- package/dist/cjs/interfaces/debounce-options.js.map +0 -1
- package/dist/cjs/scripts/addParentJob-4.js.map +0 -1
- package/dist/esm/interfaces/debounce-options.d.ts +0 -13
- package/dist/esm/interfaces/debounce-options.js +0 -2
- package/dist/esm/interfaces/debounce-options.js.map +0 -1
- package/dist/esm/scripts/addParentJob-4.js.map +0 -1
- /package/dist/esm/scripts/{addParentJob-4.d.ts → addParentJob-5.d.ts} +0 -0
- /package/dist/esm/scripts/{addPrioritizedJob-8.d.ts → addPrioritizedJob-9.d.ts} +0 -0
- /package/dist/esm/scripts/{addStandardJob-8.d.ts → addStandardJob-9.d.ts} +0 -0
@@ -114,26 +114,75 @@ local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
|
114
114
|
end
|
115
115
|
--[[
|
116
116
|
Function to debounce a job.
|
117
|
-
]]
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
117
|
+
]]
|
118
|
+
-- Includes
|
119
|
+
--[[
|
120
|
+
Function to remove job keys.
|
121
|
+
]]
|
122
|
+
local function removeJobKeys(jobKey)
|
123
|
+
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
124
|
+
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
125
|
+
end
|
126
|
+
local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
|
127
|
+
prefix)
|
128
|
+
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
129
|
+
if deduplicationId then
|
130
|
+
local ttl = deduplicationOpts['ttl']
|
131
|
+
if deduplicationOpts['replace'] and ttl and ttl > 0 then
|
132
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
133
|
+
if currentDebounceJobId then
|
134
|
+
if rcall("ZREM", delayedKey, currentDebounceJobId) > 0 then
|
135
|
+
removeJobKeys(prefix .. currentDebounceJobId)
|
136
|
+
rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDebounceJobId,
|
137
|
+
"prev", "delayed")
|
138
|
+
if deduplicationOpts['extend'] then
|
139
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
140
|
+
else
|
141
|
+
rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
|
142
|
+
end
|
143
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
144
|
+
jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
|
145
|
+
return
|
146
|
+
else
|
147
|
+
return currentDebounceJobId
|
148
|
+
end
|
149
|
+
else
|
150
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
151
|
+
return
|
152
|
+
end
|
153
|
+
else
|
154
|
+
local ttl = deduplicationOpts['ttl']
|
155
|
+
local deduplicationKeyExists
|
156
|
+
if ttl then
|
157
|
+
if deduplicationOpts['extend'] then
|
158
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
159
|
+
if currentDebounceJobId then
|
160
|
+
rcall('SET', deduplicationKey, currentDebounceJobId, 'PX', ttl)
|
161
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced",
|
162
|
+
"jobId", currentDebounceJobId, "debounceId", deduplicationId)
|
163
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
164
|
+
currentDebounceJobId, "deduplicationId", deduplicationId, "deduplicatedJobId", jobId)
|
165
|
+
return currentDebounceJobId
|
166
|
+
else
|
167
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
168
|
+
return
|
169
|
+
end
|
170
|
+
else
|
171
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
|
172
|
+
end
|
173
|
+
else
|
174
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
|
175
|
+
end
|
176
|
+
if deduplicationKeyExists then
|
177
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
178
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
|
179
|
+
currentDebounceJobId, "debounceId", deduplicationId)
|
180
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
181
|
+
currentDebounceJobId, "deduplicationId", deduplicationId, "deduplicatedJobId", jobId)
|
182
|
+
return currentDebounceJobId
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end
|
137
186
|
end
|
138
187
|
--[[
|
139
188
|
Function to get max events value or set by default 10000.
|
@@ -381,8 +430,8 @@ else
|
|
381
430
|
maxEvents, timestamp)
|
382
431
|
end
|
383
432
|
end
|
384
|
-
local deduplicationJobId = deduplicateJob(opts['de'], jobId, deduplicationKey,
|
385
|
-
eventsKey, maxEvents)
|
433
|
+
local deduplicationJobId = deduplicateJob(opts['de'], jobId, delayedKey, deduplicationKey,
|
434
|
+
eventsKey, maxEvents, args[1])
|
386
435
|
if deduplicationJobId then
|
387
436
|
return deduplicationJobId
|
388
437
|
end
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0bf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -9,8 +9,9 @@ const content = `--[[
|
|
9
9
|
Input:
|
10
10
|
KEYS[1] 'meta'
|
11
11
|
KEYS[2] 'id'
|
12
|
-
KEYS[3] '
|
13
|
-
KEYS[4]
|
12
|
+
KEYS[3] 'delayed'
|
13
|
+
KEYS[4] 'completed'
|
14
|
+
KEYS[5] events stream key
|
14
15
|
ARGV[1] msgpacked arguments array
|
15
16
|
[1] key prefix,
|
16
17
|
[2] custom id (will not generate one automatically)
|
@@ -30,8 +31,8 @@ const content = `--[[
|
|
30
31
|
]]
|
31
32
|
local metaKey = KEYS[1]
|
32
33
|
local idKey = KEYS[2]
|
33
|
-
local completedKey = KEYS[
|
34
|
-
local eventsKey = KEYS[
|
34
|
+
local completedKey = KEYS[4]
|
35
|
+
local eventsKey = KEYS[5]
|
35
36
|
local jobId
|
36
37
|
local jobIdKey
|
37
38
|
local rcall = redis.call
|
@@ -46,26 +47,75 @@ local parentData
|
|
46
47
|
-- Includes
|
47
48
|
--[[
|
48
49
|
Function to debounce a job.
|
49
|
-
]]
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
50
|
+
]]
|
51
|
+
-- Includes
|
52
|
+
--[[
|
53
|
+
Function to remove job keys.
|
54
|
+
]]
|
55
|
+
local function removeJobKeys(jobKey)
|
56
|
+
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
57
|
+
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
58
|
+
end
|
59
|
+
local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
|
60
|
+
prefix)
|
61
|
+
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
62
|
+
if deduplicationId then
|
63
|
+
local ttl = deduplicationOpts['ttl']
|
64
|
+
if deduplicationOpts['replace'] and ttl and ttl > 0 then
|
65
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
66
|
+
if currentDebounceJobId then
|
67
|
+
if rcall("ZREM", delayedKey, currentDebounceJobId) > 0 then
|
68
|
+
removeJobKeys(prefix .. currentDebounceJobId)
|
69
|
+
rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDebounceJobId,
|
70
|
+
"prev", "delayed")
|
71
|
+
if deduplicationOpts['extend'] then
|
72
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
73
|
+
else
|
74
|
+
rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
|
75
|
+
end
|
76
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
77
|
+
jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
|
78
|
+
return
|
79
|
+
else
|
80
|
+
return currentDebounceJobId
|
81
|
+
end
|
82
|
+
else
|
83
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
84
|
+
return
|
85
|
+
end
|
86
|
+
else
|
87
|
+
local ttl = deduplicationOpts['ttl']
|
88
|
+
local deduplicationKeyExists
|
89
|
+
if ttl then
|
90
|
+
if deduplicationOpts['extend'] then
|
91
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
92
|
+
if currentDebounceJobId then
|
93
|
+
rcall('SET', deduplicationKey, currentDebounceJobId, 'PX', ttl)
|
94
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced",
|
95
|
+
"jobId", currentDebounceJobId, "debounceId", deduplicationId)
|
96
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
97
|
+
currentDebounceJobId, "deduplicationId", deduplicationId, "deduplicatedJobId", jobId)
|
98
|
+
return currentDebounceJobId
|
99
|
+
else
|
100
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
101
|
+
return
|
102
|
+
end
|
103
|
+
else
|
104
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
|
105
|
+
end
|
106
|
+
else
|
107
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
|
108
|
+
end
|
109
|
+
if deduplicationKeyExists then
|
110
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
111
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
|
112
|
+
currentDebounceJobId, "debounceId", deduplicationId)
|
113
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
114
|
+
currentDebounceJobId, "deduplicationId", deduplicationId, "deduplicatedJobId", jobId)
|
115
|
+
return currentDebounceJobId
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
69
119
|
end
|
70
120
|
--[[
|
71
121
|
Function to get max events value or set by default 10000.
|
@@ -336,8 +386,8 @@ else
|
|
336
386
|
maxEvents, timestamp)
|
337
387
|
end
|
338
388
|
end
|
339
|
-
local deduplicationJobId = deduplicateJob(opts['de'], jobId,
|
340
|
-
deduplicationKey, eventsKey, maxEvents)
|
389
|
+
local deduplicationJobId = deduplicateJob(opts['de'], jobId, KEYS[3],
|
390
|
+
deduplicationKey, eventsKey, maxEvents, args[1])
|
341
391
|
if deduplicationJobId then
|
342
392
|
return deduplicationJobId
|
343
393
|
end
|
@@ -357,6 +407,6 @@ return jobId .. "" -- convert to string
|
|
357
407
|
exports.addParentJob = {
|
358
408
|
name: 'addParentJob',
|
359
409
|
content,
|
360
|
-
keys:
|
410
|
+
keys: 5,
|
361
411
|
};
|
362
|
-
//# sourceMappingURL=addParentJob-
|
412
|
+
//# sourceMappingURL=addParentJob-5.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"addParentJob-5.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-5.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkZf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -11,10 +11,11 @@ const content = `--[[
|
|
11
11
|
KEYS[2] 'meta'
|
12
12
|
KEYS[3] 'id'
|
13
13
|
KEYS[4] 'prioritized'
|
14
|
-
KEYS[5] '
|
15
|
-
KEYS[6] '
|
16
|
-
KEYS[7]
|
17
|
-
KEYS[8]
|
14
|
+
KEYS[5] 'delayed'
|
15
|
+
KEYS[6] 'completed'
|
16
|
+
KEYS[7] 'active'
|
17
|
+
KEYS[8] events stream key
|
18
|
+
KEYS[9] 'pc' priority counter
|
18
19
|
ARGV[1] msgpacked arguments array
|
19
20
|
[1] key prefix,
|
20
21
|
[2] custom id (will not generate one automatically)
|
@@ -35,10 +36,10 @@ const content = `--[[
|
|
35
36
|
local metaKey = KEYS[2]
|
36
37
|
local idKey = KEYS[3]
|
37
38
|
local priorityKey = KEYS[4]
|
38
|
-
local completedKey = KEYS[
|
39
|
-
local activeKey = KEYS[
|
40
|
-
local eventsKey = KEYS[
|
41
|
-
local priorityCounterKey = KEYS[
|
39
|
+
local completedKey = KEYS[6]
|
40
|
+
local activeKey = KEYS[7]
|
41
|
+
local eventsKey = KEYS[8]
|
42
|
+
local priorityCounterKey = KEYS[9]
|
42
43
|
local jobId
|
43
44
|
local jobIdKey
|
44
45
|
local rcall = redis.call
|
@@ -78,26 +79,75 @@ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, pr
|
|
78
79
|
end
|
79
80
|
--[[
|
80
81
|
Function to debounce a job.
|
81
|
-
]]
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
82
|
+
]]
|
83
|
+
-- Includes
|
84
|
+
--[[
|
85
|
+
Function to remove job keys.
|
86
|
+
]]
|
87
|
+
local function removeJobKeys(jobKey)
|
88
|
+
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
89
|
+
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
90
|
+
end
|
91
|
+
local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
|
92
|
+
prefix)
|
93
|
+
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
94
|
+
if deduplicationId then
|
95
|
+
local ttl = deduplicationOpts['ttl']
|
96
|
+
if deduplicationOpts['replace'] and ttl and ttl > 0 then
|
97
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
98
|
+
if currentDebounceJobId then
|
99
|
+
if rcall("ZREM", delayedKey, currentDebounceJobId) > 0 then
|
100
|
+
removeJobKeys(prefix .. currentDebounceJobId)
|
101
|
+
rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDebounceJobId,
|
102
|
+
"prev", "delayed")
|
103
|
+
if deduplicationOpts['extend'] then
|
104
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
105
|
+
else
|
106
|
+
rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
|
107
|
+
end
|
108
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
109
|
+
jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
|
110
|
+
return
|
111
|
+
else
|
112
|
+
return currentDebounceJobId
|
113
|
+
end
|
114
|
+
else
|
115
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
116
|
+
return
|
117
|
+
end
|
118
|
+
else
|
119
|
+
local ttl = deduplicationOpts['ttl']
|
120
|
+
local deduplicationKeyExists
|
121
|
+
if ttl then
|
122
|
+
if deduplicationOpts['extend'] then
|
123
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
124
|
+
if currentDebounceJobId then
|
125
|
+
rcall('SET', deduplicationKey, currentDebounceJobId, 'PX', ttl)
|
126
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced",
|
127
|
+
"jobId", currentDebounceJobId, "debounceId", deduplicationId)
|
128
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
129
|
+
currentDebounceJobId, "deduplicationId", deduplicationId, "deduplicatedJobId", jobId)
|
130
|
+
return currentDebounceJobId
|
131
|
+
else
|
132
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
133
|
+
return
|
134
|
+
end
|
135
|
+
else
|
136
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
|
137
|
+
end
|
138
|
+
else
|
139
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
|
140
|
+
end
|
141
|
+
if deduplicationKeyExists then
|
142
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
143
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
|
144
|
+
currentDebounceJobId, "debounceId", deduplicationId)
|
145
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
146
|
+
currentDebounceJobId, "deduplicationId", deduplicationId, "deduplicatedJobId", jobId)
|
147
|
+
return currentDebounceJobId
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
101
151
|
end
|
102
152
|
--[[
|
103
153
|
Function to store a job
|
@@ -343,8 +393,8 @@ else
|
|
343
393
|
maxEvents, timestamp)
|
344
394
|
end
|
345
395
|
end
|
346
|
-
local deduplicationJobId = deduplicateJob(opts['de'], jobId,
|
347
|
-
deduplicationKey, eventsKey, maxEvents)
|
396
|
+
local deduplicationJobId = deduplicateJob(opts['de'], jobId, KEYS[5],
|
397
|
+
deduplicationKey, eventsKey, maxEvents, args[1])
|
348
398
|
if deduplicationJobId then
|
349
399
|
return deduplicationJobId
|
350
400
|
end
|
@@ -367,6 +417,6 @@ return jobId .. "" -- convert to string
|
|
367
417
|
exports.addPrioritizedJob = {
|
368
418
|
name: 'addPrioritizedJob',
|
369
419
|
content,
|
370
|
-
keys:
|
420
|
+
keys: 9,
|
371
421
|
};
|
372
|
-
//# sourceMappingURL=addPrioritizedJob-
|
422
|
+
//# sourceMappingURL=addPrioritizedJob-9.js.map
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addPrioritizedJob-
|
1
|
+
{"version":3,"file":"addPrioritizedJob-9.js","sourceRoot":"","sources":["../../../src/scripts/addPrioritizedJob-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4Zf,CAAC;AACW,QAAA,iBAAiB,GAAG;IAC/B,IAAI,EAAE,mBAAmB;IACzB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -21,9 +21,10 @@ const content = `--[[
|
|
21
21
|
KEYS[3] 'meta'
|
22
22
|
KEYS[4] 'id'
|
23
23
|
KEYS[5] 'completed'
|
24
|
-
KEYS[6] '
|
25
|
-
KEYS[7]
|
26
|
-
KEYS[8]
|
24
|
+
KEYS[6] 'delayed'
|
25
|
+
KEYS[7] 'active'
|
26
|
+
KEYS[8] events stream key
|
27
|
+
KEYS[9] marker key
|
27
28
|
ARGV[1] msgpacked arguments array
|
28
29
|
[1] key prefix,
|
29
30
|
[2] custom id (will not generate one automatically)
|
@@ -41,7 +42,7 @@ const content = `--[[
|
|
41
42
|
jobId - OK
|
42
43
|
-5 - Missing parent key
|
43
44
|
]]
|
44
|
-
local eventsKey = KEYS[
|
45
|
+
local eventsKey = KEYS[8]
|
45
46
|
local jobId
|
46
47
|
local jobIdKey
|
47
48
|
local rcall = redis.call
|
@@ -72,26 +73,75 @@ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed
|
|
72
73
|
end
|
73
74
|
--[[
|
74
75
|
Function to debounce a job.
|
75
|
-
]]
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
76
|
+
]]
|
77
|
+
-- Includes
|
78
|
+
--[[
|
79
|
+
Function to remove job keys.
|
80
|
+
]]
|
81
|
+
local function removeJobKeys(jobKey)
|
82
|
+
return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
|
83
|
+
jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
|
84
|
+
end
|
85
|
+
local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
|
86
|
+
prefix)
|
87
|
+
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
88
|
+
if deduplicationId then
|
89
|
+
local ttl = deduplicationOpts['ttl']
|
90
|
+
if deduplicationOpts['replace'] and ttl and ttl > 0 then
|
91
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
92
|
+
if currentDebounceJobId then
|
93
|
+
if rcall("ZREM", delayedKey, currentDebounceJobId) > 0 then
|
94
|
+
removeJobKeys(prefix .. currentDebounceJobId)
|
95
|
+
rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDebounceJobId,
|
96
|
+
"prev", "delayed")
|
97
|
+
if deduplicationOpts['extend'] then
|
98
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
99
|
+
else
|
100
|
+
rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
|
101
|
+
end
|
102
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
103
|
+
jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
|
104
|
+
return
|
105
|
+
else
|
106
|
+
return currentDebounceJobId
|
107
|
+
end
|
108
|
+
else
|
109
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
110
|
+
return
|
111
|
+
end
|
112
|
+
else
|
113
|
+
local ttl = deduplicationOpts['ttl']
|
114
|
+
local deduplicationKeyExists
|
115
|
+
if ttl then
|
116
|
+
if deduplicationOpts['extend'] then
|
117
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
118
|
+
if currentDebounceJobId then
|
119
|
+
rcall('SET', deduplicationKey, currentDebounceJobId, 'PX', ttl)
|
120
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced",
|
121
|
+
"jobId", currentDebounceJobId, "debounceId", deduplicationId)
|
122
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
123
|
+
currentDebounceJobId, "deduplicationId", deduplicationId, "deduplicatedJobId", jobId)
|
124
|
+
return currentDebounceJobId
|
125
|
+
else
|
126
|
+
rcall('SET', deduplicationKey, jobId, 'PX', ttl)
|
127
|
+
return
|
128
|
+
end
|
129
|
+
else
|
130
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
|
131
|
+
end
|
132
|
+
else
|
133
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
|
134
|
+
end
|
135
|
+
if deduplicationKeyExists then
|
136
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
137
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
|
138
|
+
currentDebounceJobId, "debounceId", deduplicationId)
|
139
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
140
|
+
currentDebounceJobId, "deduplicationId", deduplicationId, "deduplicatedJobId", jobId)
|
141
|
+
return currentDebounceJobId
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
95
145
|
end
|
96
146
|
--[[
|
97
147
|
Function to get max events value or set by default 10000.
|
@@ -347,18 +397,18 @@ else
|
|
347
397
|
maxEvents, timestamp)
|
348
398
|
end
|
349
399
|
end
|
350
|
-
local deduplicationJobId = deduplicateJob(opts['de'], jobId,
|
351
|
-
deduplicationKey, eventsKey, maxEvents)
|
400
|
+
local deduplicationJobId = deduplicateJob(opts['de'], jobId, KEYS[6],
|
401
|
+
deduplicationKey, eventsKey, maxEvents, args[1])
|
352
402
|
if deduplicationJobId then
|
353
403
|
return deduplicationJobId
|
354
404
|
end
|
355
405
|
-- Store the job.
|
356
406
|
storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
|
357
407
|
parentKey, parentData, repeatJobKey)
|
358
|
-
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[
|
408
|
+
local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[7], KEYS[1], KEYS[2])
|
359
409
|
-- LIFO or FIFO
|
360
410
|
local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
|
361
|
-
addJobInTargetList(target, KEYS[
|
411
|
+
addJobInTargetList(target, KEYS[9], pushCmd, isPausedOrMaxed, jobId)
|
362
412
|
-- Emit waiting event
|
363
413
|
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
|
364
414
|
"jobId", jobId)
|
@@ -371,6 +421,6 @@ return jobId .. "" -- convert to string
|
|
371
421
|
exports.addStandardJob = {
|
372
422
|
name: 'addStandardJob',
|
373
423
|
content,
|
374
|
-
keys:
|
424
|
+
keys: 9,
|
375
425
|
};
|
376
|
-
//# sourceMappingURL=addStandardJob-
|
426
|
+
//# sourceMappingURL=addStandardJob-9.js.map
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addStandardJob-
|
1
|
+
{"version":3,"file":"addStandardJob-9.js","sourceRoot":"","sources":["../../../src/scripts/addStandardJob-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgaf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -4,10 +4,10 @@ const tslib_1 = require("tslib");
|
|
4
4
|
tslib_1.__exportStar(require("./addDelayedJob-6"), exports);
|
5
5
|
tslib_1.__exportStar(require("./addJobScheduler-11"), exports);
|
6
6
|
tslib_1.__exportStar(require("./addLog-2"), exports);
|
7
|
-
tslib_1.__exportStar(require("./addParentJob-
|
8
|
-
tslib_1.__exportStar(require("./addPrioritizedJob-
|
7
|
+
tslib_1.__exportStar(require("./addParentJob-5"), exports);
|
8
|
+
tslib_1.__exportStar(require("./addPrioritizedJob-9"), exports);
|
9
9
|
tslib_1.__exportStar(require("./addRepeatableJob-2"), exports);
|
10
|
-
tslib_1.__exportStar(require("./addStandardJob-
|
10
|
+
tslib_1.__exportStar(require("./addStandardJob-9"), exports);
|
11
11
|
tslib_1.__exportStar(require("./changeDelay-4"), exports);
|
12
12
|
tslib_1.__exportStar(require("./changePriority-7"), exports);
|
13
13
|
tslib_1.__exportStar(require("./cleanJobsInSet-3"), exports);
|
@@ -141,6 +141,8 @@ local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey
|
|
141
141
|
rcall("HMSET", jobKey, "processedOn", processedOn, unpack(optionalValues))
|
142
142
|
rcall("HINCRBY", jobKey, "ats", 1)
|
143
143
|
addBaseMarkerIfNeeded(markerKey, false)
|
144
|
+
-- rate limit delay must be 0 in this case to prevent adding more delay
|
145
|
+
-- when job that is moved to active needs to be processed
|
144
146
|
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
|
145
147
|
end
|
146
148
|
--[[
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyOf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|
@@ -57,7 +57,7 @@ local rcall = redis.call
|
|
57
57
|
--[[
|
58
58
|
Functions to collect metrics based on a current and previous count of jobs.
|
59
59
|
Granualarity is fixed at 1 minute.
|
60
|
-
]]
|
60
|
+
]]
|
61
61
|
--[[
|
62
62
|
Function to loop in batches.
|
63
63
|
Just a bit of warning, some commands as ZREM
|
@@ -567,6 +567,8 @@ local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey
|
|
567
567
|
rcall("HMSET", jobKey, "processedOn", processedOn, unpack(optionalValues))
|
568
568
|
rcall("HINCRBY", jobKey, "ats", 1)
|
569
569
|
addBaseMarkerIfNeeded(markerKey, false)
|
570
|
+
-- rate limit delay must be 0 in this case to prevent adding more delay
|
571
|
+
-- when job that is moved to active needs to be processed
|
570
572
|
return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
|
571
573
|
end
|
572
574
|
--[[
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAm1Bf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|