bullmq 4.14.1 → 4.14.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/dist/cjs/classes/async-fifo-queue.js +12 -5
  2. package/dist/cjs/classes/async-fifo-queue.js.map +1 -1
  3. package/dist/cjs/classes/scripts.js +57 -4
  4. package/dist/cjs/classes/scripts.js.map +1 -1
  5. package/dist/cjs/commands/addDelayedJob-7.lua +123 -0
  6. package/dist/cjs/commands/addParentJob-4.lua +100 -0
  7. package/dist/cjs/commands/addPrioritizedJob-8.lua +116 -0
  8. package/dist/cjs/commands/addStandardJob-6.lua +115 -0
  9. package/dist/cjs/commands/includes/storeJob.lua +30 -0
  10. package/dist/cjs/commands/includes/updateExistingJobsParent.lua +25 -0
  11. package/dist/cjs/commands/updateProgress-3.lua +30 -0
  12. package/dist/cjs/scripts/addDelayedJob-7.js +267 -0
  13. package/dist/cjs/scripts/addDelayedJob-7.js.map +1 -0
  14. package/dist/cjs/scripts/addParentJob-4.js +251 -0
  15. package/dist/cjs/scripts/addParentJob-4.js.map +1 -0
  16. package/dist/cjs/scripts/addPrioritizedJob-8.js +262 -0
  17. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -0
  18. package/dist/cjs/scripts/{addJob-9.js → addStandardJob-6.js} +87 -91
  19. package/dist/cjs/scripts/addStandardJob-6.js.map +1 -0
  20. package/dist/cjs/scripts/index.js +4 -1
  21. package/dist/cjs/scripts/index.js.map +1 -1
  22. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  23. package/dist/esm/classes/async-fifo-queue.d.ts +13 -1
  24. package/dist/esm/classes/async-fifo-queue.js +12 -5
  25. package/dist/esm/classes/async-fifo-queue.js.map +1 -1
  26. package/dist/esm/classes/job.d.ts +1 -1
  27. package/dist/esm/classes/scripts.d.ts +3 -0
  28. package/dist/esm/classes/scripts.js +57 -4
  29. package/dist/esm/classes/scripts.js.map +1 -1
  30. package/dist/esm/commands/addDelayedJob-7.lua +123 -0
  31. package/dist/esm/commands/addParentJob-4.lua +100 -0
  32. package/dist/esm/commands/addPrioritizedJob-8.lua +116 -0
  33. package/dist/esm/commands/addStandardJob-6.lua +115 -0
  34. package/dist/esm/commands/includes/storeJob.lua +30 -0
  35. package/dist/esm/commands/includes/updateExistingJobsParent.lua +25 -0
  36. package/dist/esm/commands/updateProgress-3.lua +30 -0
  37. package/dist/esm/scripts/addDelayedJob-7.d.ts +5 -0
  38. package/dist/esm/scripts/addDelayedJob-7.js +264 -0
  39. package/dist/esm/scripts/addDelayedJob-7.js.map +1 -0
  40. package/dist/esm/scripts/{addJob-9.d.ts → addParentJob-4.d.ts} +1 -1
  41. package/dist/esm/scripts/addParentJob-4.js +248 -0
  42. package/dist/esm/scripts/addParentJob-4.js.map +1 -0
  43. package/dist/esm/scripts/addPrioritizedJob-8.d.ts +5 -0
  44. package/dist/esm/scripts/addPrioritizedJob-8.js +259 -0
  45. package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -0
  46. package/dist/esm/scripts/addStandardJob-6.d.ts +5 -0
  47. package/dist/esm/scripts/{addJob-9.js → addStandardJob-6.js} +86 -90
  48. package/dist/esm/scripts/addStandardJob-6.js.map +1 -0
  49. package/dist/esm/scripts/index.d.ts +4 -1
  50. package/dist/esm/scripts/index.js +4 -1
  51. package/dist/esm/scripts/index.js.map +1 -1
  52. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  53. package/package.json +1 -1
  54. package/dist/cjs/commands/addJob-9.lua +0 -173
  55. package/dist/cjs/scripts/addJob-9.js.map +0 -1
  56. package/dist/esm/commands/addJob-9.lua +0 -173
  57. package/dist/esm/scripts/addJob-9.js.map +0 -1
@@ -0,0 +1,30 @@
1
+ --[[
2
+ Function to store a job
3
+ ]]
4
+ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
5
+ parentKey, parentData, repeatJobKey)
6
+ local jsonOpts = cjson.encode(opts)
7
+ local delay = opts['delay'] or 0
8
+ local priority = opts['priority'] or 0
9
+
10
+ local optionalValues = {}
11
+ if parentKey ~= nil then
12
+ table.insert(optionalValues, "parentKey")
13
+ table.insert(optionalValues, parentKey)
14
+ table.insert(optionalValues, "parent")
15
+ table.insert(optionalValues, parentData)
16
+ end
17
+
18
+ if repeatJobKey ~= nil then
19
+ table.insert(optionalValues, "rjk")
20
+ table.insert(optionalValues, repeatJobKey)
21
+ end
22
+
23
+ rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
24
+ "timestamp", timestamp, "delay", delay, "priority", priority,
25
+ unpack(optionalValues))
26
+
27
+ rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
28
+
29
+ return delay, priority
30
+ end
@@ -0,0 +1,25 @@
1
+ --- @include "updateParentDepsIfNeeded"
2
+
3
+ --[[
4
+ This function is used to update the parent's dependencies if the job
5
+ is already completed and about to be ignored. The parent must get its
6
+ dependencies updated to avoid the parent job being stuck forever in
7
+ the waiting-children state.
8
+ ]]
9
+ local function updateExistingJobsParent(parentKey, parent, parentData,
10
+ parentDependenciesKey, completedKey,
11
+ jobIdKey, jobId, timestamp)
12
+ if parentKey ~= nil then
13
+ if rcall("ZSCORE", completedKey, jobId) ~= false then
14
+ local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
15
+ updateParentDepsIfNeeded(parentKey, parent['queueKey'],
16
+ parentDependenciesKey, parent['id'],
17
+ jobIdKey, returnvalue, timestamp)
18
+ else
19
+ if parentDependenciesKey ~= nil then
20
+ rcall("SADD", parentDependenciesKey, jobIdKey)
21
+ end
22
+ end
23
+ rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
24
+ end
25
+ end
@@ -0,0 +1,30 @@
1
+ --[[
2
+ Update job progress
3
+
4
+ Input:
5
+ KEYS[1] Job id key
6
+ KEYS[2] event stream key
7
+ KEYS[3] meta key
8
+
9
+ ARGV[1] id
10
+ ARGV[2] progress
11
+
12
+ Output:
13
+ 0 - OK
14
+ -1 - Missing job.
15
+
16
+ Event:
17
+ progress(jobId, progress)
18
+ ]]
19
+ local rcall = redis.call
20
+
21
+ if rcall("EXISTS", KEYS[1]) == 1 then -- // Make sure job exists
22
+ local maxEvents = rcall("HGET", KEYS[3], "opts.maxLenEvents") or 10000
23
+
24
+ rcall("HSET", KEYS[1], "progress", ARGV[2])
25
+ rcall("XADD", KEYS[2], "MAXLEN", "~", maxEvents, "*", "event", "progress",
26
+ "jobId", ARGV[1], "data", ARGV[2]);
27
+ return 0
28
+ else
29
+ return -1
30
+ end
@@ -0,0 +1,5 @@
1
+ export declare const addDelayedJob: {
2
+ name: string;
3
+ content: string;
4
+ keys: number;
5
+ };
@@ -0,0 +1,264 @@
1
+ const content = `--[[
2
+ Adds a delayed job to the queue by doing the following:
3
+ - Increases the job counter if needed.
4
+ - Creates a new job key with the job data.
5
+ - computes timestamp.
6
+ - adds to delayed zset.
7
+ - Emits a global event 'delayed' if the job is delayed.
8
+ Input:
9
+ KEYS[1] 'wait',
10
+ KEYS[2] 'paused'
11
+ KEYS[3] 'meta'
12
+ KEYS[4] 'id'
13
+ KEYS[5] 'delayed'
14
+ KEYS[6] 'completed'
15
+ KEYS[7] events stream key
16
+ ARGV[1] msgpacked arguments array
17
+ [1] key prefix,
18
+ [2] custom id (use custom instead of one generated automatically)
19
+ [3] name
20
+ [4] timestamp
21
+ [5] parentKey?
22
+ x [6] waitChildrenKey key.
23
+ [7] parent dependencies key.
24
+ [8] parent? {id, queueKey}
25
+ [9] repeat job key
26
+ ARGV[2] Json stringified job data
27
+ ARGV[3] msgpacked options
28
+ Output:
29
+ jobId - OK
30
+ -5 - Missing parent key
31
+ ]]
32
+ local waitKey = KEYS[1]
33
+ local pausedKey = KEYS[2]
34
+ local metaKey = KEYS[3]
35
+ local idKey = KEYS[4]
36
+ local delayedKey = KEYS[5]
37
+ local completedKey = KEYS[6]
38
+ local eventsKey = KEYS[7]
39
+ local jobId
40
+ local jobIdKey
41
+ local rcall = redis.call
42
+ local args = cmsgpack.unpack(ARGV[1])
43
+ local data = ARGV[2]
44
+ local opts = cmsgpack.unpack(ARGV[3])
45
+ local parentKey = args[5]
46
+ local repeatJobKey = args[9]
47
+ local parent = args[8]
48
+ local parentData
49
+ -- Includes
50
+ --[[
51
+ Function to store a job
52
+ ]]
53
+ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
54
+ parentKey, parentData, repeatJobKey)
55
+ local jsonOpts = cjson.encode(opts)
56
+ local delay = opts['delay'] or 0
57
+ local priority = opts['priority'] or 0
58
+ local optionalValues = {}
59
+ if parentKey ~= nil then
60
+ table.insert(optionalValues, "parentKey")
61
+ table.insert(optionalValues, parentKey)
62
+ table.insert(optionalValues, "parent")
63
+ table.insert(optionalValues, parentData)
64
+ end
65
+ if repeatJobKey ~= nil then
66
+ table.insert(optionalValues, "rjk")
67
+ table.insert(optionalValues, repeatJobKey)
68
+ end
69
+ rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
70
+ "timestamp", timestamp, "delay", delay, "priority", priority,
71
+ unpack(optionalValues))
72
+ rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
73
+ return delay, priority
74
+ end
75
+ --[[
76
+ Add delay marker if needed.
77
+ ]]
78
+ -- Includes
79
+ --[[
80
+ Function to return the next delayed job timestamp.
81
+ ]]
82
+ local function getNextDelayedTimestamp(delayedKey)
83
+ local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
84
+ if #result then
85
+ local nextTimestamp = tonumber(result[2])
86
+ if (nextTimestamp ~= nil) then
87
+ nextTimestamp = nextTimestamp / 0x1000
88
+ end
89
+ return nextTimestamp
90
+ end
91
+ end
92
+ local function addDelayMarkerIfNeeded(targetKey, delayedKey)
93
+ local waitLen = rcall("LLEN", targetKey)
94
+ if waitLen <= 1 then
95
+ local nextTimestamp = getNextDelayedTimestamp(delayedKey)
96
+ if nextTimestamp ~= nil then
97
+ -- Check if there is already a marker with older timestamp
98
+ -- if there is, we need to replace it.
99
+ if waitLen == 1 then
100
+ local marker = rcall("LINDEX", targetKey, 0)
101
+ local oldTimestamp = tonumber(marker:sub(3))
102
+ if oldTimestamp and oldTimestamp > nextTimestamp then
103
+ rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
104
+ end
105
+ else
106
+ -- if there is no marker, then we need to add one
107
+ rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
108
+ end
109
+ end
110
+ end
111
+ end
112
+ --[[
113
+ Function to check for the meta.paused key to decide if we are paused or not
114
+ (since an empty list and !EXISTS are not really the same).
115
+ ]]
116
+ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
117
+ if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
118
+ return waitKey, false
119
+ else
120
+ return pausedKey, true
121
+ end
122
+ end
123
+ --[[
124
+ Validate and move or add dependencies to parent.
125
+ ]]
126
+ -- Includes
127
+ --[[
128
+ Validate and move parent to active if needed.
129
+ ]]
130
+ -- Includes
131
+ --[[
132
+ Function to add job considering priority.
133
+ ]]
134
+ -- Includes
135
+ --[[
136
+ Function priority marker to wait if needed
137
+ in order to wake up our workers and to respect priority
138
+ order as much as possible
139
+ ]]
140
+ local function addPriorityMarkerIfNeeded(waitKey)
141
+ local waitLen = rcall("LLEN", waitKey)
142
+ if waitLen == 0 then
143
+ rcall("LPUSH", waitKey, "0:0")
144
+ end
145
+ end
146
+ local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
147
+ local prioCounter = rcall("INCR", priorityCounterKey)
148
+ local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
149
+ rcall("ZADD", prioritizedKey, score, jobId)
150
+ if not paused then
151
+ addPriorityMarkerIfNeeded(waitKey)
152
+ end
153
+ end
154
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
155
+ local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
156
+ if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
157
+ rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
158
+ local parentWaitKey = parentQueueKey .. ":wait"
159
+ local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
160
+ parentQueueKey .. ":paused")
161
+ local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
162
+ local priority = tonumber(jobAttributes[1]) or 0
163
+ local delay = tonumber(jobAttributes[2]) or 0
164
+ if delay > 0 then
165
+ local delayedTimestamp = tonumber(timestamp) + delay
166
+ local score = delayedTimestamp * 0x1000
167
+ local parentDelayedKey = parentQueueKey .. ":delayed"
168
+ rcall("ZADD", parentDelayedKey, score, parentId)
169
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
170
+ "delay", delayedTimestamp)
171
+ addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
172
+ else
173
+ if priority == 0 then
174
+ rcall("RPUSH", parentTarget, parentId)
175
+ else
176
+ addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
177
+ parentId, parentQueueKey .. ":pc")
178
+ end
179
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
180
+ "prev", "waiting-children")
181
+ end
182
+ end
183
+ end
184
+ local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
185
+ parentId, jobIdKey, returnvalue, timestamp )
186
+ local processedSet = parentKey .. ":processed"
187
+ rcall("HSET", processedSet, jobIdKey, returnvalue)
188
+ moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
189
+ end
190
+ --[[
191
+ This function is used to update the parent's dependencies if the job
192
+ is already completed and about to be ignored. The parent must get its
193
+ dependencies updated to avoid the parent job being stuck forever in
194
+ the waiting-children state.
195
+ ]]
196
+ local function updateExistingJobsParent(parentKey, parent, parentData,
197
+ parentDependenciesKey, completedKey,
198
+ jobIdKey, jobId, timestamp)
199
+ if parentKey ~= nil then
200
+ if rcall("ZSCORE", completedKey, jobId) ~= false then
201
+ local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
202
+ updateParentDepsIfNeeded(parentKey, parent['queueKey'],
203
+ parentDependenciesKey, parent['id'],
204
+ jobIdKey, returnvalue, timestamp)
205
+ else
206
+ if parentDependenciesKey ~= nil then
207
+ rcall("SADD", parentDependenciesKey, jobIdKey)
208
+ end
209
+ end
210
+ rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
211
+ end
212
+ end
213
+ if parentKey ~= nil then
214
+ if rcall("EXISTS", parentKey) ~= 1 then return -5 end
215
+ parentData = cjson.encode(parent)
216
+ end
217
+ local jobCounter = rcall("INCR", idKey)
218
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents") or 10000
219
+ local parentDependenciesKey = args[7]
220
+ local timestamp = args[4]
221
+ if args[2] == "" then
222
+ jobId = jobCounter
223
+ jobIdKey = args[1] .. jobId
224
+ else
225
+ -- Refactor to: handleDuplicateJob.lua
226
+ jobId = args[2]
227
+ jobIdKey = args[1] .. jobId
228
+ if rcall("EXISTS", jobIdKey) == 1 then
229
+ updateExistingJobsParent(parentKey, parent, parentData,
230
+ parentDependenciesKey, completedKey, jobIdKey,
231
+ jobId, timestamp)
232
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
233
+ "duplicated", "jobId", jobId)
234
+ return jobId .. "" -- convert to string
235
+ end
236
+ end
237
+ -- Store the job.
238
+ local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
239
+ opts, timestamp, parentKey, parentData,
240
+ repeatJobKey)
241
+ -- Compute delayed timestamp and the score.
242
+ local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
243
+ local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
244
+ rcall("ZADD", delayedKey, score, jobId)
245
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
246
+ "jobId", jobId, "delay", delayedTimestamp)
247
+ -- If wait list is empty, and this delayed job is the next one to be processed,
248
+ -- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
249
+ local target = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
250
+ addDelayMarkerIfNeeded(target, delayedKey)
251
+ -- Check if this job is a child of another job, if so add it to the parents dependencies
252
+ -- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
253
+ -- fail in this case.
254
+ if parentDependenciesKey ~= nil then
255
+ rcall("SADD", parentDependenciesKey, jobIdKey)
256
+ end
257
+ return jobId .. "" -- convert to string
258
+ `;
259
+ export const addDelayedJob = {
260
+ name: 'addDelayedJob',
261
+ content,
262
+ keys: 7,
263
+ };
264
+ //# sourceMappingURL=addDelayedJob-7.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"addDelayedJob-7.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiQf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -1,4 +1,4 @@
1
- export declare const addJob: {
1
+ export declare const addParentJob: {
2
2
  name: string;
3
3
  content: string;
4
4
  keys: number;
@@ -0,0 +1,248 @@
1
+ const content = `--[[
2
+ Adds a parent job to the queue by doing the following:
3
+ - Increases the job counter if needed.
4
+ - Creates a new job key with the job data.
5
+ - adds the job to the waiting-children zset
6
+ Input:
7
+ KEYS[1] 'meta'
8
+ KEYS[2] 'id'
9
+ KEYS[3] 'completed'
10
+ KEYS[4] events stream key
11
+ ARGV[1] msgpacked arguments array
12
+ [1] key prefix,
13
+ [2] custom id (will not generate one automatically)
14
+ [3] name
15
+ [4] timestamp
16
+ [5] parentKey?
17
+ [6] waitChildrenKey key.
18
+ [7] parent dependencies key.
19
+ [8] parent? {id, queueKey}
20
+ [9] repeat job key
21
+ ARGV[2] Json stringified job data
22
+ ARGV[3] msgpacked options
23
+ Output:
24
+ jobId - OK
25
+ -5 - Missing parent key
26
+ ]]
27
+ local metaKey = KEYS[1]
28
+ local idKey = KEYS[2]
29
+ local completedKey = KEYS[3]
30
+ local eventsKey = KEYS[4]
31
+ local jobId
32
+ local jobIdKey
33
+ local rcall = redis.call
34
+ local args = cmsgpack.unpack(ARGV[1])
35
+ local data = ARGV[2]
36
+ local opts = cmsgpack.unpack(ARGV[3])
37
+ local parentKey = args[5]
38
+ local repeatJobKey = args[9]
39
+ local parent = args[8]
40
+ local parentData
41
+ -- Includes
42
+ --[[
43
+ Function to store a job
44
+ ]]
45
+ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
46
+ parentKey, parentData, repeatJobKey)
47
+ local jsonOpts = cjson.encode(opts)
48
+ local delay = opts['delay'] or 0
49
+ local priority = opts['priority'] or 0
50
+ local optionalValues = {}
51
+ if parentKey ~= nil then
52
+ table.insert(optionalValues, "parentKey")
53
+ table.insert(optionalValues, parentKey)
54
+ table.insert(optionalValues, "parent")
55
+ table.insert(optionalValues, parentData)
56
+ end
57
+ if repeatJobKey ~= nil then
58
+ table.insert(optionalValues, "rjk")
59
+ table.insert(optionalValues, repeatJobKey)
60
+ end
61
+ rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
62
+ "timestamp", timestamp, "delay", delay, "priority", priority,
63
+ unpack(optionalValues))
64
+ rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
65
+ return delay, priority
66
+ end
67
+ --[[
68
+ Validate and move or add dependencies to parent.
69
+ ]]
70
+ -- Includes
71
+ --[[
72
+ Validate and move parent to active if needed.
73
+ ]]
74
+ -- Includes
75
+ --[[
76
+ Add delay marker if needed.
77
+ ]]
78
+ -- Includes
79
+ --[[
80
+ Function to return the next delayed job timestamp.
81
+ ]]
82
+ local function getNextDelayedTimestamp(delayedKey)
83
+ local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
84
+ if #result then
85
+ local nextTimestamp = tonumber(result[2])
86
+ if (nextTimestamp ~= nil) then
87
+ nextTimestamp = nextTimestamp / 0x1000
88
+ end
89
+ return nextTimestamp
90
+ end
91
+ end
92
+ local function addDelayMarkerIfNeeded(targetKey, delayedKey)
93
+ local waitLen = rcall("LLEN", targetKey)
94
+ if waitLen <= 1 then
95
+ local nextTimestamp = getNextDelayedTimestamp(delayedKey)
96
+ if nextTimestamp ~= nil then
97
+ -- Check if there is already a marker with older timestamp
98
+ -- if there is, we need to replace it.
99
+ if waitLen == 1 then
100
+ local marker = rcall("LINDEX", targetKey, 0)
101
+ local oldTimestamp = tonumber(marker:sub(3))
102
+ if oldTimestamp and oldTimestamp > nextTimestamp then
103
+ rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
104
+ end
105
+ else
106
+ -- if there is no marker, then we need to add one
107
+ rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
108
+ end
109
+ end
110
+ end
111
+ end
112
+ --[[
113
+ Function to add job considering priority.
114
+ ]]
115
+ -- Includes
116
+ --[[
117
+ Function priority marker to wait if needed
118
+ in order to wake up our workers and to respect priority
119
+ order as much as possible
120
+ ]]
121
+ local function addPriorityMarkerIfNeeded(waitKey)
122
+ local waitLen = rcall("LLEN", waitKey)
123
+ if waitLen == 0 then
124
+ rcall("LPUSH", waitKey, "0:0")
125
+ end
126
+ end
127
+ local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
128
+ local prioCounter = rcall("INCR", priorityCounterKey)
129
+ local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
130
+ rcall("ZADD", prioritizedKey, score, jobId)
131
+ if not paused then
132
+ addPriorityMarkerIfNeeded(waitKey)
133
+ end
134
+ end
135
+ --[[
136
+ Function to check for the meta.paused key to decide if we are paused or not
137
+ (since an empty list and !EXISTS are not really the same).
138
+ ]]
139
+ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
140
+ if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
141
+ return waitKey, false
142
+ else
143
+ return pausedKey, true
144
+ end
145
+ end
146
+ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
147
+ local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
148
+ if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
149
+ rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
150
+ local parentWaitKey = parentQueueKey .. ":wait"
151
+ local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
152
+ parentQueueKey .. ":paused")
153
+ local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
154
+ local priority = tonumber(jobAttributes[1]) or 0
155
+ local delay = tonumber(jobAttributes[2]) or 0
156
+ if delay > 0 then
157
+ local delayedTimestamp = tonumber(timestamp) + delay
158
+ local score = delayedTimestamp * 0x1000
159
+ local parentDelayedKey = parentQueueKey .. ":delayed"
160
+ rcall("ZADD", parentDelayedKey, score, parentId)
161
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
162
+ "delay", delayedTimestamp)
163
+ addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
164
+ else
165
+ if priority == 0 then
166
+ rcall("RPUSH", parentTarget, parentId)
167
+ else
168
+ addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
169
+ parentId, parentQueueKey .. ":pc")
170
+ end
171
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
172
+ "prev", "waiting-children")
173
+ end
174
+ end
175
+ end
176
+ local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
177
+ parentId, jobIdKey, returnvalue, timestamp )
178
+ local processedSet = parentKey .. ":processed"
179
+ rcall("HSET", processedSet, jobIdKey, returnvalue)
180
+ moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
181
+ end
182
+ --[[
183
+ This function is used to update the parent's dependencies if the job
184
+ is already completed and about to be ignored. The parent must get its
185
+ dependencies updated to avoid the parent job being stuck forever in
186
+ the waiting-children state.
187
+ ]]
188
+ local function updateExistingJobsParent(parentKey, parent, parentData,
189
+ parentDependenciesKey, completedKey,
190
+ jobIdKey, jobId, timestamp)
191
+ if parentKey ~= nil then
192
+ if rcall("ZSCORE", completedKey, jobId) ~= false then
193
+ local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
194
+ updateParentDepsIfNeeded(parentKey, parent['queueKey'],
195
+ parentDependenciesKey, parent['id'],
196
+ jobIdKey, returnvalue, timestamp)
197
+ else
198
+ if parentDependenciesKey ~= nil then
199
+ rcall("SADD", parentDependenciesKey, jobIdKey)
200
+ end
201
+ end
202
+ rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
203
+ end
204
+ end
205
+ if parentKey ~= nil then
206
+ if rcall("EXISTS", parentKey) ~= 1 then return -5 end
207
+ parentData = cjson.encode(parent)
208
+ end
209
+ local jobCounter = rcall("INCR", idKey)
210
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents") or 10000
211
+ local parentDependenciesKey = args[7]
212
+ local timestamp = args[4]
213
+ if args[2] == "" then
214
+ jobId = jobCounter
215
+ jobIdKey = args[1] .. jobId
216
+ else
217
+ jobId = args[2]
218
+ jobIdKey = args[1] .. jobId
219
+ if rcall("EXISTS", jobIdKey) == 1 then
220
+ updateExistingJobsParent(parentKey, parent, parentData,
221
+ parentDependenciesKey, completedKey, jobIdKey,
222
+ jobId, timestamp)
223
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
224
+ "duplicated", "jobId", jobId)
225
+ return jobId .. "" -- convert to string
226
+ end
227
+ end
228
+ -- Store the job.
229
+ storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
230
+ parentKey, parentData, repeatJobKey)
231
+ local waitChildrenKey = args[6]
232
+ rcall("ZADD", waitChildrenKey, timestamp, jobId)
233
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
234
+ "waiting-children", "jobId", jobId)
235
+ -- Check if this job is a child of another job, if so add it to the parents dependencies
236
+ -- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
237
+ -- fail in this case.
238
+ if parentDependenciesKey ~= nil then
239
+ rcall("SADD", parentDependenciesKey, jobIdKey)
240
+ end
241
+ return jobId .. "" -- convert to string
242
+ `;
243
+ export const addParentJob = {
244
+ name: 'addParentJob',
245
+ content,
246
+ keys: 4,
247
+ };
248
+ //# sourceMappingURL=addParentJob-4.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiPf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -0,0 +1,5 @@
1
+ export declare const addPrioritizedJob: {
2
+ name: string;
3
+ content: string;
4
+ keys: number;
5
+ };