bullmq 5.44.1 → 5.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/async-fifo-queue.js.map +1 -1
- package/dist/cjs/classes/child-processor.js.map +1 -1
- package/dist/cjs/classes/child.js.map +1 -1
- package/dist/cjs/classes/flow-producer.js +15 -4
- package/dist/cjs/classes/flow-producer.js.map +1 -1
- package/dist/cjs/classes/job-scheduler.js +1 -1
- package/dist/cjs/classes/job-scheduler.js.map +1 -1
- package/dist/cjs/classes/job.js +137 -54
- package/dist/cjs/classes/job.js.map +1 -1
- package/dist/cjs/classes/queue-base.js +1 -4
- package/dist/cjs/classes/queue-base.js.map +1 -1
- package/dist/cjs/classes/queue-events-producer.js.map +1 -1
- package/dist/cjs/classes/queue-events.js.map +1 -1
- package/dist/cjs/classes/queue-getters.js +12 -15
- package/dist/cjs/classes/queue-getters.js.map +1 -1
- package/dist/cjs/classes/queue-keys.js.map +1 -1
- package/dist/cjs/classes/queue.js +12 -13
- package/dist/cjs/classes/queue.js.map +1 -1
- package/dist/cjs/classes/redis-connection.js +1 -1
- package/dist/cjs/classes/redis-connection.js.map +1 -1
- package/dist/cjs/classes/repeat.js +1 -1
- package/dist/cjs/classes/repeat.js.map +1 -1
- package/dist/cjs/classes/scripts.js +18 -7
- package/dist/cjs/classes/scripts.js.map +1 -1
- package/dist/cjs/classes/worker.js +4 -0
- package/dist/cjs/classes/worker.js.map +1 -1
- package/dist/cjs/commands/addJobScheduler-11.lua +52 -33
- package/dist/cjs/commands/includes/deduplicateJob.lua +16 -17
- package/dist/cjs/commands/includes/isLocked.lua +1 -0
- package/dist/cjs/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
- package/dist/cjs/commands/includes/moveParentToWait.lua +48 -0
- package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
- package/dist/cjs/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/cjs/commands/includes/removeJob.lua +2 -2
- package/dist/cjs/commands/includes/removeJobWithChildren.lua +94 -0
- package/dist/cjs/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/cjs/commands/includes/storeJob.lua +1 -1
- package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/cjs/commands/moveStalledJobsToWait-9.lua +18 -53
- package/dist/cjs/commands/moveToFinished-14.lua +43 -51
- package/dist/cjs/commands/moveToWaitingChildren-8.lua +5 -32
- package/dist/cjs/commands/removeJob-3.lua +14 -72
- package/dist/cjs/commands/removeUnprocessedChildren-2.lua +31 -0
- package/dist/cjs/commands/updateJobScheduler-12.lua +29 -23
- package/dist/cjs/scripts/addDelayedJob-6.js +73 -55
- package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/cjs/scripts/addJobScheduler-11.js +59 -40
- package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/cjs/scripts/addParentJob-4.js +73 -55
- package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
- package/dist/cjs/scripts/addPrioritizedJob-8.js +73 -55
- package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/cjs/scripts/addRepeatableJob-2.js +14 -9
- package/dist/cjs/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/cjs/scripts/addStandardJob-8.js +73 -55
- package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
- package/dist/cjs/scripts/cleanJobsInSet-3.js +14 -9
- package/dist/cjs/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/cjs/scripts/drain-5.js +14 -9
- package/dist/cjs/scripts/drain-5.js.map +1 -1
- package/dist/cjs/scripts/index.js +1 -0
- package/dist/cjs/scripts/index.js.map +1 -1
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js +157 -155
- package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/cjs/scripts/moveToFinished-14.js +183 -157
- package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
- package/dist/cjs/scripts/moveToWaitingChildren-8.js +145 -137
- package/dist/cjs/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/cjs/scripts/obliterate-2.js +14 -9
- package/dist/cjs/scripts/obliterate-2.js.map +1 -1
- package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
- package/dist/cjs/scripts/removeJob-3.js +97 -63
- package/dist/cjs/scripts/removeJob-3.js.map +1 -1
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js +338 -0
- package/dist/cjs/scripts/removeUnprocessedChildren-2.js.map +1 -0
- package/dist/cjs/scripts/updateJobScheduler-12.js +27 -21
- package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/types/index.js +1 -0
- package/dist/cjs/types/index.js.map +1 -1
- package/dist/cjs/types/job-progress.js +3 -0
- package/dist/cjs/types/job-progress.js.map +1 -0
- package/dist/cjs/utils.js +12 -1
- package/dist/cjs/utils.js.map +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/async-fifo-queue.js.map +1 -1
- package/dist/esm/classes/child-processor.js.map +1 -1
- package/dist/esm/classes/child.js.map +1 -1
- package/dist/esm/classes/flow-producer.js +15 -4
- package/dist/esm/classes/flow-producer.js.map +1 -1
- package/dist/esm/classes/job-scheduler.js +1 -1
- package/dist/esm/classes/job-scheduler.js.map +1 -1
- package/dist/esm/classes/job.d.ts +49 -14
- package/dist/esm/classes/job.js +138 -55
- package/dist/esm/classes/job.js.map +1 -1
- package/dist/esm/classes/queue-base.d.ts +1 -4
- package/dist/esm/classes/queue-base.js +1 -4
- package/dist/esm/classes/queue-base.js.map +1 -1
- package/dist/esm/classes/queue-events-producer.d.ts +2 -2
- package/dist/esm/classes/queue-events-producer.js.map +1 -1
- package/dist/esm/classes/queue-events.d.ts +118 -29
- package/dist/esm/classes/queue-events.js.map +1 -1
- package/dist/esm/classes/queue-getters.d.ts +11 -14
- package/dist/esm/classes/queue-getters.js +12 -15
- package/dist/esm/classes/queue-getters.js.map +1 -1
- package/dist/esm/classes/queue-keys.js.map +1 -1
- package/dist/esm/classes/queue.d.ts +14 -15
- package/dist/esm/classes/queue.js +12 -13
- package/dist/esm/classes/queue.js.map +1 -1
- package/dist/esm/classes/redis-connection.js +1 -1
- package/dist/esm/classes/redis-connection.js.map +1 -1
- package/dist/esm/classes/repeat.js +1 -1
- package/dist/esm/classes/repeat.js.map +1 -1
- package/dist/esm/classes/scripts.d.ts +4 -3
- package/dist/esm/classes/scripts.js +18 -7
- package/dist/esm/classes/scripts.js.map +1 -1
- package/dist/esm/classes/worker.d.ts +2 -1
- package/dist/esm/classes/worker.js +5 -1
- package/dist/esm/classes/worker.js.map +1 -1
- package/dist/esm/commands/addJobScheduler-11.lua +52 -33
- package/dist/esm/commands/includes/deduplicateJob.lua +16 -17
- package/dist/esm/commands/includes/isLocked.lua +1 -0
- package/dist/esm/commands/includes/moveChildFromDependenciesIfNeeded.lua +79 -0
- package/dist/esm/commands/includes/moveParentToWait.lua +48 -0
- package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +9 -50
- package/dist/esm/commands/includes/moveParentToWaitIfNoPendingDependencies.lua +13 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnFinalization.lua +23 -0
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeededOnRemoval.lua +16 -0
- package/dist/esm/commands/includes/removeJob.lua +2 -2
- package/dist/esm/commands/includes/removeJobWithChildren.lua +94 -0
- package/dist/esm/commands/includes/removeParentDependencyKey.lua +5 -5
- package/dist/esm/commands/includes/storeJob.lua +1 -1
- package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +2 -2
- package/dist/esm/commands/moveStalledJobsToWait-9.lua +18 -53
- package/dist/esm/commands/moveToFinished-14.lua +43 -51
- package/dist/esm/commands/moveToWaitingChildren-8.lua +5 -32
- package/dist/esm/commands/removeJob-3.lua +14 -72
- package/dist/esm/commands/removeUnprocessedChildren-2.lua +31 -0
- package/dist/esm/commands/updateJobScheduler-12.lua +29 -23
- package/dist/esm/interfaces/job-json.d.ts +5 -2
- package/dist/esm/interfaces/minimal-job.d.ts +29 -11
- package/dist/esm/interfaces/queue-options.d.ts +6 -2
- package/dist/esm/interfaces/repeat-options.d.ts +1 -1
- package/dist/esm/interfaces/sandboxed-job.d.ts +2 -2
- package/dist/esm/interfaces/sandboxed-options.d.ts +1 -1
- package/dist/esm/interfaces/telemetry.d.ts +18 -15
- package/dist/esm/interfaces/worker-options.d.ts +10 -10
- package/dist/esm/scripts/addDelayedJob-6.js +73 -55
- package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
- package/dist/esm/scripts/addJobScheduler-11.js +59 -40
- package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
- package/dist/esm/scripts/addParentJob-4.js +73 -55
- package/dist/esm/scripts/addParentJob-4.js.map +1 -1
- package/dist/esm/scripts/addPrioritizedJob-8.js +73 -55
- package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
- package/dist/esm/scripts/addRepeatableJob-2.js +14 -9
- package/dist/esm/scripts/addRepeatableJob-2.js.map +1 -1
- package/dist/esm/scripts/addStandardJob-8.js +73 -55
- package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
- package/dist/esm/scripts/cleanJobsInSet-3.js +14 -9
- package/dist/esm/scripts/cleanJobsInSet-3.js.map +1 -1
- package/dist/esm/scripts/drain-5.js +14 -9
- package/dist/esm/scripts/drain-5.js.map +1 -1
- package/dist/esm/scripts/index.d.ts +1 -0
- package/dist/esm/scripts/index.js +1 -0
- package/dist/esm/scripts/index.js.map +1 -1
- package/dist/esm/scripts/moveStalledJobsToWait-9.js +157 -155
- package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
- package/dist/esm/scripts/moveToFinished-14.js +183 -157
- package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
- package/dist/esm/scripts/moveToWaitingChildren-8.js +145 -137
- package/dist/esm/scripts/moveToWaitingChildren-8.js.map +1 -1
- package/dist/esm/scripts/obliterate-2.js +14 -9
- package/dist/esm/scripts/obliterate-2.js.map +1 -1
- package/dist/esm/scripts/removeChildDependency-1.js +5 -5
- package/dist/esm/scripts/removeJob-3.js +97 -63
- package/dist/esm/scripts/removeJob-3.js.map +1 -1
- package/dist/esm/scripts/removeUnprocessedChildren-2.d.ts +5 -0
- package/dist/esm/scripts/removeUnprocessedChildren-2.js +335 -0
- package/dist/esm/scripts/removeUnprocessedChildren-2.js.map +1 -0
- package/dist/esm/scripts/updateJobScheduler-12.js +27 -21
- package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/types/index.d.ts +1 -0
- package/dist/esm/types/index.js +1 -0
- package/dist/esm/types/index.js.map +1 -1
- package/dist/esm/types/job-options.d.ts +13 -2
- package/dist/esm/types/job-progress.d.ts +1 -0
- package/dist/esm/types/job-progress.js +2 -0
- package/dist/esm/types/job-progress.js.map +1 -0
- package/dist/esm/utils.d.ts +17 -0
- package/dist/esm/utils.js +11 -0
- package/dist/esm/utils.js.map +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +26 -21
- package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
- package/dist/cjs/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/cjs/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
- package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -80
- package/dist/esm/commands/includes/removeDeduplicationKey.lua +0 -11
- package/dist/esm/commands/includes/removeDeduplicationKeyIfNeeded.lua +0 -14
@@ -111,25 +111,25 @@ local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
|
|
111
111
|
end
|
112
112
|
--[[
|
113
113
|
Function to debounce a job.
|
114
|
-
]]
|
114
|
+
]]
|
115
115
|
local function deduplicateJob(deduplicationOpts, jobId, deduplicationKey, eventsKey, maxEvents)
|
116
116
|
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
117
117
|
if deduplicationId then
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
118
|
+
local ttl = deduplicationOpts['ttl']
|
119
|
+
local deduplicationKeyExists
|
120
|
+
if ttl then
|
121
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
|
122
|
+
else
|
123
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
|
124
|
+
end
|
125
|
+
if deduplicationKeyExists then
|
126
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
127
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId", currentDebounceJobId,
|
128
|
+
"debounceId", deduplicationId)
|
129
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
130
|
+
currentDebounceJobId, "deduplicationId", deduplicationId, "deduplicatedJobId", jobId)
|
131
|
+
return currentDebounceJobId
|
132
|
+
end
|
133
133
|
end
|
134
134
|
end
|
135
135
|
--[[
|
@@ -159,7 +159,16 @@ end
|
|
159
159
|
]]
|
160
160
|
-- Includes
|
161
161
|
--[[
|
162
|
-
Validate and move parent to
|
162
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
163
|
+
if no pending dependencies.
|
164
|
+
]]
|
165
|
+
-- Includes
|
166
|
+
--[[
|
167
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
168
|
+
]]
|
169
|
+
-- Includes
|
170
|
+
--[[
|
171
|
+
Move parent to a wait status (wait, prioritized or delayed)
|
163
172
|
]]
|
164
173
|
-- Includes
|
165
174
|
--[[
|
@@ -231,51 +240,60 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
231
240
|
end
|
232
241
|
return waitKey, false
|
233
242
|
end
|
234
|
-
local function
|
235
|
-
|
236
|
-
local
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
local
|
247
|
-
local
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
243
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
244
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
245
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
246
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
247
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
248
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
249
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
250
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
251
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
252
|
+
-- ignore dependencies if any left
|
253
|
+
rcall("HSET", parentKey, "igdp", 1)
|
254
|
+
if delay > 0 then
|
255
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
256
|
+
local score = delayedTimestamp * 0x1000
|
257
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
258
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
259
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
260
|
+
delayedTimestamp)
|
261
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
262
|
+
else
|
263
|
+
if priority == 0 then
|
264
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
265
|
+
parentWaitKey, parentPausedKey)
|
266
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
256
267
|
else
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
parentPausedKey)
|
261
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
262
|
-
parentId)
|
263
|
-
else
|
264
|
-
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
265
|
-
addJobWithPriority(parentMarkerKey,
|
266
|
-
parentQueueKey .. ":prioritized", priority,
|
267
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
268
|
-
end
|
269
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
270
|
-
"jobId", parentId, "prev", "waiting-children")
|
268
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
269
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
270
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
271
271
|
end
|
272
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
273
|
+
"waiting-children")
|
272
274
|
end
|
273
275
|
end
|
276
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
277
|
+
if rcall("EXISTS", parentKey) == 1 then
|
278
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
279
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
280
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
281
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
282
|
+
end
|
283
|
+
end
|
284
|
+
end
|
285
|
+
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
286
|
+
parentId, timestamp)
|
287
|
+
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
288
|
+
if doNotHavePendingDependencies then
|
289
|
+
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
290
|
+
end
|
291
|
+
end
|
274
292
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
275
293
|
parentId, jobIdKey, returnvalue, timestamp )
|
276
294
|
local processedSet = parentKey .. ":processed"
|
277
295
|
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
278
|
-
|
296
|
+
moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
279
297
|
end
|
280
298
|
local function updateExistingJobsParent(parentKey, parent, parentData,
|
281
299
|
parentDependenciesKey, completedKey,
|
@@ -327,7 +345,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
327
345
|
table.insert(optionalValues, "parent")
|
328
346
|
table.insert(optionalValues, parentData)
|
329
347
|
end
|
330
|
-
if repeatJobKey
|
348
|
+
if repeatJobKey then
|
331
349
|
table.insert(optionalValues, "rjk")
|
332
350
|
table.insert(optionalValues, repeatJobKey)
|
333
351
|
end
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2Yf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|
@@ -36,6 +36,7 @@ local waitKey = KEYS[3]
|
|
36
36
|
local pausedKey = KEYS[4]
|
37
37
|
local metaKey = KEYS[5]
|
38
38
|
local prioritizedKey = KEYS[6]
|
39
|
+
local eventsKey = KEYS[9]
|
39
40
|
local nextMillis = ARGV[1]
|
40
41
|
local jobSchedulerId = ARGV[3]
|
41
42
|
local templateOpts = cmsgpack.unpack(ARGV[5])
|
@@ -157,7 +158,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
157
158
|
table.insert(optionalValues, "parent")
|
158
159
|
table.insert(optionalValues, parentData)
|
159
160
|
end
|
160
|
-
if repeatJobKey
|
161
|
+
if repeatJobKey then
|
161
162
|
table.insert(optionalValues, "rjk")
|
162
163
|
table.insert(optionalValues, repeatJobKey)
|
163
164
|
end
|
@@ -237,13 +238,18 @@ end
|
|
237
238
|
]]
|
238
239
|
-- Includes
|
239
240
|
--[[
|
240
|
-
Function to remove deduplication key
|
241
|
+
Function to remove deduplication key if needed
|
242
|
+
when a job is being removed.
|
241
243
|
]]
|
242
|
-
local function
|
244
|
+
local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
|
245
|
+
jobKey, jobId)
|
243
246
|
local deduplicationId = rcall("HGET", jobKey, "deid")
|
244
247
|
if deduplicationId then
|
245
248
|
local deduplicationKey = prefixKey .. "de:" .. deduplicationId
|
246
|
-
rcall(
|
249
|
+
local currentJobId = rcall('GET', deduplicationKey)
|
250
|
+
if currentJobId and currentJobId == jobId then
|
251
|
+
return rcall("DEL", deduplicationKey)
|
252
|
+
end
|
247
253
|
end
|
248
254
|
end
|
249
255
|
--[[
|
@@ -269,7 +275,7 @@ end
|
|
269
275
|
local getJobKeyPrefix = function (jobKey, jobId)
|
270
276
|
return string.sub(jobKey, 0, #jobKey - #jobId)
|
271
277
|
end
|
272
|
-
local function
|
278
|
+
local function _moveParentToWait(parentPrefix, parentId, emitEvent)
|
273
279
|
local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
|
274
280
|
parentPrefix .. "wait", parentPrefix .. "paused")
|
275
281
|
addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
|
@@ -297,10 +303,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
297
303
|
rcall("DEL", parentPrefix .. "de:" .. debounceId)
|
298
304
|
end
|
299
305
|
else
|
300
|
-
|
306
|
+
_moveParentToWait(parentPrefix, parentId)
|
301
307
|
end
|
302
308
|
else
|
303
|
-
|
309
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
304
310
|
end
|
305
311
|
end
|
306
312
|
end
|
@@ -328,10 +334,10 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debou
|
|
328
334
|
rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
|
329
335
|
end
|
330
336
|
else
|
331
|
-
|
337
|
+
_moveParentToWait(parentPrefix, parentId)
|
332
338
|
end
|
333
339
|
else
|
334
|
-
|
340
|
+
_moveParentToWait(parentPrefix, parentId, true)
|
335
341
|
end
|
336
342
|
end
|
337
343
|
end
|
@@ -345,7 +351,7 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
|
|
345
351
|
local jobKey = baseKey .. jobId
|
346
352
|
removeParentDependencyKey(jobKey, hard, nil, baseKey)
|
347
353
|
if shouldRemoveDeduplicationKey then
|
348
|
-
|
354
|
+
removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
|
349
355
|
end
|
350
356
|
removeJobKeys(jobKey)
|
351
357
|
end
|
@@ -392,43 +398,56 @@ end
|
|
392
398
|
local schedulerKey = repeatKey .. ":" .. jobSchedulerId
|
393
399
|
local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
|
394
400
|
local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
|
401
|
+
local maxEvents = getOrSetMaxEvents(metaKey)
|
402
|
+
local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey, jobId,
|
403
|
+
metaKey, eventsKey)
|
404
|
+
if rcall("ZSCORE", delayedKey, jobId) then
|
405
|
+
removeJob(nextDelayedJobId, true, prefixKey, true --[[remove debounce key]] )
|
406
|
+
rcall("ZREM", delayedKey, jobId)
|
407
|
+
return true
|
408
|
+
elseif rcall("ZSCORE", prioritizedKey, jobId) then
|
409
|
+
removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
|
410
|
+
rcall("ZREM", prioritizedKey, jobId)
|
411
|
+
return true
|
412
|
+
else
|
413
|
+
local pausedOrWaitKey = waitKey
|
414
|
+
if isQueuePaused(metaKey) then
|
415
|
+
pausedOrWaitKey = pausedKey
|
416
|
+
end
|
417
|
+
if rcall("LREM", pausedOrWaitKey, 1, jobId) > 0 then
|
418
|
+
removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
|
419
|
+
return true
|
420
|
+
end
|
421
|
+
end
|
422
|
+
return false
|
423
|
+
end
|
424
|
+
if rcall("EXISTS", nextDelayedJobKey) == 1 then
|
425
|
+
if not removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
|
426
|
+
nextDelayedJobId, metaKey, eventsKey) then
|
427
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
|
428
|
+
"duplicated", "jobId", nextDelayedJobId)
|
429
|
+
return nextDelayedJobId .. "" -- convert to string
|
430
|
+
end
|
431
|
+
end
|
395
432
|
local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
|
396
|
-
if prevMillis
|
433
|
+
if prevMillis then
|
397
434
|
local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
elseif rcall("ZSCORE", prioritizedKey, currentJobId) ~= false then
|
403
|
-
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
404
|
-
rcall("ZREM", prioritizedKey, currentJobId)
|
405
|
-
else
|
406
|
-
if isQueuePaused(metaKey) then
|
407
|
-
if rcall("LREM", pausedKey, 1, currentJobId) > 0 then
|
408
|
-
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
409
|
-
end
|
410
|
-
else
|
411
|
-
if rcall("LREM", waitKey, 1, currentJobId) > 0 then
|
412
|
-
removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
|
413
|
-
end
|
414
|
-
end
|
415
|
-
end
|
435
|
+
local currentDelayedJobKey = schedulerKey .. ":" .. prevMillis
|
436
|
+
if currentJobId ~= nextDelayedJobId and rcall("EXISTS", currentDelayedJobKey) == 1 then
|
437
|
+
removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
|
438
|
+
currentJobId, metaKey, eventsKey)
|
416
439
|
end
|
417
440
|
end
|
418
441
|
local schedulerOpts = cmsgpack.unpack(ARGV[2])
|
419
442
|
storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
schedulerOpts['name'], maxEvents, ARGV[7], ARGV[4], jobSchedulerId)
|
427
|
-
if ARGV[9] ~= "" then
|
428
|
-
rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
|
429
|
-
end
|
430
|
-
return nextDelayedJobId .. "" -- convert to string
|
443
|
+
rcall("INCR", KEYS[8])
|
444
|
+
addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, ARGV[6], waitKey, pausedKey,
|
445
|
+
KEYS[11], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey,
|
446
|
+
schedulerOpts['name'], maxEvents, ARGV[7], ARGV[4], jobSchedulerId)
|
447
|
+
if ARGV[9] ~= "" then
|
448
|
+
rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
|
431
449
|
end
|
450
|
+
return nextDelayedJobId .. "" -- convert to string
|
432
451
|
`;
|
433
452
|
export const addJobScheduler = {
|
434
453
|
name: 'addJobScheduler',
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkcf,CAAC;AACF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
|
@@ -43,25 +43,25 @@ local parentData
|
|
43
43
|
-- Includes
|
44
44
|
--[[
|
45
45
|
Function to debounce a job.
|
46
|
-
]]
|
46
|
+
]]
|
47
47
|
local function deduplicateJob(deduplicationOpts, jobId, deduplicationKey, eventsKey, maxEvents)
|
48
48
|
local deduplicationId = deduplicationOpts and deduplicationOpts['id']
|
49
49
|
if deduplicationId then
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
50
|
+
local ttl = deduplicationOpts['ttl']
|
51
|
+
local deduplicationKeyExists
|
52
|
+
if ttl then
|
53
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
|
54
|
+
else
|
55
|
+
deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
|
56
|
+
end
|
57
|
+
if deduplicationKeyExists then
|
58
|
+
local currentDebounceJobId = rcall('GET', deduplicationKey)
|
59
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId", currentDebounceJobId,
|
60
|
+
"debounceId", deduplicationId)
|
61
|
+
rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
|
62
|
+
currentDebounceJobId, "deduplicationId", deduplicationId, "deduplicatedJobId", jobId)
|
63
|
+
return currentDebounceJobId
|
64
|
+
end
|
65
65
|
end
|
66
66
|
end
|
67
67
|
--[[
|
@@ -91,7 +91,16 @@ end
|
|
91
91
|
]]
|
92
92
|
-- Includes
|
93
93
|
--[[
|
94
|
-
Validate and move parent to
|
94
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized)
|
95
|
+
if no pending dependencies.
|
96
|
+
]]
|
97
|
+
-- Includes
|
98
|
+
--[[
|
99
|
+
Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
|
100
|
+
]]
|
101
|
+
-- Includes
|
102
|
+
--[[
|
103
|
+
Move parent to a wait status (wait, prioritized or delayed)
|
95
104
|
]]
|
96
105
|
-- Includes
|
97
106
|
--[[
|
@@ -187,51 +196,60 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
|
|
187
196
|
end
|
188
197
|
return waitKey, false
|
189
198
|
end
|
190
|
-
local function
|
191
|
-
|
192
|
-
local
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
local
|
203
|
-
local
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
199
|
+
local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
200
|
+
local parentWaitKey = parentQueueKey .. ":wait"
|
201
|
+
local parentPausedKey = parentQueueKey .. ":paused"
|
202
|
+
local parentActiveKey = parentQueueKey .. ":active"
|
203
|
+
local parentMetaKey = parentQueueKey .. ":meta"
|
204
|
+
local parentMarkerKey = parentQueueKey .. ":marker"
|
205
|
+
local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
|
206
|
+
local priority = tonumber(jobAttributes[1]) or 0
|
207
|
+
local delay = tonumber(jobAttributes[2]) or 0
|
208
|
+
-- ignore dependencies if any left
|
209
|
+
rcall("HSET", parentKey, "igdp", 1)
|
210
|
+
if delay > 0 then
|
211
|
+
local delayedTimestamp = tonumber(timestamp) + delay
|
212
|
+
local score = delayedTimestamp * 0x1000
|
213
|
+
local parentDelayedKey = parentQueueKey .. ":delayed"
|
214
|
+
rcall("ZADD", parentDelayedKey, score, parentId)
|
215
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
|
216
|
+
delayedTimestamp)
|
217
|
+
addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
|
218
|
+
else
|
219
|
+
if priority == 0 then
|
220
|
+
local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
|
221
|
+
parentWaitKey, parentPausedKey)
|
222
|
+
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
|
212
223
|
else
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
parentPausedKey)
|
217
|
-
addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
|
218
|
-
parentId)
|
219
|
-
else
|
220
|
-
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
221
|
-
addJobWithPriority(parentMarkerKey,
|
222
|
-
parentQueueKey .. ":prioritized", priority,
|
223
|
-
parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
|
224
|
-
end
|
225
|
-
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
|
226
|
-
"jobId", parentId, "prev", "waiting-children")
|
224
|
+
local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
|
225
|
+
addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
|
226
|
+
parentQueueKey .. ":pc", isPausedOrMaxed)
|
227
227
|
end
|
228
|
+
rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
|
229
|
+
"waiting-children")
|
228
230
|
end
|
229
231
|
end
|
232
|
+
local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
233
|
+
if rcall("EXISTS", parentKey) == 1 then
|
234
|
+
local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
|
235
|
+
if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then
|
236
|
+
rcall("ZREM", parentWaitingChildrenKey, parentId)
|
237
|
+
moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
|
238
|
+
end
|
239
|
+
end
|
240
|
+
end
|
241
|
+
local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
|
242
|
+
parentId, timestamp)
|
243
|
+
local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
|
244
|
+
if doNotHavePendingDependencies then
|
245
|
+
moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
|
246
|
+
end
|
247
|
+
end
|
230
248
|
local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
|
231
249
|
parentId, jobIdKey, returnvalue, timestamp )
|
232
250
|
local processedSet = parentKey .. ":processed"
|
233
251
|
rcall("HSET", processedSet, jobIdKey, returnvalue)
|
234
|
-
|
252
|
+
moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
|
235
253
|
end
|
236
254
|
local function updateExistingJobsParent(parentKey, parent, parentData,
|
237
255
|
parentDependenciesKey, completedKey,
|
@@ -283,7 +301,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
|
|
283
301
|
table.insert(optionalValues, "parent")
|
284
302
|
table.insert(optionalValues, parentData)
|
285
303
|
end
|
286
|
-
if repeatJobKey
|
304
|
+
if repeatJobKey then
|
287
305
|
table.insert(optionalValues, "rjk")
|
288
306
|
table.insert(optionalValues, repeatJobKey)
|
289
307
|
end
|
@@ -1 +1 @@
|
|
1
|
-
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG
|
1
|
+
{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkWf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
|