bullmq 6.5.0 → 6.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/flow-producer.js +8 -2
- package/dist/cjs/classes/redis-queue-backend.js +38 -2
- package/dist/cjs/commands/addDelayedJob-6.lua +3 -1
- package/dist/cjs/commands/addParentJob-6.lua +3 -1
- package/dist/cjs/commands/addPrioritizedJob-9.lua +3 -1
- package/dist/cjs/commands/addStandardJob-9.lua +3 -1
- package/dist/cjs/commands/includes/handleDuplicatedJob.lua +8 -3
- package/dist/cjs/commands/includes/updateExistingJobsParent.lua +16 -4
- package/dist/cjs/commands/releaseParentIfNoPendingDependencies-2.lua +34 -0
- package/dist/cjs/postgres/migrations/index.js +9 -0
- package/dist/cjs/postgres/sql-scripts.js +1 -0
- package/dist/cjs/scripts/addDelayedJob-6.js +26 -8
- package/dist/cjs/scripts/addParentJob-6.js +26 -8
- package/dist/cjs/scripts/addPrioritizedJob-9.js +26 -8
- package/dist/cjs/scripts/addStandardJob-9.js +26 -8
- package/dist/cjs/scripts/index.js +1 -0
- package/dist/cjs/scripts/releaseParentIfNoPendingDependencies-2.js +167 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/flow-producer.js +8 -2
- package/dist/esm/classes/redis-queue-backend.js +38 -2
- package/dist/esm/commands/addDelayedJob-6.lua +3 -1
- package/dist/esm/commands/addParentJob-6.lua +3 -1
- package/dist/esm/commands/addPrioritizedJob-9.lua +3 -1
- package/dist/esm/commands/addStandardJob-9.lua +3 -1
- package/dist/esm/commands/includes/handleDuplicatedJob.lua +8 -3
- package/dist/esm/commands/includes/updateExistingJobsParent.lua +16 -4
- package/dist/esm/commands/releaseParentIfNoPendingDependencies-2.lua +34 -0
- package/dist/esm/interfaces/parent.d.ts +8 -0
- package/dist/esm/postgres/migrations/index.js +9 -0
- package/dist/esm/postgres/sql-scripts.js +1 -0
- package/dist/esm/scripts/addDelayedJob-6.js +26 -8
- package/dist/esm/scripts/addParentJob-6.js +26 -8
- package/dist/esm/scripts/addPrioritizedJob-9.js +26 -8
- package/dist/esm/scripts/addStandardJob-9.js +26 -8
- package/dist/esm/scripts/index.d.ts +1 -0
- package/dist/esm/scripts/index.js +1 -0
- package/dist/esm/scripts/releaseParentIfNoPendingDependencies-2.d.ts +5 -0
- package/dist/esm/scripts/releaseParentIfNoPendingDependencies-2.js +164 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +2 -2
|
@@ -209,7 +209,7 @@ class FlowProducer extends node_events_1.EventEmitter {
|
|
|
209
209
|
const jobsOpts = (_a = queueOpts === null || queueOpts === void 0 ? void 0 : queueOpts.defaultJobOptions) !== null && _a !== void 0 ? _a : {};
|
|
210
210
|
const jobId = ((_b = node.opts) === null || _b === void 0 ? void 0 : _b.jobId) || (0, utils_1.randomUUID)();
|
|
211
211
|
return (0, utils_1.trace)(this.telemetry, enums_1.SpanKind.PRODUCER, node.queueName, 'addNode', node.queueName, async (span, srcPropagationMetadata) => {
|
|
212
|
-
var _a, _b;
|
|
212
|
+
var _a, _b, _c;
|
|
213
213
|
span === null || span === void 0 ? void 0 : span.setAttributes({
|
|
214
214
|
[enums_1.TelemetryAttributes.JobName]: node.name,
|
|
215
215
|
[enums_1.TelemetryAttributes.JobId]: jobId,
|
|
@@ -227,7 +227,11 @@ class FlowProducer extends node_events_1.EventEmitter {
|
|
|
227
227
|
};
|
|
228
228
|
}
|
|
229
229
|
}
|
|
230
|
-
const
|
|
230
|
+
const jobOpts = Object.assign(Object.assign(Object.assign({}, jobsOpts), opts), { parent: parent === null || parent === void 0 ? void 0 : parent.parentOpts, telemetry });
|
|
231
|
+
if (jobOpts.parent && ((_c = jobOpts.deduplication) === null || _c === void 0 ? void 0 : _c.keepLastIfActive)) {
|
|
232
|
+
throw new Error('keepLastIfActive deduplication cannot be used on flow nodes with a parent');
|
|
233
|
+
}
|
|
234
|
+
const job = new this.Job(queue, node.name, node.data, jobOpts, jobId);
|
|
231
235
|
const parentKey = (0, utils_1.getParentKey)(parent === null || parent === void 0 ? void 0 : parent.parentOpts);
|
|
232
236
|
if (node.children && node.children.length > 0) {
|
|
233
237
|
// Create the parent job, it will be a job in status "waiting-children".
|
|
@@ -334,6 +338,8 @@ class FlowProducer extends node_events_1.EventEmitter {
|
|
|
334
338
|
}
|
|
335
339
|
}
|
|
336
340
|
validateFlowJobs(nodes) {
|
|
341
|
+
// TODO: In the next breaking change, reject deduplication on every flow
|
|
342
|
+
// node. Parent IDs must be known before any add scripts run.
|
|
337
343
|
for (const node of nodes) {
|
|
338
344
|
const children = node.children;
|
|
339
345
|
if (children && children.length > 0) {
|
|
@@ -452,6 +452,7 @@ class RedisQueueBackend extends node_events_1.EventEmitter {
|
|
|
452
452
|
parent,
|
|
453
453
|
job.repeatJobKey,
|
|
454
454
|
job.deduplicationId ? `${queueKeys.de}:${job.deduplicationId}` : null,
|
|
455
|
+
parentKeyOpts.deferParentRelease ? 1 : null,
|
|
455
456
|
];
|
|
456
457
|
const encodedOpts = pack(optsAsJSON(opts));
|
|
457
458
|
let result;
|
|
@@ -509,13 +510,48 @@ class RedisQueueBackend extends node_events_1.EventEmitter {
|
|
|
509
510
|
* queue's key map.
|
|
510
511
|
*/
|
|
511
512
|
async addFlow(entries) {
|
|
513
|
+
var _a;
|
|
512
514
|
const client = await this.queue.client;
|
|
513
515
|
const multi = client.multi();
|
|
516
|
+
// Parents touched by this flow, deduplicated by key. A parent is released
|
|
517
|
+
// from "waiting-children" only after the whole tree has been inserted (see
|
|
518
|
+
// below), so collect them while queuing the inserts.
|
|
519
|
+
const parents = new Map();
|
|
514
520
|
for (const entry of entries) {
|
|
515
521
|
const keys = new queue_keys_1.QueueKeys(entry.prefix).getKeys(entry.queueName);
|
|
516
|
-
await this.addJobToTransaction(multi, entry.jobData, entry.jobId, entry.parentKeyOpts, keys);
|
|
522
|
+
await this.addJobToTransaction(multi, entry.jobData, entry.jobId, Object.assign(Object.assign({}, entry.parentKeyOpts), { deferParentRelease: true }), keys);
|
|
523
|
+
const parent = entry.jobData.parent;
|
|
524
|
+
const parentKey = (_a = entry.parentKeyOpts) === null || _a === void 0 ? void 0 : _a.parentKey;
|
|
525
|
+
if (parent && parentKey) {
|
|
526
|
+
parents.set(parentKey, {
|
|
527
|
+
parentId: parent.id,
|
|
528
|
+
parentKey,
|
|
529
|
+
parentQueueKey: parent.queueKey,
|
|
530
|
+
});
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
// Now that every node of the flow is in the transaction, validate each
|
|
534
|
+
// parent: one that ended up without pending dependencies (e.g. all of its
|
|
535
|
+
// children already existed and were completed) is moved out of
|
|
536
|
+
// "waiting-children". The script is a no-op otherwise.
|
|
537
|
+
for (const { parentId, parentKey, parentQueueKey } of parents.values()) {
|
|
538
|
+
this.execCommand(multi, 'releaseParentIfNoPendingDependencies', [
|
|
539
|
+
parentKey,
|
|
540
|
+
`${parentKey}:dependencies`,
|
|
541
|
+
parentQueueKey,
|
|
542
|
+
parentId,
|
|
543
|
+
Date.now(),
|
|
544
|
+
]);
|
|
545
|
+
}
|
|
546
|
+
const results = (await multi.exec());
|
|
547
|
+
// Callers index the results by entry, so drop the trailing parent-release
|
|
548
|
+
// replies.
|
|
549
|
+
for (const [err] of results.slice(entries.length)) {
|
|
550
|
+
if (err) {
|
|
551
|
+
throw err;
|
|
552
|
+
}
|
|
517
553
|
}
|
|
518
|
-
return (
|
|
554
|
+
return results.slice(0, entries.length);
|
|
519
555
|
}
|
|
520
556
|
pauseArgs(pause, emitEvent = true) {
|
|
521
557
|
let src = 'wait', dst = 'paused';
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
[7] parent? {id, queueKey}
|
|
26
26
|
[8] repeat job key
|
|
27
27
|
[9] deduplication key
|
|
28
|
+
[10] defer parent release (flow inserts)
|
|
28
29
|
|
|
29
30
|
ARGV[2] Json stringified job data
|
|
30
31
|
ARGV[3] msgpacked options
|
|
@@ -52,6 +53,7 @@ local parentKey = args[5]
|
|
|
52
53
|
local parent = args[7]
|
|
53
54
|
local repeatJobKey = args[8]
|
|
54
55
|
local deduplicationKey = args[9]
|
|
56
|
+
local deferParentRelease = args[10]
|
|
55
57
|
local parentData
|
|
56
58
|
|
|
57
59
|
-- Includes
|
|
@@ -84,7 +86,7 @@ else
|
|
|
84
86
|
if rcall("EXISTS", jobIdKey) == 1 then
|
|
85
87
|
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
|
86
88
|
parentData, parentDependenciesKey, completedKey, eventsKey,
|
|
87
|
-
maxEvents, timestamp)
|
|
89
|
+
maxEvents, timestamp, deferParentRelease)
|
|
88
90
|
end
|
|
89
91
|
end
|
|
90
92
|
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
[7] parent? {id, queueKey}
|
|
23
23
|
[8] repeat job key
|
|
24
24
|
[9] deduplication key
|
|
25
|
+
[10] defer parent release (flow inserts)
|
|
25
26
|
|
|
26
27
|
ARGV[2] Json stringified job data
|
|
27
28
|
ARGV[3] msgpacked options
|
|
@@ -50,6 +51,7 @@ local parentKey = args[5]
|
|
|
50
51
|
local parent = args[7]
|
|
51
52
|
local repeatJobKey = args[8]
|
|
52
53
|
local deduplicationKey = args[9]
|
|
54
|
+
local deferParentRelease = args[10]
|
|
53
55
|
local parentData
|
|
54
56
|
|
|
55
57
|
-- Includes
|
|
@@ -79,7 +81,7 @@ else
|
|
|
79
81
|
if rcall("EXISTS", jobIdKey) == 1 then
|
|
80
82
|
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
|
81
83
|
parentData, parentDependenciesKey, completedKey, eventsKey,
|
|
82
|
-
maxEvents, timestamp)
|
|
84
|
+
maxEvents, timestamp, deferParentRelease)
|
|
83
85
|
end
|
|
84
86
|
end
|
|
85
87
|
|
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
[7] parent? {id, queueKey}
|
|
26
26
|
[8] repeat job key
|
|
27
27
|
[9] deduplication key
|
|
28
|
+
[10] defer parent release (flow inserts)
|
|
28
29
|
|
|
29
30
|
ARGV[2] Json stringified job data
|
|
30
31
|
ARGV[3] msgpacked options
|
|
@@ -55,6 +56,7 @@ local parentKey = args[5]
|
|
|
55
56
|
local parent = args[7]
|
|
56
57
|
local repeatJobKey = args[8]
|
|
57
58
|
local deduplicationKey = args[9]
|
|
59
|
+
local deferParentRelease = args[10]
|
|
58
60
|
local parentData
|
|
59
61
|
|
|
60
62
|
-- Includes
|
|
@@ -86,7 +88,7 @@ else
|
|
|
86
88
|
if rcall("EXISTS", jobIdKey) == 1 then
|
|
87
89
|
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
|
88
90
|
parentData, parentDependenciesKey, completedKey, eventsKey,
|
|
89
|
-
maxEvents, timestamp)
|
|
91
|
+
maxEvents, timestamp, deferParentRelease)
|
|
90
92
|
end
|
|
91
93
|
end
|
|
92
94
|
|
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
[7] parent? {id, queueKey}
|
|
36
36
|
[8] repeat job key
|
|
37
37
|
[9] deduplication key
|
|
38
|
+
[10] defer parent release (flow inserts)
|
|
38
39
|
|
|
39
40
|
ARGV[2] Json stringified job data
|
|
40
41
|
ARGV[3] msgpacked options
|
|
@@ -61,6 +62,7 @@ local parentKey = args[5]
|
|
|
61
62
|
local parent = args[7]
|
|
62
63
|
local repeatJobKey = args[8]
|
|
63
64
|
local deduplicationKey = args[9]
|
|
65
|
+
local deferParentRelease = args[10]
|
|
64
66
|
local parentData
|
|
65
67
|
|
|
66
68
|
-- Includes
|
|
@@ -92,7 +94,7 @@ else
|
|
|
92
94
|
if rcall("EXISTS", jobIdKey) == 1 then
|
|
93
95
|
return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
|
|
94
96
|
parentData, parentDependenciesKey, KEYS[5], eventsKey,
|
|
95
|
-
maxEvents, timestamp)
|
|
97
|
+
maxEvents, timestamp, deferParentRelease)
|
|
96
98
|
end
|
|
97
99
|
end
|
|
98
100
|
|
|
@@ -6,16 +6,21 @@
|
|
|
6
6
|
--- @include "updateExistingJobsParent"
|
|
7
7
|
|
|
8
8
|
local function handleDuplicatedJob(jobKey, jobId, currentParentKey, currentParent,
|
|
9
|
-
parentData, parentDependenciesKey, completedKey, eventsKey, maxEvents, timestamp
|
|
9
|
+
parentData, parentDependenciesKey, completedKey, eventsKey, maxEvents, timestamp,
|
|
10
|
+
deferParentRelease)
|
|
10
11
|
local existedParentKey = rcall("HGET", jobKey, "parentKey")
|
|
11
12
|
|
|
12
|
-
if not existedParentKey or existedParentKey == currentParentKey
|
|
13
|
+
if not existedParentKey or existedParentKey == currentParentKey
|
|
14
|
+
or rcall("EXISTS", existedParentKey) == 0 then
|
|
13
15
|
updateExistingJobsParent(currentParentKey, currentParent, parentData,
|
|
14
16
|
parentDependenciesKey, completedKey, jobKey,
|
|
15
|
-
jobId, timestamp)
|
|
17
|
+
jobId, timestamp, deferParentRelease)
|
|
16
18
|
else
|
|
17
19
|
if currentParentKey ~= nil and currentParentKey ~= existedParentKey
|
|
18
20
|
and (rcall("EXISTS", existedParentKey) == 1) then
|
|
21
|
+
if deferParentRelease then
|
|
22
|
+
rcall("HSET", currentParentKey, "flowAttachmentFailed", "1")
|
|
23
|
+
end
|
|
19
24
|
return -7
|
|
20
25
|
end
|
|
21
26
|
end
|
|
@@ -3,6 +3,13 @@
|
|
|
3
3
|
is already completed and about to be ignored. The parent must get its
|
|
4
4
|
dependencies updated to avoid the parent job being stuck forever in
|
|
5
5
|
the waiting-children state.
|
|
6
|
+
|
|
7
|
+
When "deferParentRelease" is true (flow inserts), the parent is *not*
|
|
8
|
+
moved out of waiting-children here: the remaining children of the flow
|
|
9
|
+
have not been inserted yet, so the parent would be released too early.
|
|
10
|
+
Only the processed dependency is recorded; releasing the parent is
|
|
11
|
+
validated by "releaseParentIfNoPendingDependencies" once the whole flow
|
|
12
|
+
has been added.
|
|
6
13
|
]]
|
|
7
14
|
|
|
8
15
|
-- Includes
|
|
@@ -10,13 +17,18 @@
|
|
|
10
17
|
|
|
11
18
|
local function updateExistingJobsParent(parentKey, parent, parentData,
|
|
12
19
|
parentDependenciesKey, completedKey,
|
|
13
|
-
jobIdKey, jobId, timestamp
|
|
20
|
+
jobIdKey, jobId, timestamp,
|
|
21
|
+
deferParentRelease)
|
|
14
22
|
if parentKey ~= nil then
|
|
15
23
|
if rcall("ZSCORE", completedKey, jobId) then
|
|
16
24
|
local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
25
|
+
if deferParentRelease then
|
|
26
|
+
rcall("HSET", parentKey .. ":processed", jobIdKey, returnvalue)
|
|
27
|
+
else
|
|
28
|
+
updateParentDepsIfNeeded(parentKey, parent['queueKey'],
|
|
29
|
+
parentDependenciesKey, parent['id'],
|
|
30
|
+
jobIdKey, returnvalue, timestamp)
|
|
31
|
+
end
|
|
20
32
|
else
|
|
21
33
|
if parentDependenciesKey ~= nil then
|
|
22
34
|
rcall("SADD", parentDependenciesKey, jobIdKey)
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
Moves a parent out of the waiting-children state if it has no pending
|
|
3
|
+
dependencies left.
|
|
4
|
+
|
|
5
|
+
This runs at the end of a flow insert, once every node of the tree has been
|
|
6
|
+
added. During the insert itself a child that already exists and is completed
|
|
7
|
+
only records its processed dependency (see updateExistingJobsParent); the
|
|
8
|
+
parent cannot be released at that point because its remaining children have
|
|
9
|
+
not been added to the dependencies set yet.
|
|
10
|
+
|
|
11
|
+
It is a no-op unless the parent still exists and is still scored in its
|
|
12
|
+
queue's waiting-children zset, so it is safe to call for every parent
|
|
13
|
+
touched by a flow.
|
|
14
|
+
|
|
15
|
+
Input:
|
|
16
|
+
KEYS[1] parent key
|
|
17
|
+
KEYS[2] parent dependencies key
|
|
18
|
+
|
|
19
|
+
ARGV[1] parent queue key
|
|
20
|
+
ARGV[2] parent id
|
|
21
|
+
ARGV[3] timestamp
|
|
22
|
+
]]
|
|
23
|
+
local rcall = redis.call
|
|
24
|
+
|
|
25
|
+
if rcall("HGET", KEYS[1], "flowAttachmentFailed") then
|
|
26
|
+
rcall("HDEL", KEYS[1], "flowAttachmentFailed")
|
|
27
|
+
return
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
-- Includes
|
|
31
|
+
--- @include "includes/moveParentToWaitIfNoPendingDependencies"
|
|
32
|
+
|
|
33
|
+
moveParentToWaitIfNoPendingDependencies(ARGV[1], KEYS[2], KEYS[1], ARGV[2],
|
|
34
|
+
ARGV[3])
|
|
@@ -36,6 +36,15 @@ exports.MIGRATIONS = [
|
|
|
36
36
|
minClientVersion: 6,
|
|
37
37
|
load: () => (0, sql_loader_1.loadMigrationSql)('0004_relay.sql'),
|
|
38
38
|
},
|
|
39
|
+
{
|
|
40
|
+
version: 5,
|
|
41
|
+
name: '0005_flow_parent_release',
|
|
42
|
+
// Function-body fixes only: `add_flow` keeps its signature and
|
|
43
|
+
// `handle_duplicated_job` gains a defaulted trailing parameter, so the
|
|
44
|
+
// 6-argument calls made by instances on the previous code keep resolving.
|
|
45
|
+
minClientVersion: 6,
|
|
46
|
+
load: () => (0, sql_loader_1.loadMigrationSql)('0005_flow_parent_release.sql'),
|
|
47
|
+
},
|
|
39
48
|
];
|
|
40
49
|
/**
|
|
41
50
|
* The highest schema version this BullMQ build knows how to produce. Explicit
|