bullmq 6.5.0 → 6.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/cjs/classes/flow-producer.js +8 -2
  2. package/dist/cjs/classes/redis-queue-backend.js +38 -2
  3. package/dist/cjs/commands/addDelayedJob-6.lua +3 -1
  4. package/dist/cjs/commands/addParentJob-6.lua +3 -1
  5. package/dist/cjs/commands/addPrioritizedJob-9.lua +3 -1
  6. package/dist/cjs/commands/addStandardJob-9.lua +3 -1
  7. package/dist/cjs/commands/includes/handleDuplicatedJob.lua +8 -3
  8. package/dist/cjs/commands/includes/updateExistingJobsParent.lua +16 -4
  9. package/dist/cjs/commands/releaseParentIfNoPendingDependencies-2.lua +34 -0
  10. package/dist/cjs/postgres/migrations/index.js +9 -0
  11. package/dist/cjs/postgres/sql-scripts.js +1 -0
  12. package/dist/cjs/scripts/addDelayedJob-6.js +26 -8
  13. package/dist/cjs/scripts/addParentJob-6.js +26 -8
  14. package/dist/cjs/scripts/addPrioritizedJob-9.js +26 -8
  15. package/dist/cjs/scripts/addStandardJob-9.js +26 -8
  16. package/dist/cjs/scripts/index.js +1 -0
  17. package/dist/cjs/scripts/releaseParentIfNoPendingDependencies-2.js +167 -0
  18. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  19. package/dist/cjs/version.js +1 -1
  20. package/dist/esm/classes/flow-producer.js +8 -2
  21. package/dist/esm/classes/redis-queue-backend.js +38 -2
  22. package/dist/esm/commands/addDelayedJob-6.lua +3 -1
  23. package/dist/esm/commands/addParentJob-6.lua +3 -1
  24. package/dist/esm/commands/addPrioritizedJob-9.lua +3 -1
  25. package/dist/esm/commands/addStandardJob-9.lua +3 -1
  26. package/dist/esm/commands/includes/handleDuplicatedJob.lua +8 -3
  27. package/dist/esm/commands/includes/updateExistingJobsParent.lua +16 -4
  28. package/dist/esm/commands/releaseParentIfNoPendingDependencies-2.lua +34 -0
  29. package/dist/esm/interfaces/parent.d.ts +8 -0
  30. package/dist/esm/postgres/migrations/index.js +9 -0
  31. package/dist/esm/postgres/sql-scripts.js +1 -0
  32. package/dist/esm/scripts/addDelayedJob-6.js +26 -8
  33. package/dist/esm/scripts/addParentJob-6.js +26 -8
  34. package/dist/esm/scripts/addPrioritizedJob-9.js +26 -8
  35. package/dist/esm/scripts/addStandardJob-9.js +26 -8
  36. package/dist/esm/scripts/index.d.ts +1 -0
  37. package/dist/esm/scripts/index.js +1 -0
  38. package/dist/esm/scripts/releaseParentIfNoPendingDependencies-2.d.ts +5 -0
  39. package/dist/esm/scripts/releaseParentIfNoPendingDependencies-2.js +164 -0
  40. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  41. package/dist/esm/version.d.ts +1 -1
  42. package/dist/esm/version.js +1 -1
  43. package/package.json +2 -2
@@ -209,7 +209,7 @@ class FlowProducer extends node_events_1.EventEmitter {
209
209
  const jobsOpts = (_a = queueOpts === null || queueOpts === void 0 ? void 0 : queueOpts.defaultJobOptions) !== null && _a !== void 0 ? _a : {};
210
210
  const jobId = ((_b = node.opts) === null || _b === void 0 ? void 0 : _b.jobId) || (0, utils_1.randomUUID)();
211
211
  return (0, utils_1.trace)(this.telemetry, enums_1.SpanKind.PRODUCER, node.queueName, 'addNode', node.queueName, async (span, srcPropagationMetadata) => {
212
- var _a, _b;
212
+ var _a, _b, _c;
213
213
  span === null || span === void 0 ? void 0 : span.setAttributes({
214
214
  [enums_1.TelemetryAttributes.JobName]: node.name,
215
215
  [enums_1.TelemetryAttributes.JobId]: jobId,
@@ -227,7 +227,11 @@ class FlowProducer extends node_events_1.EventEmitter {
227
227
  };
228
228
  }
229
229
  }
230
- const job = new this.Job(queue, node.name, node.data, Object.assign(Object.assign(Object.assign({}, jobsOpts), opts), { parent: parent === null || parent === void 0 ? void 0 : parent.parentOpts, telemetry }), jobId);
230
+ const jobOpts = Object.assign(Object.assign(Object.assign({}, jobsOpts), opts), { parent: parent === null || parent === void 0 ? void 0 : parent.parentOpts, telemetry });
231
+ if (jobOpts.parent && ((_c = jobOpts.deduplication) === null || _c === void 0 ? void 0 : _c.keepLastIfActive)) {
232
+ throw new Error('keepLastIfActive deduplication cannot be used on flow nodes with a parent');
233
+ }
234
+ const job = new this.Job(queue, node.name, node.data, jobOpts, jobId);
231
235
  const parentKey = (0, utils_1.getParentKey)(parent === null || parent === void 0 ? void 0 : parent.parentOpts);
232
236
  if (node.children && node.children.length > 0) {
233
237
  // Create the parent job, it will be a job in status "waiting-children".
@@ -334,6 +338,8 @@ class FlowProducer extends node_events_1.EventEmitter {
334
338
  }
335
339
  }
336
340
  validateFlowJobs(nodes) {
341
+ // TODO: In the next breaking change, reject deduplication on every flow
342
+ // node. Parent IDs must be known before any add scripts run.
337
343
  for (const node of nodes) {
338
344
  const children = node.children;
339
345
  if (children && children.length > 0) {
@@ -452,6 +452,7 @@ class RedisQueueBackend extends node_events_1.EventEmitter {
452
452
  parent,
453
453
  job.repeatJobKey,
454
454
  job.deduplicationId ? `${queueKeys.de}:${job.deduplicationId}` : null,
455
+ parentKeyOpts.deferParentRelease ? 1 : null,
455
456
  ];
456
457
  const encodedOpts = pack(optsAsJSON(opts));
457
458
  let result;
@@ -509,13 +510,48 @@ class RedisQueueBackend extends node_events_1.EventEmitter {
509
510
  * queue's key map.
510
511
  */
511
512
  async addFlow(entries) {
513
+ var _a;
512
514
  const client = await this.queue.client;
513
515
  const multi = client.multi();
516
+ // Parents touched by this flow, deduplicated by key. A parent is released
517
+ // from "waiting-children" only after the whole tree has been inserted (see
518
+ // below), so collect them while queuing the inserts.
519
+ const parents = new Map();
514
520
  for (const entry of entries) {
515
521
  const keys = new queue_keys_1.QueueKeys(entry.prefix).getKeys(entry.queueName);
516
- await this.addJobToTransaction(multi, entry.jobData, entry.jobId, entry.parentKeyOpts, keys);
522
+ await this.addJobToTransaction(multi, entry.jobData, entry.jobId, Object.assign(Object.assign({}, entry.parentKeyOpts), { deferParentRelease: true }), keys);
523
+ const parent = entry.jobData.parent;
524
+ const parentKey = (_a = entry.parentKeyOpts) === null || _a === void 0 ? void 0 : _a.parentKey;
525
+ if (parent && parentKey) {
526
+ parents.set(parentKey, {
527
+ parentId: parent.id,
528
+ parentKey,
529
+ parentQueueKey: parent.queueKey,
530
+ });
531
+ }
532
+ }
533
+ // Now that every node of the flow is in the transaction, validate each
534
+ // parent: one that ended up without pending dependencies (e.g. all of its
535
+ // children already existed and were completed) is moved out of
536
+ // "waiting-children". The script is a no-op otherwise.
537
+ for (const { parentId, parentKey, parentQueueKey } of parents.values()) {
538
+ this.execCommand(multi, 'releaseParentIfNoPendingDependencies', [
539
+ parentKey,
540
+ `${parentKey}:dependencies`,
541
+ parentQueueKey,
542
+ parentId,
543
+ Date.now(),
544
+ ]);
545
+ }
546
+ const results = (await multi.exec());
547
+ // Callers index the results by entry, so drop the trailing parent-release
548
+ // replies.
549
+ for (const [err] of results.slice(entries.length)) {
550
+ if (err) {
551
+ throw err;
552
+ }
517
553
  }
518
- return (await multi.exec());
554
+ return results.slice(0, entries.length);
519
555
  }
520
556
  pauseArgs(pause, emitEvent = true) {
521
557
  let src = 'wait', dst = 'paused';
@@ -25,6 +25,7 @@
25
25
  [7] parent? {id, queueKey}
26
26
  [8] repeat job key
27
27
  [9] deduplication key
28
+ [10] defer parent release (flow inserts)
28
29
 
29
30
  ARGV[2] Json stringified job data
30
31
  ARGV[3] msgpacked options
@@ -52,6 +53,7 @@ local parentKey = args[5]
52
53
  local parent = args[7]
53
54
  local repeatJobKey = args[8]
54
55
  local deduplicationKey = args[9]
56
+ local deferParentRelease = args[10]
55
57
  local parentData
56
58
 
57
59
  -- Includes
@@ -84,7 +86,7 @@ else
84
86
  if rcall("EXISTS", jobIdKey) == 1 then
85
87
  return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
86
88
  parentData, parentDependenciesKey, completedKey, eventsKey,
87
- maxEvents, timestamp)
89
+ maxEvents, timestamp, deferParentRelease)
88
90
  end
89
91
  end
90
92
 
@@ -22,6 +22,7 @@
22
22
  [7] parent? {id, queueKey}
23
23
  [8] repeat job key
24
24
  [9] deduplication key
25
+ [10] defer parent release (flow inserts)
25
26
 
26
27
  ARGV[2] Json stringified job data
27
28
  ARGV[3] msgpacked options
@@ -50,6 +51,7 @@ local parentKey = args[5]
50
51
  local parent = args[7]
51
52
  local repeatJobKey = args[8]
52
53
  local deduplicationKey = args[9]
54
+ local deferParentRelease = args[10]
53
55
  local parentData
54
56
 
55
57
  -- Includes
@@ -79,7 +81,7 @@ else
79
81
  if rcall("EXISTS", jobIdKey) == 1 then
80
82
  return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
81
83
  parentData, parentDependenciesKey, completedKey, eventsKey,
82
- maxEvents, timestamp)
84
+ maxEvents, timestamp, deferParentRelease)
83
85
  end
84
86
  end
85
87
 
@@ -25,6 +25,7 @@
25
25
  [7] parent? {id, queueKey}
26
26
  [8] repeat job key
27
27
  [9] deduplication key
28
+ [10] defer parent release (flow inserts)
28
29
 
29
30
  ARGV[2] Json stringified job data
30
31
  ARGV[3] msgpacked options
@@ -55,6 +56,7 @@ local parentKey = args[5]
55
56
  local parent = args[7]
56
57
  local repeatJobKey = args[8]
57
58
  local deduplicationKey = args[9]
59
+ local deferParentRelease = args[10]
58
60
  local parentData
59
61
 
60
62
  -- Includes
@@ -86,7 +88,7 @@ else
86
88
  if rcall("EXISTS", jobIdKey) == 1 then
87
89
  return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
88
90
  parentData, parentDependenciesKey, completedKey, eventsKey,
89
- maxEvents, timestamp)
91
+ maxEvents, timestamp, deferParentRelease)
90
92
  end
91
93
  end
92
94
 
@@ -35,6 +35,7 @@
35
35
  [7] parent? {id, queueKey}
36
36
  [8] repeat job key
37
37
  [9] deduplication key
38
+ [10] defer parent release (flow inserts)
38
39
 
39
40
  ARGV[2] Json stringified job data
40
41
  ARGV[3] msgpacked options
@@ -61,6 +62,7 @@ local parentKey = args[5]
61
62
  local parent = args[7]
62
63
  local repeatJobKey = args[8]
63
64
  local deduplicationKey = args[9]
65
+ local deferParentRelease = args[10]
64
66
  local parentData
65
67
 
66
68
  -- Includes
@@ -92,7 +94,7 @@ else
92
94
  if rcall("EXISTS", jobIdKey) == 1 then
93
95
  return handleDuplicatedJob(jobIdKey, jobId, parentKey, parent,
94
96
  parentData, parentDependenciesKey, KEYS[5], eventsKey,
95
- maxEvents, timestamp)
97
+ maxEvents, timestamp, deferParentRelease)
96
98
  end
97
99
  end
98
100
 
@@ -6,16 +6,21 @@
6
6
  --- @include "updateExistingJobsParent"
7
7
 
8
8
  local function handleDuplicatedJob(jobKey, jobId, currentParentKey, currentParent,
9
- parentData, parentDependenciesKey, completedKey, eventsKey, maxEvents, timestamp)
9
+ parentData, parentDependenciesKey, completedKey, eventsKey, maxEvents, timestamp,
10
+ deferParentRelease)
10
11
  local existedParentKey = rcall("HGET", jobKey, "parentKey")
11
12
 
12
- if not existedParentKey or existedParentKey == currentParentKey then
13
+ if not existedParentKey or existedParentKey == currentParentKey
14
+ or rcall("EXISTS", existedParentKey) == 0 then
13
15
  updateExistingJobsParent(currentParentKey, currentParent, parentData,
14
16
  parentDependenciesKey, completedKey, jobKey,
15
- jobId, timestamp)
17
+ jobId, timestamp, deferParentRelease)
16
18
  else
17
19
  if currentParentKey ~= nil and currentParentKey ~= existedParentKey
18
20
  and (rcall("EXISTS", existedParentKey) == 1) then
21
+ if deferParentRelease then
22
+ rcall("HSET", currentParentKey, "flowAttachmentFailed", "1")
23
+ end
19
24
  return -7
20
25
  end
21
26
  end
@@ -3,6 +3,13 @@
3
3
  is already completed and about to be ignored. The parent must get its
4
4
  dependencies updated to avoid the parent job being stuck forever in
5
5
  the waiting-children state.
6
+
7
+ When "deferParentRelease" is true (flow inserts), the parent is *not*
8
+ moved out of waiting-children here: the remaining children of the flow
9
+ have not been inserted yet, so the parent would be released too early.
10
+ Only the processed dependency is recorded; releasing the parent is
11
+ validated by "releaseParentIfNoPendingDependencies" once the whole flow
12
+ has been added.
6
13
  ]]
7
14
 
8
15
  -- Includes
@@ -10,13 +17,18 @@
10
17
 
11
18
  local function updateExistingJobsParent(parentKey, parent, parentData,
12
19
  parentDependenciesKey, completedKey,
13
- jobIdKey, jobId, timestamp)
20
+ jobIdKey, jobId, timestamp,
21
+ deferParentRelease)
14
22
  if parentKey ~= nil then
15
23
  if rcall("ZSCORE", completedKey, jobId) then
16
24
  local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
17
- updateParentDepsIfNeeded(parentKey, parent['queueKey'],
18
- parentDependenciesKey, parent['id'],
19
- jobIdKey, returnvalue, timestamp)
25
+ if deferParentRelease then
26
+ rcall("HSET", parentKey .. ":processed", jobIdKey, returnvalue)
27
+ else
28
+ updateParentDepsIfNeeded(parentKey, parent['queueKey'],
29
+ parentDependenciesKey, parent['id'],
30
+ jobIdKey, returnvalue, timestamp)
31
+ end
20
32
  else
21
33
  if parentDependenciesKey ~= nil then
22
34
  rcall("SADD", parentDependenciesKey, jobIdKey)
@@ -0,0 +1,34 @@
1
+ --[[
2
+ Moves a parent out of the waiting-children state if it has no pending
3
+ dependencies left.
4
+
5
+ This runs at the end of a flow insert, once every node of the tree has been
6
+ added. During the insert itself a child that already exists and is completed
7
+ only records its processed dependency (see updateExistingJobsParent); the
8
+ parent cannot be released at that point because its remaining children have
9
+ not been added to the dependencies set yet.
10
+
11
+ It is a no-op unless the parent still exists and is still scored in its
12
+ queue's waiting-children zset, so it is safe to call for every parent
13
+ touched by a flow.
14
+
15
+ Input:
16
+ KEYS[1] parent key
17
+ KEYS[2] parent dependencies key
18
+
19
+ ARGV[1] parent queue key
20
+ ARGV[2] parent id
21
+ ARGV[3] timestamp
22
+ ]]
23
+ local rcall = redis.call
24
+
25
+ if rcall("HGET", KEYS[1], "flowAttachmentFailed") then
26
+ rcall("HDEL", KEYS[1], "flowAttachmentFailed")
27
+ return
28
+ end
29
+
30
+ -- Includes
31
+ --- @include "includes/moveParentToWaitIfNoPendingDependencies"
32
+
33
+ moveParentToWaitIfNoPendingDependencies(ARGV[1], KEYS[2], KEYS[1], ARGV[2],
34
+ ARGV[3])
@@ -36,6 +36,15 @@ exports.MIGRATIONS = [
36
36
  minClientVersion: 6,
37
37
  load: () => (0, sql_loader_1.loadMigrationSql)('0004_relay.sql'),
38
38
  },
39
+ {
40
+ version: 5,
41
+ name: '0005_flow_parent_release',
42
+ // Function-body fixes only: `add_flow` keeps its signature and
43
+ // `handle_duplicated_job` gains a defaulted trailing parameter, so the
44
+ // 6-argument calls made by instances on the previous code keep resolving.
45
+ minClientVersion: 6,
46
+ load: () => (0, sql_loader_1.loadMigrationSql)('0005_flow_parent_release.sql'),
47
+ },
39
48
  ];
40
49
  /**
41
50
  * The highest schema version this BullMQ build knows how to produce. Explicit