bullmq 6.3.19 → 6.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/dist/cjs/classes/bun-redis-client.js +3 -0
  2. package/dist/cjs/classes/index.js +3 -0
  3. package/dist/cjs/classes/ioredis-client.js +1 -0
  4. package/dist/cjs/classes/node-redis-client.js +3 -0
  5. package/dist/cjs/classes/redis-connection.js +33 -18
  6. package/dist/cjs/classes/redis-relay-backend.js +455 -0
  7. package/dist/cjs/classes/relay-errors.js +41 -0
  8. package/dist/cjs/classes/relay.js +562 -0
  9. package/dist/cjs/classes/valkey-glide-client.js +7 -2
  10. package/dist/cjs/commands/includes/refreshRelayNamespace.lua +36 -0
  11. package/dist/cjs/commands/includes/relayIndex.lua +112 -0
  12. package/dist/cjs/commands/includes/relayTime.lua +15 -0
  13. package/dist/cjs/commands/includes/relayTopics.lua +92 -0
  14. package/dist/cjs/commands/relayAckInbox-1.lua +18 -0
  15. package/dist/cjs/commands/relayDeleteRetained-1.lua +39 -0
  16. package/dist/cjs/commands/relayGet-1.lua +133 -0
  17. package/dist/cjs/commands/relayGetNamespaces-1.lua +16 -0
  18. package/dist/cjs/commands/relayHeartbeat-1.lua +35 -0
  19. package/dist/cjs/commands/relayPublish-1.lua +150 -0
  20. package/dist/cjs/commands/relayReadInbox-1.lua +31 -0
  21. package/dist/cjs/commands/relayRegisterNode-1.lua +60 -0
  22. package/dist/cjs/commands/relayRemoveEndpoint-1.lua +18 -0
  23. package/dist/cjs/commands/relaySubscribe-1.lua +69 -0
  24. package/dist/cjs/commands/relaySweep-1.lua +39 -0
  25. package/dist/cjs/commands/relayTouchNamespace-1.lua +22 -0
  26. package/dist/cjs/commands/relayTrimInbox-1.lua +65 -0
  27. package/dist/cjs/commands/relayUnregisterNode-1.lua +25 -0
  28. package/dist/cjs/interfaces/index.js +1 -0
  29. package/dist/cjs/interfaces/relay-backend.js +2 -0
  30. package/dist/cjs/postgres/index.js +4 -1
  31. package/dist/cjs/postgres/migrations/index.js +9 -2
  32. package/dist/cjs/postgres/postgres-connection.js +3 -2
  33. package/dist/cjs/postgres/postgres-relay-backend.js +342 -0
  34. package/dist/cjs/postgres/sql-scripts.js +19 -0
  35. package/dist/cjs/scripts/index.js +14 -0
  36. package/dist/cjs/scripts/relayAckInbox-1.js +23 -0
  37. package/dist/cjs/scripts/relayDeleteRetained-1.js +168 -0
  38. package/dist/cjs/scripts/relayGet-1.js +323 -0
  39. package/dist/cjs/scripts/relayGetNamespaces-1.js +34 -0
  40. package/dist/cjs/scripts/relayHeartbeat-1.js +81 -0
  41. package/dist/cjs/scripts/relayPublish-1.js +368 -0
  42. package/dist/cjs/scripts/relayReadInbox-1.js +35 -0
  43. package/dist/cjs/scripts/relayRegisterNode-1.js +285 -0
  44. package/dist/cjs/scripts/relayRemoveEndpoint-1.js +204 -0
  45. package/dist/cjs/scripts/relaySubscribe-1.js +150 -0
  46. package/dist/cjs/scripts/relaySweep-1.js +265 -0
  47. package/dist/cjs/scripts/relayTouchNamespace-1.js +38 -0
  48. package/dist/cjs/scripts/relayTrimInbox-1.js +81 -0
  49. package/dist/cjs/scripts/relayUnregisterNode-1.js +255 -0
  50. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  51. package/dist/cjs/version.js +1 -1
  52. package/dist/esm/classes/bun-redis-client.js +3 -0
  53. package/dist/esm/classes/index.d.ts +3 -0
  54. package/dist/esm/classes/index.js +3 -0
  55. package/dist/esm/classes/ioredis-client.js +1 -0
  56. package/dist/esm/classes/node-redis-client.js +3 -0
  57. package/dist/esm/classes/redis-connection.d.ts +8 -0
  58. package/dist/esm/classes/redis-connection.js +33 -18
  59. package/dist/esm/classes/redis-relay-backend.d.ts +110 -0
  60. package/dist/esm/classes/redis-relay-backend.js +450 -0
  61. package/dist/esm/classes/relay-errors.d.ts +19 -0
  62. package/dist/esm/classes/relay-errors.js +36 -0
  63. package/dist/esm/classes/relay.d.ts +183 -0
  64. package/dist/esm/classes/relay.js +558 -0
  65. package/dist/esm/classes/valkey-glide-client.js +7 -2
  66. package/dist/esm/commands/includes/refreshRelayNamespace.lua +36 -0
  67. package/dist/esm/commands/includes/relayIndex.lua +112 -0
  68. package/dist/esm/commands/includes/relayTime.lua +15 -0
  69. package/dist/esm/commands/includes/relayTopics.lua +92 -0
  70. package/dist/esm/commands/relayAckInbox-1.lua +18 -0
  71. package/dist/esm/commands/relayDeleteRetained-1.lua +39 -0
  72. package/dist/esm/commands/relayGet-1.lua +133 -0
  73. package/dist/esm/commands/relayGetNamespaces-1.lua +16 -0
  74. package/dist/esm/commands/relayHeartbeat-1.lua +35 -0
  75. package/dist/esm/commands/relayPublish-1.lua +150 -0
  76. package/dist/esm/commands/relayReadInbox-1.lua +31 -0
  77. package/dist/esm/commands/relayRegisterNode-1.lua +60 -0
  78. package/dist/esm/commands/relayRemoveEndpoint-1.lua +18 -0
  79. package/dist/esm/commands/relaySubscribe-1.lua +69 -0
  80. package/dist/esm/commands/relaySweep-1.lua +39 -0
  81. package/dist/esm/commands/relayTouchNamespace-1.lua +22 -0
  82. package/dist/esm/commands/relayTrimInbox-1.lua +65 -0
  83. package/dist/esm/commands/relayUnregisterNode-1.lua +25 -0
  84. package/dist/esm/interfaces/index.d.ts +1 -0
  85. package/dist/esm/interfaces/index.js +1 -0
  86. package/dist/esm/interfaces/redis-client.d.ts +1 -0
  87. package/dist/esm/interfaces/relay-backend.d.ts +237 -0
  88. package/dist/esm/interfaces/relay-backend.js +1 -0
  89. package/dist/esm/postgres/index.d.ts +1 -0
  90. package/dist/esm/postgres/index.js +1 -0
  91. package/dist/esm/postgres/migrations/index.d.ts +11 -1
  92. package/dist/esm/postgres/migrations/index.js +9 -2
  93. package/dist/esm/postgres/postgres-connection.js +3 -2
  94. package/dist/esm/postgres/postgres-relay-backend.d.ts +67 -0
  95. package/dist/esm/postgres/postgres-relay-backend.js +337 -0
  96. package/dist/esm/postgres/sql-scripts.js +19 -0
  97. package/dist/esm/scripts/index.d.ts +14 -0
  98. package/dist/esm/scripts/index.js +14 -0
  99. package/dist/esm/scripts/relayAckInbox-1.d.ts +5 -0
  100. package/dist/esm/scripts/relayAckInbox-1.js +20 -0
  101. package/dist/esm/scripts/relayDeleteRetained-1.d.ts +5 -0
  102. package/dist/esm/scripts/relayDeleteRetained-1.js +165 -0
  103. package/dist/esm/scripts/relayGet-1.d.ts +5 -0
  104. package/dist/esm/scripts/relayGet-1.js +320 -0
  105. package/dist/esm/scripts/relayGetNamespaces-1.d.ts +5 -0
  106. package/dist/esm/scripts/relayGetNamespaces-1.js +31 -0
  107. package/dist/esm/scripts/relayHeartbeat-1.d.ts +5 -0
  108. package/dist/esm/scripts/relayHeartbeat-1.js +78 -0
  109. package/dist/esm/scripts/relayPublish-1.d.ts +5 -0
  110. package/dist/esm/scripts/relayPublish-1.js +365 -0
  111. package/dist/esm/scripts/relayReadInbox-1.d.ts +5 -0
  112. package/dist/esm/scripts/relayReadInbox-1.js +32 -0
  113. package/dist/esm/scripts/relayRegisterNode-1.d.ts +5 -0
  114. package/dist/esm/scripts/relayRegisterNode-1.js +282 -0
  115. package/dist/esm/scripts/relayRemoveEndpoint-1.d.ts +5 -0
  116. package/dist/esm/scripts/relayRemoveEndpoint-1.js +201 -0
  117. package/dist/esm/scripts/relaySubscribe-1.d.ts +5 -0
  118. package/dist/esm/scripts/relaySubscribe-1.js +147 -0
  119. package/dist/esm/scripts/relaySweep-1.d.ts +5 -0
  120. package/dist/esm/scripts/relaySweep-1.js +262 -0
  121. package/dist/esm/scripts/relayTouchNamespace-1.d.ts +5 -0
  122. package/dist/esm/scripts/relayTouchNamespace-1.js +35 -0
  123. package/dist/esm/scripts/relayTrimInbox-1.d.ts +5 -0
  124. package/dist/esm/scripts/relayTrimInbox-1.js +78 -0
  125. package/dist/esm/scripts/relayUnregisterNode-1.d.ts +5 -0
  126. package/dist/esm/scripts/relayUnregisterNode-1.js +252 -0
  127. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  128. package/dist/esm/version.d.ts +1 -1
  129. package/dist/esm/version.js +1 -1
  130. package/package.json +8 -8
@@ -843,6 +843,9 @@ class BunRedisAdapter extends events_1.EventEmitter {
843
843
  async clientList() {
844
844
  return await this.sendCommand('CLIENT', ['LIST']);
845
845
  }
846
+ async clientInfo() {
847
+ return await this.sendCommand('CLIENT', ['INFO']);
848
+ }
846
849
  // ---------------------------------------------------------------
847
850
  // Key scanning
848
851
  // ---------------------------------------------------------------
@@ -26,4 +26,7 @@ tslib_1.__exportStar(require("./queue"), exports);
26
26
  tslib_1.__exportStar(require("./redis-connection"), exports);
27
27
  tslib_1.__exportStar(require("./sandbox"), exports);
28
28
  tslib_1.__exportStar(require("./redis-queue-backend"), exports);
29
+ tslib_1.__exportStar(require("./redis-relay-backend"), exports);
30
+ tslib_1.__exportStar(require("./relay"), exports);
31
+ tslib_1.__exportStar(require("./relay-errors"), exports);
29
32
  tslib_1.__exportStar(require("./worker"), exports);
@@ -168,6 +168,7 @@ function createIORedisClient(client) {
168
168
  // clientSetName / clientList helpers that forward to CLIENT subcommands.
169
169
  overrides.clientSetName = (name) => client.client('SETNAME', name);
170
170
  overrides.clientList = () => client.client('LIST');
171
+ overrides.clientInfo = () => client.client('INFO');
171
172
  // scan(cursor, { MATCH?, COUNT? }) — accepts either structured options or
172
173
  // ioredis varargs (used internally by `scanStream`).
173
174
  overrides.scan = (cursor, ...rest) => {
@@ -523,6 +523,9 @@ class NodeRedisAdapter extends events_1.EventEmitter {
523
523
  async clientList() {
524
524
  return await this.raw.sendCommand(['CLIENT', 'LIST']);
525
525
  }
526
+ async clientInfo() {
527
+ return await this.raw.sendCommand(['CLIENT', 'INFO']);
528
+ }
526
529
  // ---------------------------------------------------------------
527
530
  // Key scanning
528
531
  // ---------------------------------------------------------------
@@ -313,29 +313,44 @@ class RedisConnection extends events_1.EventEmitter {
313
313
  return;
314
314
  }
315
315
  const bzpopmin = blockingClient.bzpopmin;
316
- const wrappedBzpopmin = async (...args) => {
317
- await RedisConnection.reconnectClusterIfNeeded(blockingClient, reconnectTimeoutMs);
318
- try {
319
- return await bzpopmin.apply(blockingClient, args);
320
- }
321
- catch (error) {
322
- const commandError = error;
323
- if (RedisConnection.shouldReconnectClusterAfterError(blockingClient, commandError)) {
324
- try {
325
- await RedisConnection.reconnectCluster(blockingClient, reconnectTimeoutMs);
326
- }
327
- catch (_a) {
328
- // Preserve the original command failure if best-effort recovery fails.
329
- }
330
- }
331
- throw commandError;
332
- }
333
- };
316
+ const wrappedBzpopmin = (...args) => RedisConnection.runWithClusterRecovery(blockingClient, reconnectTimeoutMs, () => bzpopmin.apply(blockingClient, args));
334
317
  blockingClient[clusterOriginalBzpopmin] = bzpopmin;
335
318
  blockingClient[clusterWrappedBzpopmin] = wrappedBzpopmin;
336
319
  blockingClient[clusterPatchedForBlocking] = true;
337
320
  blockingClient.bzpopmin = wrappedBzpopmin;
338
321
  }
322
+ /**
323
+ * Runs a command of a blocking connection with the recovery that `bzpopmin`
324
+ * gets on an ioredis Cluster: an empty node pool is reconnected before the
325
+ * command, and a cluster left unusable by its failure is reconnected before
326
+ * rethrowing it. Other connections just run the command.
327
+ */
328
+ runWithClusterRecovery(command) {
329
+ var _a;
330
+ const client = this.patchedBlockingClusterClient;
331
+ if (!client) {
332
+ return command();
333
+ }
334
+ return RedisConnection.runWithClusterRecovery(client, (_a = this.extraOptions.clusterReconnectTimeoutMs) !== null && _a !== void 0 ? _a : DEFAULT_CLUSTER_RECONNECT_TIMEOUT_MS, command);
335
+ }
336
+ static async runWithClusterRecovery(client, timeoutMs, command) {
337
+ await RedisConnection.reconnectClusterIfNeeded(client, timeoutMs);
338
+ try {
339
+ return await command();
340
+ }
341
+ catch (error) {
342
+ const commandError = error;
343
+ if (RedisConnection.shouldReconnectClusterAfterError(client, commandError)) {
344
+ try {
345
+ await RedisConnection.reconnectCluster(client, timeoutMs);
346
+ }
347
+ catch (_a) {
348
+ // Preserve the original command failure if best-effort recovery fails.
349
+ }
350
+ }
351
+ throw commandError;
352
+ }
353
+ }
339
354
  disableBlockingClusterReconnect() {
340
355
  const client = this.patchedBlockingClusterClient;
341
356
  if (!client || this.disabledBlockingClusterReconnect) {
@@ -0,0 +1,455 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createRedisRelayBackend = exports.RedisRelayBackend = exports.RELAY_LAYOUT_VERSION = exports.RELAY_MINIMUM_REDIS_VERSION = void 0;
4
+ const events_1 = require("events");
5
+ const utils_1 = require("../utils");
6
+ const redis_connection_1 = require("./redis-connection");
7
+ const relay_errors_1 = require("./relay-errors");
8
+ const version_1 = require("../version");
9
+ const DEFAULT_INBOX_MAX_LENGTH = 10000;
10
+ /** Minimum Redis version for the relay (queues still run on 5.0). */
11
+ exports.RELAY_MINIMUM_REDIS_VERSION = '6.2.0';
12
+ /**
13
+ * Version of the relay's Redis data layout. Changes within a layout version
14
+ * are additive only (new keys, fields or entry kinds that older nodes
15
+ * ignore); anything else needs a new version, and nodes of different
16
+ * versions refuse to share a namespace.
17
+ */
18
+ exports.RELAY_LAYOUT_VERSION = '1';
19
+ /**
20
+ * Redis implementation of {@link IRelayBackend}. Every operation is one of the
21
+ * `relay*` Lua scripts; this class only passes arguments and decodes results.
22
+ *
23
+ * Keys of a namespace live under `{prefix}::relay:{<namespace>}`:
24
+ *
25
+ * - The empty segment after the prefix keeps the relay out of the queues'
26
+ * keyspace: every queue key (jobs included) is `{prefix}:{queue}:...` with
27
+ * a non-empty queue name, so no queue (not even one named "relay") nor any
28
+ * of its jobs can ever collide with a relay key.
29
+ * - Every key ends with a fixed word that is not a queue key suffix, never
30
+ * with a node id or topic, so tools that discover queues by key suffix
31
+ * (`*:*:id`, `*:*:meta`) never mistake relay keys for queues.
32
+ * - The namespace is a hash tag, so on Redis Cluster each namespace is one
33
+ * slot (the unit of scale) whatever the prefix.
34
+ *
35
+ * | Key | Type | Content |
36
+ * | --- | --- | --- |
37
+ * | `:layout` | hash | `layout` version |
38
+ * | `:nodes` | sorted set | node → lease deadline |
39
+ * | `:started` | sorted set | node → registration time |
40
+ * | `:{id}:node` | hash | `info`, `hb`, `backlog`, `dropped`; expires with the lease |
41
+ * | `:{id}:inbox` | stream | the node's messages; expires with the lease |
42
+ * | `:subs` | sorted set | subscription index (see `includes/relayIndex.lua`) |
43
+ * | `:cleanup` | sorted set | removed nodes whose subscriptions are being pruned |
44
+ * | `:retained` | sorted set | topic → expiry |
45
+ * | `:{topic}:ret` | hash | retained message (`m`, `ts`, `d`) |
46
+ * | `:mid` | string | message id counter (starts at the server time in µs) |
47
+ *
48
+ * Every key expires on its own when the namespace has no live node (and no
49
+ * retained message), so nothing is left behind even if no node sweeps it.
50
+ * `{prefix}::relay:namespaces` (sorted set: namespace → in use until) lists
51
+ * the namespaces in use, and expires on its own as well.
52
+ *
53
+ * Each node's inbox is a stream, read with a blocking `XREAD` on a dedicated
54
+ * connection. Entries published while that connection is reconnecting are
55
+ * read once it's back.
56
+ */
57
+ class RedisRelayBackend extends events_1.EventEmitter {
58
+ constructor(opts) {
59
+ var _a, _b, _c, _d, _e;
60
+ super();
61
+ this.initialInboxCursor = '0-0';
62
+ this.namespaceExpiresAt = 0;
63
+ this.interrupted = false;
64
+ this.namespace = opts.namespace;
65
+ this.prefix = (_a = opts.prefix) !== null && _a !== void 0 ? _a : 'bull';
66
+ this.baseKey = this.namespaceKey(opts.namespace);
67
+ this.nodesKey = `${this.baseKey}:nodes`;
68
+ this.registryKey = `${this.prefix}::relay:namespaces`;
69
+ this.nodeId = opts.nodeId;
70
+ const connOpts = opts.connection;
71
+ this.skipVersionCheck =
72
+ opts.skipVersionCheck === true ||
73
+ !!(connOpts &&
74
+ (connOpts.skipVersionCheck ||
75
+ ((_b = connOpts.options) === null || _b === void 0 ? void 0 : _b.skipVersionCheck) ||
76
+ ((_d = (_c = connOpts.options) === null || _c === void 0 ? void 0 : _c.redisOptions) === null || _d === void 0 ? void 0 : _d.skipVersionCheck)));
77
+ this.inboxMaxLength = (_e = opts.inboxMaxLength) !== null && _e !== void 0 ? _e : DEFAULT_INBOX_MAX_LENGTH;
78
+ this.connectionOpts = opts.connection;
79
+ this.connection = new redis_connection_1.RedisConnection(opts.connection, {
80
+ shared: (0, utils_1.isRedisInstance)(opts.connection),
81
+ blocking: false,
82
+ skipVersionCheck: this.skipVersionCheck,
83
+ });
84
+ this.forwardConnectionErrors(this.connection);
85
+ }
86
+ async waitUntilReady() {
87
+ await this.connection.client;
88
+ if (!this.skipVersionCheck &&
89
+ (0, utils_1.isRedisVersionLowerThan)(this.connection.redisVersion, exports.RELAY_MINIMUM_REDIS_VERSION, this.connection.databaseType)) {
90
+ throw new Error(`The relay requires Redis ${exports.RELAY_MINIMUM_REDIS_VERSION} or newer. ` +
91
+ `Current: ${this.connection.redisVersion}`);
92
+ }
93
+ }
94
+ close() {
95
+ if (!this.closing) {
96
+ this.closing = (async () => {
97
+ var _a;
98
+ this.interrupted = true;
99
+ await ((_a = this.blockingConnection) === null || _a === void 0 ? void 0 : _a.close(true).catch(() => undefined));
100
+ await this.connection.close();
101
+ })();
102
+ }
103
+ return this.closing;
104
+ }
105
+ async registerNode(nodeId, leaseMs, info = {}, inboxMaxLength = this.inboxMaxLength) {
106
+ // Reusing an expired/removed ID must finish its old indexes before it can
107
+ // receive messages again. Each call yields Redis after a bounded batch.
108
+ let result;
109
+ do {
110
+ result = (0, relay_errors_1.checkRelayResult)(await this.exec('relayRegisterNode', [
111
+ nodeId,
112
+ leaseMs,
113
+ JSON.stringify(info),
114
+ exports.RELAY_LAYOUT_VERSION,
115
+ inboxMaxLength,
116
+ ]), nodeId);
117
+ } while (result === 0);
118
+ this.touchNamespace(leaseMs);
119
+ }
120
+ async heartbeat(nodeId, leaseMs) {
121
+ const alive = (await this.exec('relayHeartbeat', [nodeId, leaseMs])) === 1;
122
+ if (alive) {
123
+ this.touchNamespace(leaseMs);
124
+ }
125
+ return alive;
126
+ }
127
+ async unregisterNode(nodeId) {
128
+ return (await this.exec('relayUnregisterNode', [nodeId])) === 1;
129
+ }
130
+ async sweep(limit) {
131
+ return this.exec('relaySweep', [limit]);
132
+ }
133
+ async subscribe(nodeId, endpointId, pattern) {
134
+ const [added, mid, ts, data] = (0, relay_errors_1.checkRelayResult)(await this.exec('relaySubscribe', [nodeId, endpointId, pattern]), pattern);
135
+ return {
136
+ added: added === 1,
137
+ retained: mid ? { mid: String(mid), ts: Number(ts), data } : undefined,
138
+ };
139
+ }
140
+ async removeEndpoint(nodeId, endpointId) {
141
+ return this.exec('relayRemoveEndpoint', [nodeId, endpointId]);
142
+ }
143
+ async publish(topic, data, opts) {
144
+ const [mid, nodes, endpoints] = (0, relay_errors_1.checkRelayResult)(await this.exec('relayPublish', [
145
+ topic,
146
+ data,
147
+ opts.ts,
148
+ opts.retainMs,
149
+ opts.maxSize,
150
+ this.inboxMaxLength,
151
+ ]), topic);
152
+ if (opts.retainMs > 0) {
153
+ this.touchNamespace(opts.retainMs);
154
+ }
155
+ return { mid: String(mid), nodes, endpoints };
156
+ }
157
+ async deleteRetained(topic) {
158
+ return ((0, relay_errors_1.checkRelayResult)(await this.exec('relayDeleteRetained', [topic]), topic) === 1);
159
+ }
160
+ async readInbox(nodeId, cursor, opts) {
161
+ if (this.interrupted) {
162
+ return { entries: [], cursor };
163
+ }
164
+ let connection;
165
+ try {
166
+ connection = await this.getBlockingConnection();
167
+ }
168
+ catch (err) {
169
+ if (this.interrupted) {
170
+ return { entries: [], cursor };
171
+ }
172
+ throw err;
173
+ }
174
+ const client = await connection.client;
175
+ if (this.interrupted) {
176
+ return { entries: [], cursor };
177
+ }
178
+ // Recovers an ioredis Cluster left with an empty node pool by an outage,
179
+ // as for the workers' blocking reads.
180
+ const read = () => connection.runWithClusterRecovery(() => client.runCommand(`relayReadInbox:${version_1.version}`, [
181
+ this.nodesKey,
182
+ this.baseKey,
183
+ nodeId,
184
+ cursor,
185
+ opts.count,
186
+ ]));
187
+ let result = await read();
188
+ if (!result && opts.blockMs > 0 && !this.interrupted) {
189
+ // Only use the blocking read as a wakeup. Fetching and marking the actual
190
+ // batch must be atomic with publish-time trimming.
191
+ await connection.runWithClusterRecovery(() => client.xread([{ key: `${this.baseKey}:${nodeId}:inbox`, id: cursor }], {
192
+ BLOCK: opts.blockMs,
193
+ COUNT: 1,
194
+ }));
195
+ if (this.interrupted) {
196
+ return { entries: [], cursor };
197
+ }
198
+ result = await read();
199
+ }
200
+ if (!result) {
201
+ return { entries: [], cursor };
202
+ }
203
+ const entries = [];
204
+ let next = cursor;
205
+ for (const [id, fields] of result[0][1]) {
206
+ next = id;
207
+ const entry = {};
208
+ for (let i = 0; i + 1 < fields.length; i += 2) {
209
+ entry[fields[i]] = fields[i + 1];
210
+ }
211
+ entries.push({
212
+ kind: entry.k,
213
+ topic: entry.t,
214
+ mid: entry.m,
215
+ ts: Number(entry.ts),
216
+ data: entry.d,
217
+ endpoints: entry.e ? entry.e.split(',') : [],
218
+ });
219
+ }
220
+ return { entries, cursor: next };
221
+ }
222
+ async trimInbox(nodeId, cursor, maxLength) {
223
+ return this.exec('relayTrimInbox', [nodeId, cursor, maxLength]);
224
+ }
225
+ async getCounts() {
226
+ const [nodes, subscriptions, patterns, retained] = await this.get('counts');
227
+ return { nodes, subscriptions, patterns, retained };
228
+ }
229
+ async getNodes(start, end) {
230
+ const rows = await this.get('nodes', start, end);
231
+ return rows.map(([id, registeredAt, leaseUntil, info, hb, backlog, dropped, subs]) => ({
232
+ id,
233
+ info: info ? JSON.parse(info) : {},
234
+ registeredAt: Number(registeredAt),
235
+ leaseUntil: Number(leaseUntil),
236
+ lastHeartbeat: hb ? Number(hb) : undefined,
237
+ backlog: Number(backlog),
238
+ dropped: Number(dropped),
239
+ subscriptions: subs,
240
+ }));
241
+ }
242
+ async getSubscriptions(filter, start, end) {
243
+ const [kind, value] = filter && 'nodeId' in filter
244
+ ? ['node', filter.nodeId]
245
+ : filter && 'pattern' in filter
246
+ ? ['pattern', filter.pattern]
247
+ : ['', ''];
248
+ const rows = await this.get('subscriptions', start, end, kind, value);
249
+ return rows.map(([nodeId, endpointId, pattern]) => ({
250
+ nodeId,
251
+ endpointId,
252
+ pattern,
253
+ }));
254
+ }
255
+ async getPatterns(start, end) {
256
+ const rows = await this.get('patterns', start, end);
257
+ return rows.map(([pattern, subscriptions]) => ({
258
+ pattern,
259
+ subscriptions,
260
+ }));
261
+ }
262
+ async getRetained(start, end) {
263
+ const rows = await this.get('retained', start, end);
264
+ return rows.map(([topic, expiresAt, mid, ts, data]) => ({
265
+ topic,
266
+ expiresAt: Number(expiresAt),
267
+ mid: String(mid),
268
+ ts: Number(ts),
269
+ data,
270
+ }));
271
+ }
272
+ async getNamespaces() {
273
+ const client = await this.connection.client;
274
+ const candidates = await client.runCommand(`relayGetNamespaces:${version_1.version}`, [this.registryKey]);
275
+ // The registry only says until when a namespace may be in use (its nodes
276
+ // can unregister and its retained messages be deleted earlier), so each
277
+ // candidate is checked against its own keys.
278
+ const used = await Promise.all(candidates.map(async (namespace) => {
279
+ const baseKey = this.namespaceKey(namespace);
280
+ return ((await client.runCommand(`relayGet:${version_1.version}`, [
281
+ `${baseKey}:nodes`,
282
+ baseKey,
283
+ 'used',
284
+ 0,
285
+ -1,
286
+ '',
287
+ '',
288
+ ])) === 1);
289
+ }));
290
+ return candidates.filter((_, i) => used[i]);
291
+ }
292
+ async getNodeConnections(nodeId) {
293
+ var _a;
294
+ const client = await this.connection.client;
295
+ // Adapter options do not consistently expose the selected database.
296
+ const db = client.isCluster
297
+ ? '0'
298
+ : (_a = (await client.clientInfo()).match(/(?:^| )db=(\d+)/)) === null || _a === void 0 ? void 0 : _a[1];
299
+ // Every node of a cluster: the connection may be to any of them.
300
+ const lists = client.isCluster && typeof client.nodes === 'function'
301
+ ? await Promise.all((client.nodes() || []).map((node) => typeof node.clientList === 'function'
302
+ ? node.clientList()
303
+ : node.client('LIST')))
304
+ : [await client.clientList()];
305
+ const name = this.inboxConnectionName(nodeId);
306
+ const connections = [];
307
+ for (const line of lists.join('\n').split('\n')) {
308
+ const fields = {};
309
+ for (const pair of line.trim().split(' ')) {
310
+ const eq = pair.indexOf('=');
311
+ if (eq > 0) {
312
+ fields[pair.slice(0, eq)] = pair.slice(eq + 1);
313
+ }
314
+ }
315
+ if (fields.name === name && fields.db === db) {
316
+ connections.push({
317
+ address: fields.addr,
318
+ age: Number(fields.age),
319
+ idle: Number(fields.idle),
320
+ });
321
+ }
322
+ }
323
+ return connections;
324
+ }
325
+ interruptInbox() {
326
+ var _a;
327
+ this.interrupted = true;
328
+ (_a = this.blockingConnection) === null || _a === void 0 ? void 0 : _a.close(true).catch(() => undefined);
329
+ }
330
+ getBlockingConnection() {
331
+ if (!this.blockingConnectionPromise) {
332
+ const promise = (async () => {
333
+ const shared = (0, utils_1.isRedisInstance)(this.connectionOpts);
334
+ const client = await this.connection.client;
335
+ // Closed meanwhile: a connection opened now would never be closed.
336
+ if (this.interrupted) {
337
+ throw new Error('Connection is closed.');
338
+ }
339
+ // Named after the node (CLIENT SETNAME, re-applied on reconnects) so
340
+ // getNodeConnections can find it.
341
+ const naming = this.nodeId
342
+ ? { connectionName: this.inboxConnectionName(this.nodeId) }
343
+ : {};
344
+ const connection = new redis_connection_1.RedisConnection(shared
345
+ ? client.duplicate(Object.assign({ maxRetriesPerRequest: null }, naming))
346
+ : Object.assign(Object.assign({}, this.connectionOpts), naming), {
347
+ shared: false,
348
+ blocking: true,
349
+ skipVersionCheck: this.skipVersionCheck,
350
+ });
351
+ this.forwardConnectionErrors(connection);
352
+ // Set before it's ready, so closing can stop it while it connects.
353
+ this.blockingConnection = connection;
354
+ try {
355
+ await connection.client;
356
+ }
357
+ catch (err) {
358
+ // Disposed of, so the next read starts over with a new connection.
359
+ if (this.blockingConnection === connection) {
360
+ this.blockingConnection = undefined;
361
+ }
362
+ await connection.close(true).catch(() => undefined);
363
+ throw err;
364
+ }
365
+ return connection;
366
+ })();
367
+ this.blockingConnectionPromise = promise;
368
+ promise.catch(() => {
369
+ if (this.blockingConnectionPromise === promise) {
370
+ this.blockingConnectionPromise = undefined;
371
+ }
372
+ });
373
+ }
374
+ return this.blockingConnectionPromise;
375
+ }
376
+ forwardConnectionErrors(connection) {
377
+ connection.on('error', err => {
378
+ if (this.listenerCount('error') > 0) {
379
+ this.emit('error', err);
380
+ }
381
+ });
382
+ }
383
+ /** Base key of a namespace of this backend's prefix. */
384
+ namespaceKey(namespace) {
385
+ return `${this.prefix}::relay:{${namespace}}`;
386
+ }
387
+ /** Redis client name of a node's inbox connection. */
388
+ inboxConnectionName(nodeId) {
389
+ return `${Buffer.from(this.baseKey).toString('base64')}:${nodeId}`;
390
+ }
391
+ get(view, start = 0, end = -1, filter = '', value = '') {
392
+ return this.exec('relayGet', [view, start, end, filter, value]);
393
+ }
394
+ /**
395
+ * Lists the namespace in the prefix's registry. The registry is outside the
396
+ * namespace's hash tag, so this is a call of its own.
397
+ */
398
+ touchNamespace(forMs) {
399
+ this.namespaceExpiresAt = Math.max(this.namespaceExpiresAt, Date.now() + forMs);
400
+ this.startNamespaceTouch();
401
+ }
402
+ startNamespaceTouch() {
403
+ if (!this.namespaceTouch) {
404
+ const requestedUntil = this.namespaceExpiresAt;
405
+ const promise = this.refreshNamespace();
406
+ this.namespaceTouch = promise;
407
+ promise
408
+ .finally(() => {
409
+ if (this.namespaceTouch === promise) {
410
+ this.namespaceTouch = undefined;
411
+ if (!this.closing && this.namespaceExpiresAt > requestedUntil) {
412
+ this.startNamespaceTouch();
413
+ }
414
+ }
415
+ })
416
+ .catch(() => undefined);
417
+ }
418
+ }
419
+ async refreshNamespace() {
420
+ for (let attempt = 0; attempt < 3 && !this.closing; attempt++) {
421
+ const expiresAt = this.namespaceExpiresAt;
422
+ try {
423
+ const client = await this.connection.client;
424
+ await client.runCommand(`relayTouchNamespace:${version_1.version}`, [
425
+ this.registryKey,
426
+ this.namespace,
427
+ Math.max(1, expiresAt - Date.now()),
428
+ ]);
429
+ return;
430
+ }
431
+ catch (err) {
432
+ if (attempt === 2) {
433
+ if (!this.closing && this.listenerCount('error') > 0) {
434
+ this.emit('error', err);
435
+ }
436
+ }
437
+ else {
438
+ await new Promise(resolve => setTimeout(resolve, 100 * 2 ** attempt));
439
+ }
440
+ }
441
+ }
442
+ }
443
+ async exec(command, args) {
444
+ const client = await this.connection.client;
445
+ return client.runCommand(`${command}:${version_1.version}`, [
446
+ this.nodesKey,
447
+ this.baseKey,
448
+ ...args,
449
+ ]);
450
+ }
451
+ }
452
+ exports.RedisRelayBackend = RedisRelayBackend;
453
+ /** The default {@link RelayBackendFactory}: Redis. */
454
+ const createRedisRelayBackend = (opts) => new RedisRelayBackend(opts);
455
+ exports.createRedisRelayBackend = createRedisRelayBackend;
@@ -0,0 +1,41 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.RelayError = exports.RelayErrorCode = void 0;
4
+ exports.checkRelayResult = checkRelayResult;
5
+ /**
6
+ * Error codes shared by the relay's Lua scripts and SQL functions.
7
+ */
8
+ var RelayErrorCode;
9
+ (function (RelayErrorCode) {
10
+ RelayErrorCode[RelayErrorCode["InvalidTopic"] = -1] = "InvalidTopic";
11
+ RelayErrorCode[RelayErrorCode["InvalidPattern"] = -2] = "InvalidPattern";
12
+ RelayErrorCode[RelayErrorCode["NodeNotRegistered"] = -3] = "NodeNotRegistered";
13
+ RelayErrorCode[RelayErrorCode["DataTooLarge"] = -4] = "DataTooLarge";
14
+ RelayErrorCode[RelayErrorCode["InvalidId"] = -5] = "InvalidId";
15
+ RelayErrorCode[RelayErrorCode["IncompatibleLayout"] = -6] = "IncompatibleLayout";
16
+ })(RelayErrorCode || (exports.RelayErrorCode = RelayErrorCode = {}));
17
+ const MESSAGES = {
18
+ [RelayErrorCode.InvalidTopic]: 'Invalid topic',
19
+ [RelayErrorCode.InvalidPattern]: 'Invalid topic pattern',
20
+ [RelayErrorCode.NodeNotRegistered]: 'Relay node is not registered',
21
+ [RelayErrorCode.DataTooLarge]: 'Message data is too large',
22
+ [RelayErrorCode.InvalidId]: 'Invalid relay node or endpoint id',
23
+ [RelayErrorCode.IncompatibleLayout]: 'The relay namespace is in use by nodes with an incompatible BullMQ version',
24
+ };
25
+ class RelayError extends Error {
26
+ constructor(code, detail) {
27
+ super(`${MESSAGES[code] || `Relay error ${code}`}${detail ? `: ${detail}` : ''}`);
28
+ this.code = code;
29
+ this.name = 'RelayError';
30
+ }
31
+ }
32
+ exports.RelayError = RelayError;
33
+ /**
34
+ * Throws a {@link RelayError} if a script result is an error code.
35
+ */
36
+ function checkRelayResult(result, detail) {
37
+ if (typeof result === 'number' && result < 0) {
38
+ throw new RelayError(result, detail);
39
+ }
40
+ return result;
41
+ }