bullmq 6.3.19 → 6.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/bun-redis-client.js +3 -0
- package/dist/cjs/classes/index.js +3 -0
- package/dist/cjs/classes/ioredis-client.js +1 -0
- package/dist/cjs/classes/node-redis-client.js +3 -0
- package/dist/cjs/classes/redis-connection.js +33 -18
- package/dist/cjs/classes/redis-relay-backend.js +455 -0
- package/dist/cjs/classes/relay-errors.js +41 -0
- package/dist/cjs/classes/relay.js +562 -0
- package/dist/cjs/classes/valkey-glide-client.js +7 -2
- package/dist/cjs/commands/includes/refreshRelayNamespace.lua +36 -0
- package/dist/cjs/commands/includes/relayIndex.lua +112 -0
- package/dist/cjs/commands/includes/relayTime.lua +15 -0
- package/dist/cjs/commands/includes/relayTopics.lua +92 -0
- package/dist/cjs/commands/relayAckInbox-1.lua +18 -0
- package/dist/cjs/commands/relayDeleteRetained-1.lua +39 -0
- package/dist/cjs/commands/relayGet-1.lua +133 -0
- package/dist/cjs/commands/relayGetNamespaces-1.lua +16 -0
- package/dist/cjs/commands/relayHeartbeat-1.lua +35 -0
- package/dist/cjs/commands/relayPublish-1.lua +150 -0
- package/dist/cjs/commands/relayReadInbox-1.lua +31 -0
- package/dist/cjs/commands/relayRegisterNode-1.lua +60 -0
- package/dist/cjs/commands/relayRemoveEndpoint-1.lua +18 -0
- package/dist/cjs/commands/relaySubscribe-1.lua +69 -0
- package/dist/cjs/commands/relaySweep-1.lua +39 -0
- package/dist/cjs/commands/relayTouchNamespace-1.lua +22 -0
- package/dist/cjs/commands/relayTrimInbox-1.lua +65 -0
- package/dist/cjs/commands/relayUnregisterNode-1.lua +25 -0
- package/dist/cjs/interfaces/index.js +1 -0
- package/dist/cjs/interfaces/relay-backend.js +2 -0
- package/dist/cjs/postgres/index.js +4 -1
- package/dist/cjs/postgres/migrations/index.js +9 -2
- package/dist/cjs/postgres/postgres-connection.js +3 -2
- package/dist/cjs/postgres/postgres-relay-backend.js +342 -0
- package/dist/cjs/postgres/sql-scripts.js +19 -0
- package/dist/cjs/scripts/index.js +14 -0
- package/dist/cjs/scripts/relayAckInbox-1.js +23 -0
- package/dist/cjs/scripts/relayDeleteRetained-1.js +168 -0
- package/dist/cjs/scripts/relayGet-1.js +323 -0
- package/dist/cjs/scripts/relayGetNamespaces-1.js +34 -0
- package/dist/cjs/scripts/relayHeartbeat-1.js +81 -0
- package/dist/cjs/scripts/relayPublish-1.js +368 -0
- package/dist/cjs/scripts/relayReadInbox-1.js +35 -0
- package/dist/cjs/scripts/relayRegisterNode-1.js +285 -0
- package/dist/cjs/scripts/relayRemoveEndpoint-1.js +204 -0
- package/dist/cjs/scripts/relaySubscribe-1.js +150 -0
- package/dist/cjs/scripts/relaySweep-1.js +265 -0
- package/dist/cjs/scripts/relayTouchNamespace-1.js +38 -0
- package/dist/cjs/scripts/relayTrimInbox-1.js +81 -0
- package/dist/cjs/scripts/relayUnregisterNode-1.js +255 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/bun-redis-client.js +3 -0
- package/dist/esm/classes/index.d.ts +3 -0
- package/dist/esm/classes/index.js +3 -0
- package/dist/esm/classes/ioredis-client.js +1 -0
- package/dist/esm/classes/node-redis-client.js +3 -0
- package/dist/esm/classes/redis-connection.d.ts +8 -0
- package/dist/esm/classes/redis-connection.js +33 -18
- package/dist/esm/classes/redis-relay-backend.d.ts +110 -0
- package/dist/esm/classes/redis-relay-backend.js +450 -0
- package/dist/esm/classes/relay-errors.d.ts +19 -0
- package/dist/esm/classes/relay-errors.js +36 -0
- package/dist/esm/classes/relay.d.ts +183 -0
- package/dist/esm/classes/relay.js +558 -0
- package/dist/esm/classes/valkey-glide-client.js +7 -2
- package/dist/esm/commands/includes/refreshRelayNamespace.lua +36 -0
- package/dist/esm/commands/includes/relayIndex.lua +112 -0
- package/dist/esm/commands/includes/relayTime.lua +15 -0
- package/dist/esm/commands/includes/relayTopics.lua +92 -0
- package/dist/esm/commands/relayAckInbox-1.lua +18 -0
- package/dist/esm/commands/relayDeleteRetained-1.lua +39 -0
- package/dist/esm/commands/relayGet-1.lua +133 -0
- package/dist/esm/commands/relayGetNamespaces-1.lua +16 -0
- package/dist/esm/commands/relayHeartbeat-1.lua +35 -0
- package/dist/esm/commands/relayPublish-1.lua +150 -0
- package/dist/esm/commands/relayReadInbox-1.lua +31 -0
- package/dist/esm/commands/relayRegisterNode-1.lua +60 -0
- package/dist/esm/commands/relayRemoveEndpoint-1.lua +18 -0
- package/dist/esm/commands/relaySubscribe-1.lua +69 -0
- package/dist/esm/commands/relaySweep-1.lua +39 -0
- package/dist/esm/commands/relayTouchNamespace-1.lua +22 -0
- package/dist/esm/commands/relayTrimInbox-1.lua +65 -0
- package/dist/esm/commands/relayUnregisterNode-1.lua +25 -0
- package/dist/esm/interfaces/index.d.ts +1 -0
- package/dist/esm/interfaces/index.js +1 -0
- package/dist/esm/interfaces/redis-client.d.ts +1 -0
- package/dist/esm/interfaces/relay-backend.d.ts +237 -0
- package/dist/esm/interfaces/relay-backend.js +1 -0
- package/dist/esm/postgres/index.d.ts +1 -0
- package/dist/esm/postgres/index.js +1 -0
- package/dist/esm/postgres/migrations/index.d.ts +11 -1
- package/dist/esm/postgres/migrations/index.js +9 -2
- package/dist/esm/postgres/postgres-connection.js +3 -2
- package/dist/esm/postgres/postgres-relay-backend.d.ts +67 -0
- package/dist/esm/postgres/postgres-relay-backend.js +337 -0
- package/dist/esm/postgres/sql-scripts.js +19 -0
- package/dist/esm/scripts/index.d.ts +14 -0
- package/dist/esm/scripts/index.js +14 -0
- package/dist/esm/scripts/relayAckInbox-1.d.ts +5 -0
- package/dist/esm/scripts/relayAckInbox-1.js +20 -0
- package/dist/esm/scripts/relayDeleteRetained-1.d.ts +5 -0
- package/dist/esm/scripts/relayDeleteRetained-1.js +165 -0
- package/dist/esm/scripts/relayGet-1.d.ts +5 -0
- package/dist/esm/scripts/relayGet-1.js +320 -0
- package/dist/esm/scripts/relayGetNamespaces-1.d.ts +5 -0
- package/dist/esm/scripts/relayGetNamespaces-1.js +31 -0
- package/dist/esm/scripts/relayHeartbeat-1.d.ts +5 -0
- package/dist/esm/scripts/relayHeartbeat-1.js +78 -0
- package/dist/esm/scripts/relayPublish-1.d.ts +5 -0
- package/dist/esm/scripts/relayPublish-1.js +365 -0
- package/dist/esm/scripts/relayReadInbox-1.d.ts +5 -0
- package/dist/esm/scripts/relayReadInbox-1.js +32 -0
- package/dist/esm/scripts/relayRegisterNode-1.d.ts +5 -0
- package/dist/esm/scripts/relayRegisterNode-1.js +282 -0
- package/dist/esm/scripts/relayRemoveEndpoint-1.d.ts +5 -0
- package/dist/esm/scripts/relayRemoveEndpoint-1.js +201 -0
- package/dist/esm/scripts/relaySubscribe-1.d.ts +5 -0
- package/dist/esm/scripts/relaySubscribe-1.js +147 -0
- package/dist/esm/scripts/relaySweep-1.d.ts +5 -0
- package/dist/esm/scripts/relaySweep-1.js +262 -0
- package/dist/esm/scripts/relayTouchNamespace-1.d.ts +5 -0
- package/dist/esm/scripts/relayTouchNamespace-1.js +35 -0
- package/dist/esm/scripts/relayTrimInbox-1.d.ts +5 -0
- package/dist/esm/scripts/relayTrimInbox-1.js +78 -0
- package/dist/esm/scripts/relayUnregisterNode-1.d.ts +5 -0
- package/dist/esm/scripts/relayUnregisterNode-1.js +252 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +8 -8
|
@@ -843,6 +843,9 @@ class BunRedisAdapter extends events_1.EventEmitter {
|
|
|
843
843
|
async clientList() {
|
|
844
844
|
return await this.sendCommand('CLIENT', ['LIST']);
|
|
845
845
|
}
|
|
846
|
+
async clientInfo() {
|
|
847
|
+
return await this.sendCommand('CLIENT', ['INFO']);
|
|
848
|
+
}
|
|
846
849
|
// ---------------------------------------------------------------
|
|
847
850
|
// Key scanning
|
|
848
851
|
// ---------------------------------------------------------------
|
|
@@ -26,4 +26,7 @@ tslib_1.__exportStar(require("./queue"), exports);
|
|
|
26
26
|
tslib_1.__exportStar(require("./redis-connection"), exports);
|
|
27
27
|
tslib_1.__exportStar(require("./sandbox"), exports);
|
|
28
28
|
tslib_1.__exportStar(require("./redis-queue-backend"), exports);
|
|
29
|
+
tslib_1.__exportStar(require("./redis-relay-backend"), exports);
|
|
30
|
+
tslib_1.__exportStar(require("./relay"), exports);
|
|
31
|
+
tslib_1.__exportStar(require("./relay-errors"), exports);
|
|
29
32
|
tslib_1.__exportStar(require("./worker"), exports);
|
|
@@ -168,6 +168,7 @@ function createIORedisClient(client) {
|
|
|
168
168
|
// clientSetName / clientList helpers that forward to CLIENT subcommands.
|
|
169
169
|
overrides.clientSetName = (name) => client.client('SETNAME', name);
|
|
170
170
|
overrides.clientList = () => client.client('LIST');
|
|
171
|
+
overrides.clientInfo = () => client.client('INFO');
|
|
171
172
|
// scan(cursor, { MATCH?, COUNT? }) — accepts either structured options or
|
|
172
173
|
// ioredis varargs (used internally by `scanStream`).
|
|
173
174
|
overrides.scan = (cursor, ...rest) => {
|
|
@@ -523,6 +523,9 @@ class NodeRedisAdapter extends events_1.EventEmitter {
|
|
|
523
523
|
async clientList() {
|
|
524
524
|
return await this.raw.sendCommand(['CLIENT', 'LIST']);
|
|
525
525
|
}
|
|
526
|
+
async clientInfo() {
|
|
527
|
+
return await this.raw.sendCommand(['CLIENT', 'INFO']);
|
|
528
|
+
}
|
|
526
529
|
// ---------------------------------------------------------------
|
|
527
530
|
// Key scanning
|
|
528
531
|
// ---------------------------------------------------------------
|
|
@@ -313,29 +313,44 @@ class RedisConnection extends events_1.EventEmitter {
|
|
|
313
313
|
return;
|
|
314
314
|
}
|
|
315
315
|
const bzpopmin = blockingClient.bzpopmin;
|
|
316
|
-
const wrappedBzpopmin =
|
|
317
|
-
await RedisConnection.reconnectClusterIfNeeded(blockingClient, reconnectTimeoutMs);
|
|
318
|
-
try {
|
|
319
|
-
return await bzpopmin.apply(blockingClient, args);
|
|
320
|
-
}
|
|
321
|
-
catch (error) {
|
|
322
|
-
const commandError = error;
|
|
323
|
-
if (RedisConnection.shouldReconnectClusterAfterError(blockingClient, commandError)) {
|
|
324
|
-
try {
|
|
325
|
-
await RedisConnection.reconnectCluster(blockingClient, reconnectTimeoutMs);
|
|
326
|
-
}
|
|
327
|
-
catch (_a) {
|
|
328
|
-
// Preserve the original command failure if best-effort recovery fails.
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
throw commandError;
|
|
332
|
-
}
|
|
333
|
-
};
|
|
316
|
+
const wrappedBzpopmin = (...args) => RedisConnection.runWithClusterRecovery(blockingClient, reconnectTimeoutMs, () => bzpopmin.apply(blockingClient, args));
|
|
334
317
|
blockingClient[clusterOriginalBzpopmin] = bzpopmin;
|
|
335
318
|
blockingClient[clusterWrappedBzpopmin] = wrappedBzpopmin;
|
|
336
319
|
blockingClient[clusterPatchedForBlocking] = true;
|
|
337
320
|
blockingClient.bzpopmin = wrappedBzpopmin;
|
|
338
321
|
}
|
|
322
|
+
/**
|
|
323
|
+
* Runs a command of a blocking connection with the recovery that `bzpopmin`
|
|
324
|
+
* gets on an ioredis Cluster: an empty node pool is reconnected before the
|
|
325
|
+
* command, and a cluster left unusable by its failure is reconnected before
|
|
326
|
+
* rethrowing it. Other connections just run the command.
|
|
327
|
+
*/
|
|
328
|
+
runWithClusterRecovery(command) {
|
|
329
|
+
var _a;
|
|
330
|
+
const client = this.patchedBlockingClusterClient;
|
|
331
|
+
if (!client) {
|
|
332
|
+
return command();
|
|
333
|
+
}
|
|
334
|
+
return RedisConnection.runWithClusterRecovery(client, (_a = this.extraOptions.clusterReconnectTimeoutMs) !== null && _a !== void 0 ? _a : DEFAULT_CLUSTER_RECONNECT_TIMEOUT_MS, command);
|
|
335
|
+
}
|
|
336
|
+
static async runWithClusterRecovery(client, timeoutMs, command) {
|
|
337
|
+
await RedisConnection.reconnectClusterIfNeeded(client, timeoutMs);
|
|
338
|
+
try {
|
|
339
|
+
return await command();
|
|
340
|
+
}
|
|
341
|
+
catch (error) {
|
|
342
|
+
const commandError = error;
|
|
343
|
+
if (RedisConnection.shouldReconnectClusterAfterError(client, commandError)) {
|
|
344
|
+
try {
|
|
345
|
+
await RedisConnection.reconnectCluster(client, timeoutMs);
|
|
346
|
+
}
|
|
347
|
+
catch (_a) {
|
|
348
|
+
// Preserve the original command failure if best-effort recovery fails.
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
throw commandError;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
339
354
|
disableBlockingClusterReconnect() {
|
|
340
355
|
const client = this.patchedBlockingClusterClient;
|
|
341
356
|
if (!client || this.disabledBlockingClusterReconnect) {
|
|
@@ -0,0 +1,455 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createRedisRelayBackend = exports.RedisRelayBackend = exports.RELAY_LAYOUT_VERSION = exports.RELAY_MINIMUM_REDIS_VERSION = void 0;
|
|
4
|
+
const events_1 = require("events");
|
|
5
|
+
const utils_1 = require("../utils");
|
|
6
|
+
const redis_connection_1 = require("./redis-connection");
|
|
7
|
+
const relay_errors_1 = require("./relay-errors");
|
|
8
|
+
const version_1 = require("../version");
|
|
9
|
+
const DEFAULT_INBOX_MAX_LENGTH = 10000;
|
|
10
|
+
/** Minimum Redis version for the relay (queues still run on 5.0). */
|
|
11
|
+
exports.RELAY_MINIMUM_REDIS_VERSION = '6.2.0';
|
|
12
|
+
/**
|
|
13
|
+
* Version of the relay's Redis data layout. Changes within a layout version
|
|
14
|
+
* are additive only (new keys, fields or entry kinds that older nodes
|
|
15
|
+
* ignore); anything else needs a new version, and nodes of different
|
|
16
|
+
* versions refuse to share a namespace.
|
|
17
|
+
*/
|
|
18
|
+
exports.RELAY_LAYOUT_VERSION = '1';
|
|
19
|
+
/**
|
|
20
|
+
* Redis implementation of {@link IRelayBackend}. Every operation is one of the
|
|
21
|
+
* `relay*` Lua scripts; this class only passes arguments and decodes results.
|
|
22
|
+
*
|
|
23
|
+
* Keys of a namespace live under `{prefix}::relay:{<namespace>}`:
|
|
24
|
+
*
|
|
25
|
+
* - The empty segment after the prefix keeps the relay out of the queues'
|
|
26
|
+
* keyspace: every queue key (jobs included) is `{prefix}:{queue}:...` with
|
|
27
|
+
* a non-empty queue name, so no queue (not even one named "relay") nor any
|
|
28
|
+
* of its jobs can ever collide with a relay key.
|
|
29
|
+
* - Every key ends with a fixed word that is not a queue key suffix, never
|
|
30
|
+
* with a node id or topic, so tools that discover queues by key suffix
|
|
31
|
+
* (`*:*:id`, `*:*:meta`) never mistake relay keys for queues.
|
|
32
|
+
* - The namespace is a hash tag, so on Redis Cluster each namespace is one
|
|
33
|
+
* slot (the unit of scale) whatever the prefix.
|
|
34
|
+
*
|
|
35
|
+
* | Key | Type | Content |
|
|
36
|
+
* | --- | --- | --- |
|
|
37
|
+
* | `:layout` | hash | `layout` version |
|
|
38
|
+
* | `:nodes` | sorted set | node → lease deadline |
|
|
39
|
+
* | `:started` | sorted set | node → registration time |
|
|
40
|
+
* | `:{id}:node` | hash | `info`, `hb`, `backlog`, `dropped`; expires with the lease |
|
|
41
|
+
* | `:{id}:inbox` | stream | the node's messages; expires with the lease |
|
|
42
|
+
* | `:subs` | sorted set | subscription index (see `includes/relayIndex.lua`) |
|
|
43
|
+
* | `:cleanup` | sorted set | removed nodes whose subscriptions are being pruned |
|
|
44
|
+
* | `:retained` | sorted set | topic → expiry |
|
|
45
|
+
* | `:{topic}:ret` | hash | retained message (`m`, `ts`, `d`) |
|
|
46
|
+
* | `:mid` | string | message id counter (starts at the server time in µs) |
|
|
47
|
+
*
|
|
48
|
+
* Every key expires on its own when the namespace has no live node (and no
|
|
49
|
+
* retained message), so nothing is left behind even if no node sweeps it.
|
|
50
|
+
* `{prefix}::relay:namespaces` (sorted set: namespace → in use until) lists
|
|
51
|
+
* the namespaces in use, and expires on its own as well.
|
|
52
|
+
*
|
|
53
|
+
* Each node's inbox is a stream, read with a blocking `XREAD` on a dedicated
|
|
54
|
+
* connection. Entries published while that connection is reconnecting are
|
|
55
|
+
* read once it's back.
|
|
56
|
+
*/
|
|
57
|
+
class RedisRelayBackend extends events_1.EventEmitter {
|
|
58
|
+
constructor(opts) {
|
|
59
|
+
var _a, _b, _c, _d, _e;
|
|
60
|
+
super();
|
|
61
|
+
this.initialInboxCursor = '0-0';
|
|
62
|
+
this.namespaceExpiresAt = 0;
|
|
63
|
+
this.interrupted = false;
|
|
64
|
+
this.namespace = opts.namespace;
|
|
65
|
+
this.prefix = (_a = opts.prefix) !== null && _a !== void 0 ? _a : 'bull';
|
|
66
|
+
this.baseKey = this.namespaceKey(opts.namespace);
|
|
67
|
+
this.nodesKey = `${this.baseKey}:nodes`;
|
|
68
|
+
this.registryKey = `${this.prefix}::relay:namespaces`;
|
|
69
|
+
this.nodeId = opts.nodeId;
|
|
70
|
+
const connOpts = opts.connection;
|
|
71
|
+
this.skipVersionCheck =
|
|
72
|
+
opts.skipVersionCheck === true ||
|
|
73
|
+
!!(connOpts &&
|
|
74
|
+
(connOpts.skipVersionCheck ||
|
|
75
|
+
((_b = connOpts.options) === null || _b === void 0 ? void 0 : _b.skipVersionCheck) ||
|
|
76
|
+
((_d = (_c = connOpts.options) === null || _c === void 0 ? void 0 : _c.redisOptions) === null || _d === void 0 ? void 0 : _d.skipVersionCheck)));
|
|
77
|
+
this.inboxMaxLength = (_e = opts.inboxMaxLength) !== null && _e !== void 0 ? _e : DEFAULT_INBOX_MAX_LENGTH;
|
|
78
|
+
this.connectionOpts = opts.connection;
|
|
79
|
+
this.connection = new redis_connection_1.RedisConnection(opts.connection, {
|
|
80
|
+
shared: (0, utils_1.isRedisInstance)(opts.connection),
|
|
81
|
+
blocking: false,
|
|
82
|
+
skipVersionCheck: this.skipVersionCheck,
|
|
83
|
+
});
|
|
84
|
+
this.forwardConnectionErrors(this.connection);
|
|
85
|
+
}
|
|
86
|
+
async waitUntilReady() {
|
|
87
|
+
await this.connection.client;
|
|
88
|
+
if (!this.skipVersionCheck &&
|
|
89
|
+
(0, utils_1.isRedisVersionLowerThan)(this.connection.redisVersion, exports.RELAY_MINIMUM_REDIS_VERSION, this.connection.databaseType)) {
|
|
90
|
+
throw new Error(`The relay requires Redis ${exports.RELAY_MINIMUM_REDIS_VERSION} or newer. ` +
|
|
91
|
+
`Current: ${this.connection.redisVersion}`);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
close() {
|
|
95
|
+
if (!this.closing) {
|
|
96
|
+
this.closing = (async () => {
|
|
97
|
+
var _a;
|
|
98
|
+
this.interrupted = true;
|
|
99
|
+
await ((_a = this.blockingConnection) === null || _a === void 0 ? void 0 : _a.close(true).catch(() => undefined));
|
|
100
|
+
await this.connection.close();
|
|
101
|
+
})();
|
|
102
|
+
}
|
|
103
|
+
return this.closing;
|
|
104
|
+
}
|
|
105
|
+
async registerNode(nodeId, leaseMs, info = {}, inboxMaxLength = this.inboxMaxLength) {
|
|
106
|
+
// Reusing an expired/removed ID must finish its old indexes before it can
|
|
107
|
+
// receive messages again. Each call yields Redis after a bounded batch.
|
|
108
|
+
let result;
|
|
109
|
+
do {
|
|
110
|
+
result = (0, relay_errors_1.checkRelayResult)(await this.exec('relayRegisterNode', [
|
|
111
|
+
nodeId,
|
|
112
|
+
leaseMs,
|
|
113
|
+
JSON.stringify(info),
|
|
114
|
+
exports.RELAY_LAYOUT_VERSION,
|
|
115
|
+
inboxMaxLength,
|
|
116
|
+
]), nodeId);
|
|
117
|
+
} while (result === 0);
|
|
118
|
+
this.touchNamespace(leaseMs);
|
|
119
|
+
}
|
|
120
|
+
async heartbeat(nodeId, leaseMs) {
|
|
121
|
+
const alive = (await this.exec('relayHeartbeat', [nodeId, leaseMs])) === 1;
|
|
122
|
+
if (alive) {
|
|
123
|
+
this.touchNamespace(leaseMs);
|
|
124
|
+
}
|
|
125
|
+
return alive;
|
|
126
|
+
}
|
|
127
|
+
async unregisterNode(nodeId) {
|
|
128
|
+
return (await this.exec('relayUnregisterNode', [nodeId])) === 1;
|
|
129
|
+
}
|
|
130
|
+
async sweep(limit) {
|
|
131
|
+
return this.exec('relaySweep', [limit]);
|
|
132
|
+
}
|
|
133
|
+
async subscribe(nodeId, endpointId, pattern) {
|
|
134
|
+
const [added, mid, ts, data] = (0, relay_errors_1.checkRelayResult)(await this.exec('relaySubscribe', [nodeId, endpointId, pattern]), pattern);
|
|
135
|
+
return {
|
|
136
|
+
added: added === 1,
|
|
137
|
+
retained: mid ? { mid: String(mid), ts: Number(ts), data } : undefined,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
async removeEndpoint(nodeId, endpointId) {
|
|
141
|
+
return this.exec('relayRemoveEndpoint', [nodeId, endpointId]);
|
|
142
|
+
}
|
|
143
|
+
async publish(topic, data, opts) {
|
|
144
|
+
const [mid, nodes, endpoints] = (0, relay_errors_1.checkRelayResult)(await this.exec('relayPublish', [
|
|
145
|
+
topic,
|
|
146
|
+
data,
|
|
147
|
+
opts.ts,
|
|
148
|
+
opts.retainMs,
|
|
149
|
+
opts.maxSize,
|
|
150
|
+
this.inboxMaxLength,
|
|
151
|
+
]), topic);
|
|
152
|
+
if (opts.retainMs > 0) {
|
|
153
|
+
this.touchNamespace(opts.retainMs);
|
|
154
|
+
}
|
|
155
|
+
return { mid: String(mid), nodes, endpoints };
|
|
156
|
+
}
|
|
157
|
+
async deleteRetained(topic) {
|
|
158
|
+
return ((0, relay_errors_1.checkRelayResult)(await this.exec('relayDeleteRetained', [topic]), topic) === 1);
|
|
159
|
+
}
|
|
160
|
+
async readInbox(nodeId, cursor, opts) {
|
|
161
|
+
if (this.interrupted) {
|
|
162
|
+
return { entries: [], cursor };
|
|
163
|
+
}
|
|
164
|
+
let connection;
|
|
165
|
+
try {
|
|
166
|
+
connection = await this.getBlockingConnection();
|
|
167
|
+
}
|
|
168
|
+
catch (err) {
|
|
169
|
+
if (this.interrupted) {
|
|
170
|
+
return { entries: [], cursor };
|
|
171
|
+
}
|
|
172
|
+
throw err;
|
|
173
|
+
}
|
|
174
|
+
const client = await connection.client;
|
|
175
|
+
if (this.interrupted) {
|
|
176
|
+
return { entries: [], cursor };
|
|
177
|
+
}
|
|
178
|
+
// Recovers an ioredis Cluster left with an empty node pool by an outage,
|
|
179
|
+
// as for the workers' blocking reads.
|
|
180
|
+
const read = () => connection.runWithClusterRecovery(() => client.runCommand(`relayReadInbox:${version_1.version}`, [
|
|
181
|
+
this.nodesKey,
|
|
182
|
+
this.baseKey,
|
|
183
|
+
nodeId,
|
|
184
|
+
cursor,
|
|
185
|
+
opts.count,
|
|
186
|
+
]));
|
|
187
|
+
let result = await read();
|
|
188
|
+
if (!result && opts.blockMs > 0 && !this.interrupted) {
|
|
189
|
+
// Only use the blocking read as a wakeup. Fetching and marking the actual
|
|
190
|
+
// batch must be atomic with publish-time trimming.
|
|
191
|
+
await connection.runWithClusterRecovery(() => client.xread([{ key: `${this.baseKey}:${nodeId}:inbox`, id: cursor }], {
|
|
192
|
+
BLOCK: opts.blockMs,
|
|
193
|
+
COUNT: 1,
|
|
194
|
+
}));
|
|
195
|
+
if (this.interrupted) {
|
|
196
|
+
return { entries: [], cursor };
|
|
197
|
+
}
|
|
198
|
+
result = await read();
|
|
199
|
+
}
|
|
200
|
+
if (!result) {
|
|
201
|
+
return { entries: [], cursor };
|
|
202
|
+
}
|
|
203
|
+
const entries = [];
|
|
204
|
+
let next = cursor;
|
|
205
|
+
for (const [id, fields] of result[0][1]) {
|
|
206
|
+
next = id;
|
|
207
|
+
const entry = {};
|
|
208
|
+
for (let i = 0; i + 1 < fields.length; i += 2) {
|
|
209
|
+
entry[fields[i]] = fields[i + 1];
|
|
210
|
+
}
|
|
211
|
+
entries.push({
|
|
212
|
+
kind: entry.k,
|
|
213
|
+
topic: entry.t,
|
|
214
|
+
mid: entry.m,
|
|
215
|
+
ts: Number(entry.ts),
|
|
216
|
+
data: entry.d,
|
|
217
|
+
endpoints: entry.e ? entry.e.split(',') : [],
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
return { entries, cursor: next };
|
|
221
|
+
}
|
|
222
|
+
async trimInbox(nodeId, cursor, maxLength) {
|
|
223
|
+
return this.exec('relayTrimInbox', [nodeId, cursor, maxLength]);
|
|
224
|
+
}
|
|
225
|
+
async getCounts() {
|
|
226
|
+
const [nodes, subscriptions, patterns, retained] = await this.get('counts');
|
|
227
|
+
return { nodes, subscriptions, patterns, retained };
|
|
228
|
+
}
|
|
229
|
+
async getNodes(start, end) {
|
|
230
|
+
const rows = await this.get('nodes', start, end);
|
|
231
|
+
return rows.map(([id, registeredAt, leaseUntil, info, hb, backlog, dropped, subs]) => ({
|
|
232
|
+
id,
|
|
233
|
+
info: info ? JSON.parse(info) : {},
|
|
234
|
+
registeredAt: Number(registeredAt),
|
|
235
|
+
leaseUntil: Number(leaseUntil),
|
|
236
|
+
lastHeartbeat: hb ? Number(hb) : undefined,
|
|
237
|
+
backlog: Number(backlog),
|
|
238
|
+
dropped: Number(dropped),
|
|
239
|
+
subscriptions: subs,
|
|
240
|
+
}));
|
|
241
|
+
}
|
|
242
|
+
async getSubscriptions(filter, start, end) {
|
|
243
|
+
const [kind, value] = filter && 'nodeId' in filter
|
|
244
|
+
? ['node', filter.nodeId]
|
|
245
|
+
: filter && 'pattern' in filter
|
|
246
|
+
? ['pattern', filter.pattern]
|
|
247
|
+
: ['', ''];
|
|
248
|
+
const rows = await this.get('subscriptions', start, end, kind, value);
|
|
249
|
+
return rows.map(([nodeId, endpointId, pattern]) => ({
|
|
250
|
+
nodeId,
|
|
251
|
+
endpointId,
|
|
252
|
+
pattern,
|
|
253
|
+
}));
|
|
254
|
+
}
|
|
255
|
+
async getPatterns(start, end) {
|
|
256
|
+
const rows = await this.get('patterns', start, end);
|
|
257
|
+
return rows.map(([pattern, subscriptions]) => ({
|
|
258
|
+
pattern,
|
|
259
|
+
subscriptions,
|
|
260
|
+
}));
|
|
261
|
+
}
|
|
262
|
+
async getRetained(start, end) {
|
|
263
|
+
const rows = await this.get('retained', start, end);
|
|
264
|
+
return rows.map(([topic, expiresAt, mid, ts, data]) => ({
|
|
265
|
+
topic,
|
|
266
|
+
expiresAt: Number(expiresAt),
|
|
267
|
+
mid: String(mid),
|
|
268
|
+
ts: Number(ts),
|
|
269
|
+
data,
|
|
270
|
+
}));
|
|
271
|
+
}
|
|
272
|
+
async getNamespaces() {
|
|
273
|
+
const client = await this.connection.client;
|
|
274
|
+
const candidates = await client.runCommand(`relayGetNamespaces:${version_1.version}`, [this.registryKey]);
|
|
275
|
+
// The registry only says until when a namespace may be in use (its nodes
|
|
276
|
+
// can unregister and its retained messages be deleted earlier), so each
|
|
277
|
+
// candidate is checked against its own keys.
|
|
278
|
+
const used = await Promise.all(candidates.map(async (namespace) => {
|
|
279
|
+
const baseKey = this.namespaceKey(namespace);
|
|
280
|
+
return ((await client.runCommand(`relayGet:${version_1.version}`, [
|
|
281
|
+
`${baseKey}:nodes`,
|
|
282
|
+
baseKey,
|
|
283
|
+
'used',
|
|
284
|
+
0,
|
|
285
|
+
-1,
|
|
286
|
+
'',
|
|
287
|
+
'',
|
|
288
|
+
])) === 1);
|
|
289
|
+
}));
|
|
290
|
+
return candidates.filter((_, i) => used[i]);
|
|
291
|
+
}
|
|
292
|
+
async getNodeConnections(nodeId) {
|
|
293
|
+
var _a;
|
|
294
|
+
const client = await this.connection.client;
|
|
295
|
+
// Adapter options do not consistently expose the selected database.
|
|
296
|
+
const db = client.isCluster
|
|
297
|
+
? '0'
|
|
298
|
+
: (_a = (await client.clientInfo()).match(/(?:^| )db=(\d+)/)) === null || _a === void 0 ? void 0 : _a[1];
|
|
299
|
+
// Every node of a cluster: the connection may be to any of them.
|
|
300
|
+
const lists = client.isCluster && typeof client.nodes === 'function'
|
|
301
|
+
? await Promise.all((client.nodes() || []).map((node) => typeof node.clientList === 'function'
|
|
302
|
+
? node.clientList()
|
|
303
|
+
: node.client('LIST')))
|
|
304
|
+
: [await client.clientList()];
|
|
305
|
+
const name = this.inboxConnectionName(nodeId);
|
|
306
|
+
const connections = [];
|
|
307
|
+
for (const line of lists.join('\n').split('\n')) {
|
|
308
|
+
const fields = {};
|
|
309
|
+
for (const pair of line.trim().split(' ')) {
|
|
310
|
+
const eq = pair.indexOf('=');
|
|
311
|
+
if (eq > 0) {
|
|
312
|
+
fields[pair.slice(0, eq)] = pair.slice(eq + 1);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
if (fields.name === name && fields.db === db) {
|
|
316
|
+
connections.push({
|
|
317
|
+
address: fields.addr,
|
|
318
|
+
age: Number(fields.age),
|
|
319
|
+
idle: Number(fields.idle),
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
return connections;
|
|
324
|
+
}
|
|
325
|
+
interruptInbox() {
|
|
326
|
+
var _a;
|
|
327
|
+
this.interrupted = true;
|
|
328
|
+
(_a = this.blockingConnection) === null || _a === void 0 ? void 0 : _a.close(true).catch(() => undefined);
|
|
329
|
+
}
|
|
330
|
+
getBlockingConnection() {
|
|
331
|
+
if (!this.blockingConnectionPromise) {
|
|
332
|
+
const promise = (async () => {
|
|
333
|
+
const shared = (0, utils_1.isRedisInstance)(this.connectionOpts);
|
|
334
|
+
const client = await this.connection.client;
|
|
335
|
+
// Closed meanwhile: a connection opened now would never be closed.
|
|
336
|
+
if (this.interrupted) {
|
|
337
|
+
throw new Error('Connection is closed.');
|
|
338
|
+
}
|
|
339
|
+
// Named after the node (CLIENT SETNAME, re-applied on reconnects) so
|
|
340
|
+
// getNodeConnections can find it.
|
|
341
|
+
const naming = this.nodeId
|
|
342
|
+
? { connectionName: this.inboxConnectionName(this.nodeId) }
|
|
343
|
+
: {};
|
|
344
|
+
const connection = new redis_connection_1.RedisConnection(shared
|
|
345
|
+
? client.duplicate(Object.assign({ maxRetriesPerRequest: null }, naming))
|
|
346
|
+
: Object.assign(Object.assign({}, this.connectionOpts), naming), {
|
|
347
|
+
shared: false,
|
|
348
|
+
blocking: true,
|
|
349
|
+
skipVersionCheck: this.skipVersionCheck,
|
|
350
|
+
});
|
|
351
|
+
this.forwardConnectionErrors(connection);
|
|
352
|
+
// Set before it's ready, so closing can stop it while it connects.
|
|
353
|
+
this.blockingConnection = connection;
|
|
354
|
+
try {
|
|
355
|
+
await connection.client;
|
|
356
|
+
}
|
|
357
|
+
catch (err) {
|
|
358
|
+
// Disposed of, so the next read starts over with a new connection.
|
|
359
|
+
if (this.blockingConnection === connection) {
|
|
360
|
+
this.blockingConnection = undefined;
|
|
361
|
+
}
|
|
362
|
+
await connection.close(true).catch(() => undefined);
|
|
363
|
+
throw err;
|
|
364
|
+
}
|
|
365
|
+
return connection;
|
|
366
|
+
})();
|
|
367
|
+
this.blockingConnectionPromise = promise;
|
|
368
|
+
promise.catch(() => {
|
|
369
|
+
if (this.blockingConnectionPromise === promise) {
|
|
370
|
+
this.blockingConnectionPromise = undefined;
|
|
371
|
+
}
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
return this.blockingConnectionPromise;
|
|
375
|
+
}
|
|
376
|
+
forwardConnectionErrors(connection) {
|
|
377
|
+
connection.on('error', err => {
|
|
378
|
+
if (this.listenerCount('error') > 0) {
|
|
379
|
+
this.emit('error', err);
|
|
380
|
+
}
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
/** Base key of a namespace of this backend's prefix. */
|
|
384
|
+
namespaceKey(namespace) {
|
|
385
|
+
return `${this.prefix}::relay:{${namespace}}`;
|
|
386
|
+
}
|
|
387
|
+
/** Redis client name of a node's inbox connection. */
|
|
388
|
+
inboxConnectionName(nodeId) {
|
|
389
|
+
return `${Buffer.from(this.baseKey).toString('base64')}:${nodeId}`;
|
|
390
|
+
}
|
|
391
|
+
get(view, start = 0, end = -1, filter = '', value = '') {
|
|
392
|
+
return this.exec('relayGet', [view, start, end, filter, value]);
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* Lists the namespace in the prefix's registry. The registry is outside the
|
|
396
|
+
* namespace's hash tag, so this is a call of its own.
|
|
397
|
+
*/
|
|
398
|
+
touchNamespace(forMs) {
|
|
399
|
+
this.namespaceExpiresAt = Math.max(this.namespaceExpiresAt, Date.now() + forMs);
|
|
400
|
+
this.startNamespaceTouch();
|
|
401
|
+
}
|
|
402
|
+
startNamespaceTouch() {
|
|
403
|
+
if (!this.namespaceTouch) {
|
|
404
|
+
const requestedUntil = this.namespaceExpiresAt;
|
|
405
|
+
const promise = this.refreshNamespace();
|
|
406
|
+
this.namespaceTouch = promise;
|
|
407
|
+
promise
|
|
408
|
+
.finally(() => {
|
|
409
|
+
if (this.namespaceTouch === promise) {
|
|
410
|
+
this.namespaceTouch = undefined;
|
|
411
|
+
if (!this.closing && this.namespaceExpiresAt > requestedUntil) {
|
|
412
|
+
this.startNamespaceTouch();
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
})
|
|
416
|
+
.catch(() => undefined);
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
async refreshNamespace() {
|
|
420
|
+
for (let attempt = 0; attempt < 3 && !this.closing; attempt++) {
|
|
421
|
+
const expiresAt = this.namespaceExpiresAt;
|
|
422
|
+
try {
|
|
423
|
+
const client = await this.connection.client;
|
|
424
|
+
await client.runCommand(`relayTouchNamespace:${version_1.version}`, [
|
|
425
|
+
this.registryKey,
|
|
426
|
+
this.namespace,
|
|
427
|
+
Math.max(1, expiresAt - Date.now()),
|
|
428
|
+
]);
|
|
429
|
+
return;
|
|
430
|
+
}
|
|
431
|
+
catch (err) {
|
|
432
|
+
if (attempt === 2) {
|
|
433
|
+
if (!this.closing && this.listenerCount('error') > 0) {
|
|
434
|
+
this.emit('error', err);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
else {
|
|
438
|
+
await new Promise(resolve => setTimeout(resolve, 100 * 2 ** attempt));
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
async exec(command, args) {
|
|
444
|
+
const client = await this.connection.client;
|
|
445
|
+
return client.runCommand(`${command}:${version_1.version}`, [
|
|
446
|
+
this.nodesKey,
|
|
447
|
+
this.baseKey,
|
|
448
|
+
...args,
|
|
449
|
+
]);
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
exports.RedisRelayBackend = RedisRelayBackend;
|
|
453
|
+
/** The default {@link RelayBackendFactory}: Redis. */
|
|
454
|
+
const createRedisRelayBackend = (opts) => new RedisRelayBackend(opts);
|
|
455
|
+
exports.createRedisRelayBackend = createRedisRelayBackend;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RelayError = exports.RelayErrorCode = void 0;
|
|
4
|
+
exports.checkRelayResult = checkRelayResult;
|
|
5
|
+
/**
|
|
6
|
+
* Error codes shared by the relay's Lua scripts and SQL functions.
|
|
7
|
+
*/
|
|
8
|
+
var RelayErrorCode;
|
|
9
|
+
(function (RelayErrorCode) {
|
|
10
|
+
RelayErrorCode[RelayErrorCode["InvalidTopic"] = -1] = "InvalidTopic";
|
|
11
|
+
RelayErrorCode[RelayErrorCode["InvalidPattern"] = -2] = "InvalidPattern";
|
|
12
|
+
RelayErrorCode[RelayErrorCode["NodeNotRegistered"] = -3] = "NodeNotRegistered";
|
|
13
|
+
RelayErrorCode[RelayErrorCode["DataTooLarge"] = -4] = "DataTooLarge";
|
|
14
|
+
RelayErrorCode[RelayErrorCode["InvalidId"] = -5] = "InvalidId";
|
|
15
|
+
RelayErrorCode[RelayErrorCode["IncompatibleLayout"] = -6] = "IncompatibleLayout";
|
|
16
|
+
})(RelayErrorCode || (exports.RelayErrorCode = RelayErrorCode = {}));
|
|
17
|
+
const MESSAGES = {
|
|
18
|
+
[RelayErrorCode.InvalidTopic]: 'Invalid topic',
|
|
19
|
+
[RelayErrorCode.InvalidPattern]: 'Invalid topic pattern',
|
|
20
|
+
[RelayErrorCode.NodeNotRegistered]: 'Relay node is not registered',
|
|
21
|
+
[RelayErrorCode.DataTooLarge]: 'Message data is too large',
|
|
22
|
+
[RelayErrorCode.InvalidId]: 'Invalid relay node or endpoint id',
|
|
23
|
+
[RelayErrorCode.IncompatibleLayout]: 'The relay namespace is in use by nodes with an incompatible BullMQ version',
|
|
24
|
+
};
|
|
25
|
+
class RelayError extends Error {
|
|
26
|
+
constructor(code, detail) {
|
|
27
|
+
super(`${MESSAGES[code] || `Relay error ${code}`}${detail ? `: ${detail}` : ''}`);
|
|
28
|
+
this.code = code;
|
|
29
|
+
this.name = 'RelayError';
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
exports.RelayError = RelayError;
|
|
33
|
+
/**
|
|
34
|
+
* Throws a {@link RelayError} if a script result is an error code.
|
|
35
|
+
*/
|
|
36
|
+
function checkRelayResult(result, detail) {
|
|
37
|
+
if (typeof result === 'number' && result < 0) {
|
|
38
|
+
throw new RelayError(result, detail);
|
|
39
|
+
}
|
|
40
|
+
return result;
|
|
41
|
+
}
|