bullmq 6.3.19 → 6.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/classes/bun-redis-client.js +3 -0
- package/dist/cjs/classes/index.js +3 -0
- package/dist/cjs/classes/ioredis-client.js +1 -0
- package/dist/cjs/classes/node-redis-client.js +3 -0
- package/dist/cjs/classes/redis-connection.js +33 -18
- package/dist/cjs/classes/redis-relay-backend.js +455 -0
- package/dist/cjs/classes/relay-errors.js +41 -0
- package/dist/cjs/classes/relay.js +562 -0
- package/dist/cjs/classes/valkey-glide-client.js +7 -2
- package/dist/cjs/commands/includes/refreshRelayNamespace.lua +36 -0
- package/dist/cjs/commands/includes/relayIndex.lua +112 -0
- package/dist/cjs/commands/includes/relayTime.lua +15 -0
- package/dist/cjs/commands/includes/relayTopics.lua +92 -0
- package/dist/cjs/commands/relayAckInbox-1.lua +18 -0
- package/dist/cjs/commands/relayDeleteRetained-1.lua +39 -0
- package/dist/cjs/commands/relayGet-1.lua +133 -0
- package/dist/cjs/commands/relayGetNamespaces-1.lua +16 -0
- package/dist/cjs/commands/relayHeartbeat-1.lua +35 -0
- package/dist/cjs/commands/relayPublish-1.lua +150 -0
- package/dist/cjs/commands/relayReadInbox-1.lua +31 -0
- package/dist/cjs/commands/relayRegisterNode-1.lua +60 -0
- package/dist/cjs/commands/relayRemoveEndpoint-1.lua +18 -0
- package/dist/cjs/commands/relaySubscribe-1.lua +69 -0
- package/dist/cjs/commands/relaySweep-1.lua +39 -0
- package/dist/cjs/commands/relayTouchNamespace-1.lua +22 -0
- package/dist/cjs/commands/relayTrimInbox-1.lua +65 -0
- package/dist/cjs/commands/relayUnregisterNode-1.lua +25 -0
- package/dist/cjs/interfaces/index.js +1 -0
- package/dist/cjs/interfaces/relay-backend.js +2 -0
- package/dist/cjs/postgres/index.js +4 -1
- package/dist/cjs/postgres/migrations/index.js +9 -2
- package/dist/cjs/postgres/postgres-connection.js +3 -2
- package/dist/cjs/postgres/postgres-relay-backend.js +342 -0
- package/dist/cjs/postgres/sql-scripts.js +19 -0
- package/dist/cjs/scripts/index.js +14 -0
- package/dist/cjs/scripts/relayAckInbox-1.js +23 -0
- package/dist/cjs/scripts/relayDeleteRetained-1.js +168 -0
- package/dist/cjs/scripts/relayGet-1.js +323 -0
- package/dist/cjs/scripts/relayGetNamespaces-1.js +34 -0
- package/dist/cjs/scripts/relayHeartbeat-1.js +81 -0
- package/dist/cjs/scripts/relayPublish-1.js +368 -0
- package/dist/cjs/scripts/relayReadInbox-1.js +35 -0
- package/dist/cjs/scripts/relayRegisterNode-1.js +285 -0
- package/dist/cjs/scripts/relayRemoveEndpoint-1.js +204 -0
- package/dist/cjs/scripts/relaySubscribe-1.js +150 -0
- package/dist/cjs/scripts/relaySweep-1.js +265 -0
- package/dist/cjs/scripts/relayTouchNamespace-1.js +38 -0
- package/dist/cjs/scripts/relayTrimInbox-1.js +81 -0
- package/dist/cjs/scripts/relayUnregisterNode-1.js +255 -0
- package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
- package/dist/cjs/version.js +1 -1
- package/dist/esm/classes/bun-redis-client.js +3 -0
- package/dist/esm/classes/index.d.ts +3 -0
- package/dist/esm/classes/index.js +3 -0
- package/dist/esm/classes/ioredis-client.js +1 -0
- package/dist/esm/classes/node-redis-client.js +3 -0
- package/dist/esm/classes/redis-connection.d.ts +8 -0
- package/dist/esm/classes/redis-connection.js +33 -18
- package/dist/esm/classes/redis-relay-backend.d.ts +110 -0
- package/dist/esm/classes/redis-relay-backend.js +450 -0
- package/dist/esm/classes/relay-errors.d.ts +19 -0
- package/dist/esm/classes/relay-errors.js +36 -0
- package/dist/esm/classes/relay.d.ts +183 -0
- package/dist/esm/classes/relay.js +558 -0
- package/dist/esm/classes/valkey-glide-client.js +7 -2
- package/dist/esm/commands/includes/refreshRelayNamespace.lua +36 -0
- package/dist/esm/commands/includes/relayIndex.lua +112 -0
- package/dist/esm/commands/includes/relayTime.lua +15 -0
- package/dist/esm/commands/includes/relayTopics.lua +92 -0
- package/dist/esm/commands/relayAckInbox-1.lua +18 -0
- package/dist/esm/commands/relayDeleteRetained-1.lua +39 -0
- package/dist/esm/commands/relayGet-1.lua +133 -0
- package/dist/esm/commands/relayGetNamespaces-1.lua +16 -0
- package/dist/esm/commands/relayHeartbeat-1.lua +35 -0
- package/dist/esm/commands/relayPublish-1.lua +150 -0
- package/dist/esm/commands/relayReadInbox-1.lua +31 -0
- package/dist/esm/commands/relayRegisterNode-1.lua +60 -0
- package/dist/esm/commands/relayRemoveEndpoint-1.lua +18 -0
- package/dist/esm/commands/relaySubscribe-1.lua +69 -0
- package/dist/esm/commands/relaySweep-1.lua +39 -0
- package/dist/esm/commands/relayTouchNamespace-1.lua +22 -0
- package/dist/esm/commands/relayTrimInbox-1.lua +65 -0
- package/dist/esm/commands/relayUnregisterNode-1.lua +25 -0
- package/dist/esm/interfaces/index.d.ts +1 -0
- package/dist/esm/interfaces/index.js +1 -0
- package/dist/esm/interfaces/redis-client.d.ts +1 -0
- package/dist/esm/interfaces/relay-backend.d.ts +237 -0
- package/dist/esm/interfaces/relay-backend.js +1 -0
- package/dist/esm/postgres/index.d.ts +1 -0
- package/dist/esm/postgres/index.js +1 -0
- package/dist/esm/postgres/migrations/index.d.ts +11 -1
- package/dist/esm/postgres/migrations/index.js +9 -2
- package/dist/esm/postgres/postgres-connection.js +3 -2
- package/dist/esm/postgres/postgres-relay-backend.d.ts +67 -0
- package/dist/esm/postgres/postgres-relay-backend.js +337 -0
- package/dist/esm/postgres/sql-scripts.js +19 -0
- package/dist/esm/scripts/index.d.ts +14 -0
- package/dist/esm/scripts/index.js +14 -0
- package/dist/esm/scripts/relayAckInbox-1.d.ts +5 -0
- package/dist/esm/scripts/relayAckInbox-1.js +20 -0
- package/dist/esm/scripts/relayDeleteRetained-1.d.ts +5 -0
- package/dist/esm/scripts/relayDeleteRetained-1.js +165 -0
- package/dist/esm/scripts/relayGet-1.d.ts +5 -0
- package/dist/esm/scripts/relayGet-1.js +320 -0
- package/dist/esm/scripts/relayGetNamespaces-1.d.ts +5 -0
- package/dist/esm/scripts/relayGetNamespaces-1.js +31 -0
- package/dist/esm/scripts/relayHeartbeat-1.d.ts +5 -0
- package/dist/esm/scripts/relayHeartbeat-1.js +78 -0
- package/dist/esm/scripts/relayPublish-1.d.ts +5 -0
- package/dist/esm/scripts/relayPublish-1.js +365 -0
- package/dist/esm/scripts/relayReadInbox-1.d.ts +5 -0
- package/dist/esm/scripts/relayReadInbox-1.js +32 -0
- package/dist/esm/scripts/relayRegisterNode-1.d.ts +5 -0
- package/dist/esm/scripts/relayRegisterNode-1.js +282 -0
- package/dist/esm/scripts/relayRemoveEndpoint-1.d.ts +5 -0
- package/dist/esm/scripts/relayRemoveEndpoint-1.js +201 -0
- package/dist/esm/scripts/relaySubscribe-1.d.ts +5 -0
- package/dist/esm/scripts/relaySubscribe-1.js +147 -0
- package/dist/esm/scripts/relaySweep-1.d.ts +5 -0
- package/dist/esm/scripts/relaySweep-1.js +262 -0
- package/dist/esm/scripts/relayTouchNamespace-1.d.ts +5 -0
- package/dist/esm/scripts/relayTouchNamespace-1.js +35 -0
- package/dist/esm/scripts/relayTrimInbox-1.d.ts +5 -0
- package/dist/esm/scripts/relayTrimInbox-1.js +78 -0
- package/dist/esm/scripts/relayUnregisterNode-1.d.ts +5 -0
- package/dist/esm/scripts/relayUnregisterNode-1.js +252 -0
- package/dist/esm/tsconfig.tsbuildinfo +1 -1
- package/dist/esm/version.d.ts +1 -1
- package/dist/esm/version.js +1 -1
- package/package.json +8 -8
|
@@ -0,0 +1,562 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.Relay = void 0;
|
|
4
|
+
const events_1 = require("events");
|
|
5
|
+
const crypto_1 = require("crypto");
|
|
6
|
+
const os_1 = require("os");
|
|
7
|
+
const redis_relay_backend_1 = require("./redis-relay-backend");
|
|
8
|
+
const version_1 = require("../version");
|
|
9
|
+
const ID_RE = /^[A-Za-z0-9_-]{1,64}$/;
|
|
10
|
+
const MAX_NAME_LENGTH = 256;
|
|
11
|
+
/**
|
|
12
|
+
* Routes messages between the nodes of a fleet: any process can publish on a
|
|
13
|
+
* topic, and every subscription matching it receives the message, wherever it
|
|
14
|
+
* runs.
|
|
15
|
+
*
|
|
16
|
+
* The routing logic (topic grammar, matching, subscriptions, node liveness and
|
|
17
|
+
* inboxes) lives in Lua scripts (Redis) or SQL functions (PostgreSQL) behind
|
|
18
|
+
* {@link IRelayBackend}; this class only runs timers, reads its inbox and
|
|
19
|
+
* calls listeners.
|
|
20
|
+
*
|
|
21
|
+
* Topics are dot-separated keypaths (`queues.emails.jobs.42.progress`).
|
|
22
|
+
* Patterns may use `*` (one segment) and `>` (one or more trailing segments).
|
|
23
|
+
*
|
|
24
|
+
* Events: `'error'` (Error), `'recovered'` (the node re-registered after its
|
|
25
|
+
* lease expired), `'swept'` (string[] of removed node ids), `'lag'` (number
|
|
26
|
+
* of unread messages, when it reaches `lagThreshold`).
|
|
27
|
+
*
|
|
28
|
+
* @experimental The API may change in minor releases while the relay is
|
|
29
|
+
* completed.
|
|
30
|
+
*/
|
|
31
|
+
class Relay extends events_1.EventEmitter {
|
|
32
|
+
constructor(opts) {
|
|
33
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
34
|
+
super();
|
|
35
|
+
this.opts = opts;
|
|
36
|
+
this.endpoints = new Map();
|
|
37
|
+
this.batchInFlight = false;
|
|
38
|
+
this.recoveryPending = false;
|
|
39
|
+
/** Incremented when a recovery replaces the node's registration. */
|
|
40
|
+
this.registration = 0;
|
|
41
|
+
this.connected = false;
|
|
42
|
+
this.startCancelled = new Promise(resolve => {
|
|
43
|
+
this.cancelStart = resolve;
|
|
44
|
+
});
|
|
45
|
+
if (!opts || !opts.connection) {
|
|
46
|
+
throw new Error('Relay requires a connection');
|
|
47
|
+
}
|
|
48
|
+
this.namespace = (_a = opts.namespace) !== null && _a !== void 0 ? _a : 'default';
|
|
49
|
+
this.nodeId = (_b = opts.nodeId) !== null && _b !== void 0 ? _b : (0, crypto_1.randomUUID)().replace(/-/g, '');
|
|
50
|
+
for (const [name, value] of [
|
|
51
|
+
['namespace', this.namespace],
|
|
52
|
+
['nodeId', this.nodeId],
|
|
53
|
+
]) {
|
|
54
|
+
if (!ID_RE.test(value)) {
|
|
55
|
+
throw new Error(`Invalid relay ${name}: ${value}`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
if (opts.name !== undefined &&
|
|
59
|
+
(typeof opts.name !== 'string' || opts.name.length > MAX_NAME_LENGTH)) {
|
|
60
|
+
throw new Error(`Invalid relay name: ${opts.name}`);
|
|
61
|
+
}
|
|
62
|
+
this.leaseDuration = (_c = opts.leaseDuration) !== null && _c !== void 0 ? _c : 30000;
|
|
63
|
+
for (const name of [
|
|
64
|
+
'leaseDuration',
|
|
65
|
+
'heartbeatInterval',
|
|
66
|
+
'sweepInterval',
|
|
67
|
+
'blockTimeout',
|
|
68
|
+
'inboxMaxLength',
|
|
69
|
+
'lagThreshold',
|
|
70
|
+
'maxMessageSize',
|
|
71
|
+
]) {
|
|
72
|
+
const value = opts[name];
|
|
73
|
+
if (value !== undefined &&
|
|
74
|
+
(!Number.isInteger(value) || value <= 0 || value > 2147483647)) {
|
|
75
|
+
throw new Error(`Invalid relay ${name}: ${value}`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
this.maxMessageSize = (_d = opts.maxMessageSize) !== null && _d !== void 0 ? _d : 512 * 1024;
|
|
79
|
+
this.blockTimeout = (_e = opts.blockTimeout) !== null && _e !== void 0 ? _e : 5000;
|
|
80
|
+
this.inboxMaxLength = (_f = opts.inboxMaxLength) !== null && _f !== void 0 ? _f : 10000;
|
|
81
|
+
this.lagThreshold =
|
|
82
|
+
(_g = opts.lagThreshold) !== null && _g !== void 0 ? _g : Math.max(1, Math.floor(this.inboxMaxLength / 2));
|
|
83
|
+
if (this.lagThreshold > this.inboxMaxLength) {
|
|
84
|
+
throw new Error('Relay lagThreshold must not exceed inboxMaxLength');
|
|
85
|
+
}
|
|
86
|
+
const factory = ((_h = opts.backend) !== null && _h !== void 0 ? _h : redis_relay_backend_1.createRedisRelayBackend);
|
|
87
|
+
this.backend = factory({
|
|
88
|
+
connection: opts.connection,
|
|
89
|
+
namespace: this.namespace,
|
|
90
|
+
nodeId: this.nodeId,
|
|
91
|
+
prefix: opts.prefix,
|
|
92
|
+
inboxMaxLength: this.inboxMaxLength,
|
|
93
|
+
skipVersionCheck: opts.skipVersionCheck,
|
|
94
|
+
});
|
|
95
|
+
if (this.backend instanceof events_1.EventEmitter) {
|
|
96
|
+
this.backend.on('error', err => this.emitError(err));
|
|
97
|
+
}
|
|
98
|
+
this.cursor = this.backend.initialInboxCursor;
|
|
99
|
+
this.ready = this.start();
|
|
100
|
+
this.ready.catch(err => this.emitError(err));
|
|
101
|
+
}
|
|
102
|
+
waitUntilReady() {
|
|
103
|
+
return this.ready;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Subscribes to a topic or pattern. For an exact topic, its retained
|
|
107
|
+
* message (if any) is delivered first, with `retained: true`.
|
|
108
|
+
*/
|
|
109
|
+
async subscribe(pattern, listener) {
|
|
110
|
+
var _a;
|
|
111
|
+
await this.ready;
|
|
112
|
+
this.assertOpen();
|
|
113
|
+
const endpointId = (0, crypto_1.randomUUID)().replace(/-/g, '');
|
|
114
|
+
const endpoint = {
|
|
115
|
+
pattern,
|
|
116
|
+
listener,
|
|
117
|
+
active: true,
|
|
118
|
+
pending: [],
|
|
119
|
+
};
|
|
120
|
+
// Registered first, so live messages that arrive while subscribing are
|
|
121
|
+
// buffered, then delivered after the retained one.
|
|
122
|
+
this.endpoints.set(endpointId, endpoint);
|
|
123
|
+
try {
|
|
124
|
+
const { retained } = await this.backend.subscribe(this.nodeId, endpointId, pattern);
|
|
125
|
+
this.assertOpen();
|
|
126
|
+
const pending = (_a = endpoint.pending) !== null && _a !== void 0 ? _a : [];
|
|
127
|
+
endpoint.pending = undefined;
|
|
128
|
+
if (retained) {
|
|
129
|
+
this.deliver(endpoint, {
|
|
130
|
+
topic: pattern,
|
|
131
|
+
mid: retained.mid,
|
|
132
|
+
ts: retained.ts,
|
|
133
|
+
data: retained.data,
|
|
134
|
+
retained: true,
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
// Skips live copies of the retained message (or older ones). Buffered
|
|
138
|
+
// messages are not compared with each other: publishes on different
|
|
139
|
+
// topics may commit out of message id order.
|
|
140
|
+
const cutoff = retained ? BigInt(retained.mid) : BigInt(0);
|
|
141
|
+
for (const message of pending) {
|
|
142
|
+
if (BigInt(message.mid) > cutoff) {
|
|
143
|
+
this.deliver(endpoint, message);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
catch (err) {
|
|
148
|
+
endpoint.pending = undefined;
|
|
149
|
+
endpoint.active = false;
|
|
150
|
+
try {
|
|
151
|
+
await this.removeEndpoint(endpointId, endpoint);
|
|
152
|
+
}
|
|
153
|
+
catch (cleanupError) {
|
|
154
|
+
this.recoveryPending = true;
|
|
155
|
+
this.emitError(cleanupError);
|
|
156
|
+
}
|
|
157
|
+
throw err;
|
|
158
|
+
}
|
|
159
|
+
return {
|
|
160
|
+
pattern,
|
|
161
|
+
unsubscribe: async () => {
|
|
162
|
+
endpoint.active = false;
|
|
163
|
+
await this.removeEndpoint(endpointId, endpoint);
|
|
164
|
+
},
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Publishes `data` (any JSON-serializable value) on a concrete topic.
|
|
169
|
+
*/
|
|
170
|
+
async publish(topic, data, opts = {}) {
|
|
171
|
+
var _a;
|
|
172
|
+
await this.ready;
|
|
173
|
+
this.assertOpen();
|
|
174
|
+
const retainMs = (_a = opts.retain) !== null && _a !== void 0 ? _a : 0;
|
|
175
|
+
if (!Number.isSafeInteger(retainMs) || retainMs < 0) {
|
|
176
|
+
throw new Error(`Invalid relay retain: ${retainMs}`);
|
|
177
|
+
}
|
|
178
|
+
const serialized = JSON.stringify(data === undefined ? null : data);
|
|
179
|
+
return this.backend.publish(topic, serialized, {
|
|
180
|
+
ts: Date.now(),
|
|
181
|
+
retainMs,
|
|
182
|
+
maxSize: this.maxMessageSize,
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Deletes the retained message of a topic: new subscribers no longer get
|
|
187
|
+
* it. Returns false if the topic had no retained message.
|
|
188
|
+
*/
|
|
189
|
+
async deleteRetained(topic) {
|
|
190
|
+
await this.ready;
|
|
191
|
+
this.assertOpen();
|
|
192
|
+
return this.backend.deleteRetained(topic);
|
|
193
|
+
}
|
|
194
|
+
/** Totals of this namespace, for dashboards. */
|
|
195
|
+
async getCounts() {
|
|
196
|
+
await this.ready;
|
|
197
|
+
return this.backend.getCounts();
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Nodes of this namespace, in registration order. `start` and `end` are
|
|
201
|
+
* inclusive positions; an `end` of -1 means "to the end".
|
|
202
|
+
*/
|
|
203
|
+
async getNodes(start = 0, end = -1) {
|
|
204
|
+
await this.ready;
|
|
205
|
+
return this.backend.getNodes(...checkRange(start, end));
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Subscriptions of this namespace (or of a node or pattern only), ordered
|
|
209
|
+
* by node and endpoint (or pattern), then pattern (or node and endpoint).
|
|
210
|
+
*/
|
|
211
|
+
async getSubscriptions(filter, start = 0, end = -1) {
|
|
212
|
+
await this.ready;
|
|
213
|
+
return this.backend.getSubscriptions(filter, ...checkRange(start, end));
|
|
214
|
+
}
|
|
215
|
+
/** Subscribed patterns, ordered by pattern, with their subscriptions. */
|
|
216
|
+
async getPatterns(start = 0, end = -1) {
|
|
217
|
+
await this.ready;
|
|
218
|
+
return this.backend.getPatterns(...checkRange(start, end));
|
|
219
|
+
}
|
|
220
|
+
/** Retained messages, by expiry (the data is as serialized). */
|
|
221
|
+
async getRetained(start = 0, end = -1) {
|
|
222
|
+
await this.ready;
|
|
223
|
+
return this.backend.getRetained(...checkRange(start, end));
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* A node's inbox connection (this node's by default), as the datastore sees
|
|
227
|
+
* it: its address and how long it has been open and idle. Empty when the
|
|
228
|
+
* node isn't connected. Lists every client of the datastore: use it on
|
|
229
|
+
* demand, not on a timer.
|
|
230
|
+
*/
|
|
231
|
+
async getNodeConnections(nodeId = this.nodeId) {
|
|
232
|
+
await this.ready;
|
|
233
|
+
if (!ID_RE.test(nodeId)) {
|
|
234
|
+
throw new Error(`Invalid relay nodeId: ${nodeId}`);
|
|
235
|
+
}
|
|
236
|
+
return this.backend.getNodeConnections(nodeId);
|
|
237
|
+
}
|
|
238
|
+
/** Namespaces in use in the datastore (or Redis prefix). */
|
|
239
|
+
async getNamespaces() {
|
|
240
|
+
await this.ready;
|
|
241
|
+
return this.backend.getNamespaces();
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Stops this node and unregisters it, removing its subscriptions.
|
|
245
|
+
*/
|
|
246
|
+
close() {
|
|
247
|
+
if (!this.closing) {
|
|
248
|
+
this.closing = this.stop();
|
|
249
|
+
}
|
|
250
|
+
return this.closing;
|
|
251
|
+
}
|
|
252
|
+
async start() {
|
|
253
|
+
var _a, _b;
|
|
254
|
+
// close() cancels a startup still waiting for the datastore.
|
|
255
|
+
await Promise.race([this.backend.waitUntilReady(), this.startCancelled]);
|
|
256
|
+
if (this.closing) {
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
this.connected = true;
|
|
260
|
+
await this.register();
|
|
261
|
+
if (this.closing) {
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
const heartbeatInterval = (_a = this.opts.heartbeatInterval) !== null && _a !== void 0 ? _a : Math.max(1, Math.floor(this.leaseDuration / 3));
|
|
265
|
+
this.heartbeatTimer = setInterval(() => this.heartbeat(true).catch(err => this.emitError(err)), heartbeatInterval);
|
|
266
|
+
this.sweepTimer = setInterval(() => this.sweep().catch(err => this.emitError(err)), (_b = this.opts.sweepInterval) !== null && _b !== void 0 ? _b : this.leaseDuration);
|
|
267
|
+
this.loop = this.readLoop();
|
|
268
|
+
}
|
|
269
|
+
async stop() {
|
|
270
|
+
var _a, _b, _c;
|
|
271
|
+
clearInterval(this.heartbeatTimer);
|
|
272
|
+
clearInterval(this.sweepTimer);
|
|
273
|
+
if (!this.connected) {
|
|
274
|
+
// Still connecting (possibly retrying forever): never registered, so
|
|
275
|
+
// there is nothing to unregister. Cancel startup and close directly.
|
|
276
|
+
this.cancelStart();
|
|
277
|
+
this.endpoints.clear();
|
|
278
|
+
await this.backend.close();
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
try {
|
|
282
|
+
await this.ready;
|
|
283
|
+
}
|
|
284
|
+
catch (_d) {
|
|
285
|
+
// closing anyway
|
|
286
|
+
}
|
|
287
|
+
this.endpoints.clear();
|
|
288
|
+
this.backend.interruptInbox();
|
|
289
|
+
await ((_a = this.loop) === null || _a === void 0 ? void 0 : _a.catch(() => undefined));
|
|
290
|
+
await ((_b = this.heartbeating) === null || _b === void 0 ? void 0 : _b.catch(() => undefined));
|
|
291
|
+
await ((_c = this.sweeping) === null || _c === void 0 ? void 0 : _c.catch(() => undefined));
|
|
292
|
+
try {
|
|
293
|
+
await this.backend.unregisterNode(this.nodeId);
|
|
294
|
+
}
|
|
295
|
+
catch (err) {
|
|
296
|
+
this.emitError(err);
|
|
297
|
+
}
|
|
298
|
+
await this.backend.close();
|
|
299
|
+
}
|
|
300
|
+
heartbeat(trimInbox = false) {
|
|
301
|
+
// A slow datastore must not pile up overlapping heartbeats (or run two
|
|
302
|
+
// recoveries at once): skip the tick while the previous one runs.
|
|
303
|
+
if (this.closing) {
|
|
304
|
+
return Promise.resolve();
|
|
305
|
+
}
|
|
306
|
+
if (this.heartbeating) {
|
|
307
|
+
return this.heartbeating;
|
|
308
|
+
}
|
|
309
|
+
this.heartbeating = (async () => {
|
|
310
|
+
const alive = await this.backend.heartbeat(this.nodeId, this.leaseDuration);
|
|
311
|
+
if ((!alive || this.recoveryPending) && !this.closing) {
|
|
312
|
+
await this.recover();
|
|
313
|
+
}
|
|
314
|
+
if (trimInbox && !this.closing) {
|
|
315
|
+
await this.checkBacklog();
|
|
316
|
+
}
|
|
317
|
+
})().finally(() => {
|
|
318
|
+
this.heartbeating = undefined;
|
|
319
|
+
});
|
|
320
|
+
return this.heartbeating;
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Removes consumed inbox entries, caps the inbox at `inboxMaxLength`, and
|
|
324
|
+
* emits `'lag'` when the unread backlog reaches `lagThreshold`.
|
|
325
|
+
*/
|
|
326
|
+
async checkBacklog() {
|
|
327
|
+
if (this.closing || this.batchInFlight) {
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
if (!this.backlogChecking) {
|
|
331
|
+
this.backlogChecking = (async () => {
|
|
332
|
+
const backlog = await this.backend.trimInbox(this.nodeId, this.cursor, this.inboxMaxLength);
|
|
333
|
+
if (backlog >= this.lagThreshold) {
|
|
334
|
+
this.emit('lag', backlog);
|
|
335
|
+
}
|
|
336
|
+
})().finally(() => {
|
|
337
|
+
this.backlogChecking = undefined;
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
return this.backlogChecking;
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* The node's lease expired or it was swept: register again and restore
|
|
344
|
+
* its subscriptions. Retained messages are not delivered again. If it fails
|
|
345
|
+
* part way, the next heartbeat retries it.
|
|
346
|
+
*/
|
|
347
|
+
async recover() {
|
|
348
|
+
this.recoveryPending = true;
|
|
349
|
+
this.registration++;
|
|
350
|
+
await this.register();
|
|
351
|
+
let renewalError;
|
|
352
|
+
let renewal;
|
|
353
|
+
const renewLease = () => {
|
|
354
|
+
if (!renewal && !this.closing && !renewalError) {
|
|
355
|
+
renewal = this.backend
|
|
356
|
+
.heartbeat(this.nodeId, this.leaseDuration)
|
|
357
|
+
.then(alive => {
|
|
358
|
+
if (!alive) {
|
|
359
|
+
throw new Error('Relay node lease expired during recovery');
|
|
360
|
+
}
|
|
361
|
+
})
|
|
362
|
+
.catch(err => {
|
|
363
|
+
renewalError !== null && renewalError !== void 0 ? renewalError : (renewalError = err);
|
|
364
|
+
})
|
|
365
|
+
.finally(() => {
|
|
366
|
+
renewal = undefined;
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
return renewal;
|
|
370
|
+
};
|
|
371
|
+
const renewalTimer = setInterval(() => void renewLease(), Math.max(1, Math.floor(this.leaseDuration / 3)));
|
|
372
|
+
try {
|
|
373
|
+
for (const [endpointId, endpoint] of this.endpoints) {
|
|
374
|
+
if (renewalError) {
|
|
375
|
+
throw renewalError;
|
|
376
|
+
}
|
|
377
|
+
if (!endpoint.active) {
|
|
378
|
+
await this.removeEndpoint(endpointId, endpoint);
|
|
379
|
+
continue;
|
|
380
|
+
}
|
|
381
|
+
await this.backend.subscribe(this.nodeId, endpointId, endpoint.pattern);
|
|
382
|
+
if (!endpoint.active) {
|
|
383
|
+
await this.removeEndpoint(endpointId, endpoint);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
if (renewal) {
|
|
387
|
+
await renewal;
|
|
388
|
+
}
|
|
389
|
+
if (renewalError) {
|
|
390
|
+
throw renewalError;
|
|
391
|
+
}
|
|
392
|
+
this.recoveryPending = false;
|
|
393
|
+
this.emit('recovered');
|
|
394
|
+
}
|
|
395
|
+
finally {
|
|
396
|
+
clearInterval(renewalTimer);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
register() {
|
|
400
|
+
return this.backend.registerNode(this.nodeId, this.leaseDuration, {
|
|
401
|
+
name: this.opts.name,
|
|
402
|
+
host: (0, os_1.hostname)(),
|
|
403
|
+
pid: process.pid,
|
|
404
|
+
version: version_1.version,
|
|
405
|
+
}, this.inboxMaxLength);
|
|
406
|
+
}
|
|
407
|
+
removeEndpoint(endpointId, endpoint) {
|
|
408
|
+
if (this.closing) {
|
|
409
|
+
return Promise.resolve();
|
|
410
|
+
}
|
|
411
|
+
if (!endpoint.removing) {
|
|
412
|
+
endpoint.removing = this.backend
|
|
413
|
+
.removeEndpoint(this.nodeId, endpointId)
|
|
414
|
+
.then(() => {
|
|
415
|
+
if (!endpoint.active) {
|
|
416
|
+
this.endpoints.delete(endpointId);
|
|
417
|
+
}
|
|
418
|
+
})
|
|
419
|
+
.finally(() => {
|
|
420
|
+
endpoint.removing = undefined;
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
return endpoint.removing;
|
|
424
|
+
}
|
|
425
|
+
sweep() {
|
|
426
|
+
if (this.closing || this.sweeping) {
|
|
427
|
+
return Promise.resolve();
|
|
428
|
+
}
|
|
429
|
+
this.sweeping = (async () => {
|
|
430
|
+
const removed = await this.backend.sweep(100);
|
|
431
|
+
if (removed.length > 0) {
|
|
432
|
+
this.emit('swept', removed);
|
|
433
|
+
}
|
|
434
|
+
})().finally(() => {
|
|
435
|
+
this.sweeping = undefined;
|
|
436
|
+
});
|
|
437
|
+
return this.sweeping;
|
|
438
|
+
}
|
|
439
|
+
async readLoop() {
|
|
440
|
+
// A batch fetched but not dispatched yet. It is kept when the heartbeat
|
|
441
|
+
// before its dispatch fails: the datastore already counts its entries as
|
|
442
|
+
// in flight, so trimming may remove them without counting them as dropped.
|
|
443
|
+
let fetched;
|
|
444
|
+
while (!this.closing) {
|
|
445
|
+
try {
|
|
446
|
+
if (!fetched) {
|
|
447
|
+
if (this.backlogChecking) {
|
|
448
|
+
await this.backlogChecking;
|
|
449
|
+
}
|
|
450
|
+
const registration = this.registration;
|
|
451
|
+
const batch = await this.backend.readInbox(this.nodeId, this.cursor, {
|
|
452
|
+
blockMs: this.blockTimeout,
|
|
453
|
+
count: 100,
|
|
454
|
+
});
|
|
455
|
+
fetched = { batch, registration };
|
|
456
|
+
}
|
|
457
|
+
const { batch, registration } = fetched;
|
|
458
|
+
this.batchInFlight = batch.entries.length > 0;
|
|
459
|
+
if (batch.entries.length > 0 && !this.closing) {
|
|
460
|
+
// A lease may have expired while reading. Recover before listeners
|
|
461
|
+
// consume messages, so they can invalidate state after a routing gap.
|
|
462
|
+
await this.heartbeat();
|
|
463
|
+
}
|
|
464
|
+
if (this.closing) {
|
|
465
|
+
break;
|
|
466
|
+
}
|
|
467
|
+
fetched = undefined;
|
|
468
|
+
// If a recovery started during or after this read, the batch may belong
|
|
469
|
+
// to the replaced registration, so it must not overwrite state reset
|
|
470
|
+
// on 'recovered'. Read again without acknowledging it: the replaced
|
|
471
|
+
// registration's entries are gone, the new one's are read again.
|
|
472
|
+
if (registration === this.registration) {
|
|
473
|
+
this.cursor = batch.cursor;
|
|
474
|
+
for (const entry of batch.entries) {
|
|
475
|
+
this.dispatch(entry);
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
this.batchInFlight = false;
|
|
479
|
+
if (batch.entries.length > 0) {
|
|
480
|
+
await this.checkBacklog();
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
catch (err) {
|
|
484
|
+
this.batchInFlight = false;
|
|
485
|
+
if (this.closing) {
|
|
486
|
+
break;
|
|
487
|
+
}
|
|
488
|
+
this.emitError(err);
|
|
489
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
490
|
+
}
|
|
491
|
+
finally {
|
|
492
|
+
this.batchInFlight = false;
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
dispatch(entry) {
|
|
497
|
+
// Kinds added by newer versions are not for this node.
|
|
498
|
+
if (entry.kind !== 'msg') {
|
|
499
|
+
return;
|
|
500
|
+
}
|
|
501
|
+
for (const endpointId of entry.endpoints) {
|
|
502
|
+
const endpoint = this.endpoints.get(endpointId);
|
|
503
|
+
if (endpoint === null || endpoint === void 0 ? void 0 : endpoint.active) {
|
|
504
|
+
const message = {
|
|
505
|
+
topic: entry.topic,
|
|
506
|
+
mid: entry.mid,
|
|
507
|
+
ts: entry.ts,
|
|
508
|
+
data: entry.data,
|
|
509
|
+
retained: false,
|
|
510
|
+
};
|
|
511
|
+
if (endpoint.pending) {
|
|
512
|
+
endpoint.pending.push(message);
|
|
513
|
+
}
|
|
514
|
+
else {
|
|
515
|
+
this.deliver(endpoint, message);
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
deliver(endpoint, message) {
|
|
521
|
+
if (!endpoint.active) {
|
|
522
|
+
return;
|
|
523
|
+
}
|
|
524
|
+
let data;
|
|
525
|
+
try {
|
|
526
|
+
data = JSON.parse(message.data);
|
|
527
|
+
}
|
|
528
|
+
catch (err) {
|
|
529
|
+
this.emitError(err);
|
|
530
|
+
return;
|
|
531
|
+
}
|
|
532
|
+
try {
|
|
533
|
+
const result = endpoint.listener(Object.assign(Object.assign({}, message), { data }));
|
|
534
|
+
if (result && typeof result.catch === 'function') {
|
|
535
|
+
result.catch(err => this.emitError(err));
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
catch (err) {
|
|
539
|
+
this.emitError(err);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
assertOpen() {
|
|
543
|
+
if (this.closing) {
|
|
544
|
+
throw new Error('Relay is closed');
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
emitError(err) {
|
|
548
|
+
if (this.listenerCount('error') > 0) {
|
|
549
|
+
this.emit('error', err);
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
exports.Relay = Relay;
|
|
554
|
+
function checkRange(start, end) {
|
|
555
|
+
if (!Number.isSafeInteger(start) ||
|
|
556
|
+
!Number.isSafeInteger(end) ||
|
|
557
|
+
start < 0 ||
|
|
558
|
+
end < -1) {
|
|
559
|
+
throw new Error(`Invalid relay range: ${start}..${end}`);
|
|
560
|
+
}
|
|
561
|
+
return [start, end];
|
|
562
|
+
}
|
|
@@ -290,9 +290,9 @@ class ValkeyGlideAdapter extends events_1.EventEmitter {
|
|
|
290
290
|
this.operationChain = new Promise(resolve => {
|
|
291
291
|
release = resolve;
|
|
292
292
|
});
|
|
293
|
-
await previous;
|
|
294
|
-
this.ensureOpen();
|
|
295
293
|
try {
|
|
294
|
+
await previous;
|
|
295
|
+
this.ensureOpen();
|
|
296
296
|
const raw = await this.ensureRaw();
|
|
297
297
|
this.ensureOpen();
|
|
298
298
|
return await fn(raw);
|
|
@@ -718,6 +718,11 @@ class ValkeyGlideAdapter extends events_1.EventEmitter {
|
|
|
718
718
|
decoder: GLIDE_STRING_DECODER,
|
|
719
719
|
}));
|
|
720
720
|
}
|
|
721
|
+
async clientInfo() {
|
|
722
|
+
return toStringValue(await this.runRawCommand(['CLIENT', 'INFO'], {
|
|
723
|
+
decoder: GLIDE_STRING_DECODER,
|
|
724
|
+
}));
|
|
725
|
+
}
|
|
721
726
|
async scan(cursor, options) {
|
|
722
727
|
const args = ['SCAN', String(cursor)];
|
|
723
728
|
if (options === null || options === void 0 ? void 0 : options.MATCH) {
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
--[[
|
|
2
|
+
Keeps every key of a relay namespace expiring on its own, so a namespace
|
|
3
|
+
whose nodes all vanish leaves nothing behind, even if no other node ever
|
|
4
|
+
sweeps it:
|
|
5
|
+
- node records and inboxes expire with their node's lease;
|
|
6
|
+
- routing keys (nodes, started, subs, cleanup) with the longest lease;
|
|
7
|
+
- :mid and :layout with the longest lease or the last retained message,
|
|
8
|
+
whichever ends later;
|
|
9
|
+
- retained messages and their index with their own expiry.
|
|
10
|
+
Called after every change to the node set or to retained messages.
|
|
11
|
+
]]
|
|
12
|
+
|
|
13
|
+
--- @include "relayTime"
|
|
14
|
+
|
|
15
|
+
local function refreshRelayNamespace(baseKey, nodesKey, now)
|
|
16
|
+
local nodesUntil = getRelayLastScore(nodesKey)
|
|
17
|
+
local routingKeys = { nodesKey, baseKey .. ":started", baseKey .. ":subs",
|
|
18
|
+
baseKey .. ":cleanup" }
|
|
19
|
+
if nodesUntil > now then
|
|
20
|
+
for i = 1, #routingKeys do
|
|
21
|
+
rcall("PEXPIREAT", routingKeys[i], nodesUntil)
|
|
22
|
+
end
|
|
23
|
+
else
|
|
24
|
+
-- No live node: every subscription left belongs to a dead one.
|
|
25
|
+
rcall("UNLINK", unpack(routingKeys))
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
local keepUntil = math.max(nodesUntil,
|
|
29
|
+
getRelayLastScore(baseKey .. ":retained"))
|
|
30
|
+
if keepUntil > now then
|
|
31
|
+
rcall("PEXPIREAT", baseKey .. ":mid", keepUntil)
|
|
32
|
+
rcall("PEXPIREAT", baseKey .. ":layout", keepUntil)
|
|
33
|
+
else
|
|
34
|
+
rcall("UNLINK", baseKey .. ":mid", baseKey .. ":layout")
|
|
35
|
+
end
|
|
36
|
+
end
|