@dokebi/websocket-hub 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AI.md +16 -8
- package/README.md +6 -6
- package/dist/delivery.d.ts +10 -0
- package/dist/delivery.js +7 -0
- package/dist/event-queue.d.ts +6 -1
- package/dist/event-queue.js +9 -4
- package/dist/index.js +21 -3
- package/dist/leader.d.ts +8 -1
- package/dist/leader.js +293 -38
- package/dist/local.js +91 -16
- package/dist/node.js +195 -43
- package/dist/notifications.d.ts +31 -0
- package/dist/notifications.js +133 -0
- package/dist/options.d.ts +16 -0
- package/dist/options.js +26 -5
- package/dist/peer-client.d.ts +7 -1
- package/dist/peer-client.js +215 -25
- package/dist/protocol.js +11 -4
- package/dist/status-cli.js +3 -2
- package/dist/status.d.ts +20 -4
- package/dist/status.js +121 -16
- package/dist/store.d.ts +11 -2
- package/dist/store.js +91 -15
- package/dist/types.d.ts +9 -0
- package/dist/types.js +2 -1
- package/dist/wire.d.ts +17 -2
- package/dist/wire.js +32 -8
- package/options.schema.json +22 -0
- package/package.json +5 -5
package/dist/node.js
CHANGED
|
@@ -4,6 +4,7 @@ import { setTimeout as delay } from 'node:timers/promises';
|
|
|
4
4
|
import { diagnostics } from '@dokebi/core/libraries';
|
|
5
5
|
import { checkEviction } from './eviction.js';
|
|
6
6
|
import { Leader } from './leader.js';
|
|
7
|
+
import { ControlNotifications } from './notifications.js';
|
|
7
8
|
import { PeerClient } from './peer-client.js';
|
|
8
9
|
import { hubStatus } from './status.js';
|
|
9
10
|
import { HubStore } from './store.js';
|
|
@@ -21,12 +22,13 @@ export class HubNode {
|
|
|
21
22
|
#leader;
|
|
22
23
|
#timer;
|
|
23
24
|
#tick;
|
|
24
|
-
#subscription;
|
|
25
25
|
#notifications;
|
|
26
26
|
#closing = false;
|
|
27
27
|
#closed = false;
|
|
28
|
-
|
|
29
|
-
#
|
|
28
|
+
/** The outage this server is in (a failover or a store failure), until a poll succeeds. */
|
|
29
|
+
#outage;
|
|
30
|
+
/** How long a failover may take before it is an error (D-0097 §2). */
|
|
31
|
+
#boundMs;
|
|
30
32
|
#configMismatch;
|
|
31
33
|
#started = false;
|
|
32
34
|
#startupCode;
|
|
@@ -39,15 +41,30 @@ export class HubNode {
|
|
|
39
41
|
this.redis = redis;
|
|
40
42
|
this.nats = nats;
|
|
41
43
|
this.store = new HubStore(redis, options);
|
|
42
|
-
this.#throttle = new diagnostics.Repeats({
|
|
44
|
+
this.#throttle = new diagnostics.Repeats({
|
|
45
|
+
logger: context.logger,
|
|
46
|
+
intervalMs: options.logRepeatWindowMs,
|
|
47
|
+
});
|
|
43
48
|
// The client logs why a ready connection dropped (revision gap, stale leader, socket closed);
|
|
44
49
|
// it is when upgrades see the hub recovering.
|
|
45
50
|
this.client = new PeerClient(options, context.logger);
|
|
51
|
+
// The old leader's lease runs out within leaseMs; electing and reaching the next one takes
|
|
52
|
+
// at most another leaseMs (the connect timeout) and a poll.
|
|
53
|
+
this.#boundMs = 2 * options.leaseMs + options.pollIntervalMs;
|
|
54
|
+
this.client.onLinkLost = (error, planned) => this.#lost(error, planned);
|
|
46
55
|
const self = this;
|
|
47
56
|
// Shared by every server of the namespace: the first to see a nonce wins (D-0038).
|
|
48
57
|
const nonces = Object.freeze({
|
|
49
58
|
shared: true,
|
|
50
59
|
setIfNotExists: (key, ttlSeconds) => this.store.claimNonce(key, ttlSeconds),
|
|
60
|
+
lostAt: () => this.store.noncesLostAt(),
|
|
61
|
+
});
|
|
62
|
+
// A Redis that lost its keys (restart without persistence) forgot the nonces: sealed frames
|
|
63
|
+
// stamped before it are refused for 2 x drift (seal), and this says why.
|
|
64
|
+
this.store.onNoncesLost = (startedAt) => context.logger.log('info', 'Hub replay nonce epoch written in Redis', {
|
|
65
|
+
code: 'HUB_NONCES_EPOCH',
|
|
66
|
+
epoch: new Date(startedAt).toISOString(),
|
|
67
|
+
fix: 'Expected on the first start on this Redis. Later it means the nonces were lost (a restart without persistence, a FLUSHALL, an eviction): sealed frames stamped before it are refused as replays for 2 x sealDriftMs.',
|
|
51
68
|
});
|
|
52
69
|
this.api = Object.freeze({
|
|
53
70
|
get state() {
|
|
@@ -101,6 +118,17 @@ export class HubNode {
|
|
|
101
118
|
const started = performance.now();
|
|
102
119
|
// D-0049: warn when this Redis can evict the leader key; never refuses to start.
|
|
103
120
|
await checkEviction(this.redis, this.context.logger);
|
|
121
|
+
// The nonce epoch before the first sealed frame: a new Redis then refuses no frame sealed just
|
|
122
|
+
// before its first claim. A failure is logged; the first claim writes it instead.
|
|
123
|
+
try {
|
|
124
|
+
await this.store.primeNonces();
|
|
125
|
+
}
|
|
126
|
+
catch (error) {
|
|
127
|
+
this.context.logger.log('error', 'Hub replay nonce epoch not written at start', {
|
|
128
|
+
code: 'HUB_NONCES_EPOCH_FAILED',
|
|
129
|
+
error: diagnostics.describeError(error),
|
|
130
|
+
});
|
|
131
|
+
}
|
|
104
132
|
const server = createServer((socket) => {
|
|
105
133
|
if (!this.#closing && this.#leader?.active)
|
|
106
134
|
this.#leader.accept(socket);
|
|
@@ -132,9 +160,9 @@ export class HubNode {
|
|
|
132
160
|
serverId: this.options.serverId,
|
|
133
161
|
};
|
|
134
162
|
if (this.nats) {
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
163
|
+
const notifications = new ControlNotifications(this.nats, `dokebi.websocket-hub.${this.options.namespace}.control`, this.context.logger, this.options.logRepeatWindowMs, () => this.#poll());
|
|
164
|
+
this.#notifications = notifications;
|
|
165
|
+
await notifications.start();
|
|
138
166
|
}
|
|
139
167
|
const deadline = performance.now() + this.options.startupTimeoutMs;
|
|
140
168
|
while (!this.client.ready) {
|
|
@@ -165,19 +193,6 @@ export class HubNode {
|
|
|
165
193
|
}, this.options.pollIntervalMs);
|
|
166
194
|
this.#timer.unref();
|
|
167
195
|
}
|
|
168
|
-
async #readNotifications(subscription) {
|
|
169
|
-
try {
|
|
170
|
-
for await (const _message of subscription) {
|
|
171
|
-
if (this.#closing)
|
|
172
|
-
break;
|
|
173
|
-
await this.#poll();
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
catch (error) {
|
|
177
|
-
if (!this.#closing)
|
|
178
|
-
this.context.logger.log('error', 'Hub control notifications unavailable; Redis discovery polling remains active', { code: 'NATS_SUBSCRIPTION_FAILED', error: diagnostics.describeError(error) });
|
|
179
|
-
}
|
|
180
|
-
}
|
|
181
196
|
#poll() {
|
|
182
197
|
if (this.#closing)
|
|
183
198
|
return Promise.resolve();
|
|
@@ -192,6 +207,15 @@ export class HubNode {
|
|
|
192
207
|
this.context.logger.log('error', 'Hub configuration differs from the servers running in this namespace', { code: 'CONFIG_MISMATCH', namespace: this.options.namespace });
|
|
193
208
|
this.#configMismatch = error;
|
|
194
209
|
}
|
|
210
|
+
else if (this.#closing) {
|
|
211
|
+
// This server is closing (a cluster stop, a rolling restart): a poll that fails now
|
|
212
|
+
// meets a leader that is leaving too; no outage to report.
|
|
213
|
+
if (this.context.logger.isEnabled('debug'))
|
|
214
|
+
this.context.logger.log('debug', 'Hub poll failed while closing', {
|
|
215
|
+
code: hubCode(error),
|
|
216
|
+
error: diagnostics.describeError(error),
|
|
217
|
+
});
|
|
218
|
+
}
|
|
195
219
|
else if (!this.#started) {
|
|
196
220
|
// Startup retries are normal (peers race to reach a leader that is still opening);
|
|
197
221
|
// the last code is reported if startup times out, each attempt at debug.
|
|
@@ -201,23 +225,8 @@ export class HubNode {
|
|
|
201
225
|
error: diagnostics.describeError(error),
|
|
202
226
|
});
|
|
203
227
|
}
|
|
204
|
-
else if (!this.#lastFailure) {
|
|
205
|
-
this.#failingSince = performance.now();
|
|
206
|
-
this.context.logger.log('error', 'Hub transport or store unavailable; recovering', {
|
|
207
|
-
code: hubCode(error),
|
|
208
|
-
error: diagnostics.describeError(error),
|
|
209
|
-
});
|
|
210
|
-
}
|
|
211
|
-
// Every further failure of the same outage (once per pollIntervalMs), so a long recovery shows
|
|
212
|
-
// its causes.
|
|
213
228
|
else
|
|
214
|
-
this
|
|
215
|
-
code: hubCode(error),
|
|
216
|
-
error: diagnostics.describeError(error),
|
|
217
|
-
ms: Math.round(performance.now() - this.#failingSince),
|
|
218
|
-
leader: this.client.leader?.serverId ?? null,
|
|
219
|
-
});
|
|
220
|
-
this.#lastFailure = true;
|
|
229
|
+
this.#failed(error);
|
|
221
230
|
const leader = this.#leader;
|
|
222
231
|
this.#leader = undefined;
|
|
223
232
|
await leader?.stop();
|
|
@@ -278,12 +287,136 @@ export class HubNode {
|
|
|
278
287
|
previous: this.client.leader?.serverId ?? null,
|
|
279
288
|
});
|
|
280
289
|
await this.client.connect(discovered);
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
290
|
+
const outage = this.#outage;
|
|
291
|
+
this.#outage = undefined;
|
|
292
|
+
if (outage && this.#started)
|
|
293
|
+
this.#recovered(outage, discovered.serverId);
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* The ready connection to the leader closed (it died: `planned` false) or the leader said it
|
|
297
|
+
* is stopping. A failover follows; it is a warn, an error only past its bound (D-0097 §2).
|
|
298
|
+
*/
|
|
299
|
+
#lost(error, planned) {
|
|
300
|
+
if (this.#closing || this.#outage) {
|
|
301
|
+
if (this.#outage)
|
|
302
|
+
this.#outage.last = error;
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
const leader = this.client.leader?.serverId ?? null;
|
|
306
|
+
this.#outage = {
|
|
307
|
+
since: performance.now(),
|
|
308
|
+
attempts: 0,
|
|
309
|
+
first: error,
|
|
310
|
+
last: error,
|
|
311
|
+
failover: true,
|
|
312
|
+
planned,
|
|
313
|
+
escalated: false,
|
|
314
|
+
previous: leader,
|
|
315
|
+
};
|
|
316
|
+
if (!this.#started || planned)
|
|
317
|
+
this.context.logger.log('debug', 'Hub leader left; moving to the next leader', {
|
|
318
|
+
code: hubCode(error),
|
|
319
|
+
leader,
|
|
320
|
+
planned,
|
|
321
|
+
});
|
|
322
|
+
else
|
|
323
|
+
this.context.logger.log('warn', 'Hub lost the leader connection; failing over', {
|
|
324
|
+
code: hubCode(error),
|
|
325
|
+
leader,
|
|
326
|
+
boundMs: this.#boundMs,
|
|
327
|
+
error: diagnostics.describeError(error),
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
/** A poll of a started hub failed: a failover attempt, or a transport or store outage. */
|
|
331
|
+
#failed(error) {
|
|
332
|
+
const code = hubCode(error);
|
|
333
|
+
const link = LINK_CODES.has(code);
|
|
334
|
+
let outage = this.#outage;
|
|
335
|
+
if (!outage) {
|
|
336
|
+
// The leader was unreachable before the drop was seen (a ping, the reconnect after it).
|
|
337
|
+
if (link)
|
|
338
|
+
this.#lost(error, false);
|
|
339
|
+
outage = this.#outage ?? {
|
|
340
|
+
since: performance.now(),
|
|
341
|
+
attempts: 0,
|
|
342
|
+
first: error,
|
|
343
|
+
last: error,
|
|
344
|
+
failover: false,
|
|
345
|
+
planned: false,
|
|
346
|
+
escalated: false,
|
|
347
|
+
previous: this.client.leader?.serverId ?? null,
|
|
348
|
+
};
|
|
349
|
+
this.#outage = outage;
|
|
350
|
+
}
|
|
351
|
+
outage.attempts++;
|
|
352
|
+
outage.last = error;
|
|
353
|
+
const ms = Math.round(performance.now() - outage.since);
|
|
354
|
+
if (!outage.escalated && outage.failover && link && ms <= this.#boundMs) {
|
|
355
|
+
this.context.logger.log('debug', 'Hub failover attempt failed; retrying', {
|
|
356
|
+
code,
|
|
357
|
+
attempts: outage.attempts,
|
|
358
|
+
ms,
|
|
359
|
+
boundMs: this.#boundMs,
|
|
360
|
+
leader: this.client.leader?.serverId ?? null,
|
|
361
|
+
error: diagnostics.describeError(error),
|
|
362
|
+
});
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
if (!outage.escalated) {
|
|
366
|
+
outage.escalated = true;
|
|
367
|
+
if (outage.failover && link)
|
|
368
|
+
this.context.logger.log('error', 'Hub failover exceeded its bound; still recovering', {
|
|
369
|
+
code,
|
|
370
|
+
attempts: outage.attempts,
|
|
371
|
+
ms,
|
|
372
|
+
boundMs: this.#boundMs,
|
|
373
|
+
leader: this.client.leader?.serverId ?? null,
|
|
374
|
+
previous: outage.previous,
|
|
375
|
+
firstError: diagnostics.describeError(outage.first),
|
|
376
|
+
error: diagnostics.describeError(error),
|
|
377
|
+
});
|
|
378
|
+
else
|
|
379
|
+
this.context.logger.log('error', 'Hub transport or store unavailable; recovering', {
|
|
380
|
+
code,
|
|
381
|
+
attempts: outage.attempts,
|
|
382
|
+
ms,
|
|
383
|
+
error: diagnostics.describeError(error),
|
|
384
|
+
});
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
// Every further failure of the same outage (once per pollIntervalMs), so a long recovery
|
|
388
|
+
// shows its causes.
|
|
389
|
+
this.context.logger.log('debug', 'Hub still recovering', {
|
|
390
|
+
code,
|
|
391
|
+
attempts: outage.attempts,
|
|
392
|
+
error: diagnostics.describeError(error),
|
|
393
|
+
ms,
|
|
394
|
+
leader: this.client.leader?.serverId ?? null,
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
/** The poll after an outage succeeded: how it ended, at the level the outage had. */
|
|
398
|
+
#recovered(outage, leader) {
|
|
399
|
+
const ms = Math.round(performance.now() - outage.since);
|
|
400
|
+
const attempts = outage.attempts + 1;
|
|
401
|
+
if (outage.escalated)
|
|
402
|
+
this.context.logger.log('info', 'Hub recovered', { ms, attempts, leader });
|
|
403
|
+
else if (outage.planned)
|
|
404
|
+
this.context.logger.log('debug', 'Hub moved to the next leader', {
|
|
405
|
+
ms,
|
|
406
|
+
attempts,
|
|
407
|
+
leader,
|
|
408
|
+
previous: outage.previous,
|
|
409
|
+
});
|
|
410
|
+
else
|
|
411
|
+
this.context.logger.log('warn', 'Hub failed over to a new leader', {
|
|
412
|
+
ms,
|
|
413
|
+
attempts,
|
|
414
|
+
leader,
|
|
415
|
+
previous: outage.previous,
|
|
416
|
+
boundMs: this.#boundMs,
|
|
417
|
+
firstError: diagnostics.describeError(outage.first),
|
|
418
|
+
lastError: diagnostics.describeError(outage.last),
|
|
285
419
|
});
|
|
286
|
-
this.#lastFailure = false;
|
|
287
420
|
}
|
|
288
421
|
async close() {
|
|
289
422
|
if (this.#closed)
|
|
@@ -292,7 +425,7 @@ export class HubNode {
|
|
|
292
425
|
this.#closing = true;
|
|
293
426
|
if (this.#timer)
|
|
294
427
|
clearInterval(this.#timer);
|
|
295
|
-
this.#
|
|
428
|
+
this.#notifications?.close();
|
|
296
429
|
await this.#tick;
|
|
297
430
|
const errors = [];
|
|
298
431
|
// If the leader went away first (whole cluster stopping), `bye` cannot reach it; `leave`
|
|
@@ -336,7 +469,7 @@ export class HubNode {
|
|
|
336
469
|
}
|
|
337
470
|
if (this.#server)
|
|
338
471
|
await new Promise((resolve) => this.#server?.close(() => resolve()));
|
|
339
|
-
await this.#notifications;
|
|
472
|
+
await this.#notifications?.done;
|
|
340
473
|
this.#closed = true;
|
|
341
474
|
this.#throttle.flush();
|
|
342
475
|
this.context.logger.log('debug', 'Hub closed', {
|
|
@@ -347,6 +480,25 @@ export class HubNode {
|
|
|
347
480
|
throw new AggregateError(errors, 'Hub cleanup failed');
|
|
348
481
|
}
|
|
349
482
|
}
|
|
483
|
+
/**
|
|
484
|
+
* Failures of the leader link: the expected attempts of a failover (the dead leader refuses,
|
|
485
|
+
* resets or times out; a peer reaches a server that is not, or no longer, the leader).
|
|
486
|
+
*/
|
|
487
|
+
const LINK_CODES = new Set([
|
|
488
|
+
'DISCONNECTED',
|
|
489
|
+
'RECOVERING',
|
|
490
|
+
'OUTCOME_UNKNOWN',
|
|
491
|
+
'STALE_LEADER',
|
|
492
|
+
'STALE_PEER',
|
|
493
|
+
'CONNECT_TIMEOUT',
|
|
494
|
+
'ECONNREFUSED',
|
|
495
|
+
'ECONNRESET',
|
|
496
|
+
'ECONNABORTED',
|
|
497
|
+
'EPIPE',
|
|
498
|
+
'ETIMEDOUT',
|
|
499
|
+
'EHOSTUNREACH',
|
|
500
|
+
'ENETUNREACH',
|
|
501
|
+
]);
|
|
350
502
|
/** The leader is gone or going: `bye` cannot be answered, `leave` takes its place. */
|
|
351
503
|
const GONE = new Set(['DISCONNECTED', 'STALE_LEADER', 'CLOSED', 'RECOVERING']);
|
|
352
504
|
/** A short code for logs: hub/redis/nats error codes or system errno codes, never messages. */
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { Logger } from '@dokebi/core';
|
|
2
|
+
import type { NatsConnectionApi } from '@dokebi/nats';
|
|
3
|
+
/**
|
|
4
|
+
* The hub's leader-change notices over NATS (D-0029, optional): each notice asks for one poll of
|
|
5
|
+
* Redis, which finds the new leader at once instead of after `pollIntervalMs`. Notices are read
|
|
6
|
+
* without waiting for the poll they ask for (notices that come while one runs ask for one more),
|
|
7
|
+
* so this subscription never fills the connection's buffer, and a subscription that ended
|
|
8
|
+
* recoverably is opened again with an immediate poll, since a lost notice may have named a new
|
|
9
|
+
* leader.
|
|
10
|
+
*/
|
|
11
|
+
export declare class ControlNotifications {
|
|
12
|
+
#private;
|
|
13
|
+
readonly nats: NatsConnectionApi;
|
|
14
|
+
readonly subject: string;
|
|
15
|
+
readonly logger: Logger;
|
|
16
|
+
/** The hub's logRepeatWindowMs: repeated resubscriptions are logged once per window. */
|
|
17
|
+
readonly repeatWindowMs: number;
|
|
18
|
+
/** One Redis poll (coalesced by the hub node). */
|
|
19
|
+
readonly poll: () => Promise<void>;
|
|
20
|
+
constructor(nats: NatsConnectionApi, subject: string, logger: Logger,
|
|
21
|
+
/** The hub's logRepeatWindowMs: repeated resubscriptions are logged once per window. */
|
|
22
|
+
repeatWindowMs: number,
|
|
23
|
+
/** One Redis poll (coalesced by the hub node). */
|
|
24
|
+
poll: () => Promise<void>);
|
|
25
|
+
/** Subscribes and waits until NATS took the subscription; reading runs in the background. */
|
|
26
|
+
start(): Promise<void>;
|
|
27
|
+
/** Ends the subscription; `done` settles once reading stopped. */
|
|
28
|
+
close(): void;
|
|
29
|
+
/** Reading and the notice polls it asked for, finished (after close). */
|
|
30
|
+
get done(): Promise<void>;
|
|
31
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { diagnostics } from '@dokebi/core/libraries';
|
|
2
|
+
/**
|
|
3
|
+
* Subscription ends that a new subscription heals (audit R4): the connection's shared subscription
|
|
4
|
+
* buffer was full when a message came (SUBSCRIPTION_OVERFLOW, a burst on another subscription of
|
|
5
|
+
* the same connection), or one message on the control subject was larger than the connection
|
|
6
|
+
* takes (MAX_PAYLOAD). Any other end (the connection closed, the subscription refused) stays as it
|
|
7
|
+
* was: Redis polling goes on, and the error is logged.
|
|
8
|
+
*/
|
|
9
|
+
const RECOVERABLE = new Set(['SUBSCRIPTION_OVERFLOW', 'MAX_PAYLOAD']);
|
|
10
|
+
function code(error) {
|
|
11
|
+
const value = error?.code;
|
|
12
|
+
return typeof value === 'string' && /^[A-Z][A-Z0-9_]{0,63}$/.test(value) ? value : 'UNKNOWN';
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* The hub's leader-change notices over NATS (D-0029, optional): each notice asks for one poll of
|
|
16
|
+
* Redis, which finds the new leader at once instead of after `pollIntervalMs`. Notices are read
|
|
17
|
+
* without waiting for the poll they ask for (notices that come while one runs ask for one more),
|
|
18
|
+
* so this subscription never fills the connection's buffer, and a subscription that ended
|
|
19
|
+
* recoverably is opened again with an immediate poll, since a lost notice may have named a new
|
|
20
|
+
* leader.
|
|
21
|
+
*/
|
|
22
|
+
export class ControlNotifications {
|
|
23
|
+
nats;
|
|
24
|
+
subject;
|
|
25
|
+
logger;
|
|
26
|
+
repeatWindowMs;
|
|
27
|
+
poll;
|
|
28
|
+
#subscription;
|
|
29
|
+
#closing = false;
|
|
30
|
+
#wanted = false;
|
|
31
|
+
#polling;
|
|
32
|
+
#failures = 0;
|
|
33
|
+
#loggedAt = Number.NEGATIVE_INFINITY;
|
|
34
|
+
#reading = Promise.resolve();
|
|
35
|
+
constructor(nats, subject, logger,
|
|
36
|
+
/** The hub's logRepeatWindowMs: repeated resubscriptions are logged once per window. */
|
|
37
|
+
repeatWindowMs,
|
|
38
|
+
/** One Redis poll (coalesced by the hub node). */
|
|
39
|
+
poll) {
|
|
40
|
+
this.nats = nats;
|
|
41
|
+
this.subject = subject;
|
|
42
|
+
this.logger = logger;
|
|
43
|
+
this.repeatWindowMs = repeatWindowMs;
|
|
44
|
+
this.poll = poll;
|
|
45
|
+
}
|
|
46
|
+
/** Subscribes and waits until NATS took the subscription; reading runs in the background. */
|
|
47
|
+
async start() {
|
|
48
|
+
const subscription = this.nats.subscribe(this.subject);
|
|
49
|
+
this.#subscription = subscription;
|
|
50
|
+
await subscription.ready;
|
|
51
|
+
this.#reading = this.#read();
|
|
52
|
+
}
|
|
53
|
+
/** Ends the subscription; `done` settles once reading stopped. */
|
|
54
|
+
close() {
|
|
55
|
+
this.#closing = true;
|
|
56
|
+
this.#subscription?.unsubscribe();
|
|
57
|
+
}
|
|
58
|
+
/** Reading and the notice polls it asked for, finished (after close). */
|
|
59
|
+
get done() {
|
|
60
|
+
return this.#reading.then(() => this.#polling);
|
|
61
|
+
}
|
|
62
|
+
#notice() {
|
|
63
|
+
this.#wanted = true;
|
|
64
|
+
this.#polling ??= (async () => {
|
|
65
|
+
try {
|
|
66
|
+
while (this.#wanted && !this.#closing) {
|
|
67
|
+
this.#wanted = false;
|
|
68
|
+
try {
|
|
69
|
+
await this.poll();
|
|
70
|
+
}
|
|
71
|
+
catch (error) {
|
|
72
|
+
this.logger.log('error', 'Hub poll after a control notice failed', {
|
|
73
|
+
code: 'HUB_NOTICE_POLL_FAILED',
|
|
74
|
+
error: diagnostics.describeError(error),
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
this.#polling = undefined;
|
|
81
|
+
}
|
|
82
|
+
})();
|
|
83
|
+
}
|
|
84
|
+
async #read() {
|
|
85
|
+
for (;;) {
|
|
86
|
+
const subscription = this.#subscription;
|
|
87
|
+
if (!subscription || this.#closing)
|
|
88
|
+
return;
|
|
89
|
+
try {
|
|
90
|
+
for await (const _message of subscription) {
|
|
91
|
+
if (this.#closing)
|
|
92
|
+
return;
|
|
93
|
+
this.#notice();
|
|
94
|
+
}
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
if (this.#closing)
|
|
99
|
+
return;
|
|
100
|
+
const reason = code(error);
|
|
101
|
+
if (!RECOVERABLE.has(reason) || this.nats.state !== 'connected') {
|
|
102
|
+
this.logger.log('error', 'Hub control notifications unavailable; Redis discovery polling remains active', {
|
|
103
|
+
code: 'NATS_SUBSCRIPTION_FAILED',
|
|
104
|
+
reason,
|
|
105
|
+
state: this.nats.state,
|
|
106
|
+
error: diagnostics.describeError(error),
|
|
107
|
+
});
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
this.#failures++;
|
|
111
|
+
const now = performance.now();
|
|
112
|
+
if (now - this.#loggedAt >= this.repeatWindowMs) {
|
|
113
|
+
this.logger.log('error', 'Hub control notifications lost; subscribing again', {
|
|
114
|
+
code: 'NATS_SUBSCRIPTION_FAILED',
|
|
115
|
+
reason,
|
|
116
|
+
repeated: this.#failures - 1,
|
|
117
|
+
error: diagnostics.describeError(error),
|
|
118
|
+
});
|
|
119
|
+
this.#loggedAt = now;
|
|
120
|
+
this.#failures = 0;
|
|
121
|
+
}
|
|
122
|
+
const next = this.nats.subscribe(this.subject);
|
|
123
|
+
this.#subscription = next;
|
|
124
|
+
// Not awaited in the read loop: a subscription that is refused ends its iteration with
|
|
125
|
+
// that error, which the next turn logs.
|
|
126
|
+
// dokebi-check: allow empty-catch the same failure ends the iteration below and is logged there
|
|
127
|
+
next.ready.catch(() => { });
|
|
128
|
+
// A lost notice may have named a new leader: look now, not after pollIntervalMs.
|
|
129
|
+
this.#notice();
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
package/dist/options.d.ts
CHANGED
|
@@ -1,4 +1,17 @@
|
|
|
1
1
|
import { type OptionsOf } from '@dokebi/core';
|
|
2
|
+
/**
|
|
3
|
+
* The largest size limits any hub server may set (D-0136). A server reads what other hub
|
|
4
|
+
* servers send against these, not against its own options: servers whose limits differ (a
|
|
5
|
+
* rolling deploy that changes them) run in one namespace, and each applies its own limits to
|
|
6
|
+
* its own clients and, as the leader, to the shared presence.
|
|
7
|
+
*/
|
|
8
|
+
export declare const PEER_LIMITS: Readonly<{
|
|
9
|
+
frameBytes: 4194304;
|
|
10
|
+
payloadBytes: 262144;
|
|
11
|
+
connections: 4096;
|
|
12
|
+
owners: 1024;
|
|
13
|
+
totalConnections: 100000;
|
|
14
|
+
}>;
|
|
2
15
|
/**
|
|
3
16
|
* Options of 0.2 that D-0057 removed with client ACKs. A dokebi.yaml that still has them starts:
|
|
4
17
|
* core logs warn `OPTION_REMOVED` once per key at setup and the value is ignored.
|
|
@@ -7,10 +20,13 @@ export declare const REMOVED_OPTIONS: readonly ["deliveryTimeoutMs", "retryMs",
|
|
|
7
20
|
export declare const options: import("@dokebi/core").OptionsDeclaration<{
|
|
8
21
|
readonly advertiseHost: string;
|
|
9
22
|
readonly allowInCluster: boolean;
|
|
23
|
+
readonly broadcastShare: number;
|
|
10
24
|
readonly host: string;
|
|
11
25
|
readonly leaseMs: number;
|
|
26
|
+
readonly logRepeatWindowMs: number;
|
|
12
27
|
readonly maxConnections: number;
|
|
13
28
|
readonly maxFrameBytes: number;
|
|
29
|
+
readonly maxNonces: number;
|
|
14
30
|
readonly maxOwners: number;
|
|
15
31
|
readonly maxPayloadBytes: number;
|
|
16
32
|
readonly maxPending: number;
|
package/dist/options.js
CHANGED
|
@@ -12,6 +12,19 @@ const HOST = /^[^\s\x00-\x1f]+$/;
|
|
|
12
12
|
const REMOVED = 'Removed in 0.3.0 with client ACKs (D-0057). Ignored: the hub logs warn OPTION_REMOVED once and starts. Delete it from dokebi.yaml.';
|
|
13
13
|
const ms = (minimum, maximum, fallback) => o.integer({ minimum, maximum, default: fallback });
|
|
14
14
|
const bytes = (value) => Buffer.byteLength(value);
|
|
15
|
+
/**
|
|
16
|
+
* The largest size limits any hub server may set (D-0136). A server reads what other hub
|
|
17
|
+
* servers send against these, not against its own options: servers whose limits differ (a
|
|
18
|
+
* rolling deploy that changes them) run in one namespace, and each applies its own limits to
|
|
19
|
+
* its own clients and, as the leader, to the shared presence.
|
|
20
|
+
*/
|
|
21
|
+
export const PEER_LIMITS = Object.freeze({
|
|
22
|
+
frameBytes: 4_194_304,
|
|
23
|
+
payloadBytes: 262_144,
|
|
24
|
+
connections: 4096,
|
|
25
|
+
owners: 1024,
|
|
26
|
+
totalConnections: 100_000,
|
|
27
|
+
});
|
|
15
28
|
/**
|
|
16
29
|
* Options of 0.2 that D-0057 removed with client ACKs. A dokebi.yaml that still has them starts:
|
|
17
30
|
* core logs warn `OPTION_REMOVED` once per key at setup and the value is ignored.
|
|
@@ -69,14 +82,22 @@ export const options = defineOptions({
|
|
|
69
82
|
ownerTtlMs: ms(1500, 120000, 15000),
|
|
70
83
|
startupTimeoutMs: ms(1000, 120000, 15000),
|
|
71
84
|
startupMs: o.alias('startupTimeoutMs'),
|
|
72
|
-
maxFrameBytes: ms(16384,
|
|
73
|
-
maxPayloadBytes: ms(1,
|
|
74
|
-
maxConnections: ms(1,
|
|
75
|
-
maxOwners: ms(1,
|
|
76
|
-
maxTotalConnections: ms(1,
|
|
85
|
+
maxFrameBytes: ms(16384, PEER_LIMITS.frameBytes, 1048576),
|
|
86
|
+
maxPayloadBytes: ms(1, PEER_LIMITS.payloadBytes, 65536),
|
|
87
|
+
maxConnections: ms(1, PEER_LIMITS.connections, 128),
|
|
88
|
+
maxOwners: ms(1, PEER_LIMITS.owners, 64),
|
|
89
|
+
maxTotalConnections: ms(1, PEER_LIMITS.totalConnections, 8192),
|
|
77
90
|
maxPending: ms(1, 4096, 128),
|
|
78
91
|
maxPendingBytes: ms(16384, 67108864, 4194304),
|
|
79
92
|
queueBytes: o.alias('maxPendingBytes'),
|
|
93
|
+
broadcastShare: o.number({
|
|
94
|
+
minimum: 0.05,
|
|
95
|
+
maximum: 1,
|
|
96
|
+
default: 0.5,
|
|
97
|
+
description: 'Share of the event queue (maxPending events, maxPendingBytes) broadcasts may fill before they are refused with BUSY; the rest stays for presence and kick events, which are never refused (D-0114). 1 lets broadcasts fill it.',
|
|
98
|
+
}),
|
|
99
|
+
maxNonces: ms(1, 10_000_000, 100_000),
|
|
100
|
+
logRepeatWindowMs: ms(1000, 3_600_000, 10_000),
|
|
80
101
|
deliveryTimeoutMs: o.removed({ type: 'integer', message: REMOVED }),
|
|
81
102
|
retryMs: o.removed({ type: 'integer', message: REMOVED }),
|
|
82
103
|
maxDeliveries: o.removed({ type: 'integer', message: REMOVED }),
|
package/dist/peer-client.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type Logger } from '@dokebi/core';
|
|
2
2
|
import { EventQueue } from './event-queue.js';
|
|
3
|
-
import type
|
|
3
|
+
import { type HubOptions } from './options.js';
|
|
4
4
|
import { type HubConnection, type HubKickReceipt, type HubLeader, type HubQuery, type HubReceipt, type HubRecord, type HubRegistration, type HubSnapshot } from './types.js';
|
|
5
5
|
export declare class PeerClient {
|
|
6
6
|
#private;
|
|
@@ -11,6 +11,12 @@ export declare class PeerClient {
|
|
|
11
11
|
/** Session of the last hello; the owner record in Redis carries the same value. */
|
|
12
12
|
get session(): string | undefined;
|
|
13
13
|
readonly events: EventQueue;
|
|
14
|
+
/**
|
|
15
|
+
* Told when a ready connection to the leader closed under this server (`DISCONNECTED`: the
|
|
16
|
+
* leader died or restarted) or the leader said it is stopping (`planned`). The hub node reports
|
|
17
|
+
* it as a failover (D-0097 §2: warn, error only past its bound). Without it the drop is a warn.
|
|
18
|
+
*/
|
|
19
|
+
onLinkLost: ((error: unknown, planned: boolean) => void) | undefined;
|
|
14
20
|
constructor(options: HubOptions,
|
|
15
21
|
/** The hub node passes the plugin logger; the console default keeps failures visible. */
|
|
16
22
|
logger?: Logger);
|