@dokebi/websocket-hub 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AI.md +16 -8
- package/README.md +6 -6
- package/dist/delivery.d.ts +10 -0
- package/dist/delivery.js +7 -0
- package/dist/event-queue.d.ts +6 -1
- package/dist/event-queue.js +9 -4
- package/dist/index.js +21 -3
- package/dist/leader.d.ts +8 -1
- package/dist/leader.js +293 -38
- package/dist/local.js +91 -16
- package/dist/node.js +195 -43
- package/dist/notifications.d.ts +31 -0
- package/dist/notifications.js +133 -0
- package/dist/options.d.ts +16 -0
- package/dist/options.js +26 -5
- package/dist/peer-client.d.ts +7 -1
- package/dist/peer-client.js +215 -25
- package/dist/protocol.js +11 -4
- package/dist/status-cli.js +3 -2
- package/dist/status.d.ts +20 -4
- package/dist/status.js +121 -16
- package/dist/store.d.ts +11 -2
- package/dist/store.js +91 -15
- package/dist/types.d.ts +9 -0
- package/dist/types.js +2 -1
- package/dist/wire.d.ts +17 -2
- package/dist/wire.js +32 -8
- package/options.schema.json +22 -0
- package/package.json +5 -5
package/dist/peer-client.js
CHANGED
|
@@ -2,7 +2,9 @@ import { randomUUID } from 'node:crypto';
|
|
|
2
2
|
import { createConnection } from 'node:net';
|
|
3
3
|
import { ConsoleLogger } from '@dokebi/core';
|
|
4
4
|
import { diagnostics } from '@dokebi/core/libraries';
|
|
5
|
+
import { REPLACED } from './delivery.js';
|
|
5
6
|
import { EventQueue } from './event-queue.js';
|
|
7
|
+
import { PEER_LIMITS } from './options.js';
|
|
6
8
|
import { array, epoch, identifier, matches, metadata, PROTOCOL, payload, query, record, records, sequence, } from './protocol.js';
|
|
7
9
|
import { HubError, } from './types.js';
|
|
8
10
|
import { Wire } from './wire.js';
|
|
@@ -20,6 +22,11 @@ export class PeerClient {
|
|
|
20
22
|
events;
|
|
21
23
|
#owned = new Map();
|
|
22
24
|
#tokens = new Map();
|
|
25
|
+
/**
|
|
26
|
+
* Connections forgotten as replaced since the roster flush in progress copied the list: its
|
|
27
|
+
* commit must not bring them back (cleared when a flush copies the list, so it stays small).
|
|
28
|
+
*/
|
|
29
|
+
#gone = new Set();
|
|
23
30
|
#peers = new Map();
|
|
24
31
|
#revision = 0;
|
|
25
32
|
#broadcasts = 0;
|
|
@@ -49,13 +56,19 @@ export class PeerClient {
|
|
|
49
56
|
#readyWaiters = new Set();
|
|
50
57
|
/** Failures and decisions on per-message paths, logged once per interval with a count. */
|
|
51
58
|
#throttle;
|
|
59
|
+
/**
|
|
60
|
+
* Told when a ready connection to the leader closed under this server (`DISCONNECTED`: the
|
|
61
|
+
* leader died or restarted) or the leader said it is stopping (`planned`). The hub node reports
|
|
62
|
+
* it as a failover (D-0097 §2: warn, error only past its bound). Without it the drop is a warn.
|
|
63
|
+
*/
|
|
64
|
+
onLinkLost;
|
|
52
65
|
constructor(options,
|
|
53
66
|
/** The hub node passes the plugin logger; the console default keeps failures visible. */
|
|
54
67
|
logger = new ConsoleLogger()) {
|
|
55
68
|
this.options = options;
|
|
56
69
|
this.logger = logger;
|
|
57
70
|
this.events = new EventQueue(options.maxPending, options.maxPendingBytes);
|
|
58
|
-
this.#throttle = new diagnostics.Repeats({ logger });
|
|
71
|
+
this.#throttle = new diagnostics.Repeats({ logger, intervalMs: options.logRepeatWindowMs });
|
|
59
72
|
}
|
|
60
73
|
get ready() {
|
|
61
74
|
return this.#ready && !this.#closing && !!this.#wire && !this.#wire.socket.destroyed;
|
|
@@ -71,11 +84,23 @@ export class PeerClient {
|
|
|
71
84
|
const code = error instanceof HubError ? error.code : diagnostics.errorCode(error, 'UNKNOWN');
|
|
72
85
|
// RECOVERING and CLOSED are this server's own decisions (reconnect, shutdown); any other
|
|
73
86
|
// cause (revision gap, stale leader, socket closed) is a failure.
|
|
74
|
-
if (code === 'RECOVERING' || code === 'CLOSED' || this.#leaving || this.#leaderLeaving)
|
|
87
|
+
if (code === 'RECOVERING' || code === 'CLOSED' || this.#leaving || this.#leaderLeaving) {
|
|
75
88
|
this.logger.log('debug', 'Hub connection dropped', {
|
|
76
89
|
code,
|
|
77
90
|
leader: this.#leader?.serverId ?? null,
|
|
78
91
|
});
|
|
92
|
+
if (this.#leaderLeaving && !this.#leaving && code !== 'CLOSED')
|
|
93
|
+
this.onLinkLost?.(error, true);
|
|
94
|
+
}
|
|
95
|
+
else if (code === 'DISCONNECTED') {
|
|
96
|
+
// The socket closed under us: the leader went away, a failover follows (D-0097 §2).
|
|
97
|
+
if (this.onLinkLost)
|
|
98
|
+
this.onLinkLost(error, false);
|
|
99
|
+
else
|
|
100
|
+
this.#throttle.log('warn', 'Hub connection dropped', code, error, {
|
|
101
|
+
leader: this.#leader?.serverId ?? null,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
79
104
|
else
|
|
80
105
|
this.#throttle.log('error', 'Hub connection dropped', code, error, {
|
|
81
106
|
leader: this.#leader?.serverId ?? null,
|
|
@@ -131,17 +156,32 @@ export class PeerClient {
|
|
|
131
156
|
if (this.#wire === wire)
|
|
132
157
|
this.#fail(new HubError('DISCONNECTED', socketError === undefined ? undefined : { cause: socketError }));
|
|
133
158
|
}, (code, error) => {
|
|
134
|
-
|
|
159
|
+
// A socket error is followed by its close: it becomes the cause of that DISCONNECTED,
|
|
160
|
+
// which is reported (the failover's first error, D-0097 §2). A refused frame is a fault.
|
|
161
|
+
if (established && code !== 'PEER_SOCKET_ERROR') {
|
|
135
162
|
this.#throttle.log('error', 'Hub peer connection failed', code, error, {
|
|
136
163
|
leader: leader.serverId,
|
|
137
164
|
});
|
|
138
165
|
return;
|
|
139
166
|
}
|
|
140
167
|
socketError = error;
|
|
168
|
+
if (established) {
|
|
169
|
+
if (this.logger.isEnabled('debug'))
|
|
170
|
+
this.logger.log('debug', 'Hub leader socket failed; the drop follows', {
|
|
171
|
+
code,
|
|
172
|
+
leader: leader.serverId,
|
|
173
|
+
error: diagnostics.describeError(error),
|
|
174
|
+
});
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
141
177
|
this.#throttle.log('debug', 'Hub peer connection failed before it was established', code, error, { leader: leader.serverId });
|
|
142
|
-
}
|
|
178
|
+
},
|
|
179
|
+
// The leader builds frames up to its own maxFrameBytes, which may exceed ours (D-0136).
|
|
180
|
+
PEER_LIMITS.frameBytes);
|
|
143
181
|
this.#wire = wire;
|
|
144
|
-
const timer = setTimeout(
|
|
182
|
+
const timer = setTimeout(
|
|
183
|
+
// A coded error: the failover counts it as a failed reconnect (D-0097 §2).
|
|
184
|
+
() => socket.destroy(new HubError('CONNECT_TIMEOUT')), this.options.leaseMs);
|
|
145
185
|
try {
|
|
146
186
|
await new Promise((resolve, reject) => {
|
|
147
187
|
socket.once('connect', resolve);
|
|
@@ -164,6 +204,9 @@ export class PeerClient {
|
|
|
164
204
|
serverId: this.options.serverId,
|
|
165
205
|
secret: this.options.secret ?? '',
|
|
166
206
|
connections: [...this.#owned.values()],
|
|
207
|
+
// The newest registration of a clientId wins: this server names what a set adds and
|
|
208
|
+
// closes what the leader replaces (audit 2026-10-26).
|
|
209
|
+
newestWins: true,
|
|
167
210
|
}));
|
|
168
211
|
}
|
|
169
212
|
catch (error) {
|
|
@@ -180,13 +223,17 @@ export class PeerClient {
|
|
|
180
223
|
throw error;
|
|
181
224
|
}
|
|
182
225
|
const peers = new Map();
|
|
183
|
-
|
|
226
|
+
// Other servers' lists follow their own limits and the leader's (D-0136), not ours.
|
|
227
|
+
for (const row of array(value.owners, PEER_LIMITS.owners)) {
|
|
184
228
|
const pair = array(row, 2);
|
|
185
229
|
const owner = epoch(pair[0]);
|
|
186
|
-
peers.set(owner, records(pair[1], owner,
|
|
230
|
+
peers.set(owner, records(pair[1], owner, PEER_LIMITS.connections));
|
|
187
231
|
}
|
|
188
232
|
if (this.#wire !== wire || socket.destroyed || this.#closing)
|
|
189
233
|
throw new HubError('DISCONNECTED');
|
|
234
|
+
// Connections of this server that newer registrations elsewhere replaced while its link was
|
|
235
|
+
// down (a tab that reconnected elsewhere while its socket here was half open): they close.
|
|
236
|
+
this.#replaced(value.replaced, 'hello');
|
|
190
237
|
// The snapshot replaces the mirror at once; report each owner whose list differs, and
|
|
191
238
|
// owners that are gone, as presence events so consumers (websocket presence) resync after
|
|
192
239
|
// a reconnect or a leader change instead of keeping connections that left meanwhile.
|
|
@@ -287,7 +334,7 @@ export class PeerClient {
|
|
|
287
334
|
}
|
|
288
335
|
if (rev !== this.#revision + 1)
|
|
289
336
|
throw new HubError('REVISION_GAP');
|
|
290
|
-
const owner = epoch(f.owner), values = records(f.connections, owner,
|
|
337
|
+
const owner = epoch(f.owner), values = records(f.connections, owner, PEER_LIMITS.connections);
|
|
291
338
|
this.#revision = rev;
|
|
292
339
|
if (values.length)
|
|
293
340
|
this.#peers.set(owner, values);
|
|
@@ -302,17 +349,23 @@ export class PeerClient {
|
|
|
302
349
|
this.events.push({ kind: 'presence', revision: rev, ownerEpoch: owner, connections: values });
|
|
303
350
|
return;
|
|
304
351
|
}
|
|
305
|
-
const connections = array(f.connections,
|
|
352
|
+
const connections = array(f.connections, PEER_LIMITS.connections).map(identifier), id = identifier(f.id);
|
|
306
353
|
let event;
|
|
307
354
|
if (f.kind === 'message')
|
|
308
355
|
event = {
|
|
309
356
|
kind: 'message',
|
|
310
357
|
id,
|
|
311
358
|
connections,
|
|
312
|
-
|
|
359
|
+
// The producer's server and the leader checked their maxPayloadBytes; ours may be smaller.
|
|
360
|
+
data: payload(f.data, PEER_LIMITS.payloadBytes),
|
|
313
361
|
};
|
|
314
|
-
else if (f.kind === 'kick')
|
|
362
|
+
else if (f.kind === 'kick') {
|
|
315
363
|
event = { kind: 'kick', id, connections, code: Number(f.code), reason: identifier(f.reason) };
|
|
364
|
+
// Newer registrations elsewhere took these: they leave this server's list at once, so no
|
|
365
|
+
// later set of this server carries them back (the gateway closes them on the event).
|
|
366
|
+
if (f.replaced === true)
|
|
367
|
+
this.#forget(connections, 'kick');
|
|
368
|
+
}
|
|
316
369
|
else
|
|
317
370
|
throw new HubError('PROTOCOL_ERROR');
|
|
318
371
|
// Broadcasts are at most once (D-0057) and producers no longer wait for ACKs, so a burst can
|
|
@@ -370,6 +423,7 @@ export class PeerClient {
|
|
|
370
423
|
throw new HubError('RECOVERING');
|
|
371
424
|
return fn();
|
|
372
425
|
});
|
|
426
|
+
// dokebi-check: allow empty-catch the chain only orders the next call; the caller gets the same promise with its failure
|
|
373
427
|
this.#tail = result
|
|
374
428
|
.then(() => { }, () => { })
|
|
375
429
|
.finally(() => {
|
|
@@ -382,24 +436,51 @@ export class PeerClient {
|
|
|
382
436
|
* refuse only this change; the batch goes on. `offline` changes (a connection leaving) also
|
|
383
437
|
* work while recovering: they only have to leave the local roster.
|
|
384
438
|
*/
|
|
385
|
-
#change(apply, offline = false) {
|
|
439
|
+
#change(apply, offline = false, added) {
|
|
386
440
|
if (!offline && !this.ready)
|
|
387
441
|
return Promise.reject(new HubError('RECOVERING'));
|
|
388
442
|
if (this.#roster.length >= this.options.maxConnections + this.options.maxPending)
|
|
389
443
|
return Promise.reject(new HubError('BUSY'));
|
|
390
444
|
return new Promise((resolve, reject) => {
|
|
391
|
-
this.#roster.push({
|
|
445
|
+
this.#roster.push({
|
|
446
|
+
apply,
|
|
447
|
+
offline,
|
|
448
|
+
resolve,
|
|
449
|
+
reject,
|
|
450
|
+
...(added === undefined ? {} : { added }),
|
|
451
|
+
});
|
|
392
452
|
// The first change of a batch queues the flush; later ones join it until it starts.
|
|
393
453
|
if (this.#roster.length === 1)
|
|
394
454
|
void this.#operation(() => this.#flushRoster(), true);
|
|
395
455
|
});
|
|
396
456
|
}
|
|
457
|
+
/**
|
|
458
|
+
* Sends the queued roster changes as one batch. Never rejects: the call is not awaited (it only
|
|
459
|
+
* orders the next operation), so a failure after the leader answered (the Replaced kick did not
|
|
460
|
+
* fit the event queue, an answer that is not one) is logged here and settles every change of
|
|
461
|
+
* the batch that is still waiting (audit R5 review: the registration stayed pending for good,
|
|
462
|
+
* and the websocket gateway's drain waited for it until its deadline).
|
|
463
|
+
*/
|
|
397
464
|
async #flushRoster() {
|
|
398
465
|
const batch = this.#roster;
|
|
399
466
|
this.#roster = [];
|
|
467
|
+
try {
|
|
468
|
+
await this.#sendRoster(batch);
|
|
469
|
+
}
|
|
470
|
+
catch (error) {
|
|
471
|
+
this.#throttle.log('error', 'Hub roster change failed after the leader answered; its changes are refused', 'ROSTER_COMMIT_FAILED', error, { changes: batch.length });
|
|
472
|
+
// Settled changes stay as they are (a Promise settles once); the rest are refused. The
|
|
473
|
+
// leader saved a list this server did not take: reconnect, so the hello resyncs it.
|
|
474
|
+
this.#rosterFailed(batch, error, false);
|
|
475
|
+
if (!this.#closing)
|
|
476
|
+
this.disconnect();
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
async #sendRoster(batch) {
|
|
400
480
|
const started = performance.now();
|
|
401
481
|
const online = this.ready;
|
|
402
482
|
const next = new Map(this.#owned);
|
|
483
|
+
this.#gone.clear();
|
|
403
484
|
const applied = [];
|
|
404
485
|
for (const change of batch) {
|
|
405
486
|
if (!online && !change.offline) {
|
|
@@ -416,22 +497,36 @@ export class PeerClient {
|
|
|
416
497
|
}
|
|
417
498
|
if (!applied.length)
|
|
418
499
|
return;
|
|
500
|
+
const added = applied.flatMap((change) => (change.added === undefined ? [] : [change.added]));
|
|
501
|
+
let answer;
|
|
419
502
|
if (online)
|
|
420
503
|
try {
|
|
421
|
-
await this.#rpc('set', { connections: [...next.values()] });
|
|
504
|
+
answer = await this.#rpc('set', { connections: [...next.values()], added });
|
|
422
505
|
}
|
|
423
506
|
catch (error) {
|
|
424
|
-
//
|
|
425
|
-
//
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
507
|
+
// The leader refused the list (a duplicate clientId, a limit) and the link is up: one
|
|
508
|
+
// change caused it, so each is sent on its own and only the refused ones fail. Otherwise
|
|
509
|
+
// a tab reconnecting with a clientId still held elsewhere took every registration of its
|
|
510
|
+
// batch down with it (closed 4409 or 1013), as after a failover when many reconnect.
|
|
511
|
+
if (applied.length > 1 && this.ready && error instanceof HubError) {
|
|
512
|
+
if (this.logger.isEnabled('debug'))
|
|
513
|
+
this.logger.log('debug', 'Hub roster refused; sending its changes one by one', {
|
|
514
|
+
code: error.code,
|
|
515
|
+
changes: applied.length,
|
|
516
|
+
});
|
|
517
|
+
await this.#flushEach(applied);
|
|
518
|
+
return;
|
|
519
|
+
}
|
|
520
|
+
this.#rosterFailed(applied, error);
|
|
433
521
|
return;
|
|
434
522
|
}
|
|
523
|
+
// What the leader replaced, and what kick frames replaced while the set was on its way (in
|
|
524
|
+
// the same read as its answer): the commit must not keep them.
|
|
525
|
+
const replaced = answer?.replaced;
|
|
526
|
+
for (const connection of this.#replaced(replaced, 'set', new Set(added)))
|
|
527
|
+
next.delete(connection);
|
|
528
|
+
for (const connection of this.#gone)
|
|
529
|
+
next.delete(connection);
|
|
435
530
|
this.#owned.clear();
|
|
436
531
|
for (const [key, value] of next)
|
|
437
532
|
this.#owned.set(key, value);
|
|
@@ -446,6 +541,101 @@ export class PeerClient {
|
|
|
446
541
|
ms: Math.round(performance.now() - started),
|
|
447
542
|
});
|
|
448
543
|
}
|
|
544
|
+
/**
|
|
545
|
+
* A batch whose `set` failed: leaving connections leave locally anyway and the client
|
|
546
|
+
* reconnects to resync (as a single close did); every other change is refused with the cause.
|
|
547
|
+
*/
|
|
548
|
+
#rosterFailed(changes, error, reconnect = true) {
|
|
549
|
+
const leaving = changes.filter((change) => change.offline);
|
|
550
|
+
for (const change of leaving)
|
|
551
|
+
change.apply(this.#owned);
|
|
552
|
+
for (const change of changes)
|
|
553
|
+
change.reject(error);
|
|
554
|
+
if (leaving.length && reconnect)
|
|
555
|
+
this.disconnect();
|
|
556
|
+
}
|
|
557
|
+
/** The changes of a refused batch, one `set` each (changes re-apply to the current list). */
|
|
558
|
+
async #flushEach(changes) {
|
|
559
|
+
for (const [index, change] of changes.entries()) {
|
|
560
|
+
const next = new Map(this.#owned);
|
|
561
|
+
this.#gone.clear();
|
|
562
|
+
try {
|
|
563
|
+
change.apply(next);
|
|
564
|
+
}
|
|
565
|
+
catch (error) {
|
|
566
|
+
change.reject(error);
|
|
567
|
+
continue;
|
|
568
|
+
}
|
|
569
|
+
const added = change.added === undefined ? [] : [change.added];
|
|
570
|
+
let answer;
|
|
571
|
+
try {
|
|
572
|
+
answer = await this.#rpc('set', { connections: [...next.values()], added });
|
|
573
|
+
}
|
|
574
|
+
catch (error) {
|
|
575
|
+
if (this.ready && error instanceof HubError && !change.offline) {
|
|
576
|
+
change.reject(error);
|
|
577
|
+
continue;
|
|
578
|
+
}
|
|
579
|
+
// The link went down (or a leave was refused): as a failed batch, for the rest too.
|
|
580
|
+
this.#rosterFailed(changes.slice(index), error);
|
|
581
|
+
return;
|
|
582
|
+
}
|
|
583
|
+
const replaced = answer?.replaced;
|
|
584
|
+
for (const connection of this.#replaced(replaced, 'set', new Set(added)))
|
|
585
|
+
next.delete(connection);
|
|
586
|
+
for (const connection of this.#gone)
|
|
587
|
+
next.delete(connection);
|
|
588
|
+
this.#owned.clear();
|
|
589
|
+
for (const [key, value] of next)
|
|
590
|
+
this.#owned.set(key, value);
|
|
591
|
+
change.resolve();
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
/**
|
|
595
|
+
* Connections the leader replaced (its answer to a hello or a set): forgotten here and closed
|
|
596
|
+
* by the gateway (a kick event, 4409 Replaced). Returns them, so a set's list drops them too.
|
|
597
|
+
*/
|
|
598
|
+
#replaced(value, source, added = new Set()) {
|
|
599
|
+
if (value === undefined)
|
|
600
|
+
return [];
|
|
601
|
+
const connections = array(value, PEER_LIMITS.connections).map(identifier);
|
|
602
|
+
if (!connections.length)
|
|
603
|
+
return connections;
|
|
604
|
+
// A connection a kick frame closed already is forgotten; only those still held here (or
|
|
605
|
+
// registered by this set) are closed now, once.
|
|
606
|
+
const open = connections.filter((connection) => this.#owned.has(connection) || added.has(connection));
|
|
607
|
+
this.#forget(connections, source);
|
|
608
|
+
if (!open.length)
|
|
609
|
+
return connections;
|
|
610
|
+
try {
|
|
611
|
+
this.events.push({
|
|
612
|
+
kind: 'kick',
|
|
613
|
+
id: randomUUID(),
|
|
614
|
+
connections: Object.freeze(open),
|
|
615
|
+
code: REPLACED.code,
|
|
616
|
+
reason: REPLACED.reason,
|
|
617
|
+
});
|
|
618
|
+
}
|
|
619
|
+
catch (error) {
|
|
620
|
+
this.events.finish(error);
|
|
621
|
+
this.#closing = true;
|
|
622
|
+
throw error;
|
|
623
|
+
}
|
|
624
|
+
return connections;
|
|
625
|
+
}
|
|
626
|
+
/** Drops replaced connections from the roster and their handles (they no longer change it). */
|
|
627
|
+
#forget(connections, source) {
|
|
628
|
+
for (const connection of connections) {
|
|
629
|
+
this.#owned.delete(connection);
|
|
630
|
+
this.#tokens.delete(connection);
|
|
631
|
+
this.#gone.add(connection);
|
|
632
|
+
}
|
|
633
|
+
if (this.logger.isEnabled('debug'))
|
|
634
|
+
this.logger.log('debug', 'Hub connections replaced by newer registrations', {
|
|
635
|
+
connections: connections.length,
|
|
636
|
+
source,
|
|
637
|
+
});
|
|
638
|
+
}
|
|
449
639
|
async register(input) {
|
|
450
640
|
const value = record(input, this.ownerEpoch);
|
|
451
641
|
await this.#change((next) => {
|
|
@@ -454,7 +644,7 @@ export class PeerClient {
|
|
|
454
644
|
if (next.size >= this.options.maxConnections)
|
|
455
645
|
throw new HubError('CONNECTION_LIMIT');
|
|
456
646
|
next.set(value.connection, value);
|
|
457
|
-
});
|
|
647
|
+
}, false, value.connection);
|
|
458
648
|
{
|
|
459
649
|
const token = Symbol();
|
|
460
650
|
this.#tokens.set(value.connection, token);
|
|
@@ -538,7 +728,7 @@ export class PeerClient {
|
|
|
538
728
|
const value = result;
|
|
539
729
|
return Object.freeze({
|
|
540
730
|
id: epoch(value.id),
|
|
541
|
-
recipients: Object.freeze(array(value.recipients,
|
|
731
|
+
recipients: Object.freeze(array(value.recipients, PEER_LIMITS.totalConnections).map((key) => {
|
|
542
732
|
if (typeof key !== 'string')
|
|
543
733
|
throw new HubError('PROTOCOL_ERROR');
|
|
544
734
|
return key;
|
package/dist/protocol.js
CHANGED
|
@@ -8,6 +8,12 @@ import { integer, keys, object } from './values.js';
|
|
|
8
8
|
*/
|
|
9
9
|
export const PROTOCOL = 2;
|
|
10
10
|
const decoder = new TextDecoder('utf-8', { fatal: true });
|
|
11
|
+
/**
|
|
12
|
+
* A lone surrogate: the one thing a JS string can hold that UTF-8 cannot, so the string would not
|
|
13
|
+
* survive the wire unchanged. Same answer as comparing a UTF-8 round trip, without two copies per
|
|
14
|
+
* identifier (identifiers are checked for every record of every presence frame).
|
|
15
|
+
*/
|
|
16
|
+
const LONE_SURROGATE = /\p{Cs}/u;
|
|
11
17
|
export function encode(value, limit) {
|
|
12
18
|
try {
|
|
13
19
|
const data = Buffer.from(JSON.stringify(value), 'utf8');
|
|
@@ -28,6 +34,7 @@ export function decode(data, limit) {
|
|
|
28
34
|
return object(JSON.parse(decoder.decode(data)));
|
|
29
35
|
}
|
|
30
36
|
catch {
|
|
37
|
+
// dokebi-check: allow missing-cause a JSON.parse message quotes the frame, and a hello frame holds the secret
|
|
31
38
|
throw new HubError('PROTOCOL_ERROR');
|
|
32
39
|
}
|
|
33
40
|
}
|
|
@@ -35,7 +42,7 @@ export function identifier(value) {
|
|
|
35
42
|
if (typeof value !== 'string' ||
|
|
36
43
|
value.length === 0 ||
|
|
37
44
|
Buffer.byteLength(value, 'utf8') > 256 ||
|
|
38
|
-
|
|
45
|
+
LONE_SURROGATE.test(value))
|
|
39
46
|
throw new HubError('INVALID_ARGUMENT');
|
|
40
47
|
return value;
|
|
41
48
|
}
|
|
@@ -69,7 +76,7 @@ export function metadata(value = {}) {
|
|
|
69
76
|
if (Buffer.byteLength(identifier(key), 'utf8') > 64 ||
|
|
70
77
|
typeof item !== 'string' ||
|
|
71
78
|
Buffer.byteLength(item, 'utf8') > 1024 ||
|
|
72
|
-
|
|
79
|
+
LONE_SURROGATE.test(item))
|
|
73
80
|
throw new HubError('INVALID_ARGUMENT');
|
|
74
81
|
Object.defineProperty(result, key, { value: item, enumerable: true });
|
|
75
82
|
}
|
|
@@ -131,8 +138,8 @@ export function sequence(value) {
|
|
|
131
138
|
try {
|
|
132
139
|
return integer(value, 0, Number.MAX_SAFE_INTEGER - 1);
|
|
133
140
|
}
|
|
134
|
-
catch {
|
|
135
|
-
throw new HubError('PROTOCOL_ERROR');
|
|
141
|
+
catch (error) {
|
|
142
|
+
throw new HubError('PROTOCOL_ERROR', { cause: error });
|
|
136
143
|
}
|
|
137
144
|
}
|
|
138
145
|
export function payload(value, maxBytes) {
|
package/dist/status-cli.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
// dokebi-check: allow console-output a command line tool that prints the hub status (D-0151)
|
|
2
3
|
// `dokebi-hub-status` — "how is the hub right now?" asked from outside every application process
|
|
3
4
|
// (D-0038 · original `x hubserver:status`). It reads the leader record and the owner records from
|
|
4
5
|
// Redis only; it does not start a hub, register anything or join the election.
|
|
@@ -12,7 +13,7 @@ import { hubStatus } from './status.js';
|
|
|
12
13
|
import { HubStore } from './store.js';
|
|
13
14
|
const help = `Usage: dokebi-hub-status [options]
|
|
14
15
|
|
|
15
|
-
--config PATH YAML file (default: dokebi.yaml)
|
|
16
|
+
--config PATH YAML file (default: $DOKEBI_CONFIG, else dokebi.yaml)
|
|
16
17
|
--json Emit the status as JSON
|
|
17
18
|
--help Show this help
|
|
18
19
|
|
|
@@ -54,7 +55,7 @@ async function main() {
|
|
|
54
55
|
process.stdout.write(help);
|
|
55
56
|
return;
|
|
56
57
|
}
|
|
57
|
-
const document = await loadConfig(resolve(values.config ?? 'dokebi.yaml'));
|
|
58
|
+
const document = await loadConfig(resolve(values.config ?? (process.env.DOKEBI_CONFIG || 'dokebi.yaml')));
|
|
58
59
|
const options = parseOptions(document.options['dokebi-websocket-hub'] ?? {});
|
|
59
60
|
let status;
|
|
60
61
|
if (options.mode === 'local')
|
package/dist/status.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { Logger } from '@dokebi/core';
|
|
2
|
+
import { type HubApi, type HubLease, type HubStatus } from './types.js';
|
|
2
3
|
/**
|
|
3
4
|
* The one place that turns facts into a status (D-0038 · original x-hubserver `hubStatus`).
|
|
4
5
|
* `HubApi.status()` and `dokebi-hub-status` both call it, so a health check, a dashboard and the
|
|
@@ -12,11 +13,26 @@ export declare function hubStatus(input: {
|
|
|
12
13
|
/** cluster only: what Redis said, or `unreachable` when it did not answer. */
|
|
13
14
|
lease?: HubLease | 'unreachable';
|
|
14
15
|
}): HubStatus;
|
|
15
|
-
/**
|
|
16
|
+
/**
|
|
17
|
+
* In-process nonce store for local mode: correct for one process only (`shared: false`).
|
|
18
|
+
*
|
|
19
|
+
* Full of **live** nonces it refuses a new one with `NONCE_STORE_FULL` instead of forgetting a
|
|
20
|
+
* live one (D-0075, fail closed: a forgotten nonce lets its captured frame be replayed). Seal turns
|
|
21
|
+
* the refusal into a retryable `SEAL_REPLAY_CAPACITY`; room returns as the nonces expire.
|
|
22
|
+
*/
|
|
16
23
|
export declare class LocalNonceStore {
|
|
17
24
|
#private;
|
|
18
|
-
readonly max: number;
|
|
19
25
|
readonly shared = false;
|
|
20
|
-
|
|
26
|
+
readonly max: number;
|
|
27
|
+
constructor(options?: {
|
|
28
|
+
max?: number;
|
|
29
|
+
now?: () => number;
|
|
30
|
+
/** Warned once per `warnIntervalMs` (10 s), with counts, while the store is full (D-0047). */
|
|
31
|
+
logger?: Pick<Logger, 'log'> | undefined;
|
|
32
|
+
/** The hub's `logRepeatWindowMs` (D-0114). */
|
|
33
|
+
warnIntervalMs?: number;
|
|
34
|
+
});
|
|
35
|
+
/** When this store started empty (its creation): frames stamped before it may be replays. */
|
|
36
|
+
lostAt(): number;
|
|
21
37
|
setIfNotExists(key: string, ttlSeconds: number): Promise<boolean>;
|
|
22
38
|
}
|
package/dist/status.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { HubError } from './types.js';
|
|
1
2
|
/**
|
|
2
3
|
* The one place that turns facts into a status (D-0038 · original x-hubserver `hubStatus`).
|
|
3
4
|
* `HubApi.status()` and `dokebi-hub-status` both call it, so a health check, a dashboard and the
|
|
@@ -32,31 +33,135 @@ export function hubStatus(input) {
|
|
|
32
33
|
connections: lease.connections,
|
|
33
34
|
});
|
|
34
35
|
}
|
|
35
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* A min-heap of [expires, key] pairs by expiry (ISSUE-0006 recheck: a scan of the oldest
|
|
38
|
+
* insertions missed expired entries further back when TTLs differ). Pairs of keys that were
|
|
39
|
+
* renewed or removed since are skipped when they reach the top.
|
|
40
|
+
*/
|
|
41
|
+
class ExpiryHeap {
|
|
42
|
+
#items = [];
|
|
43
|
+
push(expires, key) {
|
|
44
|
+
const items = this.#items;
|
|
45
|
+
items.push([expires, key]);
|
|
46
|
+
let index = items.length - 1;
|
|
47
|
+
while (index > 0) {
|
|
48
|
+
const parent = (index - 1) >> 1;
|
|
49
|
+
const above = items[parent];
|
|
50
|
+
const here = items[index];
|
|
51
|
+
if (above[0] <= here[0])
|
|
52
|
+
break;
|
|
53
|
+
items[parent] = here;
|
|
54
|
+
items[index] = above;
|
|
55
|
+
index = parent;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
peek() {
|
|
59
|
+
return this.#items[0];
|
|
60
|
+
}
|
|
61
|
+
pop() {
|
|
62
|
+
const items = this.#items;
|
|
63
|
+
const last = items.pop();
|
|
64
|
+
if (last === undefined || items.length === 0)
|
|
65
|
+
return;
|
|
66
|
+
items[0] = last;
|
|
67
|
+
let index = 0;
|
|
68
|
+
for (;;) {
|
|
69
|
+
const left = 2 * index + 1;
|
|
70
|
+
const right = left + 1;
|
|
71
|
+
let smallest = index;
|
|
72
|
+
if (left < items.length &&
|
|
73
|
+
items[left][0] < items[smallest][0])
|
|
74
|
+
smallest = left;
|
|
75
|
+
if (right < items.length &&
|
|
76
|
+
items[right][0] < items[smallest][0])
|
|
77
|
+
smallest = right;
|
|
78
|
+
if (smallest === index)
|
|
79
|
+
break;
|
|
80
|
+
const moved = items[smallest];
|
|
81
|
+
items[smallest] = items[index];
|
|
82
|
+
items[index] = moved;
|
|
83
|
+
index = smallest;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
const WARN_EVERY_MS = 10_000;
|
|
88
|
+
const FULL_FIX = 'More sealed frames arrived within 2 x drift than the local nonce store keeps: use mode: cluster (Redis, no such limit), or lower the seal drift window (dokebi-websocket sealDriftMs).';
|
|
89
|
+
/**
|
|
90
|
+
* In-process nonce store for local mode: correct for one process only (`shared: false`).
|
|
91
|
+
*
|
|
92
|
+
* Full of **live** nonces it refuses a new one with `NONCE_STORE_FULL` instead of forgetting a
|
|
93
|
+
* live one (D-0075, fail closed: a forgotten nonce lets its captured frame be replayed). Seal turns
|
|
94
|
+
* the refusal into a retryable `SEAL_REPLAY_CAPACITY`; room returns as the nonces expire.
|
|
95
|
+
*/
|
|
36
96
|
export class LocalNonceStore {
|
|
37
|
-
max;
|
|
38
97
|
shared = false;
|
|
98
|
+
max;
|
|
39
99
|
#entries = new Map();
|
|
100
|
+
#expiry = new ExpiryHeap();
|
|
101
|
+
#now;
|
|
102
|
+
#logger;
|
|
40
103
|
#inserts = 0;
|
|
41
|
-
|
|
42
|
-
|
|
104
|
+
/** The store starts empty: what a previous process accepted is unknown here. */
|
|
105
|
+
#createdAt;
|
|
106
|
+
#warnedAt = Number.NEGATIVE_INFINITY;
|
|
107
|
+
#refused = 0;
|
|
108
|
+
#warnEveryMs;
|
|
109
|
+
constructor(options = {}) {
|
|
110
|
+
this.max = options.max ?? 100_000;
|
|
111
|
+
this.#now = options.now ?? Date.now;
|
|
112
|
+
this.#logger = options.logger;
|
|
113
|
+
this.#warnEveryMs = options.warnIntervalMs ?? WARN_EVERY_MS;
|
|
114
|
+
this.#createdAt = this.#now();
|
|
115
|
+
}
|
|
116
|
+
/** When this store started empty (its creation): frames stamped before it may be replays. */
|
|
117
|
+
lostAt() {
|
|
118
|
+
return this.#createdAt;
|
|
43
119
|
}
|
|
44
120
|
async setIfNotExists(key, ttlSeconds) {
|
|
45
|
-
const now =
|
|
121
|
+
const now = this.#now();
|
|
46
122
|
const existing = this.#entries.get(key);
|
|
47
|
-
if (existing !== undefined
|
|
48
|
-
|
|
123
|
+
if (existing !== undefined) {
|
|
124
|
+
if (existing > now)
|
|
125
|
+
return false;
|
|
126
|
+
// Expired: deleted first, so the renewed entry moves to the end (insertion order = age).
|
|
127
|
+
this.#entries.delete(key);
|
|
128
|
+
}
|
|
129
|
+
// Full: expired entries make room (earliest expiry first). A live nonce is never forgotten.
|
|
130
|
+
if (this.#entries.size >= this.max && !this.#evictExpired(now))
|
|
131
|
+
this.#full(now);
|
|
49
132
|
this.#entries.set(key, now + ttlSeconds * 1000);
|
|
133
|
+
this.#expiry.push(now + ttlSeconds * 1000, key);
|
|
50
134
|
if (++this.#inserts % 500 === 0)
|
|
51
|
-
|
|
52
|
-
if (expires <= now)
|
|
53
|
-
this.#entries.delete(entry);
|
|
54
|
-
// Beyond the cap the oldest go first: a thinner defence beats running out of memory.
|
|
55
|
-
for (const entry of this.#entries.keys()) {
|
|
56
|
-
if (this.#entries.size <= this.max)
|
|
57
|
-
break;
|
|
58
|
-
this.#entries.delete(entry);
|
|
59
|
-
}
|
|
135
|
+
this.#evictExpired(now);
|
|
60
136
|
return true;
|
|
61
137
|
}
|
|
138
|
+
/** Deletes every expired entry, earliest expiry first (each pair is popped once); true when that made room. */
|
|
139
|
+
#evictExpired(now) {
|
|
140
|
+
for (let top = this.#expiry.peek(); top !== undefined && top[0] <= now; top = this.#expiry.peek()) {
|
|
141
|
+
this.#expiry.pop();
|
|
142
|
+
// A renewed key has a newer pair; only the pair that matches the stored expiry deletes it.
|
|
143
|
+
if (this.#entries.get(top[1]) === top[0])
|
|
144
|
+
this.#entries.delete(top[1]);
|
|
145
|
+
}
|
|
146
|
+
return this.#entries.size < this.max;
|
|
147
|
+
}
|
|
148
|
+
#full(now) {
|
|
149
|
+
// Per frame: counted, and said once per interval with the count (D-0047).
|
|
150
|
+
this.#refused += 1;
|
|
151
|
+
if (this.#logger && now - this.#warnedAt >= this.#warnEveryMs) {
|
|
152
|
+
this.#logger.log('warn', 'Replay nonce store full; sealed frames refused', {
|
|
153
|
+
code: 'NONCE_STORE_FULL',
|
|
154
|
+
max: this.max,
|
|
155
|
+
live: this.#entries.size,
|
|
156
|
+
repeated: this.#refused - 1,
|
|
157
|
+
fix: FULL_FIX,
|
|
158
|
+
});
|
|
159
|
+
this.#warnedAt = now;
|
|
160
|
+
this.#refused = 0;
|
|
161
|
+
}
|
|
162
|
+
throw new HubError('NONCE_STORE_FULL', {
|
|
163
|
+
fix: FULL_FIX,
|
|
164
|
+
details: { max: this.max },
|
|
165
|
+
});
|
|
166
|
+
}
|
|
62
167
|
}
|