@dokebi/websocket-hub 0.4.1 → 0.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AI.md +17 -10
- package/README.md +4 -4
- package/dist/delivery.d.ts +10 -0
- package/dist/delivery.js +7 -0
- package/dist/index.js +21 -3
- package/dist/leader.d.ts +8 -1
- package/dist/leader.js +304 -42
- package/dist/local.js +38 -12
- package/dist/node.js +195 -43
- package/dist/notifications.d.ts +31 -0
- package/dist/notifications.js +133 -0
- package/dist/options.d.ts +16 -0
- package/dist/options.js +26 -5
- package/dist/peer-client.d.ts +7 -1
- package/dist/peer-client.js +215 -25
- package/dist/protocol.js +11 -4
- package/dist/status-cli.js +3 -2
- package/dist/status.d.ts +5 -1
- package/dist/status.js +10 -1
- package/dist/store.d.ts +11 -2
- package/dist/store.js +91 -15
- package/dist/types.d.ts +8 -0
- package/dist/types.js +2 -1
- package/dist/wire.d.ts +17 -2
- package/dist/wire.js +32 -8
- package/options.schema.json +22 -0
- package/package.json +5 -5
package/dist/leader.js
CHANGED
|
@@ -1,11 +1,27 @@
|
|
|
1
1
|
import { randomUUID, timingSafeEqual } from 'node:crypto';
|
|
2
2
|
import { diagnostics } from '@dokebi/core/libraries';
|
|
3
|
-
import { recipientKey } from './delivery.js';
|
|
3
|
+
import { REPLACED, recipientKey } from './delivery.js';
|
|
4
4
|
import { HubIndex } from './hub-index.js';
|
|
5
|
-
import {
|
|
5
|
+
import { PEER_LIMITS } from './options.js';
|
|
6
|
+
import { array, epoch, identifier, PROTOCOL, payload, query, records } from './protocol.js';
|
|
6
7
|
import { HubError } from './types.js';
|
|
7
8
|
import { integer } from './values.js';
|
|
8
|
-
import { Wire } from './wire.js';
|
|
9
|
+
import { frame, Wire } from './wire.js';
|
|
10
|
+
/**
|
|
11
|
+
* UTF-8 bytes of `JSON.stringify({ owners: [[owner, list], …] })` (what a reconnecting peer's hello
|
|
12
|
+
* answer carries) from each owner's id and the bytes of its list's JSON: `{"owners":[` and `]}`,
|
|
13
|
+
* then `["<owner>",<list>]` per owner, separated by commas. Owner ids are epochs (UUIDs): no
|
|
14
|
+
* escaping.
|
|
15
|
+
*/
|
|
16
|
+
export function snapshotBytes(lists) {
|
|
17
|
+
let bytes = 13;
|
|
18
|
+
let count = 0;
|
|
19
|
+
for (const [owner, listBytes] of lists) {
|
|
20
|
+
bytes += Buffer.byteLength(owner) + 5 + listBytes;
|
|
21
|
+
count++;
|
|
22
|
+
}
|
|
23
|
+
return bytes + Math.max(0, count - 1);
|
|
24
|
+
}
|
|
9
25
|
export class Leader {
|
|
10
26
|
identity;
|
|
11
27
|
store;
|
|
@@ -14,8 +30,17 @@ export class Leader {
|
|
|
14
30
|
#peers = new Map();
|
|
15
31
|
#sockets = new Set();
|
|
16
32
|
#owners = new Map();
|
|
33
|
+
/** UTF-8 bytes of each owner's connection list as JSON, for the snapshot bound; follows #owners. */
|
|
34
|
+
#listBytes = new Map();
|
|
17
35
|
/** Every owner's connections by tenant, member and room (D-0068); follows #owners. */
|
|
18
36
|
#index = new HubIndex();
|
|
37
|
+
/**
|
|
38
|
+
* Connections the leader took from a server that may not know the newest-wins rule (a server
|
|
39
|
+
* before it, or one not connected here): such a server keeps sending them until its socket has
|
|
40
|
+
* closed, and its list must not be refused for them. Pruned as its lists leave them out, and
|
|
41
|
+
* dropped with the owner (`#remove`) or when it says hello with the rule.
|
|
42
|
+
*/
|
|
43
|
+
#replacedOf = new Map();
|
|
19
44
|
#revision = 0;
|
|
20
45
|
#tail = Promise.resolve();
|
|
21
46
|
#pending = 0;
|
|
@@ -28,7 +53,7 @@ export class Leader {
|
|
|
28
53
|
this.store = store;
|
|
29
54
|
this.options = options;
|
|
30
55
|
this.logger = logger;
|
|
31
|
-
this.#throttle = new diagnostics.Repeats({ logger });
|
|
56
|
+
this.#throttle = new diagnostics.Repeats({ logger, intervalMs: options.logRepeatWindowMs });
|
|
32
57
|
}
|
|
33
58
|
/**
|
|
34
59
|
* A frame to a peer could not be written (its socket is gone, its send queue passed maxPendingBytes,
|
|
@@ -74,8 +99,11 @@ export class Leader {
|
|
|
74
99
|
const snapshot = await this.store.snapshot(this.identity.epoch);
|
|
75
100
|
this.#owners = snapshot.owners;
|
|
76
101
|
this.#index.clear();
|
|
77
|
-
|
|
102
|
+
this.#listBytes.clear();
|
|
103
|
+
for (const [ownerId, owner] of this.#owners) {
|
|
78
104
|
this.#index.setOwner(ownerId, owner.connections);
|
|
105
|
+
this.#listBytes.set(ownerId, Buffer.byteLength(JSON.stringify(owner.connections)));
|
|
106
|
+
}
|
|
79
107
|
this.#revision = snapshot.revision;
|
|
80
108
|
await this.#expire(snapshot.now);
|
|
81
109
|
this.logger.log('debug', 'Hub leader started', {
|
|
@@ -94,8 +122,15 @@ export class Leader {
|
|
|
94
122
|
throw new HubError('STALE_LEADER');
|
|
95
123
|
return now;
|
|
96
124
|
}
|
|
97
|
-
|
|
98
|
-
|
|
125
|
+
/**
|
|
126
|
+
* Runs `fn` after the requests before it. `maxPending` bounds what peers ask for (BUSY); the
|
|
127
|
+
* leader's `own` work — its lease tick, and removing a server whose socket closed — waits its
|
|
128
|
+
* turn whatever the queue holds: refused as BUSY it read as a lost lease, and the node stopped
|
|
129
|
+
* its leader and dropped every peer in the middle of the load (audit R9-g11-3). It is bounded
|
|
130
|
+
* by itself: one tick at a time (the node), one removal per peer socket (`maxOwners`).
|
|
131
|
+
*/
|
|
132
|
+
#queue(fn, own = false) {
|
|
133
|
+
if (!own && this.#pending >= this.options.maxPending) {
|
|
99
134
|
this.#throttle.log('debug', 'Hub leader queue full; request refused', 'BUSY', undefined, {
|
|
100
135
|
pending: this.#pending,
|
|
101
136
|
});
|
|
@@ -107,6 +142,7 @@ export class Leader {
|
|
|
107
142
|
throw new HubError('STALE_LEADER');
|
|
108
143
|
return fn();
|
|
109
144
|
});
|
|
145
|
+
// dokebi-check: allow empty-catch the chain only orders the next call; the caller gets the same promise with its failure
|
|
110
146
|
this.#tail = work
|
|
111
147
|
.then(() => { }, () => { })
|
|
112
148
|
.finally(() => {
|
|
@@ -158,7 +194,7 @@ export class Leader {
|
|
|
158
194
|
void this.#queue(async () => {
|
|
159
195
|
await this.#renew();
|
|
160
196
|
await this.#remove(peer.owner, peer.session);
|
|
161
|
-
}).catch((error) => {
|
|
197
|
+
}, true).catch((error) => {
|
|
162
198
|
// Lease lost or Redis down: another server takes over and expires the owner.
|
|
163
199
|
this.logger.log('error', 'Hub leader could not remove a disconnected server', {
|
|
164
200
|
code: 'OWNER_REMOVE_FAILED',
|
|
@@ -262,50 +298,234 @@ export class Leader {
|
|
|
262
298
|
throw new HubError('STALE_LEADER');
|
|
263
299
|
peer.wire.send({ ...value, epoch: this.identity.epoch });
|
|
264
300
|
}
|
|
301
|
+
/** One presence frame, serialized once and written to every peer (the same bytes for all). */
|
|
265
302
|
#notify(owner, connections) {
|
|
303
|
+
if (!this.#peers.size)
|
|
304
|
+
return;
|
|
305
|
+
let bytes;
|
|
306
|
+
try {
|
|
307
|
+
if (!this.active)
|
|
308
|
+
throw new HubError('STALE_LEADER');
|
|
309
|
+
bytes = frame({
|
|
310
|
+
kind: 'presence',
|
|
311
|
+
revision: this.#revision,
|
|
312
|
+
owner,
|
|
313
|
+
connections,
|
|
314
|
+
epoch: this.identity.epoch,
|
|
315
|
+
}, this.options.maxFrameBytes);
|
|
316
|
+
}
|
|
317
|
+
catch (error) {
|
|
318
|
+
// As when each peer's send failed: every peer is dropped and resyncs on reconnect.
|
|
319
|
+
for (const peer of this.#peers.values())
|
|
320
|
+
this.#sendFailed(peer, 'presence', error);
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
266
323
|
for (const peer of this.#peers.values()) {
|
|
267
324
|
try {
|
|
268
|
-
|
|
325
|
+
if (!this.active)
|
|
326
|
+
throw new HubError('STALE_LEADER');
|
|
327
|
+
peer.wire.sendFrame(bytes);
|
|
269
328
|
}
|
|
270
329
|
catch (error) {
|
|
271
330
|
this.#sendFailed(peer, 'presence', error);
|
|
272
331
|
}
|
|
273
332
|
}
|
|
274
333
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
334
|
+
/**
|
|
335
|
+
* One `(tenant, member, clientId)` at a time (D-0025), and the newest registration wins (audit
|
|
336
|
+
* 2026-10-26): a `set` that adds a connection whose clientId another server holds takes it, and
|
|
337
|
+
* that server's record goes (its server closes it, 4409 Replaced). Records the saving server
|
|
338
|
+
* only carries over (a hello after its link dropped, or a record the leader replaced while the
|
|
339
|
+
* server's `set` was on its way) lose to a newer holder instead, so a reconnecting server is
|
|
340
|
+
* never refused as a whole and a replaced tab never comes back. Checked through the index
|
|
341
|
+
* (D-0068): O(list). A server before the rule (`added` unknown) is refused as before
|
|
342
|
+
* (DUPLICATE_CLIENT), so its lists cannot flap between two holders.
|
|
343
|
+
*/
|
|
344
|
+
#resolve(ownerId, connections, saving) {
|
|
345
|
+
const keep = [];
|
|
346
|
+
const replaced = [];
|
|
347
|
+
const evicted = new Map();
|
|
348
|
+
const { added, hello } = saving;
|
|
349
|
+
if (added === undefined) {
|
|
350
|
+
// A server before the rule: the connections the leader gave to newer registrations leave
|
|
351
|
+
// its list (it was told to close them; it does not read `replaced`), the rest is refused as
|
|
352
|
+
// before. Without this, every list it sent until those sockets closed was refused.
|
|
353
|
+
const taken = this.#replacedOf.get(ownerId);
|
|
354
|
+
const kept = taken
|
|
355
|
+
? connections.filter((record) => {
|
|
356
|
+
if (!taken.has(record.connection))
|
|
357
|
+
return true;
|
|
358
|
+
replaced.push(record.connection);
|
|
359
|
+
return false;
|
|
360
|
+
})
|
|
361
|
+
: [...connections];
|
|
362
|
+
if (taken) {
|
|
363
|
+
const listed = new Set(connections.map((record) => record.connection));
|
|
364
|
+
for (const connection of taken)
|
|
365
|
+
if (!listed.has(connection))
|
|
366
|
+
taken.delete(connection);
|
|
367
|
+
if (!taken.size)
|
|
368
|
+
this.#replacedOf.delete(ownerId);
|
|
369
|
+
}
|
|
370
|
+
this.#refuseDuplicates(ownerId, kept);
|
|
371
|
+
return { keep: kept, replaced, evicted };
|
|
372
|
+
}
|
|
373
|
+
// What the leader saved for this server: a record that is neither in it nor added was
|
|
374
|
+
// replaced meanwhile (the server had not heard of it when it sent this list).
|
|
375
|
+
const saved = hello
|
|
376
|
+
? undefined
|
|
377
|
+
: new Set((this.#owners.get(ownerId)?.connections ?? []).map((record) => record.connection));
|
|
378
|
+
const isAdded = (record) => added.has(record.connection);
|
|
379
|
+
const position = new Map();
|
|
380
|
+
for (const record of connections) {
|
|
381
|
+
const fresh = isAdded(record);
|
|
382
|
+
if (saved && !fresh && !saved.has(record.connection)) {
|
|
383
|
+
replaced.push(record.connection);
|
|
384
|
+
continue;
|
|
385
|
+
}
|
|
386
|
+
if (record.clientId === undefined) {
|
|
387
|
+
keep.push(record);
|
|
388
|
+
continue;
|
|
389
|
+
}
|
|
390
|
+
const key = recipientKey(record);
|
|
391
|
+
const earlier = position.get(key);
|
|
392
|
+
if (earlier !== undefined) {
|
|
393
|
+
// Twice in one list: the added one wins, else the later one.
|
|
394
|
+
const other = keep[earlier];
|
|
395
|
+
if (fresh || !isAdded(other)) {
|
|
396
|
+
replaced.push(other.connection);
|
|
397
|
+
keep[earlier] = record;
|
|
398
|
+
}
|
|
399
|
+
else
|
|
400
|
+
replaced.push(record.connection);
|
|
401
|
+
continue;
|
|
402
|
+
}
|
|
403
|
+
const holder = this.#index.client(record);
|
|
404
|
+
if (holder && holder.ownerEpoch !== ownerId) {
|
|
405
|
+
if (!fresh) {
|
|
406
|
+
replaced.push(record.connection);
|
|
407
|
+
continue;
|
|
408
|
+
}
|
|
409
|
+
const lost = evicted.get(holder.ownerEpoch);
|
|
410
|
+
if (lost)
|
|
411
|
+
lost.push(holder);
|
|
412
|
+
else
|
|
413
|
+
evicted.set(holder.ownerEpoch, [holder]);
|
|
286
414
|
}
|
|
415
|
+
position.set(key, keep.length);
|
|
416
|
+
keep.push(record);
|
|
417
|
+
}
|
|
418
|
+
return { keep, replaced, evicted };
|
|
419
|
+
}
|
|
420
|
+
/** The rule before newest-wins, for a server that does not take it (rolling deploys). */
|
|
421
|
+
#refuseDuplicates(ownerId, connections) {
|
|
422
|
+
const own = new Set();
|
|
423
|
+
for (const record of connections) {
|
|
424
|
+
const key = recipientKey(record);
|
|
425
|
+
if (own.has(key))
|
|
426
|
+
throw new HubError('DUPLICATE_CLIENT');
|
|
427
|
+
own.add(key);
|
|
428
|
+
const holder = this.#index.client(record);
|
|
429
|
+
if (holder && holder.ownerEpoch !== ownerId)
|
|
430
|
+
throw new HubError('DUPLICATE_CLIENT');
|
|
287
431
|
}
|
|
288
|
-
|
|
432
|
+
}
|
|
433
|
+
/**
|
|
434
|
+
* Saves one server's list (the newest-wins rule first, `#resolve`); resolves with the
|
|
435
|
+
* connections of that server that lost to newer registrations, which it closes.
|
|
436
|
+
*/
|
|
437
|
+
async #save(ownerId, value, saving) {
|
|
438
|
+
const { keep, replaced, evicted } = this.#resolve(ownerId, value.connections, saving);
|
|
439
|
+
let freed = 0;
|
|
440
|
+
for (const lost of evicted.values())
|
|
441
|
+
freed += lost.length;
|
|
442
|
+
const total = [...this.#owners.entries()].reduce((n, [key, owner]) => n + (key === ownerId ? 0 : owner.connections.length), keep.length - freed);
|
|
289
443
|
if (total > this.options.maxTotalConnections ||
|
|
290
444
|
(!this.#owners.has(ownerId) && this.#owners.size >= this.options.maxOwners))
|
|
291
445
|
throw new HubError('CONNECTION_LIMIT');
|
|
292
446
|
// A reconnect must always be able to carry the full authoritative snapshot in one bounded frame.
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
const
|
|
447
|
+
// Its size comes from each owner's list size, kept as lists are saved, instead of serializing
|
|
448
|
+
// every owner's list on every change (the saved owner's JSON is made once, for Redis too). The
|
|
449
|
+
// lists that lose replaced records only shrink: their current size is an upper bound.
|
|
450
|
+
const list = JSON.stringify(keep);
|
|
451
|
+
const listBytes = Buffer.byteLength(list);
|
|
452
|
+
const lists = [];
|
|
453
|
+
for (const [id, bytes] of this.#listBytes)
|
|
454
|
+
if (id !== ownerId)
|
|
455
|
+
lists.push([id, bytes]);
|
|
456
|
+
lists.push([ownerId, listBytes]);
|
|
457
|
+
if (snapshotBytes(lists) > this.options.maxFrameBytes - 4096)
|
|
458
|
+
throw new HubError('MESSAGE_TOO_LARGE');
|
|
459
|
+
// The older holders first: their connections leave before the newer ones join.
|
|
460
|
+
for (const [holderId, lost] of evicted)
|
|
461
|
+
await this.#evict(holderId, lost, saving.now);
|
|
462
|
+
await this.#commit(ownerId, { ...value, connections: keep }, list, this.options.ownerTtlMs);
|
|
463
|
+
if (replaced.length && this.logger.isEnabled('debug'))
|
|
464
|
+
this.logger.log('debug', 'Hub records of a server lost to newer registrations', {
|
|
465
|
+
owner: ownerId,
|
|
466
|
+
replaced: replaced.length,
|
|
467
|
+
hello: saving.hello,
|
|
468
|
+
});
|
|
469
|
+
return replaced;
|
|
470
|
+
}
|
|
471
|
+
/**
|
|
472
|
+
* Removes records of `holderId` that newer registrations replaced: its list is saved without
|
|
473
|
+
* them (its expiry kept: a dead server's records do not live longer for it) and, when it is
|
|
474
|
+
* connected here, it is told to close them (4409 Replaced).
|
|
475
|
+
*/
|
|
476
|
+
async #evict(holderId, lost, now) {
|
|
477
|
+
const owner = this.#owners.get(holderId);
|
|
478
|
+
if (!owner)
|
|
479
|
+
return;
|
|
480
|
+
const gone = new Set(lost.map((record) => record.connection));
|
|
481
|
+
const connections = owner.connections.filter((record) => !gone.has(record.connection));
|
|
482
|
+
const remainingMs = Math.max(1, owner.expires - now);
|
|
483
|
+
await this.#commit(holderId, { ...owner, connections }, JSON.stringify(connections), remainingMs);
|
|
484
|
+
const destination = this.#peers.get(holderId);
|
|
485
|
+
if (!destination?.newestWins) {
|
|
486
|
+
const taken = this.#replacedOf.get(holderId) ?? new Set();
|
|
487
|
+
for (const connection of gone)
|
|
488
|
+
taken.add(connection);
|
|
489
|
+
this.#replacedOf.set(holderId, taken);
|
|
490
|
+
}
|
|
491
|
+
this.logger.log('debug', 'Hub connections replaced by newer registrations', {
|
|
492
|
+
holder: holderId,
|
|
493
|
+
connections: gone.size,
|
|
494
|
+
connected: destination !== undefined,
|
|
495
|
+
});
|
|
496
|
+
if (!destination)
|
|
497
|
+
return;
|
|
498
|
+
try {
|
|
499
|
+
this.#send(destination, {
|
|
500
|
+
kind: 'kick',
|
|
501
|
+
id: randomUUID(),
|
|
502
|
+
connections: [...gone],
|
|
503
|
+
code: REPLACED.code,
|
|
504
|
+
reason: REPLACED.reason,
|
|
505
|
+
replaced: true,
|
|
506
|
+
});
|
|
507
|
+
}
|
|
508
|
+
catch (error) {
|
|
509
|
+
// Dropped: its hello carries the records again, and they lose there (it closes them then).
|
|
510
|
+
this.#sendFailed(destination, 'kick', error);
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
/** Writes one owner's list to Redis, then the index and the presence of every peer. */
|
|
514
|
+
async #commit(ownerId, value, list, ttlMs) {
|
|
515
|
+
const result = (await this.store.run('save', this.identity.epoch, String(Math.ceil(ttlMs)), JSON.stringify({ ...value, connections: list }), ownerId));
|
|
299
516
|
this.#revision = result[0];
|
|
300
|
-
value
|
|
301
|
-
this.#owners.set(ownerId,
|
|
302
|
-
this.#
|
|
303
|
-
this.#
|
|
517
|
+
const saved = { ...value, expires: result[1] };
|
|
518
|
+
this.#owners.set(ownerId, saved);
|
|
519
|
+
this.#listBytes.set(ownerId, Buffer.byteLength(list));
|
|
520
|
+
this.#index.setOwner(ownerId, saved.connections);
|
|
521
|
+
this.#notify(ownerId, saved.connections);
|
|
304
522
|
}
|
|
305
523
|
async #remove(ownerId, session) {
|
|
306
524
|
const rev = Number(await this.store.run('remove', this.identity.epoch, '', session, ownerId));
|
|
307
525
|
if (rev) {
|
|
308
526
|
this.#owners.delete(ownerId);
|
|
527
|
+
this.#listBytes.delete(ownerId);
|
|
528
|
+
this.#replacedOf.delete(ownerId);
|
|
309
529
|
this.#index.removeOwner(ownerId);
|
|
310
530
|
this.#revision = rev;
|
|
311
531
|
this.logger.log('debug', 'Hub server removed', { owner: ownerId, revision: rev });
|
|
@@ -326,7 +546,7 @@ export class Leader {
|
|
|
326
546
|
}
|
|
327
547
|
async #handle(peer, f) {
|
|
328
548
|
const id = integer(f.id, 1, Number.MAX_SAFE_INTEGER);
|
|
329
|
-
await this.#renew();
|
|
549
|
+
const now = await this.#renew();
|
|
330
550
|
let result = {};
|
|
331
551
|
if (f.op === 'hello') {
|
|
332
552
|
if (peer.owner)
|
|
@@ -345,18 +565,32 @@ export class Leader {
|
|
|
345
565
|
throw new HubError('PROTOCOL_MISMATCH');
|
|
346
566
|
}
|
|
347
567
|
const owner = epoch(f.owner), session = epoch(f.session);
|
|
348
|
-
|
|
568
|
+
// Each server bounds its own list by its own maxConnections (D-0136); the leader's limits
|
|
569
|
+
// are the shared ones: maxOwners, maxTotalConnections and the snapshot frame.
|
|
570
|
+
const values = records(f.connections, owner, PEER_LIMITS.connections);
|
|
349
571
|
const value = {
|
|
350
572
|
serverId: identifier(f.serverId),
|
|
351
573
|
session,
|
|
352
574
|
expires: 0,
|
|
353
575
|
connections: values,
|
|
354
576
|
};
|
|
577
|
+
// A server that takes the newest-wins rule says so; its records lose to newer holders.
|
|
578
|
+
const newestWins = f.newestWins === true;
|
|
579
|
+
// It reads `replaced` itself from here on (its hello answer and its sets).
|
|
580
|
+
if (newestWins)
|
|
581
|
+
this.#replacedOf.delete(owner);
|
|
355
582
|
// Do not include the joining peer in deltas until it receives its initial snapshot.
|
|
356
|
-
await this.#save(owner, value
|
|
583
|
+
const replaced = await this.#save(owner, value, {
|
|
584
|
+
hello: true,
|
|
585
|
+
added: newestWins ? new Set() : undefined,
|
|
586
|
+
now,
|
|
587
|
+
});
|
|
357
588
|
const previous = this.#peers.get(owner);
|
|
358
589
|
peer.owner = owner;
|
|
359
590
|
peer.session = session;
|
|
591
|
+
peer.newestWins = newestWins;
|
|
592
|
+
// Authenticated: its frames may be as large as its own maxFrameBytes allows (D-0136).
|
|
593
|
+
peer.wire.readLimit = PEER_LIMITS.frameBytes;
|
|
360
594
|
this.#peers.set(owner, peer);
|
|
361
595
|
previous?.wire.close();
|
|
362
596
|
this.logger.log('debug', 'Hub server registered', {
|
|
@@ -364,12 +598,16 @@ export class Leader {
|
|
|
364
598
|
serverId: value.serverId,
|
|
365
599
|
connections: values.length,
|
|
366
600
|
replaced: previous !== undefined,
|
|
601
|
+
lost: replaced.length,
|
|
602
|
+
newestWins,
|
|
367
603
|
servers: this.#owners.size,
|
|
368
604
|
});
|
|
369
605
|
result = {
|
|
370
606
|
protocol: PROTOCOL,
|
|
371
607
|
revision: this.#revision,
|
|
372
608
|
owners: [...this.#owners].map(([key, v]) => [key, v.connections]),
|
|
609
|
+
// Its connections that newer registrations replaced: it closes them (4409 Replaced).
|
|
610
|
+
...(newestWins ? { replaced } : {}),
|
|
373
611
|
};
|
|
374
612
|
}
|
|
375
613
|
else {
|
|
@@ -380,8 +618,18 @@ export class Leader {
|
|
|
380
618
|
if (!owner || owner.session !== peer.session)
|
|
381
619
|
throw new HubError('STALE_PEER');
|
|
382
620
|
if (f.op === 'set') {
|
|
383
|
-
const connections = records(f.connections, ownerId,
|
|
384
|
-
|
|
621
|
+
const connections = records(f.connections, ownerId, PEER_LIMITS.connections);
|
|
622
|
+
// The connections this set registers (newest wins); a server before the rule sends none.
|
|
623
|
+
const added = peer.newestWins
|
|
624
|
+
? new Set(array(f.added, PEER_LIMITS.connections).map(identifier))
|
|
625
|
+
: undefined;
|
|
626
|
+
const replaced = await this.#save(ownerId, { ...owner, connections }, {
|
|
627
|
+
hello: false,
|
|
628
|
+
added,
|
|
629
|
+
now,
|
|
630
|
+
});
|
|
631
|
+
if (added)
|
|
632
|
+
result = { replaced };
|
|
385
633
|
if (this.logger.isEnabled('debug'))
|
|
386
634
|
this.logger.log('debug', 'Hub connections changed', {
|
|
387
635
|
owner: ownerId,
|
|
@@ -405,11 +653,17 @@ export class Leader {
|
|
|
405
653
|
if (reason && Buffer.byteLength(reason) > 123)
|
|
406
654
|
throw new HubError('INVALID_ARGUMENT');
|
|
407
655
|
let kicked = 0;
|
|
656
|
+
// Every connected holder gets its kick; one that is not connected (after a failover, a
|
|
657
|
+
// server that has not reconnected or died with the old leader) or whose frame failed
|
|
658
|
+
// makes the answer OUTCOME_UNKNOWN, after the others were sent (audit R5: the loop
|
|
659
|
+
// stopped at the first one, and the holders after it were never told).
|
|
660
|
+
let unreached = 0;
|
|
408
661
|
for (const [key, selected] of this.#index.select(filter).owners) {
|
|
409
|
-
kicked += selected.length;
|
|
410
662
|
const destination = this.#peers.get(key);
|
|
411
|
-
if (!destination)
|
|
412
|
-
|
|
663
|
+
if (!destination) {
|
|
664
|
+
unreached += selected.length;
|
|
665
|
+
continue;
|
|
666
|
+
}
|
|
413
667
|
try {
|
|
414
668
|
this.#send(destination, {
|
|
415
669
|
kind: 'kick',
|
|
@@ -418,13 +672,21 @@ export class Leader {
|
|
|
418
672
|
code,
|
|
419
673
|
reason,
|
|
420
674
|
});
|
|
675
|
+
kicked += selected.length;
|
|
421
676
|
}
|
|
422
677
|
catch (error) {
|
|
423
678
|
this.#sendFailed(destination, 'kick', error);
|
|
424
|
-
|
|
679
|
+
unreached += selected.length;
|
|
425
680
|
}
|
|
426
681
|
}
|
|
427
|
-
this.logger.log('debug', 'Hub kick sent', {
|
|
682
|
+
this.logger.log('debug', 'Hub kick sent', {
|
|
683
|
+
id: messageId,
|
|
684
|
+
connections: kicked,
|
|
685
|
+
unreached,
|
|
686
|
+
code,
|
|
687
|
+
});
|
|
688
|
+
if (unreached)
|
|
689
|
+
throw new HubError('OUTCOME_UNKNOWN');
|
|
428
690
|
result = { authorityEpoch: this.identity.epoch, id: messageId };
|
|
429
691
|
}
|
|
430
692
|
else
|
|
@@ -436,7 +698,7 @@ export class Leader {
|
|
|
436
698
|
return this.#queue(async () => {
|
|
437
699
|
const now = await this.#renew();
|
|
438
700
|
await this.#expire(now);
|
|
439
|
-
});
|
|
701
|
+
}, true);
|
|
440
702
|
}
|
|
441
703
|
async stop() {
|
|
442
704
|
if (this.#active)
|
package/dist/local.js
CHANGED
|
@@ -1,16 +1,12 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { ConsoleLogger } from '@dokebi/core';
|
|
3
3
|
import { diagnostics } from '@dokebi/core/libraries';
|
|
4
|
+
import { REPLACED } from './delivery.js';
|
|
4
5
|
import { EventQueue } from './event-queue.js';
|
|
5
6
|
import { HubIndex } from './hub-index.js';
|
|
6
7
|
import { identifier, metadata, query, record } from './protocol.js';
|
|
7
8
|
import { hubStatus, LocalNonceStore } from './status.js';
|
|
8
9
|
import { HubError, } from './types.js';
|
|
9
|
-
/**
|
|
10
|
-
* Share of the event queue (maxPending events, maxPendingBytes) that broadcasts may fill before they
|
|
11
|
-
* are refused with BUSY; the rest stays for presence and kick events, which cannot be refused.
|
|
12
|
-
*/
|
|
13
|
-
const BROADCAST_SHARE = 0.5;
|
|
14
10
|
/** The JSON of a presence event around its connections: kind, revision, ownerEpoch, brackets. */
|
|
15
11
|
const PRESENCE_ENVELOPE_BYTES = 96;
|
|
16
12
|
/** The JSON size of one record, as the event queue's own sizing would count it. */
|
|
@@ -53,10 +49,14 @@ export class LocalHub {
|
|
|
53
49
|
this.options = options;
|
|
54
50
|
this.logger = logger;
|
|
55
51
|
this.worker = worker;
|
|
56
|
-
this.#throttle = new diagnostics.Repeats({ logger });
|
|
52
|
+
this.#throttle = new diagnostics.Repeats({ logger, intervalMs: options.logRepeatWindowMs });
|
|
57
53
|
this.#events = new EventQueue(options.maxPending, options.maxPendingBytes);
|
|
58
54
|
const self = this;
|
|
59
|
-
const nonces = new LocalNonceStore({
|
|
55
|
+
const nonces = new LocalNonceStore({
|
|
56
|
+
logger,
|
|
57
|
+
max: options.maxNonces,
|
|
58
|
+
warnIntervalMs: options.logRepeatWindowMs,
|
|
59
|
+
});
|
|
60
60
|
let warned = false;
|
|
61
61
|
this.api = Object.freeze({
|
|
62
62
|
get state() {
|
|
@@ -141,6 +141,24 @@ export class LocalHub {
|
|
|
141
141
|
connections,
|
|
142
142
|
}, PRESENCE_ENVELOPE_BYTES + this.#rosterBytes + connections.length);
|
|
143
143
|
}
|
|
144
|
+
/** Removes a connection a newer registration replaced and asks its socket to close. */
|
|
145
|
+
#replace(record) {
|
|
146
|
+
this.#records.delete(record.connection);
|
|
147
|
+
this.#index.remove(record);
|
|
148
|
+
this.#untrack(record);
|
|
149
|
+
this.#changed();
|
|
150
|
+
this.#push({
|
|
151
|
+
kind: 'kick',
|
|
152
|
+
id: randomUUID(),
|
|
153
|
+
connections: Object.freeze([record.connection]),
|
|
154
|
+
code: REPLACED.code,
|
|
155
|
+
reason: REPLACED.reason,
|
|
156
|
+
});
|
|
157
|
+
if (this.logger.isEnabled('debug'))
|
|
158
|
+
this.logger.log('debug', 'Hub connection replaced by a newer registration', {
|
|
159
|
+
connection: record.connection,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
144
162
|
#ensureOpen() {
|
|
145
163
|
if (this.#closed)
|
|
146
164
|
throw new HubError('CLOSED');
|
|
@@ -150,11 +168,16 @@ export class LocalHub {
|
|
|
150
168
|
const value = record(input, this.ownerEpoch);
|
|
151
169
|
if (this.#records.has(value.connection))
|
|
152
170
|
throw new HubError('DUPLICATE_CONNECTION');
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
171
|
+
// The newest registration of a (tenant, member, clientId) wins (audit 2026-10-26): the older
|
|
172
|
+
// connection leaves the roster now and its socket is closed (4409 Replaced). It counts as
|
|
173
|
+
// gone for the limits below.
|
|
174
|
+
const held = value.clientId === undefined ? undefined : this.#index.client(value);
|
|
175
|
+
const room = held ? 1 : 0;
|
|
176
|
+
if (this.#records.size - room >= this.options.maxConnections ||
|
|
177
|
+
this.#records.size - room >= this.options.maxTotalConnections)
|
|
157
178
|
throw new HubError('CONNECTION_LIMIT');
|
|
179
|
+
if (held)
|
|
180
|
+
this.#replace(held);
|
|
158
181
|
this.#records.set(value.connection, value);
|
|
159
182
|
this.#index.add(value);
|
|
160
183
|
this.#track(value);
|
|
@@ -240,7 +263,10 @@ export class LocalHub {
|
|
|
240
263
|
id,
|
|
241
264
|
connections: Object.freeze(connections),
|
|
242
265
|
data: Uint8Array.from(data),
|
|
243
|
-
},
|
|
266
|
+
},
|
|
267
|
+
// The share of the queue broadcasts may fill (`broadcastShare`, D-0114); the rest stays
|
|
268
|
+
// for presence and kick events, which cannot be refused.
|
|
269
|
+
this.options.broadcastShare)) {
|
|
244
270
|
// The gateway has not taken the earlier messages yet: the caller backs off.
|
|
245
271
|
this.#throttle.log('debug', 'Hub broadcast refused; event queue busy', 'BUSY', undefined, {
|
|
246
272
|
maxPending: this.options.maxPending,
|