@dokebi/websocket-hub 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AI.md +16 -9
- package/README.md +4 -4
- package/dist/delivery.d.ts +10 -0
- package/dist/delivery.js +7 -0
- package/dist/index.js +21 -3
- package/dist/leader.d.ts +8 -1
- package/dist/leader.js +293 -38
- package/dist/local.js +38 -12
- package/dist/node.js +195 -43
- package/dist/notifications.d.ts +31 -0
- package/dist/notifications.js +133 -0
- package/dist/options.d.ts +16 -0
- package/dist/options.js +26 -5
- package/dist/peer-client.d.ts +7 -1
- package/dist/peer-client.js +215 -25
- package/dist/protocol.js +11 -4
- package/dist/status-cli.js +3 -2
- package/dist/status.d.ts +5 -1
- package/dist/status.js +10 -1
- package/dist/store.d.ts +11 -2
- package/dist/store.js +91 -15
- package/dist/types.d.ts +8 -0
- package/dist/types.js +2 -1
- package/dist/wire.d.ts +17 -2
- package/dist/wire.js +32 -8
- package/options.schema.json +22 -0
- package/package.json +5 -5
package/dist/leader.js
CHANGED
|
@@ -1,11 +1,27 @@
|
|
|
1
1
|
import { randomUUID, timingSafeEqual } from 'node:crypto';
|
|
2
2
|
import { diagnostics } from '@dokebi/core/libraries';
|
|
3
|
-
import { recipientKey } from './delivery.js';
|
|
3
|
+
import { REPLACED, recipientKey } from './delivery.js';
|
|
4
4
|
import { HubIndex } from './hub-index.js';
|
|
5
|
-
import {
|
|
5
|
+
import { PEER_LIMITS } from './options.js';
|
|
6
|
+
import { array, epoch, identifier, PROTOCOL, payload, query, records } from './protocol.js';
|
|
6
7
|
import { HubError } from './types.js';
|
|
7
8
|
import { integer } from './values.js';
|
|
8
|
-
import { Wire } from './wire.js';
|
|
9
|
+
import { frame, Wire } from './wire.js';
|
|
10
|
+
/**
|
|
11
|
+
* UTF-8 bytes of `JSON.stringify({ owners: [[owner, list], …] })` (what a reconnecting peer's hello
|
|
12
|
+
* answer carries) from each owner's id and the bytes of its list's JSON: `{"owners":[` and `]}`,
|
|
13
|
+
* then `["<owner>",<list>]` per owner, separated by commas. Owner ids are epochs (UUIDs): no
|
|
14
|
+
* escaping.
|
|
15
|
+
*/
|
|
16
|
+
export function snapshotBytes(lists) {
|
|
17
|
+
let bytes = 13;
|
|
18
|
+
let count = 0;
|
|
19
|
+
for (const [owner, listBytes] of lists) {
|
|
20
|
+
bytes += Buffer.byteLength(owner) + 5 + listBytes;
|
|
21
|
+
count++;
|
|
22
|
+
}
|
|
23
|
+
return bytes + Math.max(0, count - 1);
|
|
24
|
+
}
|
|
9
25
|
export class Leader {
|
|
10
26
|
identity;
|
|
11
27
|
store;
|
|
@@ -14,8 +30,17 @@ export class Leader {
|
|
|
14
30
|
#peers = new Map();
|
|
15
31
|
#sockets = new Set();
|
|
16
32
|
#owners = new Map();
|
|
33
|
+
/** UTF-8 bytes of each owner's connection list as JSON, for the snapshot bound; follows #owners. */
|
|
34
|
+
#listBytes = new Map();
|
|
17
35
|
/** Every owner's connections by tenant, member and room (D-0068); follows #owners. */
|
|
18
36
|
#index = new HubIndex();
|
|
37
|
+
/**
|
|
38
|
+
* Connections the leader took from a server that may not know the newest-wins rule (a server
|
|
39
|
+
* before it, or one not connected here): such a server keeps sending them until its socket has
|
|
40
|
+
* closed, and its list must not be refused for them. Pruned as its lists leave them out, and
|
|
41
|
+
* dropped with the owner (`#remove`) or when it says hello with the rule.
|
|
42
|
+
*/
|
|
43
|
+
#replacedOf = new Map();
|
|
19
44
|
#revision = 0;
|
|
20
45
|
#tail = Promise.resolve();
|
|
21
46
|
#pending = 0;
|
|
@@ -28,7 +53,7 @@ export class Leader {
|
|
|
28
53
|
this.store = store;
|
|
29
54
|
this.options = options;
|
|
30
55
|
this.logger = logger;
|
|
31
|
-
this.#throttle = new diagnostics.Repeats({ logger });
|
|
56
|
+
this.#throttle = new diagnostics.Repeats({ logger, intervalMs: options.logRepeatWindowMs });
|
|
32
57
|
}
|
|
33
58
|
/**
|
|
34
59
|
* A frame to a peer could not be written (its socket is gone, its send queue passed maxPendingBytes,
|
|
@@ -74,8 +99,11 @@ export class Leader {
|
|
|
74
99
|
const snapshot = await this.store.snapshot(this.identity.epoch);
|
|
75
100
|
this.#owners = snapshot.owners;
|
|
76
101
|
this.#index.clear();
|
|
77
|
-
|
|
102
|
+
this.#listBytes.clear();
|
|
103
|
+
for (const [ownerId, owner] of this.#owners) {
|
|
78
104
|
this.#index.setOwner(ownerId, owner.connections);
|
|
105
|
+
this.#listBytes.set(ownerId, Buffer.byteLength(JSON.stringify(owner.connections)));
|
|
106
|
+
}
|
|
79
107
|
this.#revision = snapshot.revision;
|
|
80
108
|
await this.#expire(snapshot.now);
|
|
81
109
|
this.logger.log('debug', 'Hub leader started', {
|
|
@@ -107,6 +135,7 @@ export class Leader {
|
|
|
107
135
|
throw new HubError('STALE_LEADER');
|
|
108
136
|
return fn();
|
|
109
137
|
});
|
|
138
|
+
// dokebi-check: allow empty-catch the chain only orders the next call; the caller gets the same promise with its failure
|
|
110
139
|
this.#tail = work
|
|
111
140
|
.then(() => { }, () => { })
|
|
112
141
|
.finally(() => {
|
|
@@ -262,50 +291,234 @@ export class Leader {
|
|
|
262
291
|
throw new HubError('STALE_LEADER');
|
|
263
292
|
peer.wire.send({ ...value, epoch: this.identity.epoch });
|
|
264
293
|
}
|
|
294
|
+
/** One presence frame, serialized once and written to every peer (the same bytes for all). */
|
|
265
295
|
#notify(owner, connections) {
|
|
296
|
+
if (!this.#peers.size)
|
|
297
|
+
return;
|
|
298
|
+
let bytes;
|
|
299
|
+
try {
|
|
300
|
+
if (!this.active)
|
|
301
|
+
throw new HubError('STALE_LEADER');
|
|
302
|
+
bytes = frame({
|
|
303
|
+
kind: 'presence',
|
|
304
|
+
revision: this.#revision,
|
|
305
|
+
owner,
|
|
306
|
+
connections,
|
|
307
|
+
epoch: this.identity.epoch,
|
|
308
|
+
}, this.options.maxFrameBytes);
|
|
309
|
+
}
|
|
310
|
+
catch (error) {
|
|
311
|
+
// As when each peer's send failed: every peer is dropped and resyncs on reconnect.
|
|
312
|
+
for (const peer of this.#peers.values())
|
|
313
|
+
this.#sendFailed(peer, 'presence', error);
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
266
316
|
for (const peer of this.#peers.values()) {
|
|
267
317
|
try {
|
|
268
|
-
|
|
318
|
+
if (!this.active)
|
|
319
|
+
throw new HubError('STALE_LEADER');
|
|
320
|
+
peer.wire.sendFrame(bytes);
|
|
269
321
|
}
|
|
270
322
|
catch (error) {
|
|
271
323
|
this.#sendFailed(peer, 'presence', error);
|
|
272
324
|
}
|
|
273
325
|
}
|
|
274
326
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
327
|
+
/**
|
|
328
|
+
* One `(tenant, member, clientId)` at a time (D-0025), and the newest registration wins (audit
|
|
329
|
+
* 2026-10-26): a `set` that adds a connection whose clientId another server holds takes it, and
|
|
330
|
+
* that server's record goes (its server closes it, 4409 Replaced). Records the saving server
|
|
331
|
+
* only carries over (a hello after its link dropped, or a record the leader replaced while the
|
|
332
|
+
* server's `set` was on its way) lose to a newer holder instead, so a reconnecting server is
|
|
333
|
+
* never refused as a whole and a replaced tab never comes back. Checked through the index
|
|
334
|
+
* (D-0068): O(list). A server before the rule (`added` unknown) is refused as before
|
|
335
|
+
* (DUPLICATE_CLIENT), so its lists cannot flap between two holders.
|
|
336
|
+
*/
|
|
337
|
+
#resolve(ownerId, connections, saving) {
|
|
338
|
+
const keep = [];
|
|
339
|
+
const replaced = [];
|
|
340
|
+
const evicted = new Map();
|
|
341
|
+
const { added, hello } = saving;
|
|
342
|
+
if (added === undefined) {
|
|
343
|
+
// A server before the rule: the connections the leader gave to newer registrations leave
|
|
344
|
+
// its list (it was told to close them; it does not read `replaced`), the rest is refused as
|
|
345
|
+
// before. Without this, every list it sent until those sockets closed was refused.
|
|
346
|
+
const taken = this.#replacedOf.get(ownerId);
|
|
347
|
+
const kept = taken
|
|
348
|
+
? connections.filter((record) => {
|
|
349
|
+
if (!taken.has(record.connection))
|
|
350
|
+
return true;
|
|
351
|
+
replaced.push(record.connection);
|
|
352
|
+
return false;
|
|
353
|
+
})
|
|
354
|
+
: [...connections];
|
|
355
|
+
if (taken) {
|
|
356
|
+
const listed = new Set(connections.map((record) => record.connection));
|
|
357
|
+
for (const connection of taken)
|
|
358
|
+
if (!listed.has(connection))
|
|
359
|
+
taken.delete(connection);
|
|
360
|
+
if (!taken.size)
|
|
361
|
+
this.#replacedOf.delete(ownerId);
|
|
286
362
|
}
|
|
363
|
+
this.#refuseDuplicates(ownerId, kept);
|
|
364
|
+
return { keep: kept, replaced, evicted };
|
|
287
365
|
}
|
|
288
|
-
|
|
366
|
+
// What the leader saved for this server: a record that is neither in it nor added was
|
|
367
|
+
// replaced meanwhile (the server had not heard of it when it sent this list).
|
|
368
|
+
const saved = hello
|
|
369
|
+
? undefined
|
|
370
|
+
: new Set((this.#owners.get(ownerId)?.connections ?? []).map((record) => record.connection));
|
|
371
|
+
const isAdded = (record) => added.has(record.connection);
|
|
372
|
+
const position = new Map();
|
|
373
|
+
for (const record of connections) {
|
|
374
|
+
const fresh = isAdded(record);
|
|
375
|
+
if (saved && !fresh && !saved.has(record.connection)) {
|
|
376
|
+
replaced.push(record.connection);
|
|
377
|
+
continue;
|
|
378
|
+
}
|
|
379
|
+
if (record.clientId === undefined) {
|
|
380
|
+
keep.push(record);
|
|
381
|
+
continue;
|
|
382
|
+
}
|
|
383
|
+
const key = recipientKey(record);
|
|
384
|
+
const earlier = position.get(key);
|
|
385
|
+
if (earlier !== undefined) {
|
|
386
|
+
// Twice in one list: the added one wins, else the later one.
|
|
387
|
+
const other = keep[earlier];
|
|
388
|
+
if (fresh || !isAdded(other)) {
|
|
389
|
+
replaced.push(other.connection);
|
|
390
|
+
keep[earlier] = record;
|
|
391
|
+
}
|
|
392
|
+
else
|
|
393
|
+
replaced.push(record.connection);
|
|
394
|
+
continue;
|
|
395
|
+
}
|
|
396
|
+
const holder = this.#index.client(record);
|
|
397
|
+
if (holder && holder.ownerEpoch !== ownerId) {
|
|
398
|
+
if (!fresh) {
|
|
399
|
+
replaced.push(record.connection);
|
|
400
|
+
continue;
|
|
401
|
+
}
|
|
402
|
+
const lost = evicted.get(holder.ownerEpoch);
|
|
403
|
+
if (lost)
|
|
404
|
+
lost.push(holder);
|
|
405
|
+
else
|
|
406
|
+
evicted.set(holder.ownerEpoch, [holder]);
|
|
407
|
+
}
|
|
408
|
+
position.set(key, keep.length);
|
|
409
|
+
keep.push(record);
|
|
410
|
+
}
|
|
411
|
+
return { keep, replaced, evicted };
|
|
412
|
+
}
|
|
413
|
+
/** The rule before newest-wins, for a server that does not take it (rolling deploys). */
|
|
414
|
+
#refuseDuplicates(ownerId, connections) {
|
|
415
|
+
const own = new Set();
|
|
416
|
+
for (const record of connections) {
|
|
417
|
+
const key = recipientKey(record);
|
|
418
|
+
if (own.has(key))
|
|
419
|
+
throw new HubError('DUPLICATE_CLIENT');
|
|
420
|
+
own.add(key);
|
|
421
|
+
const holder = this.#index.client(record);
|
|
422
|
+
if (holder && holder.ownerEpoch !== ownerId)
|
|
423
|
+
throw new HubError('DUPLICATE_CLIENT');
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
/**
|
|
427
|
+
* Saves one server's list (the newest-wins rule first, `#resolve`); resolves with the
|
|
428
|
+
* connections of that server that lost to newer registrations, which it closes.
|
|
429
|
+
*/
|
|
430
|
+
async #save(ownerId, value, saving) {
|
|
431
|
+
const { keep, replaced, evicted } = this.#resolve(ownerId, value.connections, saving);
|
|
432
|
+
let freed = 0;
|
|
433
|
+
for (const lost of evicted.values())
|
|
434
|
+
freed += lost.length;
|
|
435
|
+
const total = [...this.#owners.entries()].reduce((n, [key, owner]) => n + (key === ownerId ? 0 : owner.connections.length), keep.length - freed);
|
|
289
436
|
if (total > this.options.maxTotalConnections ||
|
|
290
437
|
(!this.#owners.has(ownerId) && this.#owners.size >= this.options.maxOwners))
|
|
291
438
|
throw new HubError('CONNECTION_LIMIT');
|
|
292
439
|
// A reconnect must always be able to carry the full authoritative snapshot in one bounded frame.
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
const
|
|
440
|
+
// Its size comes from each owner's list size, kept as lists are saved, instead of serializing
|
|
441
|
+
// every owner's list on every change (the saved owner's JSON is made once, for Redis too). The
|
|
442
|
+
// lists that lose replaced records only shrink: their current size is an upper bound.
|
|
443
|
+
const list = JSON.stringify(keep);
|
|
444
|
+
const listBytes = Buffer.byteLength(list);
|
|
445
|
+
const lists = [];
|
|
446
|
+
for (const [id, bytes] of this.#listBytes)
|
|
447
|
+
if (id !== ownerId)
|
|
448
|
+
lists.push([id, bytes]);
|
|
449
|
+
lists.push([ownerId, listBytes]);
|
|
450
|
+
if (snapshotBytes(lists) > this.options.maxFrameBytes - 4096)
|
|
451
|
+
throw new HubError('MESSAGE_TOO_LARGE');
|
|
452
|
+
// The older holders first: their connections leave before the newer ones join.
|
|
453
|
+
for (const [holderId, lost] of evicted)
|
|
454
|
+
await this.#evict(holderId, lost, saving.now);
|
|
455
|
+
await this.#commit(ownerId, { ...value, connections: keep }, list, this.options.ownerTtlMs);
|
|
456
|
+
if (replaced.length && this.logger.isEnabled('debug'))
|
|
457
|
+
this.logger.log('debug', 'Hub records of a server lost to newer registrations', {
|
|
458
|
+
owner: ownerId,
|
|
459
|
+
replaced: replaced.length,
|
|
460
|
+
hello: saving.hello,
|
|
461
|
+
});
|
|
462
|
+
return replaced;
|
|
463
|
+
}
|
|
464
|
+
/**
|
|
465
|
+
* Removes records of `holderId` that newer registrations replaced: its list is saved without
|
|
466
|
+
* them (its expiry kept: a dead server's records do not live longer for it) and, when it is
|
|
467
|
+
* connected here, it is told to close them (4409 Replaced).
|
|
468
|
+
*/
|
|
469
|
+
async #evict(holderId, lost, now) {
|
|
470
|
+
const owner = this.#owners.get(holderId);
|
|
471
|
+
if (!owner)
|
|
472
|
+
return;
|
|
473
|
+
const gone = new Set(lost.map((record) => record.connection));
|
|
474
|
+
const connections = owner.connections.filter((record) => !gone.has(record.connection));
|
|
475
|
+
const remainingMs = Math.max(1, owner.expires - now);
|
|
476
|
+
await this.#commit(holderId, { ...owner, connections }, JSON.stringify(connections), remainingMs);
|
|
477
|
+
const destination = this.#peers.get(holderId);
|
|
478
|
+
if (!destination?.newestWins) {
|
|
479
|
+
const taken = this.#replacedOf.get(holderId) ?? new Set();
|
|
480
|
+
for (const connection of gone)
|
|
481
|
+
taken.add(connection);
|
|
482
|
+
this.#replacedOf.set(holderId, taken);
|
|
483
|
+
}
|
|
484
|
+
this.logger.log('debug', 'Hub connections replaced by newer registrations', {
|
|
485
|
+
holder: holderId,
|
|
486
|
+
connections: gone.size,
|
|
487
|
+
connected: destination !== undefined,
|
|
488
|
+
});
|
|
489
|
+
if (!destination)
|
|
490
|
+
return;
|
|
491
|
+
try {
|
|
492
|
+
this.#send(destination, {
|
|
493
|
+
kind: 'kick',
|
|
494
|
+
id: randomUUID(),
|
|
495
|
+
connections: [...gone],
|
|
496
|
+
code: REPLACED.code,
|
|
497
|
+
reason: REPLACED.reason,
|
|
498
|
+
replaced: true,
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
catch (error) {
|
|
502
|
+
// Dropped: its hello carries the records again, and they lose there (it closes them then).
|
|
503
|
+
this.#sendFailed(destination, 'kick', error);
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
/** Writes one owner's list to Redis, then the index and the presence of every peer. */
|
|
507
|
+
async #commit(ownerId, value, list, ttlMs) {
|
|
508
|
+
const result = (await this.store.run('save', this.identity.epoch, String(Math.ceil(ttlMs)), JSON.stringify({ ...value, connections: list }), ownerId));
|
|
299
509
|
this.#revision = result[0];
|
|
300
|
-
value
|
|
301
|
-
this.#owners.set(ownerId,
|
|
302
|
-
this.#
|
|
303
|
-
this.#
|
|
510
|
+
const saved = { ...value, expires: result[1] };
|
|
511
|
+
this.#owners.set(ownerId, saved);
|
|
512
|
+
this.#listBytes.set(ownerId, Buffer.byteLength(list));
|
|
513
|
+
this.#index.setOwner(ownerId, saved.connections);
|
|
514
|
+
this.#notify(ownerId, saved.connections);
|
|
304
515
|
}
|
|
305
516
|
async #remove(ownerId, session) {
|
|
306
517
|
const rev = Number(await this.store.run('remove', this.identity.epoch, '', session, ownerId));
|
|
307
518
|
if (rev) {
|
|
308
519
|
this.#owners.delete(ownerId);
|
|
520
|
+
this.#listBytes.delete(ownerId);
|
|
521
|
+
this.#replacedOf.delete(ownerId);
|
|
309
522
|
this.#index.removeOwner(ownerId);
|
|
310
523
|
this.#revision = rev;
|
|
311
524
|
this.logger.log('debug', 'Hub server removed', { owner: ownerId, revision: rev });
|
|
@@ -326,7 +539,7 @@ export class Leader {
|
|
|
326
539
|
}
|
|
327
540
|
async #handle(peer, f) {
|
|
328
541
|
const id = integer(f.id, 1, Number.MAX_SAFE_INTEGER);
|
|
329
|
-
await this.#renew();
|
|
542
|
+
const now = await this.#renew();
|
|
330
543
|
let result = {};
|
|
331
544
|
if (f.op === 'hello') {
|
|
332
545
|
if (peer.owner)
|
|
@@ -345,18 +558,32 @@ export class Leader {
|
|
|
345
558
|
throw new HubError('PROTOCOL_MISMATCH');
|
|
346
559
|
}
|
|
347
560
|
const owner = epoch(f.owner), session = epoch(f.session);
|
|
348
|
-
|
|
561
|
+
// Each server bounds its own list by its own maxConnections (D-0136); the leader's limits
|
|
562
|
+
// are the shared ones: maxOwners, maxTotalConnections and the snapshot frame.
|
|
563
|
+
const values = records(f.connections, owner, PEER_LIMITS.connections);
|
|
349
564
|
const value = {
|
|
350
565
|
serverId: identifier(f.serverId),
|
|
351
566
|
session,
|
|
352
567
|
expires: 0,
|
|
353
568
|
connections: values,
|
|
354
569
|
};
|
|
570
|
+
// A server that takes the newest-wins rule says so; its records lose to newer holders.
|
|
571
|
+
const newestWins = f.newestWins === true;
|
|
572
|
+
// It reads `replaced` itself from here on (its hello answer and its sets).
|
|
573
|
+
if (newestWins)
|
|
574
|
+
this.#replacedOf.delete(owner);
|
|
355
575
|
// Do not include the joining peer in deltas until it receives its initial snapshot.
|
|
356
|
-
await this.#save(owner, value
|
|
576
|
+
const replaced = await this.#save(owner, value, {
|
|
577
|
+
hello: true,
|
|
578
|
+
added: newestWins ? new Set() : undefined,
|
|
579
|
+
now,
|
|
580
|
+
});
|
|
357
581
|
const previous = this.#peers.get(owner);
|
|
358
582
|
peer.owner = owner;
|
|
359
583
|
peer.session = session;
|
|
584
|
+
peer.newestWins = newestWins;
|
|
585
|
+
// Authenticated: its frames may be as large as its own maxFrameBytes allows (D-0136).
|
|
586
|
+
peer.wire.readLimit = PEER_LIMITS.frameBytes;
|
|
360
587
|
this.#peers.set(owner, peer);
|
|
361
588
|
previous?.wire.close();
|
|
362
589
|
this.logger.log('debug', 'Hub server registered', {
|
|
@@ -364,12 +591,16 @@ export class Leader {
|
|
|
364
591
|
serverId: value.serverId,
|
|
365
592
|
connections: values.length,
|
|
366
593
|
replaced: previous !== undefined,
|
|
594
|
+
lost: replaced.length,
|
|
595
|
+
newestWins,
|
|
367
596
|
servers: this.#owners.size,
|
|
368
597
|
});
|
|
369
598
|
result = {
|
|
370
599
|
protocol: PROTOCOL,
|
|
371
600
|
revision: this.#revision,
|
|
372
601
|
owners: [...this.#owners].map(([key, v]) => [key, v.connections]),
|
|
602
|
+
// Its connections that newer registrations replaced: it closes them (4409 Replaced).
|
|
603
|
+
...(newestWins ? { replaced } : {}),
|
|
373
604
|
};
|
|
374
605
|
}
|
|
375
606
|
else {
|
|
@@ -380,8 +611,18 @@ export class Leader {
|
|
|
380
611
|
if (!owner || owner.session !== peer.session)
|
|
381
612
|
throw new HubError('STALE_PEER');
|
|
382
613
|
if (f.op === 'set') {
|
|
383
|
-
const connections = records(f.connections, ownerId,
|
|
384
|
-
|
|
614
|
+
const connections = records(f.connections, ownerId, PEER_LIMITS.connections);
|
|
615
|
+
// The connections this set registers (newest wins); a server before the rule sends none.
|
|
616
|
+
const added = peer.newestWins
|
|
617
|
+
? new Set(array(f.added, PEER_LIMITS.connections).map(identifier))
|
|
618
|
+
: undefined;
|
|
619
|
+
const replaced = await this.#save(ownerId, { ...owner, connections }, {
|
|
620
|
+
hello: false,
|
|
621
|
+
added,
|
|
622
|
+
now,
|
|
623
|
+
});
|
|
624
|
+
if (added)
|
|
625
|
+
result = { replaced };
|
|
385
626
|
if (this.logger.isEnabled('debug'))
|
|
386
627
|
this.logger.log('debug', 'Hub connections changed', {
|
|
387
628
|
owner: ownerId,
|
|
@@ -405,11 +646,17 @@ export class Leader {
|
|
|
405
646
|
if (reason && Buffer.byteLength(reason) > 123)
|
|
406
647
|
throw new HubError('INVALID_ARGUMENT');
|
|
407
648
|
let kicked = 0;
|
|
649
|
+
// Every connected holder gets its kick; one that is not connected (after a failover, a
|
|
650
|
+
// server that has not reconnected or died with the old leader) or whose frame failed
|
|
651
|
+
// makes the answer OUTCOME_UNKNOWN, after the others were sent (audit R5: the loop
|
|
652
|
+
// stopped at the first one, and the holders after it were never told).
|
|
653
|
+
let unreached = 0;
|
|
408
654
|
for (const [key, selected] of this.#index.select(filter).owners) {
|
|
409
|
-
kicked += selected.length;
|
|
410
655
|
const destination = this.#peers.get(key);
|
|
411
|
-
if (!destination)
|
|
412
|
-
|
|
656
|
+
if (!destination) {
|
|
657
|
+
unreached += selected.length;
|
|
658
|
+
continue;
|
|
659
|
+
}
|
|
413
660
|
try {
|
|
414
661
|
this.#send(destination, {
|
|
415
662
|
kind: 'kick',
|
|
@@ -418,13 +665,21 @@ export class Leader {
|
|
|
418
665
|
code,
|
|
419
666
|
reason,
|
|
420
667
|
});
|
|
668
|
+
kicked += selected.length;
|
|
421
669
|
}
|
|
422
670
|
catch (error) {
|
|
423
671
|
this.#sendFailed(destination, 'kick', error);
|
|
424
|
-
|
|
672
|
+
unreached += selected.length;
|
|
425
673
|
}
|
|
426
674
|
}
|
|
427
|
-
this.logger.log('debug', 'Hub kick sent', {
|
|
675
|
+
this.logger.log('debug', 'Hub kick sent', {
|
|
676
|
+
id: messageId,
|
|
677
|
+
connections: kicked,
|
|
678
|
+
unreached,
|
|
679
|
+
code,
|
|
680
|
+
});
|
|
681
|
+
if (unreached)
|
|
682
|
+
throw new HubError('OUTCOME_UNKNOWN');
|
|
428
683
|
result = { authorityEpoch: this.identity.epoch, id: messageId };
|
|
429
684
|
}
|
|
430
685
|
else
|
package/dist/local.js
CHANGED
|
@@ -1,16 +1,12 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { ConsoleLogger } from '@dokebi/core';
|
|
3
3
|
import { diagnostics } from '@dokebi/core/libraries';
|
|
4
|
+
import { REPLACED } from './delivery.js';
|
|
4
5
|
import { EventQueue } from './event-queue.js';
|
|
5
6
|
import { HubIndex } from './hub-index.js';
|
|
6
7
|
import { identifier, metadata, query, record } from './protocol.js';
|
|
7
8
|
import { hubStatus, LocalNonceStore } from './status.js';
|
|
8
9
|
import { HubError, } from './types.js';
|
|
9
|
-
/**
|
|
10
|
-
* Share of the event queue (maxPending events, maxPendingBytes) that broadcasts may fill before they
|
|
11
|
-
* are refused with BUSY; the rest stays for presence and kick events, which cannot be refused.
|
|
12
|
-
*/
|
|
13
|
-
const BROADCAST_SHARE = 0.5;
|
|
14
10
|
/** The JSON of a presence event around its connections: kind, revision, ownerEpoch, brackets. */
|
|
15
11
|
const PRESENCE_ENVELOPE_BYTES = 96;
|
|
16
12
|
/** The JSON size of one record, as the event queue's own sizing would count it. */
|
|
@@ -53,10 +49,14 @@ export class LocalHub {
|
|
|
53
49
|
this.options = options;
|
|
54
50
|
this.logger = logger;
|
|
55
51
|
this.worker = worker;
|
|
56
|
-
this.#throttle = new diagnostics.Repeats({ logger });
|
|
52
|
+
this.#throttle = new diagnostics.Repeats({ logger, intervalMs: options.logRepeatWindowMs });
|
|
57
53
|
this.#events = new EventQueue(options.maxPending, options.maxPendingBytes);
|
|
58
54
|
const self = this;
|
|
59
|
-
const nonces = new LocalNonceStore({
|
|
55
|
+
const nonces = new LocalNonceStore({
|
|
56
|
+
logger,
|
|
57
|
+
max: options.maxNonces,
|
|
58
|
+
warnIntervalMs: options.logRepeatWindowMs,
|
|
59
|
+
});
|
|
60
60
|
let warned = false;
|
|
61
61
|
this.api = Object.freeze({
|
|
62
62
|
get state() {
|
|
@@ -141,6 +141,24 @@ export class LocalHub {
|
|
|
141
141
|
connections,
|
|
142
142
|
}, PRESENCE_ENVELOPE_BYTES + this.#rosterBytes + connections.length);
|
|
143
143
|
}
|
|
144
|
+
/** Removes a connection a newer registration replaced and asks its socket to close. */
|
|
145
|
+
#replace(record) {
|
|
146
|
+
this.#records.delete(record.connection);
|
|
147
|
+
this.#index.remove(record);
|
|
148
|
+
this.#untrack(record);
|
|
149
|
+
this.#changed();
|
|
150
|
+
this.#push({
|
|
151
|
+
kind: 'kick',
|
|
152
|
+
id: randomUUID(),
|
|
153
|
+
connections: Object.freeze([record.connection]),
|
|
154
|
+
code: REPLACED.code,
|
|
155
|
+
reason: REPLACED.reason,
|
|
156
|
+
});
|
|
157
|
+
if (this.logger.isEnabled('debug'))
|
|
158
|
+
this.logger.log('debug', 'Hub connection replaced by a newer registration', {
|
|
159
|
+
connection: record.connection,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
144
162
|
#ensureOpen() {
|
|
145
163
|
if (this.#closed)
|
|
146
164
|
throw new HubError('CLOSED');
|
|
@@ -150,11 +168,16 @@ export class LocalHub {
|
|
|
150
168
|
const value = record(input, this.ownerEpoch);
|
|
151
169
|
if (this.#records.has(value.connection))
|
|
152
170
|
throw new HubError('DUPLICATE_CONNECTION');
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
171
|
+
// The newest registration of a (tenant, member, clientId) wins (audit 2026-10-26): the older
|
|
172
|
+
// connection leaves the roster now and its socket is closed (4409 Replaced). It counts as
|
|
173
|
+
// gone for the limits below.
|
|
174
|
+
const held = value.clientId === undefined ? undefined : this.#index.client(value);
|
|
175
|
+
const room = held ? 1 : 0;
|
|
176
|
+
if (this.#records.size - room >= this.options.maxConnections ||
|
|
177
|
+
this.#records.size - room >= this.options.maxTotalConnections)
|
|
157
178
|
throw new HubError('CONNECTION_LIMIT');
|
|
179
|
+
if (held)
|
|
180
|
+
this.#replace(held);
|
|
158
181
|
this.#records.set(value.connection, value);
|
|
159
182
|
this.#index.add(value);
|
|
160
183
|
this.#track(value);
|
|
@@ -240,7 +263,10 @@ export class LocalHub {
|
|
|
240
263
|
id,
|
|
241
264
|
connections: Object.freeze(connections),
|
|
242
265
|
data: Uint8Array.from(data),
|
|
243
|
-
},
|
|
266
|
+
},
|
|
267
|
+
// The share of the queue broadcasts may fill (`broadcastShare`, D-0114); the rest stays
|
|
268
|
+
// for presence and kick events, which cannot be refused.
|
|
269
|
+
this.options.broadcastShare)) {
|
|
244
270
|
// The gateway has not taken the earlier messages yet: the caller backs off.
|
|
245
271
|
this.#throttle.log('debug', 'Hub broadcast refused; event queue busy', 'BUSY', undefined, {
|
|
246
272
|
maxPending: this.options.maxPending,
|