@livedesk/hub 0.1.27 → 0.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/agents/agent-manager.js +27 -11
- package/src/agents/codex-agent-runtime.js +325 -41
- package/src/frame-packet-contract.mjs +427 -0
- package/src/live-capture-transition-retry.mjs +297 -0
- package/src/mode4-atlas-pool.js +17 -3
- package/src/mode4-atlas-worker.js +134 -21
- package/src/mode4-atlas.js +180 -16
- package/src/remote-audio-subscription-contract.mjs +109 -0
- package/src/remote-audio-subscription-contract.test.mjs +72 -0
- package/src/remote-hub.js +6405 -4236
- package/src/server.js +3234 -2206
- package/src/transport/agent-binary-ingress.js +810 -0
- package/src/transport/relay-hub-control.js +388 -57
- package/src/transport/udp-hub-transport.js +222 -46
- package/src/transport/udp-protocol.js +302 -70
|
@@ -10,6 +10,7 @@ export const RELAY_CONTROL_DIRECTIONS = Object.freeze({
|
|
|
10
10
|
});
|
|
11
11
|
|
|
12
12
|
const RELAY_MAX_LINE_BYTES = 768 * 1024;
|
|
13
|
+
const RELAY_MAX_RECEIVE_CHUNKS = 256;
|
|
13
14
|
const RELAY_NONCE_BYTES = 12;
|
|
14
15
|
const RELAY_TAG_BYTES = 16;
|
|
15
16
|
const RELAY_PUBLIC_KEY_BYTES = 65;
|
|
@@ -40,7 +41,7 @@ function boundedInteger(value, minimum, maximum, fallback) {
|
|
|
40
41
|
}
|
|
41
42
|
|
|
42
43
|
function encodeBase64Url(value) {
|
|
43
|
-
return Buffer.from(value).toString('base64url');
|
|
44
|
+
return (Buffer.isBuffer(value) ? value : Buffer.from(value)).toString('base64url');
|
|
44
45
|
}
|
|
45
46
|
|
|
46
47
|
function decodeCanonicalBase64Url(value, expectedBytes = 0, maximumBytes = 1024) {
|
|
@@ -112,7 +113,9 @@ function requireSequence(sequence) {
|
|
|
112
113
|
}
|
|
113
114
|
|
|
114
115
|
function normalizeSessionKey(sessionKey) {
|
|
115
|
-
const key = Buffer.
|
|
116
|
+
const key = Buffer.isBuffer(sessionKey)
|
|
117
|
+
? sessionKey
|
|
118
|
+
: Buffer.from(sessionKey || []);
|
|
116
119
|
if (key.length !== 32) {
|
|
117
120
|
throw relayError('relay-session-key-invalid');
|
|
118
121
|
}
|
|
@@ -224,18 +227,31 @@ export function encryptRelayControlPayload({
|
|
|
224
227
|
nonce = crypto.randomBytes(RELAY_NONCE_BYTES)
|
|
225
228
|
} = {}) {
|
|
226
229
|
const key = normalizeSessionKey(sessionKey);
|
|
227
|
-
const cleartext = Buffer.
|
|
230
|
+
const cleartext = Buffer.isBuffer(plaintext)
|
|
231
|
+
? plaintext
|
|
232
|
+
: Buffer.from(plaintext || []);
|
|
228
233
|
if (cleartext.length < 1 || cleartext.length > RELAY_CONTROL_MAX_PLAINTEXT_BYTES) {
|
|
229
234
|
throw relayError('relay-plaintext-size-invalid');
|
|
230
235
|
}
|
|
231
|
-
const nonceBytes = Buffer.
|
|
236
|
+
const nonceBytes = Buffer.isBuffer(nonce)
|
|
237
|
+
? nonce
|
|
238
|
+
: Buffer.from(nonce || []);
|
|
232
239
|
if (nonceBytes.length !== RELAY_NONCE_BYTES) {
|
|
233
240
|
throw relayError('relay-nonce-invalid');
|
|
234
241
|
}
|
|
235
242
|
const cipher = crypto.createCipheriv('aes-256-gcm', key, nonceBytes, { authTagLength: RELAY_TAG_BYTES });
|
|
236
243
|
cipher.setAAD(controlAad(roomId, peerId, direction, sequence));
|
|
237
|
-
const
|
|
238
|
-
|
|
244
|
+
const update = cipher.update(cleartext);
|
|
245
|
+
const final = cipher.final();
|
|
246
|
+
const ciphertext = final.length > 0
|
|
247
|
+
? Buffer.concat([update, final], update.length + final.length)
|
|
248
|
+
: update;
|
|
249
|
+
const tag = cipher.getAuthTag();
|
|
250
|
+
const envelope = Buffer.allocUnsafe(nonceBytes.length + ciphertext.length + tag.length);
|
|
251
|
+
nonceBytes.copy(envelope, 0);
|
|
252
|
+
ciphertext.copy(envelope, nonceBytes.length);
|
|
253
|
+
tag.copy(envelope, nonceBytes.length + ciphertext.length);
|
|
254
|
+
return encodeBase64Url(envelope);
|
|
239
255
|
}
|
|
240
256
|
|
|
241
257
|
export function decryptRelayControlPayload({
|
|
@@ -266,7 +282,11 @@ export function decryptRelayControlPayload({
|
|
|
266
282
|
const decipher = crypto.createDecipheriv('aes-256-gcm', key, nonce, { authTagLength: RELAY_TAG_BYTES });
|
|
267
283
|
decipher.setAAD(controlAad(roomId, peerId, direction, sequence));
|
|
268
284
|
decipher.setAuthTag(tag);
|
|
269
|
-
const
|
|
285
|
+
const update = decipher.update(ciphertext);
|
|
286
|
+
const final = decipher.final();
|
|
287
|
+
const plaintext = final.length > 0
|
|
288
|
+
? Buffer.concat([update, final], update.length + final.length)
|
|
289
|
+
: update;
|
|
270
290
|
if (plaintext.length < 1 || plaintext.length > RELAY_CONTROL_MAX_PLAINTEXT_BYTES) {
|
|
271
291
|
throw relayError('relay-plaintext-size-invalid');
|
|
272
292
|
}
|
|
@@ -356,7 +376,13 @@ export class HubRelayVirtualSocket extends EventEmitter {
|
|
|
356
376
|
}
|
|
357
377
|
|
|
358
378
|
_deliver(plaintext) {
|
|
359
|
-
if (
|
|
379
|
+
if (this.destroyed) return;
|
|
380
|
+
if (Buffer.isBuffer(plaintext)) {
|
|
381
|
+
this.emit('data', plaintext);
|
|
382
|
+
return;
|
|
383
|
+
}
|
|
384
|
+
this.owner.recordPlaintextDeliveryCopy();
|
|
385
|
+
this.emit('data', Buffer.from(plaintext));
|
|
360
386
|
}
|
|
361
387
|
|
|
362
388
|
_drain() {
|
|
@@ -437,7 +463,9 @@ export function createHubRelayControl({
|
|
|
437
463
|
DEFAULT_CONNECT_TIMEOUT_MS);
|
|
438
464
|
const peers = new Map();
|
|
439
465
|
const retiredPeerIds = new Map();
|
|
466
|
+
const connectionOwners = new Map();
|
|
440
467
|
const sendQueue = [];
|
|
468
|
+
let sendQueueHead = 0;
|
|
441
469
|
const peerQueueStats = new Map();
|
|
442
470
|
const counters = {
|
|
443
471
|
connections: 0,
|
|
@@ -446,7 +474,17 @@ export function createHubRelayControl({
|
|
|
446
474
|
rejectedPeers: 0,
|
|
447
475
|
encryptedMessagesReceived: 0,
|
|
448
476
|
encryptedMessagesSent: 0,
|
|
449
|
-
queueDrops: 0
|
|
477
|
+
queueDrops: 0,
|
|
478
|
+
receiveLineAssemblyCopies: 0,
|
|
479
|
+
receiveLineAssemblyBytes: 0,
|
|
480
|
+
plaintextDeliveryCopies: 0,
|
|
481
|
+
peerQueuePurges: 0,
|
|
482
|
+
peerQueuePurgedBytes: 0,
|
|
483
|
+
staleConnectionEventDrops: 0,
|
|
484
|
+
socketOwnersCreated: 0,
|
|
485
|
+
socketOwnersClosed: 0,
|
|
486
|
+
peerOwnersCreated: 0,
|
|
487
|
+
peerOwnersClosed: 0
|
|
450
488
|
};
|
|
451
489
|
|
|
452
490
|
let desired = false;
|
|
@@ -456,11 +494,24 @@ export function createHubRelayControl({
|
|
|
456
494
|
let connectionGeneration = 0;
|
|
457
495
|
let reconnectTimer = null;
|
|
458
496
|
let connectAttemptTimer = null;
|
|
497
|
+
let closeSocketTimer = null;
|
|
498
|
+
let closeTimedOut = false;
|
|
459
499
|
let reconnectAttempt = 0;
|
|
460
|
-
let
|
|
500
|
+
let receiveChunks = [];
|
|
501
|
+
let receiveBytes = 0;
|
|
502
|
+
let receiveChunkHighWater = 0;
|
|
503
|
+
let receiveByteHighWater = 0;
|
|
461
504
|
let queuedBytes = 0;
|
|
505
|
+
let queuedMessageHighWater = 0;
|
|
506
|
+
let queuedByteHighWater = 0;
|
|
507
|
+
let socketWritableByteHighWater = 0;
|
|
508
|
+
let totalOutboundByteHighWater = 0;
|
|
509
|
+
let socketOwnerHighWater = 0;
|
|
510
|
+
let peerOwnerHighWater = 0;
|
|
511
|
+
let timerOwnerHighWater = 0;
|
|
462
512
|
let awaitingDrain = false;
|
|
463
513
|
let lastError = '';
|
|
514
|
+
let lastDisconnectReason = '';
|
|
464
515
|
let lastConnectedAt = '';
|
|
465
516
|
let lastTransitionAt = '';
|
|
466
517
|
let state = enabled ? 'idle' : 'disabled';
|
|
@@ -471,6 +522,40 @@ export function createHubRelayControl({
|
|
|
471
522
|
if (error) lastError = safeText(error, 120);
|
|
472
523
|
}
|
|
473
524
|
|
|
525
|
+
function activeTimerCount() {
|
|
526
|
+
return Number(Boolean(reconnectTimer))
|
|
527
|
+
+ Number(Boolean(connectAttemptTimer))
|
|
528
|
+
+ Number(Boolean(closeSocketTimer));
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
function recordTimerOwnerHighWater() {
|
|
532
|
+
timerOwnerHighWater = Math.max(
|
|
533
|
+
timerOwnerHighWater,
|
|
534
|
+
activeTimerCount());
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
function addConnectionOwner(socket, generation) {
|
|
538
|
+
if (connectionOwners.has(socket)) {
|
|
539
|
+
throw relayError('relay-socket-owner-duplicate');
|
|
540
|
+
}
|
|
541
|
+
connectionOwners.set(socket, {
|
|
542
|
+
socket,
|
|
543
|
+
generation,
|
|
544
|
+
phase: 'current'
|
|
545
|
+
});
|
|
546
|
+
counters.socketOwnersCreated += 1;
|
|
547
|
+
socketOwnerHighWater = Math.max(
|
|
548
|
+
socketOwnerHighWater,
|
|
549
|
+
connectionOwners.size);
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
function markConnectionOwnerClosing(socket, generation) {
|
|
553
|
+
const owner = connectionOwners.get(socket);
|
|
554
|
+
if (!owner || owner.generation !== generation) return false;
|
|
555
|
+
owner.phase = 'closing';
|
|
556
|
+
return true;
|
|
557
|
+
}
|
|
558
|
+
|
|
474
559
|
function queueStats(peerId) {
|
|
475
560
|
if (!peerId) return { messages: 0, bytes: 0, backpressured: false };
|
|
476
561
|
let stats = peerQueueStats.get(peerId);
|
|
@@ -490,13 +575,73 @@ export function createHubRelayControl({
|
|
|
490
575
|
stats.bytes = Math.max(0, stats.bytes - item.buffer.length);
|
|
491
576
|
}
|
|
492
577
|
|
|
578
|
+
function queuedMessageCount() {
|
|
579
|
+
return sendQueue.length - sendQueueHead;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
function socketWritableBytes(socket = connection) {
|
|
583
|
+
const value = Number(socket?.writableLength);
|
|
584
|
+
return Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
function recordOutboundHighWater(socket = connection) {
|
|
588
|
+
const writableBytes = socketWritableBytes(socket);
|
|
589
|
+
socketWritableByteHighWater = Math.max(
|
|
590
|
+
socketWritableByteHighWater,
|
|
591
|
+
writableBytes);
|
|
592
|
+
totalOutboundByteHighWater = Math.max(
|
|
593
|
+
totalOutboundByteHighWater,
|
|
594
|
+
queuedBytes + writableBytes);
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
function dequeueQueuedItem() {
|
|
598
|
+
if (queuedMessageCount() <= 0) return null;
|
|
599
|
+
const item = sendQueue[sendQueueHead];
|
|
600
|
+
sendQueue[sendQueueHead] = null;
|
|
601
|
+
sendQueueHead += 1;
|
|
602
|
+
if (sendQueueHead >= 64 && sendQueueHead * 2 >= sendQueue.length) {
|
|
603
|
+
sendQueue.splice(0, sendQueueHead);
|
|
604
|
+
sendQueueHead = 0;
|
|
605
|
+
}
|
|
606
|
+
return item;
|
|
607
|
+
}
|
|
608
|
+
|
|
493
609
|
function clearQueue() {
|
|
494
610
|
sendQueue.length = 0;
|
|
611
|
+
sendQueueHead = 0;
|
|
495
612
|
queuedBytes = 0;
|
|
496
613
|
awaitingDrain = false;
|
|
497
614
|
peerQueueStats.clear();
|
|
498
615
|
}
|
|
499
616
|
|
|
617
|
+
function purgePeerQueue(peerId) {
|
|
618
|
+
if (!peerId || queuedMessageCount() === 0) return;
|
|
619
|
+
let writeIndex = 0;
|
|
620
|
+
let purgedBytes = 0;
|
|
621
|
+
for (let readIndex = sendQueueHead; readIndex < sendQueue.length; readIndex += 1) {
|
|
622
|
+
const item = sendQueue[readIndex];
|
|
623
|
+
if (item.peerId === peerId) {
|
|
624
|
+
purgedBytes += item.buffer.length;
|
|
625
|
+
releaseQueuedItem(item);
|
|
626
|
+
continue;
|
|
627
|
+
}
|
|
628
|
+
sendQueue[writeIndex] = item;
|
|
629
|
+
writeIndex += 1;
|
|
630
|
+
}
|
|
631
|
+
for (let index = writeIndex; index < sendQueue.length; index += 1) {
|
|
632
|
+
sendQueue[index] = null;
|
|
633
|
+
}
|
|
634
|
+
if (purgedBytes > 0) {
|
|
635
|
+
sendQueue.length = writeIndex;
|
|
636
|
+
sendQueueHead = 0;
|
|
637
|
+
counters.peerQueuePurges += 1;
|
|
638
|
+
counters.peerQueuePurgedBytes += purgedBytes;
|
|
639
|
+
} else if (sendQueueHead > 0) {
|
|
640
|
+
sendQueue.splice(0, sendQueueHead);
|
|
641
|
+
sendQueueHead = 0;
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
|
|
500
645
|
function emitPeerDrains() {
|
|
501
646
|
if (awaitingDrain) return;
|
|
502
647
|
for (const [peerId, stats] of peerQueueStats) {
|
|
@@ -513,8 +658,9 @@ export function createHubRelayControl({
|
|
|
513
658
|
function flushQueue() {
|
|
514
659
|
const socket = connection;
|
|
515
660
|
if (!socket || socket.destroyed || !socket.writable || awaitingDrain) return;
|
|
516
|
-
while (
|
|
517
|
-
const item =
|
|
661
|
+
while (queuedMessageCount() > 0 && !awaitingDrain) {
|
|
662
|
+
const item = dequeueQueuedItem();
|
|
663
|
+
if (!item) break;
|
|
518
664
|
releaseQueuedItem(item);
|
|
519
665
|
let writable = false;
|
|
520
666
|
try {
|
|
@@ -524,6 +670,7 @@ export function createHubRelayControl({
|
|
|
524
670
|
return;
|
|
525
671
|
}
|
|
526
672
|
if (!writable) awaitingDrain = true;
|
|
673
|
+
recordOutboundHighWater(socket);
|
|
527
674
|
}
|
|
528
675
|
emitPeerDrains();
|
|
529
676
|
}
|
|
@@ -538,8 +685,9 @@ export function createHubRelayControl({
|
|
|
538
685
|
}
|
|
539
686
|
if (buffer.length > RELAY_MAX_LINE_BYTES) return false;
|
|
540
687
|
const stats = queueStats(peerId);
|
|
541
|
-
|
|
542
|
-
|
|
688
|
+
const writableBytes = socketWritableBytes();
|
|
689
|
+
if (queuedMessageCount() >= maxQueueMessages
|
|
690
|
+
|| queuedBytes + writableBytes + buffer.length > maxQueueBytes
|
|
543
691
|
|| (peerId && (stats.messages >= maxPeerQueueMessages
|
|
544
692
|
|| stats.bytes + buffer.length > maxPeerQueueBytes))) {
|
|
545
693
|
counters.queueDrops += 1;
|
|
@@ -548,6 +696,9 @@ export function createHubRelayControl({
|
|
|
548
696
|
}
|
|
549
697
|
sendQueue.push({ buffer, peerId });
|
|
550
698
|
queuedBytes += buffer.length;
|
|
699
|
+
queuedMessageHighWater = Math.max(queuedMessageHighWater, queuedMessageCount());
|
|
700
|
+
queuedByteHighWater = Math.max(queuedByteHighWater, queuedBytes);
|
|
701
|
+
recordOutboundHighWater();
|
|
551
702
|
if (peerId) {
|
|
552
703
|
stats.messages += 1;
|
|
553
704
|
stats.bytes += buffer.length;
|
|
@@ -594,7 +745,9 @@ export function createHubRelayControl({
|
|
|
594
745
|
return false;
|
|
595
746
|
}
|
|
596
747
|
peers.delete(id);
|
|
748
|
+
counters.peerOwnersClosed += 1;
|
|
597
749
|
rememberRetiredPeer(id);
|
|
750
|
+
purgePeerQueue(id);
|
|
598
751
|
peerQueueStats.delete(id);
|
|
599
752
|
if (notify) sendRelayClose(id, reason);
|
|
600
753
|
if (Buffer.isBuffer(peer.sessionKey)) peer.sessionKey.fill(0);
|
|
@@ -620,7 +773,9 @@ export function createHubRelayControl({
|
|
|
620
773
|
function sendPeerPlaintext(peerId, plaintext) {
|
|
621
774
|
const peer = peers.get(String(peerId || ''));
|
|
622
775
|
if (!peer || peer.socket.destroyed || peer.handshakeState !== 'established') return false;
|
|
623
|
-
const cleartext = Buffer.
|
|
776
|
+
const cleartext = Buffer.isBuffer(plaintext)
|
|
777
|
+
? plaintext
|
|
778
|
+
: Buffer.from(plaintext || []);
|
|
624
779
|
if (cleartext.length < 1 || cleartext.length > RELAY_CONTROL_MAX_PLAINTEXT_BYTES) {
|
|
625
780
|
rejectPeer(peer.peerId, 'relay-plaintext-size-invalid');
|
|
626
781
|
return false;
|
|
@@ -725,6 +880,8 @@ export function createHubRelayControl({
|
|
|
725
880
|
};
|
|
726
881
|
peer.socket = new HubRelayVirtualSocket(api, peerId, remoteAddress, port);
|
|
727
882
|
peers.set(peerId, peer);
|
|
883
|
+
counters.peerOwnersCreated += 1;
|
|
884
|
+
peerOwnerHighWater = Math.max(peerOwnerHighWater, peers.size);
|
|
728
885
|
const sent = enqueueEnvelope({
|
|
729
886
|
type: 'relay.handshake',
|
|
730
887
|
protocol: RELAY_CONTROL_PROTOCOL,
|
|
@@ -763,7 +920,7 @@ export function createHubRelayControl({
|
|
|
763
920
|
function handleClientData(message) {
|
|
764
921
|
let peerId = '';
|
|
765
922
|
try {
|
|
766
|
-
peerId =
|
|
923
|
+
peerId = safeText(message.peerId, 64);
|
|
767
924
|
const peer = peers.get(peerId);
|
|
768
925
|
if (!peer || peer.handshakeState !== 'established') {
|
|
769
926
|
throw relayError('relay-handshake-required');
|
|
@@ -826,45 +983,117 @@ export function createHubRelayControl({
|
|
|
826
983
|
return;
|
|
827
984
|
}
|
|
828
985
|
if (message.type === 'relay.close') {
|
|
986
|
+
const reason = safeText(message.reason, 80) || 'relay-server-closed';
|
|
987
|
+
if (!peerId) {
|
|
988
|
+
lastError = reason;
|
|
989
|
+
lastDisconnectReason = reason;
|
|
990
|
+
connection?.destroy();
|
|
991
|
+
return;
|
|
992
|
+
}
|
|
829
993
|
try {
|
|
830
|
-
closePeer(requirePeerId(message.peerId),
|
|
994
|
+
closePeer(requirePeerId(message.peerId), reason, { notify: false });
|
|
831
995
|
} catch {
|
|
832
996
|
// Invalid untrusted routing fields are ignored.
|
|
833
997
|
}
|
|
834
998
|
}
|
|
835
999
|
}
|
|
836
1000
|
|
|
837
|
-
function
|
|
1001
|
+
function clearReceiveChunks() {
|
|
1002
|
+
receiveChunks = [];
|
|
1003
|
+
receiveBytes = 0;
|
|
1004
|
+
}
|
|
1005
|
+
|
|
1006
|
+
function retainReceiveTail(incoming, offset) {
|
|
1007
|
+
const tail = incoming.subarray(offset);
|
|
1008
|
+
if (tail.length === 0) return true;
|
|
1009
|
+
if (receiveBytes + tail.length > RELAY_MAX_LINE_BYTES) return false;
|
|
1010
|
+
// A tail following a complete line must not retain the complete large
|
|
1011
|
+
// TCP chunk. A chunk containing no line is already exact ownership and
|
|
1012
|
+
// can be retained without copying.
|
|
1013
|
+
const retained = offset > 0 ? Buffer.from(tail) : tail;
|
|
1014
|
+
if (receiveChunks.length >= RELAY_MAX_RECEIVE_CHUNKS) return false;
|
|
1015
|
+
receiveChunks.push(retained);
|
|
1016
|
+
receiveBytes += retained.length;
|
|
1017
|
+
receiveChunkHighWater = Math.max(receiveChunkHighWater, receiveChunks.length);
|
|
1018
|
+
receiveByteHighWater = Math.max(receiveByteHighWater, receiveBytes);
|
|
1019
|
+
return true;
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
function assembleReceiveLine(segment) {
|
|
1023
|
+
if (receiveBytes === 0) return segment;
|
|
1024
|
+
const totalBytes = receiveBytes + segment.length;
|
|
1025
|
+
const line = Buffer.allocUnsafe(totalBytes);
|
|
1026
|
+
let offset = 0;
|
|
1027
|
+
for (const chunk of receiveChunks) {
|
|
1028
|
+
chunk.copy(line, offset);
|
|
1029
|
+
offset += chunk.length;
|
|
1030
|
+
}
|
|
1031
|
+
segment.copy(line, offset);
|
|
1032
|
+
counters.receiveLineAssemblyCopies += 1;
|
|
1033
|
+
counters.receiveLineAssemblyBytes += totalBytes;
|
|
1034
|
+
clearReceiveChunks();
|
|
1035
|
+
return line;
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
function isCurrentConnectionOwner(socket, generation) {
|
|
1039
|
+
return desired
|
|
1040
|
+
&& !closing
|
|
1041
|
+
&& connection === socket
|
|
1042
|
+
&& generation === connectionGeneration
|
|
1043
|
+
&& !socket.destroyed;
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
function processRelayLine(socket, generation, line) {
|
|
1047
|
+
if (!isCurrentConnectionOwner(socket, generation)) {
|
|
1048
|
+
counters.staleConnectionEventDrops += 1;
|
|
1049
|
+
return false;
|
|
1050
|
+
}
|
|
1051
|
+
if (line.length === 0) return true;
|
|
1052
|
+
let message;
|
|
1053
|
+
try {
|
|
1054
|
+
message = JSON.parse(line.toString('utf8'));
|
|
1055
|
+
} catch {
|
|
1056
|
+
lastError = 'relay-invalid-json';
|
|
1057
|
+
socket.destroy();
|
|
1058
|
+
return false;
|
|
1059
|
+
}
|
|
1060
|
+
if (!isCurrentConnectionOwner(socket, generation)) {
|
|
1061
|
+
counters.staleConnectionEventDrops += 1;
|
|
1062
|
+
return false;
|
|
1063
|
+
}
|
|
1064
|
+
handleRelayEnvelope(message);
|
|
1065
|
+
return true;
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
function onConnectionData(socket, generation, chunk) {
|
|
1069
|
+
if (!isCurrentConnectionOwner(socket, generation)) {
|
|
1070
|
+
counters.staleConnectionEventDrops += 1;
|
|
1071
|
+
return;
|
|
1072
|
+
}
|
|
838
1073
|
const incoming = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
1074
|
+
let offset = 0;
|
|
1075
|
+
while (offset < incoming.length) {
|
|
1076
|
+
if (!isCurrentConnectionOwner(socket, generation)) {
|
|
1077
|
+
counters.staleConnectionEventDrops += 1;
|
|
1078
|
+
return;
|
|
1079
|
+
}
|
|
1080
|
+
const newline = incoming.indexOf(0x0a, offset);
|
|
844
1081
|
if (newline < 0) {
|
|
845
|
-
if (
|
|
1082
|
+
if (!retainReceiveTail(incoming, offset)) {
|
|
846
1083
|
lastError = 'relay-line-too-large';
|
|
847
|
-
|
|
1084
|
+
socket.destroy();
|
|
848
1085
|
}
|
|
849
1086
|
return;
|
|
850
1087
|
}
|
|
851
|
-
|
|
1088
|
+
const segmentLength = newline - offset;
|
|
1089
|
+
if (receiveBytes + segmentLength > RELAY_MAX_LINE_BYTES) {
|
|
852
1090
|
lastError = 'relay-line-too-large';
|
|
853
|
-
|
|
854
|
-
return;
|
|
855
|
-
}
|
|
856
|
-
const line = receiveBuffer.subarray(0, newline);
|
|
857
|
-
receiveBuffer = receiveBuffer.subarray(newline + 1);
|
|
858
|
-
if (line.length === 0) continue;
|
|
859
|
-
let message;
|
|
860
|
-
try {
|
|
861
|
-
message = JSON.parse(line.toString('utf8'));
|
|
862
|
-
} catch {
|
|
863
|
-
lastError = 'relay-invalid-json';
|
|
864
|
-
connection?.destroy();
|
|
1091
|
+
socket.destroy();
|
|
865
1092
|
return;
|
|
866
1093
|
}
|
|
867
|
-
|
|
1094
|
+
const line = assembleReceiveLine(incoming.subarray(offset, newline));
|
|
1095
|
+
if (!processRelayLine(socket, generation, line)) return;
|
|
1096
|
+
offset = newline + 1;
|
|
868
1097
|
}
|
|
869
1098
|
}
|
|
870
1099
|
|
|
@@ -883,6 +1112,7 @@ export function createHubRelayControl({
|
|
|
883
1112
|
connectNow();
|
|
884
1113
|
}, delay);
|
|
885
1114
|
reconnectTimer.unref?.();
|
|
1115
|
+
recordTimerOwnerHighWater();
|
|
886
1116
|
}
|
|
887
1117
|
|
|
888
1118
|
function clearConnectAttemptTimer() {
|
|
@@ -891,10 +1121,21 @@ export function createHubRelayControl({
|
|
|
891
1121
|
}
|
|
892
1122
|
|
|
893
1123
|
function handleConnectionClosed(socket, generation) {
|
|
1124
|
+
const owner = connectionOwners.get(socket);
|
|
1125
|
+
if (!owner || owner.generation !== generation) return;
|
|
1126
|
+
connectionOwners.delete(socket);
|
|
1127
|
+
counters.socketOwnersClosed += 1;
|
|
1128
|
+
if (owner.phase === 'closing') return;
|
|
894
1129
|
if (connection !== socket || generation !== connectionGeneration) return;
|
|
895
1130
|
clearConnectAttemptTimer();
|
|
896
1131
|
connection = null;
|
|
897
|
-
|
|
1132
|
+
const disconnectReason = safeText(lastError, 120) || 'relay-socket-closed';
|
|
1133
|
+
lastDisconnectReason = disconnectReason;
|
|
1134
|
+
logWarn(
|
|
1135
|
+
'relay',
|
|
1136
|
+
`Encrypted relay control disconnected reason=${disconnectReason} peers=${peers.size}`
|
|
1137
|
+
);
|
|
1138
|
+
clearReceiveChunks();
|
|
898
1139
|
clearQueue();
|
|
899
1140
|
closeAllPeers('relay-connection-lost');
|
|
900
1141
|
if (desired && !closing) scheduleReconnect();
|
|
@@ -914,6 +1155,7 @@ export function createHubRelayControl({
|
|
|
914
1155
|
return;
|
|
915
1156
|
}
|
|
916
1157
|
connection = socket;
|
|
1158
|
+
addConnectionOwner(socket, generation);
|
|
917
1159
|
socket.setNoDelay?.(true);
|
|
918
1160
|
socket.setKeepAlive?.(true, 10_000);
|
|
919
1161
|
socket.once('connect', () => {
|
|
@@ -931,7 +1173,7 @@ export function createHubRelayControl({
|
|
|
931
1173
|
});
|
|
932
1174
|
if (!registered) socket.destroy();
|
|
933
1175
|
});
|
|
934
|
-
socket.on('data', onConnectionData);
|
|
1176
|
+
socket.on('data', chunk => onConnectionData(socket, generation, chunk));
|
|
935
1177
|
socket.on('drain', () => {
|
|
936
1178
|
if (connection !== socket || generation !== connectionGeneration) return;
|
|
937
1179
|
awaitingDrain = false;
|
|
@@ -953,10 +1195,14 @@ export function createHubRelayControl({
|
|
|
953
1195
|
socket.destroy();
|
|
954
1196
|
}, connectTimeoutMs);
|
|
955
1197
|
connectAttemptTimer.unref?.();
|
|
1198
|
+
recordTimerOwnerHighWater();
|
|
956
1199
|
}
|
|
957
1200
|
|
|
958
1201
|
async function start() {
|
|
959
1202
|
if (!enabled || started) return getStatus();
|
|
1203
|
+
if (closing || closeTimedOut || connectionOwners.size > 0) {
|
|
1204
|
+
throw relayError('relay-close-nonterminal');
|
|
1205
|
+
}
|
|
960
1206
|
started = true;
|
|
961
1207
|
desired = true;
|
|
962
1208
|
closing = false;
|
|
@@ -965,59 +1211,138 @@ export function createHubRelayControl({
|
|
|
965
1211
|
}
|
|
966
1212
|
|
|
967
1213
|
async function close() {
|
|
1214
|
+
const closeDeadlineAt = Date.now() + 1_000;
|
|
968
1215
|
desired = false;
|
|
969
1216
|
started = false;
|
|
970
1217
|
closing = true;
|
|
1218
|
+
closeTimedOut = false;
|
|
1219
|
+
transition('closing');
|
|
971
1220
|
if (reconnectTimer) clearTimeout(reconnectTimer);
|
|
972
1221
|
reconnectTimer = null;
|
|
973
1222
|
clearConnectAttemptTimer();
|
|
974
1223
|
closeAllPeers('hub-shutdown');
|
|
975
1224
|
clearQueue();
|
|
976
1225
|
const socket = connection;
|
|
1226
|
+
const generation = connectionGeneration;
|
|
1227
|
+
if (socket) markConnectionOwnerClosing(socket, generation);
|
|
977
1228
|
connection = null;
|
|
978
1229
|
connectionGeneration += 1;
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
1230
|
+
clearReceiveChunks();
|
|
1231
|
+
const closingOwners = [...connectionOwners.values()];
|
|
1232
|
+
if (closingOwners.length > 0) {
|
|
1233
|
+
const ownersClosed = new Promise((resolve, reject) => {
|
|
982
1234
|
let settled = false;
|
|
1235
|
+
const onClose = () => {
|
|
1236
|
+
if (connectionOwners.size === 0) settle();
|
|
1237
|
+
};
|
|
983
1238
|
const settle = () => {
|
|
984
1239
|
if (settled) return;
|
|
985
1240
|
settled = true;
|
|
986
|
-
clearTimeout(
|
|
1241
|
+
if (closeSocketTimer) clearTimeout(closeSocketTimer);
|
|
1242
|
+
closeSocketTimer = null;
|
|
1243
|
+
for (const owner of closingOwners) {
|
|
1244
|
+
owner.socket.off?.('close', onClose);
|
|
1245
|
+
}
|
|
987
1246
|
resolve();
|
|
988
1247
|
};
|
|
989
|
-
const
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
1248
|
+
const fail = () => {
|
|
1249
|
+
if (settled) return;
|
|
1250
|
+
settled = true;
|
|
1251
|
+
closeSocketTimer = null;
|
|
1252
|
+
for (const owner of closingOwners) {
|
|
1253
|
+
owner.socket.off?.('close', onClose);
|
|
1254
|
+
}
|
|
1255
|
+
reject(relayError('relay-close-timeout'));
|
|
1256
|
+
};
|
|
1257
|
+
for (const owner of closingOwners) {
|
|
1258
|
+
owner.phase = 'closing';
|
|
1259
|
+
owner.socket.once('close', onClose);
|
|
1260
|
+
}
|
|
1261
|
+
closeSocketTimer = setTimeout(() => {
|
|
1262
|
+
fail();
|
|
1263
|
+
}, Math.max(1, closeDeadlineAt - Date.now()));
|
|
1264
|
+
recordTimerOwnerHighWater();
|
|
1265
|
+
if (connectionOwners.size === 0) settle();
|
|
1266
|
+
});
|
|
1267
|
+
for (const owner of closingOwners) {
|
|
995
1268
|
try {
|
|
996
|
-
socket.end();
|
|
1269
|
+
owner.socket.end();
|
|
997
1270
|
} catch {
|
|
998
|
-
|
|
999
|
-
settle();
|
|
1271
|
+
// The exact owner remains in the ledger until close.
|
|
1000
1272
|
}
|
|
1001
|
-
|
|
1273
|
+
try {
|
|
1274
|
+
owner.socket.destroy();
|
|
1275
|
+
} catch {
|
|
1276
|
+
// Timeout remains explicit and nonterminal.
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
try {
|
|
1280
|
+
await ownersClosed;
|
|
1281
|
+
} catch (error) {
|
|
1282
|
+
closeTimedOut = true;
|
|
1283
|
+
lastError = 'relay-close-timeout';
|
|
1284
|
+
transition('close-timeout', lastError);
|
|
1285
|
+
throw error;
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
if (connectionOwners.size !== 0) {
|
|
1289
|
+
closeTimedOut = true;
|
|
1290
|
+
lastError = 'relay-close-nonterminal';
|
|
1291
|
+
transition('close-timeout', lastError);
|
|
1292
|
+
throw relayError(lastError);
|
|
1002
1293
|
}
|
|
1294
|
+
closeTimedOut = false;
|
|
1003
1295
|
closing = false;
|
|
1296
|
+
lastError = '';
|
|
1004
1297
|
transition(enabled ? 'idle' : 'disabled');
|
|
1005
1298
|
}
|
|
1006
1299
|
|
|
1007
1300
|
function getStatus() {
|
|
1301
|
+
const writableBytes = socketWritableBytes();
|
|
1302
|
+
recordOutboundHighWater();
|
|
1008
1303
|
return {
|
|
1009
1304
|
enabled,
|
|
1010
1305
|
started,
|
|
1011
1306
|
state,
|
|
1012
1307
|
connected: !!connection && !connection.destroyed && state === 'registered',
|
|
1308
|
+
closing,
|
|
1309
|
+
closeTimedOut,
|
|
1310
|
+
activeSocketCount: connectionOwners.size,
|
|
1311
|
+
currentSocketCount: connection && connectionOwners.has(connection) ? 1 : 0,
|
|
1312
|
+
closingSocketCount: [...connectionOwners.values()]
|
|
1313
|
+
.filter(owner => owner.phase === 'closing').length,
|
|
1314
|
+
activeTimerCount: activeTimerCount(),
|
|
1315
|
+
timerOwnerHighWater,
|
|
1316
|
+
socketOwnerHighWater,
|
|
1317
|
+
socketOwnersCreated: counters.socketOwnersCreated,
|
|
1318
|
+
socketOwnersClosed: counters.socketOwnersClosed,
|
|
1013
1319
|
protocol: RELAY_CONTROL_PROTOCOL,
|
|
1014
1320
|
endpoint: `${host}:${port}`,
|
|
1015
1321
|
host,
|
|
1016
1322
|
port,
|
|
1017
1323
|
peerCount: peers.size,
|
|
1324
|
+
peerOwnerHighWater,
|
|
1325
|
+
peerOwnersCreated: counters.peerOwnersCreated,
|
|
1326
|
+
peerOwnersClosed: counters.peerOwnersClosed,
|
|
1018
1327
|
peerQueueCount: peerQueueStats.size,
|
|
1019
|
-
queuedMessages:
|
|
1328
|
+
queuedMessages: queuedMessageCount(),
|
|
1020
1329
|
queuedBytes,
|
|
1330
|
+
socketWritableBytes: writableBytes,
|
|
1331
|
+
totalOutboundBufferedBytes: queuedBytes + writableBytes,
|
|
1332
|
+
queuedMessageHighWater,
|
|
1333
|
+
queuedByteHighWater,
|
|
1334
|
+
socketWritableByteHighWater,
|
|
1335
|
+
totalOutboundByteHighWater,
|
|
1336
|
+
receiveBufferedChunks: receiveChunks.length,
|
|
1337
|
+
receiveBufferedBytes: receiveBytes,
|
|
1338
|
+
receiveChunkHighWater,
|
|
1339
|
+
receiveByteHighWater,
|
|
1340
|
+
receiveLineAssemblyCopyCount: counters.receiveLineAssemblyCopies,
|
|
1341
|
+
receiveLineAssemblyBytes: counters.receiveLineAssemblyBytes,
|
|
1342
|
+
plaintextDeliveryCopyCount: counters.plaintextDeliveryCopies,
|
|
1343
|
+
peerQueuePurgeCount: counters.peerQueuePurges,
|
|
1344
|
+
peerQueuePurgedBytes: counters.peerQueuePurgedBytes,
|
|
1345
|
+
staleConnectionEventDropCount: counters.staleConnectionEventDrops,
|
|
1021
1346
|
maxPeers,
|
|
1022
1347
|
maxQueueMessages,
|
|
1023
1348
|
maxQueueBytes,
|
|
@@ -1029,17 +1354,23 @@ export function createHubRelayControl({
|
|
|
1029
1354
|
queueDropCount: counters.queueDrops,
|
|
1030
1355
|
lastConnectedAt,
|
|
1031
1356
|
lastTransitionAt,
|
|
1357
|
+
lastDisconnectReason,
|
|
1032
1358
|
lastError
|
|
1033
1359
|
};
|
|
1034
1360
|
}
|
|
1035
1361
|
|
|
1362
|
+
function recordPlaintextDeliveryCopy() {
|
|
1363
|
+
counters.plaintextDeliveryCopies += 1;
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1036
1366
|
const api = {
|
|
1037
1367
|
start,
|
|
1038
1368
|
close,
|
|
1039
1369
|
getStatus,
|
|
1040
1370
|
sendPeerPlaintext,
|
|
1041
1371
|
closePeer,
|
|
1042
|
-
isPeerBackpressured
|
|
1372
|
+
isPeerBackpressured,
|
|
1373
|
+
recordPlaintextDeliveryCopy
|
|
1043
1374
|
};
|
|
1044
1375
|
return api;
|
|
1045
1376
|
}
|