@livedesk/hub 0.1.26 → 0.1.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/agents/agent-manager.js +27 -11
- package/src/agents/codex-agent-runtime.js +325 -41
- package/src/frame-packet-contract.mjs +427 -0
- package/src/live-capture-transition-retry.mjs +297 -0
- package/src/mode4-atlas-pool.js +17 -3
- package/src/mode4-atlas-worker.js +134 -21
- package/src/mode4-atlas.js +180 -16
- package/src/remote-audio-subscription-contract.mjs +109 -0
- package/src/remote-audio-subscription-contract.test.mjs +72 -0
- package/src/remote-hub.js +6374 -3890
- package/src/server.js +3234 -2202
- package/src/transport/agent-binary-ingress.js +810 -0
- package/src/transport/relay-hub-control.js +372 -56
- package/src/transport/udp-hub-transport.js +215 -51
- package/src/transport/udp-protocol.js +302 -70
|
@@ -10,6 +10,7 @@ export const RELAY_CONTROL_DIRECTIONS = Object.freeze({
|
|
|
10
10
|
});
|
|
11
11
|
|
|
12
12
|
const RELAY_MAX_LINE_BYTES = 768 * 1024;
|
|
13
|
+
const RELAY_MAX_RECEIVE_CHUNKS = 256;
|
|
13
14
|
const RELAY_NONCE_BYTES = 12;
|
|
14
15
|
const RELAY_TAG_BYTES = 16;
|
|
15
16
|
const RELAY_PUBLIC_KEY_BYTES = 65;
|
|
@@ -40,7 +41,7 @@ function boundedInteger(value, minimum, maximum, fallback) {
|
|
|
40
41
|
}
|
|
41
42
|
|
|
42
43
|
function encodeBase64Url(value) {
|
|
43
|
-
return Buffer.from(value).toString('base64url');
|
|
44
|
+
return (Buffer.isBuffer(value) ? value : Buffer.from(value)).toString('base64url');
|
|
44
45
|
}
|
|
45
46
|
|
|
46
47
|
function decodeCanonicalBase64Url(value, expectedBytes = 0, maximumBytes = 1024) {
|
|
@@ -112,7 +113,9 @@ function requireSequence(sequence) {
|
|
|
112
113
|
}
|
|
113
114
|
|
|
114
115
|
function normalizeSessionKey(sessionKey) {
|
|
115
|
-
const key = Buffer.
|
|
116
|
+
const key = Buffer.isBuffer(sessionKey)
|
|
117
|
+
? sessionKey
|
|
118
|
+
: Buffer.from(sessionKey || []);
|
|
116
119
|
if (key.length !== 32) {
|
|
117
120
|
throw relayError('relay-session-key-invalid');
|
|
118
121
|
}
|
|
@@ -224,18 +227,31 @@ export function encryptRelayControlPayload({
|
|
|
224
227
|
nonce = crypto.randomBytes(RELAY_NONCE_BYTES)
|
|
225
228
|
} = {}) {
|
|
226
229
|
const key = normalizeSessionKey(sessionKey);
|
|
227
|
-
const cleartext = Buffer.
|
|
230
|
+
const cleartext = Buffer.isBuffer(plaintext)
|
|
231
|
+
? plaintext
|
|
232
|
+
: Buffer.from(plaintext || []);
|
|
228
233
|
if (cleartext.length < 1 || cleartext.length > RELAY_CONTROL_MAX_PLAINTEXT_BYTES) {
|
|
229
234
|
throw relayError('relay-plaintext-size-invalid');
|
|
230
235
|
}
|
|
231
|
-
const nonceBytes = Buffer.
|
|
236
|
+
const nonceBytes = Buffer.isBuffer(nonce)
|
|
237
|
+
? nonce
|
|
238
|
+
: Buffer.from(nonce || []);
|
|
232
239
|
if (nonceBytes.length !== RELAY_NONCE_BYTES) {
|
|
233
240
|
throw relayError('relay-nonce-invalid');
|
|
234
241
|
}
|
|
235
242
|
const cipher = crypto.createCipheriv('aes-256-gcm', key, nonceBytes, { authTagLength: RELAY_TAG_BYTES });
|
|
236
243
|
cipher.setAAD(controlAad(roomId, peerId, direction, sequence));
|
|
237
|
-
const
|
|
238
|
-
|
|
244
|
+
const update = cipher.update(cleartext);
|
|
245
|
+
const final = cipher.final();
|
|
246
|
+
const ciphertext = final.length > 0
|
|
247
|
+
? Buffer.concat([update, final], update.length + final.length)
|
|
248
|
+
: update;
|
|
249
|
+
const tag = cipher.getAuthTag();
|
|
250
|
+
const envelope = Buffer.allocUnsafe(nonceBytes.length + ciphertext.length + tag.length);
|
|
251
|
+
nonceBytes.copy(envelope, 0);
|
|
252
|
+
ciphertext.copy(envelope, nonceBytes.length);
|
|
253
|
+
tag.copy(envelope, nonceBytes.length + ciphertext.length);
|
|
254
|
+
return encodeBase64Url(envelope);
|
|
239
255
|
}
|
|
240
256
|
|
|
241
257
|
export function decryptRelayControlPayload({
|
|
@@ -266,7 +282,11 @@ export function decryptRelayControlPayload({
|
|
|
266
282
|
const decipher = crypto.createDecipheriv('aes-256-gcm', key, nonce, { authTagLength: RELAY_TAG_BYTES });
|
|
267
283
|
decipher.setAAD(controlAad(roomId, peerId, direction, sequence));
|
|
268
284
|
decipher.setAuthTag(tag);
|
|
269
|
-
const
|
|
285
|
+
const update = decipher.update(ciphertext);
|
|
286
|
+
const final = decipher.final();
|
|
287
|
+
const plaintext = final.length > 0
|
|
288
|
+
? Buffer.concat([update, final], update.length + final.length)
|
|
289
|
+
: update;
|
|
270
290
|
if (plaintext.length < 1 || plaintext.length > RELAY_CONTROL_MAX_PLAINTEXT_BYTES) {
|
|
271
291
|
throw relayError('relay-plaintext-size-invalid');
|
|
272
292
|
}
|
|
@@ -356,7 +376,13 @@ export class HubRelayVirtualSocket extends EventEmitter {
|
|
|
356
376
|
}
|
|
357
377
|
|
|
358
378
|
_deliver(plaintext) {
|
|
359
|
-
if (
|
|
379
|
+
if (this.destroyed) return;
|
|
380
|
+
if (Buffer.isBuffer(plaintext)) {
|
|
381
|
+
this.emit('data', plaintext);
|
|
382
|
+
return;
|
|
383
|
+
}
|
|
384
|
+
this.owner.recordPlaintextDeliveryCopy();
|
|
385
|
+
this.emit('data', Buffer.from(plaintext));
|
|
360
386
|
}
|
|
361
387
|
|
|
362
388
|
_drain() {
|
|
@@ -437,7 +463,9 @@ export function createHubRelayControl({
|
|
|
437
463
|
DEFAULT_CONNECT_TIMEOUT_MS);
|
|
438
464
|
const peers = new Map();
|
|
439
465
|
const retiredPeerIds = new Map();
|
|
466
|
+
const connectionOwners = new Map();
|
|
440
467
|
const sendQueue = [];
|
|
468
|
+
let sendQueueHead = 0;
|
|
441
469
|
const peerQueueStats = new Map();
|
|
442
470
|
const counters = {
|
|
443
471
|
connections: 0,
|
|
@@ -446,7 +474,17 @@ export function createHubRelayControl({
|
|
|
446
474
|
rejectedPeers: 0,
|
|
447
475
|
encryptedMessagesReceived: 0,
|
|
448
476
|
encryptedMessagesSent: 0,
|
|
449
|
-
queueDrops: 0
|
|
477
|
+
queueDrops: 0,
|
|
478
|
+
receiveLineAssemblyCopies: 0,
|
|
479
|
+
receiveLineAssemblyBytes: 0,
|
|
480
|
+
plaintextDeliveryCopies: 0,
|
|
481
|
+
peerQueuePurges: 0,
|
|
482
|
+
peerQueuePurgedBytes: 0,
|
|
483
|
+
staleConnectionEventDrops: 0,
|
|
484
|
+
socketOwnersCreated: 0,
|
|
485
|
+
socketOwnersClosed: 0,
|
|
486
|
+
peerOwnersCreated: 0,
|
|
487
|
+
peerOwnersClosed: 0
|
|
450
488
|
};
|
|
451
489
|
|
|
452
490
|
let desired = false;
|
|
@@ -456,9 +494,21 @@ export function createHubRelayControl({
|
|
|
456
494
|
let connectionGeneration = 0;
|
|
457
495
|
let reconnectTimer = null;
|
|
458
496
|
let connectAttemptTimer = null;
|
|
497
|
+
let closeSocketTimer = null;
|
|
498
|
+
let closeTimedOut = false;
|
|
459
499
|
let reconnectAttempt = 0;
|
|
460
|
-
let
|
|
500
|
+
let receiveChunks = [];
|
|
501
|
+
let receiveBytes = 0;
|
|
502
|
+
let receiveChunkHighWater = 0;
|
|
503
|
+
let receiveByteHighWater = 0;
|
|
461
504
|
let queuedBytes = 0;
|
|
505
|
+
let queuedMessageHighWater = 0;
|
|
506
|
+
let queuedByteHighWater = 0;
|
|
507
|
+
let socketWritableByteHighWater = 0;
|
|
508
|
+
let totalOutboundByteHighWater = 0;
|
|
509
|
+
let socketOwnerHighWater = 0;
|
|
510
|
+
let peerOwnerHighWater = 0;
|
|
511
|
+
let timerOwnerHighWater = 0;
|
|
462
512
|
let awaitingDrain = false;
|
|
463
513
|
let lastError = '';
|
|
464
514
|
let lastConnectedAt = '';
|
|
@@ -471,6 +521,40 @@ export function createHubRelayControl({
|
|
|
471
521
|
if (error) lastError = safeText(error, 120);
|
|
472
522
|
}
|
|
473
523
|
|
|
524
|
+
function activeTimerCount() {
|
|
525
|
+
return Number(Boolean(reconnectTimer))
|
|
526
|
+
+ Number(Boolean(connectAttemptTimer))
|
|
527
|
+
+ Number(Boolean(closeSocketTimer));
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
function recordTimerOwnerHighWater() {
|
|
531
|
+
timerOwnerHighWater = Math.max(
|
|
532
|
+
timerOwnerHighWater,
|
|
533
|
+
activeTimerCount());
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
function addConnectionOwner(socket, generation) {
|
|
537
|
+
if (connectionOwners.has(socket)) {
|
|
538
|
+
throw relayError('relay-socket-owner-duplicate');
|
|
539
|
+
}
|
|
540
|
+
connectionOwners.set(socket, {
|
|
541
|
+
socket,
|
|
542
|
+
generation,
|
|
543
|
+
phase: 'current'
|
|
544
|
+
});
|
|
545
|
+
counters.socketOwnersCreated += 1;
|
|
546
|
+
socketOwnerHighWater = Math.max(
|
|
547
|
+
socketOwnerHighWater,
|
|
548
|
+
connectionOwners.size);
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
function markConnectionOwnerClosing(socket, generation) {
|
|
552
|
+
const owner = connectionOwners.get(socket);
|
|
553
|
+
if (!owner || owner.generation !== generation) return false;
|
|
554
|
+
owner.phase = 'closing';
|
|
555
|
+
return true;
|
|
556
|
+
}
|
|
557
|
+
|
|
474
558
|
function queueStats(peerId) {
|
|
475
559
|
if (!peerId) return { messages: 0, bytes: 0, backpressured: false };
|
|
476
560
|
let stats = peerQueueStats.get(peerId);
|
|
@@ -490,13 +574,73 @@ export function createHubRelayControl({
|
|
|
490
574
|
stats.bytes = Math.max(0, stats.bytes - item.buffer.length);
|
|
491
575
|
}
|
|
492
576
|
|
|
577
|
+
function queuedMessageCount() {
|
|
578
|
+
return sendQueue.length - sendQueueHead;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
function socketWritableBytes(socket = connection) {
|
|
582
|
+
const value = Number(socket?.writableLength);
|
|
583
|
+
return Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
function recordOutboundHighWater(socket = connection) {
|
|
587
|
+
const writableBytes = socketWritableBytes(socket);
|
|
588
|
+
socketWritableByteHighWater = Math.max(
|
|
589
|
+
socketWritableByteHighWater,
|
|
590
|
+
writableBytes);
|
|
591
|
+
totalOutboundByteHighWater = Math.max(
|
|
592
|
+
totalOutboundByteHighWater,
|
|
593
|
+
queuedBytes + writableBytes);
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
function dequeueQueuedItem() {
|
|
597
|
+
if (queuedMessageCount() <= 0) return null;
|
|
598
|
+
const item = sendQueue[sendQueueHead];
|
|
599
|
+
sendQueue[sendQueueHead] = null;
|
|
600
|
+
sendQueueHead += 1;
|
|
601
|
+
if (sendQueueHead >= 64 && sendQueueHead * 2 >= sendQueue.length) {
|
|
602
|
+
sendQueue.splice(0, sendQueueHead);
|
|
603
|
+
sendQueueHead = 0;
|
|
604
|
+
}
|
|
605
|
+
return item;
|
|
606
|
+
}
|
|
607
|
+
|
|
493
608
|
function clearQueue() {
|
|
494
609
|
sendQueue.length = 0;
|
|
610
|
+
sendQueueHead = 0;
|
|
495
611
|
queuedBytes = 0;
|
|
496
612
|
awaitingDrain = false;
|
|
497
613
|
peerQueueStats.clear();
|
|
498
614
|
}
|
|
499
615
|
|
|
616
|
+
function purgePeerQueue(peerId) {
|
|
617
|
+
if (!peerId || queuedMessageCount() === 0) return;
|
|
618
|
+
let writeIndex = 0;
|
|
619
|
+
let purgedBytes = 0;
|
|
620
|
+
for (let readIndex = sendQueueHead; readIndex < sendQueue.length; readIndex += 1) {
|
|
621
|
+
const item = sendQueue[readIndex];
|
|
622
|
+
if (item.peerId === peerId) {
|
|
623
|
+
purgedBytes += item.buffer.length;
|
|
624
|
+
releaseQueuedItem(item);
|
|
625
|
+
continue;
|
|
626
|
+
}
|
|
627
|
+
sendQueue[writeIndex] = item;
|
|
628
|
+
writeIndex += 1;
|
|
629
|
+
}
|
|
630
|
+
for (let index = writeIndex; index < sendQueue.length; index += 1) {
|
|
631
|
+
sendQueue[index] = null;
|
|
632
|
+
}
|
|
633
|
+
if (purgedBytes > 0) {
|
|
634
|
+
sendQueue.length = writeIndex;
|
|
635
|
+
sendQueueHead = 0;
|
|
636
|
+
counters.peerQueuePurges += 1;
|
|
637
|
+
counters.peerQueuePurgedBytes += purgedBytes;
|
|
638
|
+
} else if (sendQueueHead > 0) {
|
|
639
|
+
sendQueue.splice(0, sendQueueHead);
|
|
640
|
+
sendQueueHead = 0;
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
|
|
500
644
|
function emitPeerDrains() {
|
|
501
645
|
if (awaitingDrain) return;
|
|
502
646
|
for (const [peerId, stats] of peerQueueStats) {
|
|
@@ -513,8 +657,9 @@ export function createHubRelayControl({
|
|
|
513
657
|
function flushQueue() {
|
|
514
658
|
const socket = connection;
|
|
515
659
|
if (!socket || socket.destroyed || !socket.writable || awaitingDrain) return;
|
|
516
|
-
while (
|
|
517
|
-
const item =
|
|
660
|
+
while (queuedMessageCount() > 0 && !awaitingDrain) {
|
|
661
|
+
const item = dequeueQueuedItem();
|
|
662
|
+
if (!item) break;
|
|
518
663
|
releaseQueuedItem(item);
|
|
519
664
|
let writable = false;
|
|
520
665
|
try {
|
|
@@ -524,6 +669,7 @@ export function createHubRelayControl({
|
|
|
524
669
|
return;
|
|
525
670
|
}
|
|
526
671
|
if (!writable) awaitingDrain = true;
|
|
672
|
+
recordOutboundHighWater(socket);
|
|
527
673
|
}
|
|
528
674
|
emitPeerDrains();
|
|
529
675
|
}
|
|
@@ -538,8 +684,9 @@ export function createHubRelayControl({
|
|
|
538
684
|
}
|
|
539
685
|
if (buffer.length > RELAY_MAX_LINE_BYTES) return false;
|
|
540
686
|
const stats = queueStats(peerId);
|
|
541
|
-
|
|
542
|
-
|
|
687
|
+
const writableBytes = socketWritableBytes();
|
|
688
|
+
if (queuedMessageCount() >= maxQueueMessages
|
|
689
|
+
|| queuedBytes + writableBytes + buffer.length > maxQueueBytes
|
|
543
690
|
|| (peerId && (stats.messages >= maxPeerQueueMessages
|
|
544
691
|
|| stats.bytes + buffer.length > maxPeerQueueBytes))) {
|
|
545
692
|
counters.queueDrops += 1;
|
|
@@ -548,6 +695,9 @@ export function createHubRelayControl({
|
|
|
548
695
|
}
|
|
549
696
|
sendQueue.push({ buffer, peerId });
|
|
550
697
|
queuedBytes += buffer.length;
|
|
698
|
+
queuedMessageHighWater = Math.max(queuedMessageHighWater, queuedMessageCount());
|
|
699
|
+
queuedByteHighWater = Math.max(queuedByteHighWater, queuedBytes);
|
|
700
|
+
recordOutboundHighWater();
|
|
551
701
|
if (peerId) {
|
|
552
702
|
stats.messages += 1;
|
|
553
703
|
stats.bytes += buffer.length;
|
|
@@ -594,7 +744,9 @@ export function createHubRelayControl({
|
|
|
594
744
|
return false;
|
|
595
745
|
}
|
|
596
746
|
peers.delete(id);
|
|
747
|
+
counters.peerOwnersClosed += 1;
|
|
597
748
|
rememberRetiredPeer(id);
|
|
749
|
+
purgePeerQueue(id);
|
|
598
750
|
peerQueueStats.delete(id);
|
|
599
751
|
if (notify) sendRelayClose(id, reason);
|
|
600
752
|
if (Buffer.isBuffer(peer.sessionKey)) peer.sessionKey.fill(0);
|
|
@@ -620,7 +772,9 @@ export function createHubRelayControl({
|
|
|
620
772
|
function sendPeerPlaintext(peerId, plaintext) {
|
|
621
773
|
const peer = peers.get(String(peerId || ''));
|
|
622
774
|
if (!peer || peer.socket.destroyed || peer.handshakeState !== 'established') return false;
|
|
623
|
-
const cleartext = Buffer.
|
|
775
|
+
const cleartext = Buffer.isBuffer(plaintext)
|
|
776
|
+
? plaintext
|
|
777
|
+
: Buffer.from(plaintext || []);
|
|
624
778
|
if (cleartext.length < 1 || cleartext.length > RELAY_CONTROL_MAX_PLAINTEXT_BYTES) {
|
|
625
779
|
rejectPeer(peer.peerId, 'relay-plaintext-size-invalid');
|
|
626
780
|
return false;
|
|
@@ -725,6 +879,8 @@ export function createHubRelayControl({
|
|
|
725
879
|
};
|
|
726
880
|
peer.socket = new HubRelayVirtualSocket(api, peerId, remoteAddress, port);
|
|
727
881
|
peers.set(peerId, peer);
|
|
882
|
+
counters.peerOwnersCreated += 1;
|
|
883
|
+
peerOwnerHighWater = Math.max(peerOwnerHighWater, peers.size);
|
|
728
884
|
const sent = enqueueEnvelope({
|
|
729
885
|
type: 'relay.handshake',
|
|
730
886
|
protocol: RELAY_CONTROL_PROTOCOL,
|
|
@@ -763,7 +919,7 @@ export function createHubRelayControl({
|
|
|
763
919
|
function handleClientData(message) {
|
|
764
920
|
let peerId = '';
|
|
765
921
|
try {
|
|
766
|
-
peerId =
|
|
922
|
+
peerId = safeText(message.peerId, 64);
|
|
767
923
|
const peer = peers.get(peerId);
|
|
768
924
|
if (!peer || peer.handshakeState !== 'established') {
|
|
769
925
|
throw relayError('relay-handshake-required');
|
|
@@ -834,37 +990,102 @@ export function createHubRelayControl({
|
|
|
834
990
|
}
|
|
835
991
|
}
|
|
836
992
|
|
|
837
|
-
function
|
|
993
|
+
function clearReceiveChunks() {
|
|
994
|
+
receiveChunks = [];
|
|
995
|
+
receiveBytes = 0;
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
function retainReceiveTail(incoming, offset) {
|
|
999
|
+
const tail = incoming.subarray(offset);
|
|
1000
|
+
if (tail.length === 0) return true;
|
|
1001
|
+
if (receiveBytes + tail.length > RELAY_MAX_LINE_BYTES) return false;
|
|
1002
|
+
// A tail following a complete line must not retain the complete large
|
|
1003
|
+
// TCP chunk. A chunk containing no line is already exact ownership and
|
|
1004
|
+
// can be retained without copying.
|
|
1005
|
+
const retained = offset > 0 ? Buffer.from(tail) : tail;
|
|
1006
|
+
if (receiveChunks.length >= RELAY_MAX_RECEIVE_CHUNKS) return false;
|
|
1007
|
+
receiveChunks.push(retained);
|
|
1008
|
+
receiveBytes += retained.length;
|
|
1009
|
+
receiveChunkHighWater = Math.max(receiveChunkHighWater, receiveChunks.length);
|
|
1010
|
+
receiveByteHighWater = Math.max(receiveByteHighWater, receiveBytes);
|
|
1011
|
+
return true;
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
function assembleReceiveLine(segment) {
|
|
1015
|
+
if (receiveBytes === 0) return segment;
|
|
1016
|
+
const totalBytes = receiveBytes + segment.length;
|
|
1017
|
+
const line = Buffer.allocUnsafe(totalBytes);
|
|
1018
|
+
let offset = 0;
|
|
1019
|
+
for (const chunk of receiveChunks) {
|
|
1020
|
+
chunk.copy(line, offset);
|
|
1021
|
+
offset += chunk.length;
|
|
1022
|
+
}
|
|
1023
|
+
segment.copy(line, offset);
|
|
1024
|
+
counters.receiveLineAssemblyCopies += 1;
|
|
1025
|
+
counters.receiveLineAssemblyBytes += totalBytes;
|
|
1026
|
+
clearReceiveChunks();
|
|
1027
|
+
return line;
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
function isCurrentConnectionOwner(socket, generation) {
|
|
1031
|
+
return desired
|
|
1032
|
+
&& !closing
|
|
1033
|
+
&& connection === socket
|
|
1034
|
+
&& generation === connectionGeneration
|
|
1035
|
+
&& !socket.destroyed;
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
function processRelayLine(socket, generation, line) {
|
|
1039
|
+
if (!isCurrentConnectionOwner(socket, generation)) {
|
|
1040
|
+
counters.staleConnectionEventDrops += 1;
|
|
1041
|
+
return false;
|
|
1042
|
+
}
|
|
1043
|
+
if (line.length === 0) return true;
|
|
1044
|
+
let message;
|
|
1045
|
+
try {
|
|
1046
|
+
message = JSON.parse(line.toString('utf8'));
|
|
1047
|
+
} catch {
|
|
1048
|
+
lastError = 'relay-invalid-json';
|
|
1049
|
+
socket.destroy();
|
|
1050
|
+
return false;
|
|
1051
|
+
}
|
|
1052
|
+
if (!isCurrentConnectionOwner(socket, generation)) {
|
|
1053
|
+
counters.staleConnectionEventDrops += 1;
|
|
1054
|
+
return false;
|
|
1055
|
+
}
|
|
1056
|
+
handleRelayEnvelope(message);
|
|
1057
|
+
return true;
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
function onConnectionData(socket, generation, chunk) {
|
|
1061
|
+
if (!isCurrentConnectionOwner(socket, generation)) {
|
|
1062
|
+
counters.staleConnectionEventDrops += 1;
|
|
1063
|
+
return;
|
|
1064
|
+
}
|
|
838
1065
|
const incoming = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
1066
|
+
let offset = 0;
|
|
1067
|
+
while (offset < incoming.length) {
|
|
1068
|
+
if (!isCurrentConnectionOwner(socket, generation)) {
|
|
1069
|
+
counters.staleConnectionEventDrops += 1;
|
|
1070
|
+
return;
|
|
1071
|
+
}
|
|
1072
|
+
const newline = incoming.indexOf(0x0a, offset);
|
|
844
1073
|
if (newline < 0) {
|
|
845
|
-
if (
|
|
1074
|
+
if (!retainReceiveTail(incoming, offset)) {
|
|
846
1075
|
lastError = 'relay-line-too-large';
|
|
847
|
-
|
|
1076
|
+
socket.destroy();
|
|
848
1077
|
}
|
|
849
1078
|
return;
|
|
850
1079
|
}
|
|
851
|
-
|
|
1080
|
+
const segmentLength = newline - offset;
|
|
1081
|
+
if (receiveBytes + segmentLength > RELAY_MAX_LINE_BYTES) {
|
|
852
1082
|
lastError = 'relay-line-too-large';
|
|
853
|
-
|
|
854
|
-
return;
|
|
855
|
-
}
|
|
856
|
-
const line = receiveBuffer.subarray(0, newline);
|
|
857
|
-
receiveBuffer = receiveBuffer.subarray(newline + 1);
|
|
858
|
-
if (line.length === 0) continue;
|
|
859
|
-
let message;
|
|
860
|
-
try {
|
|
861
|
-
message = JSON.parse(line.toString('utf8'));
|
|
862
|
-
} catch {
|
|
863
|
-
lastError = 'relay-invalid-json';
|
|
864
|
-
connection?.destroy();
|
|
1083
|
+
socket.destroy();
|
|
865
1084
|
return;
|
|
866
1085
|
}
|
|
867
|
-
|
|
1086
|
+
const line = assembleReceiveLine(incoming.subarray(offset, newline));
|
|
1087
|
+
if (!processRelayLine(socket, generation, line)) return;
|
|
1088
|
+
offset = newline + 1;
|
|
868
1089
|
}
|
|
869
1090
|
}
|
|
870
1091
|
|
|
@@ -883,6 +1104,7 @@ export function createHubRelayControl({
|
|
|
883
1104
|
connectNow();
|
|
884
1105
|
}, delay);
|
|
885
1106
|
reconnectTimer.unref?.();
|
|
1107
|
+
recordTimerOwnerHighWater();
|
|
886
1108
|
}
|
|
887
1109
|
|
|
888
1110
|
function clearConnectAttemptTimer() {
|
|
@@ -891,10 +1113,15 @@ export function createHubRelayControl({
|
|
|
891
1113
|
}
|
|
892
1114
|
|
|
893
1115
|
function handleConnectionClosed(socket, generation) {
|
|
1116
|
+
const owner = connectionOwners.get(socket);
|
|
1117
|
+
if (!owner || owner.generation !== generation) return;
|
|
1118
|
+
connectionOwners.delete(socket);
|
|
1119
|
+
counters.socketOwnersClosed += 1;
|
|
1120
|
+
if (owner.phase === 'closing') return;
|
|
894
1121
|
if (connection !== socket || generation !== connectionGeneration) return;
|
|
895
1122
|
clearConnectAttemptTimer();
|
|
896
1123
|
connection = null;
|
|
897
|
-
|
|
1124
|
+
clearReceiveChunks();
|
|
898
1125
|
clearQueue();
|
|
899
1126
|
closeAllPeers('relay-connection-lost');
|
|
900
1127
|
if (desired && !closing) scheduleReconnect();
|
|
@@ -914,6 +1141,7 @@ export function createHubRelayControl({
|
|
|
914
1141
|
return;
|
|
915
1142
|
}
|
|
916
1143
|
connection = socket;
|
|
1144
|
+
addConnectionOwner(socket, generation);
|
|
917
1145
|
socket.setNoDelay?.(true);
|
|
918
1146
|
socket.setKeepAlive?.(true, 10_000);
|
|
919
1147
|
socket.once('connect', () => {
|
|
@@ -931,7 +1159,7 @@ export function createHubRelayControl({
|
|
|
931
1159
|
});
|
|
932
1160
|
if (!registered) socket.destroy();
|
|
933
1161
|
});
|
|
934
|
-
socket.on('data', onConnectionData);
|
|
1162
|
+
socket.on('data', chunk => onConnectionData(socket, generation, chunk));
|
|
935
1163
|
socket.on('drain', () => {
|
|
936
1164
|
if (connection !== socket || generation !== connectionGeneration) return;
|
|
937
1165
|
awaitingDrain = false;
|
|
@@ -953,10 +1181,14 @@ export function createHubRelayControl({
|
|
|
953
1181
|
socket.destroy();
|
|
954
1182
|
}, connectTimeoutMs);
|
|
955
1183
|
connectAttemptTimer.unref?.();
|
|
1184
|
+
recordTimerOwnerHighWater();
|
|
956
1185
|
}
|
|
957
1186
|
|
|
958
1187
|
async function start() {
|
|
959
1188
|
if (!enabled || started) return getStatus();
|
|
1189
|
+
if (closing || closeTimedOut || connectionOwners.size > 0) {
|
|
1190
|
+
throw relayError('relay-close-nonterminal');
|
|
1191
|
+
}
|
|
960
1192
|
started = true;
|
|
961
1193
|
desired = true;
|
|
962
1194
|
closing = false;
|
|
@@ -965,59 +1197,138 @@ export function createHubRelayControl({
|
|
|
965
1197
|
}
|
|
966
1198
|
|
|
967
1199
|
async function close() {
|
|
1200
|
+
const closeDeadlineAt = Date.now() + 1_000;
|
|
968
1201
|
desired = false;
|
|
969
1202
|
started = false;
|
|
970
1203
|
closing = true;
|
|
1204
|
+
closeTimedOut = false;
|
|
1205
|
+
transition('closing');
|
|
971
1206
|
if (reconnectTimer) clearTimeout(reconnectTimer);
|
|
972
1207
|
reconnectTimer = null;
|
|
973
1208
|
clearConnectAttemptTimer();
|
|
974
1209
|
closeAllPeers('hub-shutdown');
|
|
975
1210
|
clearQueue();
|
|
976
1211
|
const socket = connection;
|
|
1212
|
+
const generation = connectionGeneration;
|
|
1213
|
+
if (socket) markConnectionOwnerClosing(socket, generation);
|
|
977
1214
|
connection = null;
|
|
978
1215
|
connectionGeneration += 1;
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
1216
|
+
clearReceiveChunks();
|
|
1217
|
+
const closingOwners = [...connectionOwners.values()];
|
|
1218
|
+
if (closingOwners.length > 0) {
|
|
1219
|
+
const ownersClosed = new Promise((resolve, reject) => {
|
|
982
1220
|
let settled = false;
|
|
1221
|
+
const onClose = () => {
|
|
1222
|
+
if (connectionOwners.size === 0) settle();
|
|
1223
|
+
};
|
|
983
1224
|
const settle = () => {
|
|
984
1225
|
if (settled) return;
|
|
985
1226
|
settled = true;
|
|
986
|
-
clearTimeout(
|
|
1227
|
+
if (closeSocketTimer) clearTimeout(closeSocketTimer);
|
|
1228
|
+
closeSocketTimer = null;
|
|
1229
|
+
for (const owner of closingOwners) {
|
|
1230
|
+
owner.socket.off?.('close', onClose);
|
|
1231
|
+
}
|
|
987
1232
|
resolve();
|
|
988
1233
|
};
|
|
989
|
-
const
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
1234
|
+
const fail = () => {
|
|
1235
|
+
if (settled) return;
|
|
1236
|
+
settled = true;
|
|
1237
|
+
closeSocketTimer = null;
|
|
1238
|
+
for (const owner of closingOwners) {
|
|
1239
|
+
owner.socket.off?.('close', onClose);
|
|
1240
|
+
}
|
|
1241
|
+
reject(relayError('relay-close-timeout'));
|
|
1242
|
+
};
|
|
1243
|
+
for (const owner of closingOwners) {
|
|
1244
|
+
owner.phase = 'closing';
|
|
1245
|
+
owner.socket.once('close', onClose);
|
|
1246
|
+
}
|
|
1247
|
+
closeSocketTimer = setTimeout(() => {
|
|
1248
|
+
fail();
|
|
1249
|
+
}, Math.max(1, closeDeadlineAt - Date.now()));
|
|
1250
|
+
recordTimerOwnerHighWater();
|
|
1251
|
+
if (connectionOwners.size === 0) settle();
|
|
1252
|
+
});
|
|
1253
|
+
for (const owner of closingOwners) {
|
|
995
1254
|
try {
|
|
996
|
-
socket.end();
|
|
1255
|
+
owner.socket.end();
|
|
997
1256
|
} catch {
|
|
998
|
-
|
|
999
|
-
settle();
|
|
1257
|
+
// The exact owner remains in the ledger until close.
|
|
1000
1258
|
}
|
|
1001
|
-
|
|
1259
|
+
try {
|
|
1260
|
+
owner.socket.destroy();
|
|
1261
|
+
} catch {
|
|
1262
|
+
// Timeout remains explicit and nonterminal.
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
try {
|
|
1266
|
+
await ownersClosed;
|
|
1267
|
+
} catch (error) {
|
|
1268
|
+
closeTimedOut = true;
|
|
1269
|
+
lastError = 'relay-close-timeout';
|
|
1270
|
+
transition('close-timeout', lastError);
|
|
1271
|
+
throw error;
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
if (connectionOwners.size !== 0) {
|
|
1275
|
+
closeTimedOut = true;
|
|
1276
|
+
lastError = 'relay-close-nonterminal';
|
|
1277
|
+
transition('close-timeout', lastError);
|
|
1278
|
+
throw relayError(lastError);
|
|
1002
1279
|
}
|
|
1280
|
+
closeTimedOut = false;
|
|
1003
1281
|
closing = false;
|
|
1282
|
+
lastError = '';
|
|
1004
1283
|
transition(enabled ? 'idle' : 'disabled');
|
|
1005
1284
|
}
|
|
1006
1285
|
|
|
1007
1286
|
function getStatus() {
|
|
1287
|
+
const writableBytes = socketWritableBytes();
|
|
1288
|
+
recordOutboundHighWater();
|
|
1008
1289
|
return {
|
|
1009
1290
|
enabled,
|
|
1010
1291
|
started,
|
|
1011
1292
|
state,
|
|
1012
1293
|
connected: !!connection && !connection.destroyed && state === 'registered',
|
|
1294
|
+
closing,
|
|
1295
|
+
closeTimedOut,
|
|
1296
|
+
activeSocketCount: connectionOwners.size,
|
|
1297
|
+
currentSocketCount: connection && connectionOwners.has(connection) ? 1 : 0,
|
|
1298
|
+
closingSocketCount: [...connectionOwners.values()]
|
|
1299
|
+
.filter(owner => owner.phase === 'closing').length,
|
|
1300
|
+
activeTimerCount: activeTimerCount(),
|
|
1301
|
+
timerOwnerHighWater,
|
|
1302
|
+
socketOwnerHighWater,
|
|
1303
|
+
socketOwnersCreated: counters.socketOwnersCreated,
|
|
1304
|
+
socketOwnersClosed: counters.socketOwnersClosed,
|
|
1013
1305
|
protocol: RELAY_CONTROL_PROTOCOL,
|
|
1014
1306
|
endpoint: `${host}:${port}`,
|
|
1015
1307
|
host,
|
|
1016
1308
|
port,
|
|
1017
1309
|
peerCount: peers.size,
|
|
1310
|
+
peerOwnerHighWater,
|
|
1311
|
+
peerOwnersCreated: counters.peerOwnersCreated,
|
|
1312
|
+
peerOwnersClosed: counters.peerOwnersClosed,
|
|
1018
1313
|
peerQueueCount: peerQueueStats.size,
|
|
1019
|
-
queuedMessages:
|
|
1314
|
+
queuedMessages: queuedMessageCount(),
|
|
1020
1315
|
queuedBytes,
|
|
1316
|
+
socketWritableBytes: writableBytes,
|
|
1317
|
+
totalOutboundBufferedBytes: queuedBytes + writableBytes,
|
|
1318
|
+
queuedMessageHighWater,
|
|
1319
|
+
queuedByteHighWater,
|
|
1320
|
+
socketWritableByteHighWater,
|
|
1321
|
+
totalOutboundByteHighWater,
|
|
1322
|
+
receiveBufferedChunks: receiveChunks.length,
|
|
1323
|
+
receiveBufferedBytes: receiveBytes,
|
|
1324
|
+
receiveChunkHighWater,
|
|
1325
|
+
receiveByteHighWater,
|
|
1326
|
+
receiveLineAssemblyCopyCount: counters.receiveLineAssemblyCopies,
|
|
1327
|
+
receiveLineAssemblyBytes: counters.receiveLineAssemblyBytes,
|
|
1328
|
+
plaintextDeliveryCopyCount: counters.plaintextDeliveryCopies,
|
|
1329
|
+
peerQueuePurgeCount: counters.peerQueuePurges,
|
|
1330
|
+
peerQueuePurgedBytes: counters.peerQueuePurgedBytes,
|
|
1331
|
+
staleConnectionEventDropCount: counters.staleConnectionEventDrops,
|
|
1021
1332
|
maxPeers,
|
|
1022
1333
|
maxQueueMessages,
|
|
1023
1334
|
maxQueueBytes,
|
|
@@ -1033,13 +1344,18 @@ export function createHubRelayControl({
|
|
|
1033
1344
|
};
|
|
1034
1345
|
}
|
|
1035
1346
|
|
|
1347
|
+
function recordPlaintextDeliveryCopy() {
|
|
1348
|
+
counters.plaintextDeliveryCopies += 1;
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1036
1351
|
const api = {
|
|
1037
1352
|
start,
|
|
1038
1353
|
close,
|
|
1039
1354
|
getStatus,
|
|
1040
1355
|
sendPeerPlaintext,
|
|
1041
1356
|
closePeer,
|
|
1042
|
-
isPeerBackpressured
|
|
1357
|
+
isPeerBackpressured,
|
|
1358
|
+
recordPlaintextDeliveryCopy
|
|
1043
1359
|
};
|
|
1044
1360
|
return api;
|
|
1045
1361
|
}
|