@streamr/dht 100.0.0-pretestnet.3 → 100.0.0-pretestnet.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/connection/ConnectionLockRpcLocal.js +1 -1
- package/dist/src/connection/ConnectionLockRpcLocal.js.map +1 -1
- package/dist/src/connection/ConnectionLockRpcRemote.js +3 -3
- package/dist/src/connection/ConnectionLockRpcRemote.js.map +1 -1
- package/dist/src/connection/ConnectionManager.d.ts +4 -8
- package/dist/src/connection/ConnectionManager.js +30 -31
- package/dist/src/connection/ConnectionManager.js.map +1 -1
- package/dist/src/connection/ConnectivityChecker.d.ts +0 -4
- package/dist/src/connection/ConnectivityChecker.js +13 -18
- package/dist/src/connection/ConnectivityChecker.js.map +1 -1
- package/dist/src/connection/ConnectorFacade.d.ts +6 -3
- package/dist/src/connection/ConnectorFacade.js +33 -23
- package/dist/src/connection/ConnectorFacade.js.map +1 -1
- package/dist/src/connection/Handshaker.d.ts +6 -6
- package/dist/src/connection/Handshaker.js +10 -13
- package/dist/src/connection/Handshaker.js.map +1 -1
- package/dist/src/connection/ManagedConnection.d.ts +7 -10
- package/dist/src/connection/ManagedConnection.js +43 -79
- package/dist/src/connection/ManagedConnection.js.map +1 -1
- package/dist/src/connection/ManagedWebrtcConnection.js.map +1 -1
- package/dist/src/connection/simulator/SimulatorConnection.js +16 -15
- package/dist/src/connection/simulator/SimulatorConnection.js.map +1 -1
- package/dist/src/connection/simulator/SimulatorConnector.d.ts +2 -2
- package/dist/src/connection/simulator/SimulatorConnector.js +10 -11
- package/dist/src/connection/simulator/SimulatorConnector.js.map +1 -1
- package/dist/src/connection/webrtc/NodeWebrtcConnection.js +8 -9
- package/dist/src/connection/webrtc/NodeWebrtcConnection.js.map +1 -1
- package/dist/src/connection/webrtc/WebrtcConnector.d.ts +1 -1
- package/dist/src/connection/webrtc/WebrtcConnector.js +6 -6
- package/dist/src/connection/webrtc/WebrtcConnector.js.map +1 -1
- package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.d.ts +1 -1
- package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.js +4 -4
- package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.js.map +1 -1
- package/dist/src/connection/websocket/ClientWebsocket.js +3 -0
- package/dist/src/connection/websocket/ClientWebsocket.js.map +1 -1
- package/dist/src/connection/websocket/ServerWebsocket.js +2 -0
- package/dist/src/connection/websocket/ServerWebsocket.js.map +1 -1
- package/dist/src/connection/websocket/WebsocketConnector.d.ts +5 -5
- package/dist/src/connection/websocket/WebsocketConnector.js +37 -35
- package/dist/src/connection/websocket/WebsocketConnector.js.map +1 -1
- package/dist/src/connection/websocket/WebsocketConnectorRpcLocal.d.ts +1 -1
- package/dist/src/connection/websocket/WebsocketConnectorRpcLocal.js +1 -1
- package/dist/src/connection/websocket/WebsocketConnectorRpcLocal.js.map +1 -1
- package/dist/src/connection/websocket/WebsocketConnectorRpcRemote.js +1 -1
- package/dist/src/connection/websocket/WebsocketConnectorRpcRemote.js.map +1 -1
- package/dist/src/dht/DhtNode.js +18 -15
- package/dist/src/dht/DhtNode.js.map +1 -1
- package/dist/src/dht/DhtNodeRpcLocal.js +2 -2
- package/dist/src/dht/DhtNodeRpcLocal.js.map +1 -1
- package/dist/src/dht/DhtNodeRpcRemote.js +4 -4
- package/dist/src/dht/DhtNodeRpcRemote.js.map +1 -1
- package/dist/src/dht/discovery/DiscoverySession.js +1 -1
- package/dist/src/dht/discovery/DiscoverySession.js.map +1 -1
- package/dist/src/dht/discovery/PeerDiscovery.js +1 -1
- package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
- package/dist/src/dht/find/FindRpcLocal.js +2 -2
- package/dist/src/dht/find/FindRpcLocal.js.map +1 -1
- package/dist/src/dht/routing/FindRpcRemote.js +3 -3
- package/dist/src/dht/routing/FindRpcRemote.js.map +1 -1
- package/dist/src/dht/routing/Router.js +3 -3
- package/dist/src/dht/routing/Router.js.map +1 -1
- package/dist/src/dht/routing/RouterRpcLocal.js +4 -4
- package/dist/src/dht/routing/RouterRpcLocal.js.map +1 -1
- package/dist/src/dht/routing/RouterRpcRemote.js +6 -6
- package/dist/src/dht/routing/RouterRpcRemote.js.map +1 -1
- package/dist/src/dht/routing/RoutingSession.js +1 -1
- package/dist/src/dht/routing/RoutingSession.js.map +1 -1
- package/dist/src/dht/store/StoreRpcLocal.js +1 -1
- package/dist/src/dht/store/StoreRpcLocal.js.map +1 -1
- package/dist/src/dht/store/StoreRpcRemote.js +3 -3
- package/dist/src/dht/store/StoreRpcRemote.js.map +1 -1
- package/dist/src/helpers/peerIdFromPeerDescriptor.d.ts +1 -0
- package/dist/src/helpers/peerIdFromPeerDescriptor.js +7 -1
- package/dist/src/helpers/peerIdFromPeerDescriptor.js.map +1 -1
- package/dist/src/proto/packages/dht/protos/DhtRpc.d.ts +21 -12
- package/dist/src/proto/packages/dht/protos/DhtRpc.js +20 -7
- package/dist/src/proto/packages/dht/protos/DhtRpc.js.map +1 -1
- package/package.json +5 -5
- package/protos/DhtRpc.proto +9 -5
- package/src/connection/ConnectionLockRpcLocal.ts +2 -2
- package/src/connection/ConnectionLockRpcRemote.ts +4 -5
- package/src/connection/ConnectionManager.ts +30 -36
- package/src/connection/ConnectivityChecker.ts +13 -15
- package/src/connection/ConnectorFacade.ts +36 -25
- package/src/connection/Handshaker.ts +15 -18
- package/src/connection/ManagedConnection.ts +63 -87
- package/src/connection/ManagedWebrtcConnection.ts +4 -2
- package/src/connection/simulator/SimulatorConnection.ts +17 -16
- package/src/connection/simulator/SimulatorConnector.ts +12 -13
- package/src/connection/webrtc/BrowserWebrtcConnection.ts +2 -2
- package/src/connection/webrtc/NodeWebrtcConnection.ts +9 -10
- package/src/connection/webrtc/WebrtcConnector.ts +7 -6
- package/src/connection/webrtc/WebrtcConnectorRpcLocal.ts +5 -5
- package/src/connection/websocket/ClientWebsocket.ts +3 -0
- package/src/connection/websocket/ServerWebsocket.ts +2 -0
- package/src/connection/websocket/WebsocketConnector.ts +63 -39
- package/src/connection/websocket/WebsocketConnectorRpcLocal.ts +2 -2
- package/src/connection/websocket/WebsocketConnectorRpcRemote.ts +2 -2
- package/src/dht/DhtNode.ts +24 -16
- package/src/dht/DhtNodeRpcLocal.ts +3 -3
- package/src/dht/DhtNodeRpcRemote.ts +5 -5
- package/src/dht/discovery/DiscoverySession.ts +2 -2
- package/src/dht/discovery/PeerDiscovery.ts +2 -2
- package/src/dht/find/FindRpcLocal.ts +3 -3
- package/src/dht/routing/FindRpcRemote.ts +4 -4
- package/src/dht/routing/Router.ts +10 -5
- package/src/dht/routing/RouterRpcLocal.ts +5 -5
- package/src/dht/routing/RouterRpcRemote.ts +7 -8
- package/src/dht/routing/RoutingSession.ts +2 -2
- package/src/dht/store/StoreRpcLocal.ts +2 -2
- package/src/dht/store/StoreRpcRemote.ts +4 -4
- package/src/helpers/peerIdFromPeerDescriptor.ts +6 -0
- package/src/proto/packages/dht/protos/DhtRpc.ts +25 -17
- package/test/benchmark/Find.test.ts +2 -2
- package/test/benchmark/KademliaCorrectness.test.ts +2 -2
- package/test/end-to-end/Layer0MixedConnectionTypes.test.ts +4 -3
- package/test/integration/ConnectionLocking.test.ts +7 -4
- package/test/integration/ConnectionManager.test.ts +72 -4
- package/test/integration/Find.test.ts +3 -6
- package/test/integration/MigrateData.test.ts +4 -4
- package/test/integration/MultipleEntryPointJoining.test.ts +2 -2
- package/test/integration/ScaleDownDht.test.ts +4 -4
- package/test/integration/Store.test.ts +3 -6
- package/test/integration/StoreAndDelete.test.ts +3 -6
- package/test/integration/WebrtcConnectionManagement.test.ts +1 -1
- package/test/utils/utils.ts +8 -2
package/protos/DhtRpc.proto
CHANGED
|
@@ -196,14 +196,18 @@ message ConnectivityResponse {
|
|
|
196
196
|
}
|
|
197
197
|
|
|
198
198
|
message HandshakeRequest {
|
|
199
|
-
|
|
200
|
-
PeerDescriptor
|
|
199
|
+
PeerDescriptor sourcePeerDescriptor = 1;
|
|
200
|
+
optional PeerDescriptor targetPeerDescriptor = 2;
|
|
201
201
|
}
|
|
202
202
|
|
|
203
203
|
message HandshakeResponse {
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
204
|
+
PeerDescriptor sourcePeerDescriptor = 1;
|
|
205
|
+
optional HandshakeError error = 2;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
enum HandshakeError {
|
|
209
|
+
DUPLICATE_CONNECTION = 0;
|
|
210
|
+
INVALID_TARGET_PEER_DESCRIPTOR = 1;
|
|
207
211
|
}
|
|
208
212
|
|
|
209
213
|
// Wraps all messages
|
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
import { IConnectionLockRpc } from '../proto/packages/dht/protos/DhtRpc.server'
|
|
18
18
|
import { DhtCallContext } from '../rpc-protocol/DhtCallContext'
|
|
19
19
|
import { PeerIDKey } from '../helpers/PeerID'
|
|
20
|
-
import {
|
|
20
|
+
import { getNodeIdOrUnknownFromPeerDescriptor } from './ConnectionManager'
|
|
21
21
|
import { LockID } from './ConnectionLockHandler'
|
|
22
22
|
|
|
23
23
|
interface ConnectionLockRpcLocalConfig {
|
|
@@ -62,7 +62,7 @@ export class ConnectionLockRpcLocal implements IConnectionLockRpc {
|
|
|
62
62
|
|
|
63
63
|
async gracefulDisconnect(disconnectNotice: DisconnectNotice, context: ServerCallContext): Promise<Empty> {
|
|
64
64
|
const senderPeerDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
|
|
65
|
-
logger.trace(
|
|
65
|
+
logger.trace(getNodeIdOrUnknownFromPeerDescriptor(senderPeerDescriptor) + ' received gracefulDisconnect notice')
|
|
66
66
|
|
|
67
67
|
if (disconnectNotice.disconnectMode === DisconnectMode.LEAVING) {
|
|
68
68
|
this.config.closeConnection(senderPeerDescriptor, true, 'graceful leave notified')
|
|
@@ -2,9 +2,8 @@ import { Logger } from '@streamr/utils'
|
|
|
2
2
|
import { ProtoRpcClient } from '@streamr/proto-rpc'
|
|
3
3
|
import { IConnectionLockRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
|
|
4
4
|
import { LockRequest, UnlockRequest, PeerDescriptor, DisconnectNotice, DisconnectMode } from '../proto/packages/dht/protos/DhtRpc'
|
|
5
|
-
|
|
6
5
|
import * as Err from '../helpers/errors'
|
|
7
|
-
import {
|
|
6
|
+
import { getNodeIdFromPeerDescriptor } from '../helpers/peerIdFromPeerDescriptor'
|
|
8
7
|
import { Remote } from '../dht/contact/Remote'
|
|
9
8
|
import { LockID } from './ConnectionLockHandler'
|
|
10
9
|
|
|
@@ -21,7 +20,7 @@ export class ConnectionLockRpcRemote extends Remote<IConnectionLockRpcClient> {
|
|
|
21
20
|
}
|
|
22
21
|
|
|
23
22
|
public async lockRequest(lockId: LockID): Promise<boolean> {
|
|
24
|
-
logger.trace(`Requesting locked connection to ${
|
|
23
|
+
logger.trace(`Requesting locked connection to ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())}`)
|
|
25
24
|
const request: LockRequest = {
|
|
26
25
|
lockId
|
|
27
26
|
}
|
|
@@ -36,7 +35,7 @@ export class ConnectionLockRpcRemote extends Remote<IConnectionLockRpcClient> {
|
|
|
36
35
|
}
|
|
37
36
|
|
|
38
37
|
public unlockRequest(lockId: LockID): void {
|
|
39
|
-
logger.trace(`Requesting connection to be unlocked from ${
|
|
38
|
+
logger.trace(`Requesting connection to be unlocked from ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())}`)
|
|
40
39
|
const request: UnlockRequest = {
|
|
41
40
|
lockId
|
|
42
41
|
}
|
|
@@ -49,7 +48,7 @@ export class ConnectionLockRpcRemote extends Remote<IConnectionLockRpcClient> {
|
|
|
49
48
|
}
|
|
50
49
|
|
|
51
50
|
public async gracefulDisconnect(disconnectMode: DisconnectMode): Promise<void> {
|
|
52
|
-
logger.trace(`Notifying a graceful disconnect to ${
|
|
51
|
+
logger.trace(`Notifying a graceful disconnect to ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())}`)
|
|
53
52
|
const request: DisconnectNotice = {
|
|
54
53
|
disconnectMode
|
|
55
54
|
}
|
|
@@ -8,6 +8,7 @@ import { PeerIDKey } from '../helpers/PeerID'
|
|
|
8
8
|
import * as Err from '../helpers/errors'
|
|
9
9
|
import {
|
|
10
10
|
areEqualPeerDescriptors,
|
|
11
|
+
getNodeIdFromPeerDescriptor,
|
|
11
12
|
keyFromPeerDescriptor,
|
|
12
13
|
peerIdFromPeerDescriptor
|
|
13
14
|
} from '../helpers/peerIdFromPeerDescriptor'
|
|
@@ -66,10 +67,6 @@ enum ConnectionManagerState {
|
|
|
66
67
|
STOPPED = 'stopped'
|
|
67
68
|
}
|
|
68
69
|
|
|
69
|
-
interface ConnectionManagerEvents {
|
|
70
|
-
newConnection: (connection: ManagedConnection) => void
|
|
71
|
-
}
|
|
72
|
-
|
|
73
70
|
export interface ConnectionLocker {
|
|
74
71
|
lockConnection(targetDescriptor: PeerDescriptor, serviceId: ServiceId): void
|
|
75
72
|
unlockConnection(targetDescriptor: PeerDescriptor, serviceId: ServiceId): void
|
|
@@ -87,8 +84,6 @@ export interface TlsCertificate {
|
|
|
87
84
|
certFileName: string
|
|
88
85
|
}
|
|
89
86
|
|
|
90
|
-
export type Events = TransportEvents & ConnectionManagerEvents
|
|
91
|
-
|
|
92
87
|
const INTERNAL_SERVICE_ID = 'system/connection-manager'
|
|
93
88
|
|
|
94
89
|
// Form an string representation from a peer description which can be undefined. This output
|
|
@@ -100,15 +95,15 @@ const INTERNAL_SERVICE_ID = 'system/connection-manager'
|
|
|
100
95
|
// - if we create stricter types for incoming messages (message.sourceDescriptor or
|
|
101
96
|
// disconnectNotice.peerDescriptor)
|
|
102
97
|
// - if ManagedConnection#peerDescriptor is never undefined
|
|
103
|
-
export const
|
|
98
|
+
export const getNodeIdOrUnknownFromPeerDescriptor = (peerDescriptor: PeerDescriptor | undefined): string => {
|
|
104
99
|
if (peerDescriptor !== undefined) {
|
|
105
|
-
return
|
|
100
|
+
return getNodeIdFromPeerDescriptor(peerDescriptor)
|
|
106
101
|
} else {
|
|
107
102
|
return 'unknown'
|
|
108
103
|
}
|
|
109
104
|
}
|
|
110
105
|
|
|
111
|
-
export class ConnectionManager extends EventEmitter<
|
|
106
|
+
export class ConnectionManager extends EventEmitter<TransportEvents> implements ITransport, ConnectionLocker {
|
|
112
107
|
|
|
113
108
|
private config: ConnectionManagerConfig
|
|
114
109
|
private readonly metricsContext: MetricsContext
|
|
@@ -126,7 +121,7 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
126
121
|
this.config = config
|
|
127
122
|
this.onData = this.onData.bind(this)
|
|
128
123
|
this.send = this.send.bind(this)
|
|
129
|
-
this.
|
|
124
|
+
this.onNewConnection = this.onNewConnection.bind(this)
|
|
130
125
|
this.metricsContext = this.config.metricsContext ?? new MetricsContext()
|
|
131
126
|
this.metrics = {
|
|
132
127
|
sendMessagesPerSecond: new RateMetric(),
|
|
@@ -165,14 +160,14 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
165
160
|
const disconnectionCandidates = new SortedContactList<Contact>(peerIdFromPeerDescriptor(this.getLocalPeerDescriptor()), 100000)
|
|
166
161
|
this.connections.forEach((connection) => {
|
|
167
162
|
if (!this.locks.isLocked(connection.peerIdKey) && Date.now() - connection.getLastUsed() > lastUsedLimit) {
|
|
168
|
-
logger.trace('disconnecting in timeout interval: ' +
|
|
163
|
+
logger.trace('disconnecting in timeout interval: ' + getNodeIdOrUnknownFromPeerDescriptor(connection.getPeerDescriptor()))
|
|
169
164
|
disconnectionCandidates.addContact(new Contact(connection.getPeerDescriptor()!))
|
|
170
165
|
}
|
|
171
166
|
})
|
|
172
167
|
const sortedCandidates = disconnectionCandidates.getAllContacts()
|
|
173
168
|
const targetNum = this.connections.size - maxConnections
|
|
174
169
|
for (let i = 0; i < sortedCandidates.length && i < targetNum; i++) {
|
|
175
|
-
logger.trace('garbageCollecting ' +
|
|
170
|
+
logger.trace('garbageCollecting ' + getNodeIdFromPeerDescriptor(sortedCandidates[sortedCandidates.length - 1 - i].getPeerDescriptor()))
|
|
176
171
|
this.gracefullyDisconnectAsync(sortedCandidates[sortedCandidates.length - 1 - i].getPeerDescriptor(),
|
|
177
172
|
DisconnectMode.NORMAL).catch((_e) => { })
|
|
178
173
|
}
|
|
@@ -185,7 +180,7 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
185
180
|
this.state = ConnectionManagerState.RUNNING
|
|
186
181
|
logger.trace(`Starting ConnectionManager...`)
|
|
187
182
|
await this.connectorFacade.start(
|
|
188
|
-
(connection: ManagedConnection) => this.
|
|
183
|
+
(connection: ManagedConnection) => this.onNewConnection(connection),
|
|
189
184
|
(peerDescriptor: PeerDescriptor) => this.canConnect(peerDescriptor),
|
|
190
185
|
this
|
|
191
186
|
)
|
|
@@ -259,7 +254,7 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
259
254
|
if (this.isConnectionToSelf(peerDescriptor)) {
|
|
260
255
|
throw new Err.CannotConnectToSelf('Cannot send to self')
|
|
261
256
|
}
|
|
262
|
-
logger.trace(`Sending message to: ${
|
|
257
|
+
logger.trace(`Sending message to: ${getNodeIdFromPeerDescriptor(peerDescriptor)}`)
|
|
263
258
|
message = {
|
|
264
259
|
...message,
|
|
265
260
|
sourceDescriptor: this.getLocalPeerDescriptor()
|
|
@@ -268,7 +263,7 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
268
263
|
let connection = this.connections.get(peerIdKey)
|
|
269
264
|
if (!connection && !doNotConnect) {
|
|
270
265
|
connection = this.connectorFacade.createConnection(peerDescriptor)
|
|
271
|
-
this.
|
|
266
|
+
this.onNewConnection(connection)
|
|
272
267
|
} else if (!connection) {
|
|
273
268
|
throw new Err.SendFailed('No connection to target, doNotConnect flag is true')
|
|
274
269
|
}
|
|
@@ -328,7 +323,7 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
328
323
|
return
|
|
329
324
|
}
|
|
330
325
|
if (this.duplicateMessageDetector.isMostLikelyDuplicate(message.messageId)) {
|
|
331
|
-
logger.trace('handleMessage filtered duplicate ' +
|
|
326
|
+
logger.trace('handleMessage filtered duplicate ' + getNodeIdFromPeerDescriptor(message.sourceDescriptor!)
|
|
332
327
|
+ ' ' + message.serviceId + ' ' + message.messageId)
|
|
333
328
|
return
|
|
334
329
|
}
|
|
@@ -336,7 +331,8 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
336
331
|
if (message.serviceId === INTERNAL_SERVICE_ID) {
|
|
337
332
|
this.rpcCommunicator?.handleMessageFromPeer(message)
|
|
338
333
|
} else {
|
|
339
|
-
logger.trace('emit "message" ' +
|
|
334
|
+
logger.trace('emit "message" ' + getNodeIdFromPeerDescriptor(message.sourceDescriptor!)
|
|
335
|
+
+ ' ' + message.serviceId + ' ' + message.messageId)
|
|
340
336
|
this.emit('message', message)
|
|
341
337
|
}
|
|
342
338
|
}
|
|
@@ -366,47 +362,46 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
366
362
|
private onConnected(connection: ManagedConnection) {
|
|
367
363
|
const peerDescriptor = connection.getPeerDescriptor()!
|
|
368
364
|
this.emit('connected', peerDescriptor)
|
|
369
|
-
logger.trace(
|
|
365
|
+
logger.trace(getNodeIdFromPeerDescriptor(peerDescriptor) + ' onConnected() ' + connection.connectionType)
|
|
370
366
|
this.onConnectionCountChange()
|
|
371
367
|
}
|
|
372
368
|
|
|
373
369
|
private onDisconnected(connection: ManagedConnection, gracefulLeave: boolean) {
|
|
374
|
-
logger.trace(
|
|
370
|
+
logger.trace(getNodeIdOrUnknownFromPeerDescriptor(connection.getPeerDescriptor()) + ' onDisconnected() gracefulLeave: ' + gracefulLeave)
|
|
375
371
|
|
|
376
372
|
const peerIdKey = keyFromPeerDescriptor(connection.getPeerDescriptor()!)
|
|
377
373
|
const storedConnection = this.connections.get(peerIdKey)
|
|
378
374
|
if (storedConnection && storedConnection.connectionId.equals(connection.connectionId)) {
|
|
379
375
|
this.locks.clearAllLocks(peerIdKey)
|
|
380
376
|
this.connections.delete(peerIdKey)
|
|
381
|
-
logger.trace(
|
|
377
|
+
logger.trace(getNodeIdOrUnknownFromPeerDescriptor(connection.getPeerDescriptor())
|
|
382
378
|
+ ' deleted connection in onDisconnected() gracefulLeave: ' + gracefulLeave)
|
|
383
379
|
this.emit('disconnected', connection.getPeerDescriptor()!, gracefulLeave)
|
|
384
380
|
this.onConnectionCountChange()
|
|
385
381
|
} else {
|
|
386
|
-
logger.trace(
|
|
382
|
+
logger.trace(getNodeIdOrUnknownFromPeerDescriptor(connection.getPeerDescriptor())
|
|
387
383
|
+ ' onDisconnected() did nothing, no such connection in connectionManager')
|
|
388
384
|
if (storedConnection) {
|
|
389
|
-
logger.trace(
|
|
385
|
+
logger.trace(getNodeIdOrUnknownFromPeerDescriptor(connection.getPeerDescriptor())
|
|
390
386
|
+ ' connectionIds do not match ' + storedConnection.connectionId + ' ' + connection.connectionId)
|
|
391
387
|
}
|
|
392
388
|
}
|
|
393
389
|
|
|
394
390
|
}
|
|
395
391
|
|
|
396
|
-
private
|
|
392
|
+
private onNewConnection(connection: ManagedConnection): boolean {
|
|
397
393
|
if (this.state === ConnectionManagerState.STOPPED) {
|
|
398
394
|
return false
|
|
399
395
|
}
|
|
400
|
-
logger.trace('
|
|
396
|
+
logger.trace('onNewConnection()')
|
|
401
397
|
connection.offeredAsIncoming = true
|
|
402
|
-
if (!this.
|
|
398
|
+
if (!this.acceptNewConnection(connection)) {
|
|
403
399
|
return false
|
|
404
400
|
}
|
|
405
401
|
connection.on('managedData', this.onData)
|
|
406
402
|
connection.on('disconnected', (gracefulLeave: boolean) => {
|
|
407
403
|
this.onDisconnected(connection, gracefulLeave)
|
|
408
404
|
})
|
|
409
|
-
this.emit('newConnection', connection)
|
|
410
405
|
if (connection.isHandshakeCompleted()) {
|
|
411
406
|
this.onConnected(connection)
|
|
412
407
|
} else {
|
|
@@ -417,17 +412,17 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
417
412
|
return true
|
|
418
413
|
}
|
|
419
414
|
|
|
420
|
-
private
|
|
421
|
-
logger.trace(
|
|
415
|
+
private acceptNewConnection(newConnection: ManagedConnection): boolean {
|
|
416
|
+
logger.trace(getNodeIdFromPeerDescriptor(newConnection.getPeerDescriptor()!) + ' acceptIncomingConnection()')
|
|
422
417
|
const newPeerID = peerIdFromPeerDescriptor(newConnection.getPeerDescriptor()!)
|
|
423
418
|
const peerIdKey = keyFromPeerDescriptor(newConnection.getPeerDescriptor()!)
|
|
424
419
|
if (this.connections.has(peerIdKey)) {
|
|
425
420
|
if (newPeerID.hasSmallerHashThan(peerIdFromPeerDescriptor(this.getLocalPeerDescriptor()))) {
|
|
426
|
-
logger.trace(
|
|
421
|
+
logger.trace(getNodeIdOrUnknownFromPeerDescriptor(newConnection.getPeerDescriptor())
|
|
427
422
|
+ ' acceptIncomingConnection() replace current connection')
|
|
428
423
|
// replace the current connection
|
|
429
424
|
const oldConnection = this.connections.get(newPeerID.toKey())!
|
|
430
|
-
logger.trace('replaced: ' +
|
|
425
|
+
logger.trace('replaced: ' + getNodeIdFromPeerDescriptor(newConnection.getPeerDescriptor()!))
|
|
431
426
|
const buffer = oldConnection.stealOutputBuffer()
|
|
432
427
|
|
|
433
428
|
for (const data of buffer) {
|
|
@@ -437,19 +432,18 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
437
432
|
oldConnection.reportBufferSentByOtherConnection()
|
|
438
433
|
oldConnection.replacedByOtherConnection = true
|
|
439
434
|
} else {
|
|
440
|
-
newConnection.rejectedAsIncoming = true
|
|
441
435
|
return false
|
|
442
436
|
}
|
|
443
437
|
}
|
|
444
438
|
|
|
445
|
-
logger.trace(
|
|
439
|
+
logger.trace(getNodeIdFromPeerDescriptor(newConnection.getPeerDescriptor()!) + ' added to connections at acceptIncomingConnection')
|
|
446
440
|
this.connections.set(peerIdKey, newConnection)
|
|
447
441
|
|
|
448
442
|
return true
|
|
449
443
|
}
|
|
450
444
|
|
|
451
445
|
private async closeConnection(peerDescriptor: PeerDescriptor, gracefulLeave: boolean, reason?: string): Promise<void> {
|
|
452
|
-
logger.trace(
|
|
446
|
+
logger.trace(getNodeIdFromPeerDescriptor(peerDescriptor) + ' ' + 'closeConnection() ' + reason)
|
|
453
447
|
const id = keyFromPeerDescriptor(peerDescriptor)
|
|
454
448
|
this.locks.clearAllLocks(id)
|
|
455
449
|
if (this.connections.has(id)) {
|
|
@@ -457,7 +451,7 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
457
451
|
await connectionToClose.close(gracefulLeave)
|
|
458
452
|
|
|
459
453
|
} else {
|
|
460
|
-
logger.trace(
|
|
454
|
+
logger.trace(getNodeIdFromPeerDescriptor(peerDescriptor) + ' ' + 'closeConnection() this.connections did not have the id')
|
|
461
455
|
this.emit('disconnected', peerDescriptor, false)
|
|
462
456
|
}
|
|
463
457
|
}
|
|
@@ -546,7 +540,7 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
546
540
|
}
|
|
547
541
|
|
|
548
542
|
private async doGracefullyDisconnectAsync(targetDescriptor: PeerDescriptor, disconnectMode: DisconnectMode): Promise<void> {
|
|
549
|
-
logger.trace(
|
|
543
|
+
logger.trace(getNodeIdFromPeerDescriptor(targetDescriptor) + ' gracefullyDisconnectAsync()')
|
|
550
544
|
const rpcRemote = new ConnectionLockRpcRemote(
|
|
551
545
|
this.getLocalPeerDescriptor(),
|
|
552
546
|
targetDescriptor,
|
|
@@ -555,7 +549,7 @@ export class ConnectionManager extends EventEmitter<Events> implements ITranspor
|
|
|
555
549
|
try {
|
|
556
550
|
await rpcRemote.gracefulDisconnect(disconnectMode)
|
|
557
551
|
} catch (ex) {
|
|
558
|
-
logger.trace(
|
|
552
|
+
logger.trace(getNodeIdFromPeerDescriptor(targetDescriptor) + ' remote.gracefulDisconnect() failed' + ex)
|
|
559
553
|
}
|
|
560
554
|
}
|
|
561
555
|
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import {
|
|
2
|
-
ConnectivityMethod,
|
|
3
2
|
ConnectivityRequest, ConnectivityResponse,
|
|
4
3
|
Message, MessageType, PeerDescriptor
|
|
5
4
|
} from '../proto/packages/dht/protos/DhtRpc'
|
|
@@ -18,7 +17,6 @@ const logger = new Logger(module)
|
|
|
18
17
|
// checks. This is attached to all ServerWebsockets to listen to
|
|
19
18
|
// ConnectivityRequest messages.
|
|
20
19
|
|
|
21
|
-
export enum ConnectionMode { REQUEST = 'connectivityRequest', PROBE = 'connectivityProbe' }
|
|
22
20
|
export class ConnectivityChecker {
|
|
23
21
|
|
|
24
22
|
private static readonly CONNECTIVITY_CHECKER_SERVICE_ID = 'system/connectivity-checker'
|
|
@@ -39,18 +37,19 @@ export class ConnectivityChecker {
|
|
|
39
37
|
throw new Err.ConnectionFailed('ConnectivityChecker is destroyed')
|
|
40
38
|
}
|
|
41
39
|
let outgoingConnection: IConnection
|
|
40
|
+
const wsServerInfo = {
|
|
41
|
+
host: entryPoint.websocket!.host,
|
|
42
|
+
port: entryPoint.websocket!.port,
|
|
43
|
+
tls: entryPoint.websocket!.tls,
|
|
44
|
+
}
|
|
45
|
+
const url = connectivityMethodToWebsocketUrl(wsServerInfo, 'connectivityRequest')
|
|
42
46
|
try {
|
|
43
47
|
outgoingConnection = await this.connectAsync({
|
|
44
|
-
|
|
45
|
-
host: entryPoint.websocket!.host,
|
|
46
|
-
port: entryPoint.websocket!.port,
|
|
47
|
-
tls: entryPoint.websocket!.tls,
|
|
48
|
-
},
|
|
49
|
-
mode: ConnectionMode.REQUEST,
|
|
48
|
+
url,
|
|
50
49
|
selfSigned
|
|
51
50
|
})
|
|
52
51
|
} catch (e) {
|
|
53
|
-
throw new Err.ConnectionFailed(`Failed to connect to the entrypoint ${
|
|
52
|
+
throw new Err.ConnectionFailed(`Failed to connect to the entrypoint ${url}`, e)
|
|
54
53
|
}
|
|
55
54
|
// send connectivity request
|
|
56
55
|
const connectivityRequestMessage: ConnectivityRequest = { port: this.websocketPort, host: this.host, tls: this.tls, selfSigned }
|
|
@@ -140,10 +139,10 @@ export class ConnectivityChecker {
|
|
|
140
139
|
port: connectivityRequest.port,
|
|
141
140
|
tls: connectivityRequest.tls
|
|
142
141
|
}
|
|
143
|
-
|
|
142
|
+
const url = connectivityMethodToWebsocketUrl(wsServerInfo, 'connectivityProbe')
|
|
143
|
+
logger.trace(`Attempting Connectivity Check to ${url}`)
|
|
144
144
|
outgoingConnection = await this.connectAsync({
|
|
145
|
-
|
|
146
|
-
mode: ConnectionMode.PROBE,
|
|
145
|
+
url,
|
|
147
146
|
selfSigned: connectivityRequest.selfSigned
|
|
148
147
|
})
|
|
149
148
|
} catch (err) {
|
|
@@ -175,11 +174,10 @@ export class ConnectivityChecker {
|
|
|
175
174
|
}
|
|
176
175
|
|
|
177
176
|
// eslint-disable-next-line class-methods-use-this
|
|
178
|
-
private async connectAsync({
|
|
179
|
-
{
|
|
177
|
+
private async connectAsync({ url, selfSigned, timeoutMs = 1000, }:
|
|
178
|
+
{ url: string, selfSigned: boolean, timeoutMs?: number }
|
|
180
179
|
): Promise<IConnection> {
|
|
181
180
|
const socket = new ClientWebsocket()
|
|
182
|
-
const url = `${connectivityMethodToWebsocketUrl(wsServerInfo)}?${mode}=true`
|
|
183
181
|
let result: RunAndRaceEventsReturnType<ConnectionEvents>
|
|
184
182
|
try {
|
|
185
183
|
result = await runAndRaceEvents3<ConnectionEvents>([
|
|
@@ -9,13 +9,13 @@ import { ManagedConnection } from './ManagedConnection'
|
|
|
9
9
|
import { Simulator } from './simulator/Simulator'
|
|
10
10
|
import { SimulatorConnector } from './simulator/SimulatorConnector'
|
|
11
11
|
import { IceServer, WebrtcConnector } from './webrtc/WebrtcConnector'
|
|
12
|
-
import { WebsocketConnector } from './websocket/WebsocketConnector'
|
|
12
|
+
import { WebsocketConnector, WebsocketConnectorConfig } from './websocket/WebsocketConnector'
|
|
13
13
|
|
|
14
14
|
export interface ConnectorFacade {
|
|
15
15
|
createConnection: (peerDescriptor: PeerDescriptor) => ManagedConnection
|
|
16
16
|
getLocalPeerDescriptor: () => PeerDescriptor | undefined
|
|
17
17
|
start: (
|
|
18
|
-
|
|
18
|
+
onNewConnection: (connection: ManagedConnection) => boolean,
|
|
19
19
|
canConnect: (peerDescriptor: PeerDescriptor) => boolean,
|
|
20
20
|
autoCertifierTransport: ITransport
|
|
21
21
|
) => Promise<void>
|
|
@@ -57,7 +57,7 @@ export class DefaultConnectorFacade implements ConnectorFacade {
|
|
|
57
57
|
}
|
|
58
58
|
|
|
59
59
|
async start(
|
|
60
|
-
|
|
60
|
+
onNewConnection: (connection: ManagedConnection) => boolean,
|
|
61
61
|
canConnect: (peerDescriptor: PeerDescriptor) => boolean,
|
|
62
62
|
autoCertifierTransport: ITransport
|
|
63
63
|
): Promise<void> {
|
|
@@ -66,7 +66,7 @@ export class DefaultConnectorFacade implements ConnectorFacade {
|
|
|
66
66
|
transport: this.config.transport!,
|
|
67
67
|
// TODO should we use canConnect also for WebrtcConnector? (NET-1142)
|
|
68
68
|
canConnect: (peerDescriptor: PeerDescriptor) => canConnect(peerDescriptor),
|
|
69
|
-
|
|
69
|
+
onNewConnection,
|
|
70
70
|
portRange: this.config.websocketPortRange,
|
|
71
71
|
host: this.config.websocketHost,
|
|
72
72
|
entrypoints: this.config.entryPoints,
|
|
@@ -89,7 +89,7 @@ export class DefaultConnectorFacade implements ConnectorFacade {
|
|
|
89
89
|
externalIp: this.config.externalIp,
|
|
90
90
|
portRange: this.config.webrtcPortRange,
|
|
91
91
|
maxMessageSize: this.config.maxMessageSize
|
|
92
|
-
},
|
|
92
|
+
}, onNewConnection)
|
|
93
93
|
await this.websocketConnector.start()
|
|
94
94
|
// TODO: generate a PeerDescriptor in a single function. Requires changes to the createOwnPeerDescriptor
|
|
95
95
|
// function in the config. Currently it's given by the DhtNode and it sets the PeerDescriptor for the
|
|
@@ -97,34 +97,45 @@ export class DefaultConnectorFacade implements ConnectorFacade {
|
|
|
97
97
|
// LocalPeerDescriptor could be stored in one place and passed from there to the connectors
|
|
98
98
|
const temporarilySelfSigned = (!this.config.tlsCertificate && this.config.websocketServerEnableTls === true)
|
|
99
99
|
const connectivityResponse = await this.websocketConnector.checkConnectivity(temporarilySelfSigned)
|
|
100
|
-
|
|
101
|
-
this.localPeerDescriptor
|
|
102
|
-
this.websocketConnector.setLocalPeerDescriptor(localPeerDescriptor)
|
|
100
|
+
const localPeerDescriptor = this.config.createLocalPeerDescriptor(connectivityResponse)
|
|
101
|
+
this.setLocalPeerDescriptor(localPeerDescriptor)
|
|
103
102
|
if (localPeerDescriptor.websocket && !this.config.tlsCertificate && this.config.websocketServerEnableTls) {
|
|
104
103
|
try {
|
|
105
104
|
await this.websocketConnector!.autoCertify()
|
|
106
105
|
const connectivityResponse = await this.websocketConnector!.checkConnectivity(false)
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
106
|
+
const autocertifiedLocalPeerDescriptor = this.config.createLocalPeerDescriptor(connectivityResponse)
|
|
107
|
+
if (autocertifiedLocalPeerDescriptor.websocket !== undefined) {
|
|
108
|
+
this.setLocalPeerDescriptor(autocertifiedLocalPeerDescriptor)
|
|
109
|
+
} else {
|
|
110
|
+
logger.warn('Connectivity check failed after auto-certification, disabling WebSocket server TLS')
|
|
111
|
+
await this.restartWebsocketConnector({
|
|
112
|
+
...webSocketConnectorConfig,
|
|
113
|
+
serverEnableTls: false
|
|
114
|
+
})
|
|
111
115
|
}
|
|
112
|
-
this.websocketConnector!.setLocalPeerDescriptor(localPeerDescriptor)
|
|
113
116
|
} catch (err) {
|
|
114
|
-
logger.warn('Failed to
|
|
115
|
-
await this.
|
|
116
|
-
this.websocketConnector = new WebsocketConnector({
|
|
117
|
+
logger.warn('Failed to auto-certify, disabling WebSocket server TLS')
|
|
118
|
+
await this.restartWebsocketConnector({
|
|
117
119
|
...webSocketConnectorConfig,
|
|
118
|
-
serverEnableTls: false
|
|
120
|
+
serverEnableTls: false
|
|
119
121
|
})
|
|
120
|
-
await this.websocketConnector.start()
|
|
121
|
-
const connectivityResponse = await this.websocketConnector.checkConnectivity(false)
|
|
122
|
-
localPeerDescriptor = this.config.createLocalPeerDescriptor(connectivityResponse)
|
|
123
|
-
this.localPeerDescriptor = localPeerDescriptor
|
|
124
|
-
this.websocketConnector.setLocalPeerDescriptor(localPeerDescriptor)
|
|
125
122
|
}
|
|
126
123
|
}
|
|
127
|
-
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
private setLocalPeerDescriptor(peerDescriptor: PeerDescriptor) {
|
|
127
|
+
this.localPeerDescriptor = peerDescriptor
|
|
128
|
+
this.websocketConnector!.setLocalPeerDescriptor(peerDescriptor)
|
|
129
|
+
this.webrtcConnector!.setLocalPeerDescriptor(peerDescriptor)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
async restartWebsocketConnector(webSocketConnectorConfig: WebsocketConnectorConfig): Promise<void> {
|
|
133
|
+
await this.websocketConnector!.destroy()
|
|
134
|
+
this.websocketConnector = new WebsocketConnector(webSocketConnectorConfig)
|
|
135
|
+
await this.websocketConnector.start()
|
|
136
|
+
const connectivityResponse = await this.websocketConnector.checkConnectivity(false)
|
|
137
|
+
const localPeerDescriptor = this.config.createLocalPeerDescriptor(connectivityResponse)
|
|
138
|
+
this.setLocalPeerDescriptor(localPeerDescriptor)
|
|
128
139
|
}
|
|
129
140
|
|
|
130
141
|
createConnection(peerDescriptor: PeerDescriptor): ManagedConnection {
|
|
@@ -156,12 +167,12 @@ export class SimulatorConnectorFacade implements ConnectorFacade {
|
|
|
156
167
|
this.simulator = simulator
|
|
157
168
|
}
|
|
158
169
|
|
|
159
|
-
async start(
|
|
170
|
+
async start(onNewConnection: (connection: ManagedConnection) => boolean): Promise<void> {
|
|
160
171
|
logger.trace(`Creating SimulatorConnector`)
|
|
161
172
|
this.simulatorConnector = new SimulatorConnector(
|
|
162
173
|
this.localPeerDescriptor,
|
|
163
174
|
this.simulator,
|
|
164
|
-
|
|
175
|
+
onNewConnection
|
|
165
176
|
)
|
|
166
177
|
this.simulator.addConnector(this.simulatorConnector)
|
|
167
178
|
}
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import { Logger } from '@streamr/utils'
|
|
2
2
|
import { EventEmitter } from 'eventemitter3'
|
|
3
3
|
import { v4 } from 'uuid'
|
|
4
|
-
import { Message, HandshakeRequest, HandshakeResponse, MessageType, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
|
|
4
|
+
import { Message, HandshakeRequest, HandshakeResponse, MessageType, PeerDescriptor, HandshakeError } from '../proto/packages/dht/protos/DhtRpc'
|
|
5
5
|
import { IConnection } from './IConnection'
|
|
6
6
|
|
|
7
7
|
const logger = new Logger(module)
|
|
8
8
|
|
|
9
9
|
interface HandshakerEvents {
|
|
10
|
-
handshakeRequest: (
|
|
11
|
-
handshakeCompleted: (
|
|
12
|
-
handshakeFailed: (
|
|
10
|
+
handshakeRequest: (source: PeerDescriptor, target?: PeerDescriptor) => void
|
|
11
|
+
handshakeCompleted: (remote: PeerDescriptor) => void
|
|
12
|
+
handshakeFailed: (error?: HandshakeError) => void
|
|
13
13
|
}
|
|
14
14
|
|
|
15
15
|
export class Handshaker extends EventEmitter<HandshakerEvents> {
|
|
@@ -34,15 +34,15 @@ export class Handshaker extends EventEmitter<HandshakerEvents> {
|
|
|
34
34
|
if (message.body.oneofKind === 'handshakeRequest') {
|
|
35
35
|
logger.trace('handshake request received')
|
|
36
36
|
const handshake = message.body.handshakeRequest
|
|
37
|
-
this.emit('handshakeRequest', handshake.
|
|
37
|
+
this.emit('handshakeRequest', handshake.sourcePeerDescriptor!, handshake.targetPeerDescriptor)
|
|
38
38
|
}
|
|
39
39
|
if (message.body.oneofKind === 'handshakeResponse') {
|
|
40
40
|
logger.trace('handshake response received')
|
|
41
41
|
const handshake = message.body.handshakeResponse
|
|
42
|
-
if (handshake.
|
|
43
|
-
this.emit('handshakeFailed', handshake.
|
|
42
|
+
if (handshake.error !== undefined) {
|
|
43
|
+
this.emit('handshakeFailed', handshake.error)
|
|
44
44
|
} else {
|
|
45
|
-
this.emit('handshakeCompleted', handshake.
|
|
45
|
+
this.emit('handshakeCompleted', handshake.sourcePeerDescriptor!)
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
48
|
} catch (err) {
|
|
@@ -51,10 +51,10 @@ export class Handshaker extends EventEmitter<HandshakerEvents> {
|
|
|
51
51
|
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
-
public sendHandshakeRequest(): void {
|
|
54
|
+
public sendHandshakeRequest(remotePeerDescriptor?: PeerDescriptor): void {
|
|
55
55
|
const outgoingHandshake: HandshakeRequest = {
|
|
56
|
-
|
|
57
|
-
|
|
56
|
+
sourcePeerDescriptor: this.localPeerDescriptor,
|
|
57
|
+
targetPeerDescriptor: remotePeerDescriptor
|
|
58
58
|
}
|
|
59
59
|
const msg: Message = {
|
|
60
60
|
serviceId: Handshaker.HANDSHAKER_SERVICE_ID,
|
|
@@ -69,13 +69,10 @@ export class Handshaker extends EventEmitter<HandshakerEvents> {
|
|
|
69
69
|
logger.trace('handshake request sent')
|
|
70
70
|
}
|
|
71
71
|
|
|
72
|
-
public sendHandshakeResponse(error?:
|
|
72
|
+
public sendHandshakeResponse(error?: HandshakeError): void {
|
|
73
73
|
const outgoingHandshakeResponse: HandshakeResponse = {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
}
|
|
77
|
-
if (error) {
|
|
78
|
-
outgoingHandshakeResponse.responseError = error
|
|
74
|
+
sourcePeerDescriptor: this.localPeerDescriptor,
|
|
75
|
+
error
|
|
79
76
|
}
|
|
80
77
|
const msg: Message = {
|
|
81
78
|
serviceId: Handshaker.HANDSHAKER_SERVICE_ID,
|
|
@@ -87,6 +84,6 @@ export class Handshaker extends EventEmitter<HandshakerEvents> {
|
|
|
87
84
|
}
|
|
88
85
|
}
|
|
89
86
|
this.connection.send(Message.toBinary(msg))
|
|
90
|
-
logger.trace('handshake
|
|
87
|
+
logger.trace('handshake response sent')
|
|
91
88
|
}
|
|
92
89
|
}
|