@streamr/dht 0.0.1-tatum.6 → 0.0.1-tatum.8

package/src/connection/WebSocket/{WebSocketConnector.ts → WebSocketConnectorRpcLocal.ts}

@@ -2,18 +2,18 @@ import { ClientWebSocket } from './ClientWebSocket'
 import { IConnection, ConnectionType } from '../IConnection'
 import { ITransport } from '../../transport/ITransport'
 import { ListeningRpcCommunicator } from '../../transport/ListeningRpcCommunicator'
-import { RemoteWebSocketConnector } from './RemoteWebSocketConnector'
+import { WebSocketConnectorRpcRemote } from './WebSocketConnectorRpcRemote'
 import {
     ConnectivityMethod,
     ConnectivityResponse,
+    NodeType,
     PeerDescriptor,
     WebSocketConnectionRequest,
     WebSocketConnectionResponse
 } from '../../proto/packages/dht/protos/DhtRpc'
-import { WebSocketConnectorServiceClient } from '../../proto/packages/dht/protos/DhtRpc.client'
-import { Logger, wait } from '@streamr/utils'
-import { IWebSocketConnectorService } from '../../proto/packages/dht/protos/DhtRpc.server'
-import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import { WebSocketConnectorRpcClient } from '../../proto/packages/dht/protos/DhtRpc.client'
+import { Logger, binaryToHex, wait } from '@streamr/utils'
+import { IWebSocketConnectorRpc } from '../../proto/packages/dht/protos/DhtRpc.server'
 import { ManagedConnection } from '../ManagedConnection'
 import { WebSocketServer } from './WebSocketServer'
 import { ConnectivityChecker } from '../ConnectivityChecker'
@@ -24,7 +24,10 @@ import { toProtoRpcClient } from '@streamr/proto-rpc'
 import { Handshaker } from '../Handshaker'
 import { keyFromPeerDescriptor, peerIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
 import { ParsedUrlQuery } from 'querystring'
-import { sample } from 'lodash'
+import { range, sample } from 'lodash'
+import { isPrivateIPv4 } from '../../helpers/AddressTools'
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import { DhtCallContext } from '../../rpc-protocol/DhtCallContext'
 
 const logger = new Logger(module)
 
@@ -32,47 +35,55 @@ export const connectivityMethodToWebSocketUrl = (ws: ConnectivityMethod): string
     return (ws.tls ? 'wss://' : 'ws://') + ws.host + ':' + ws.port
 }
 
+const canOpenConnectionFromBrowser = (websocketServer: ConnectivityMethod) => {
+    const hasPrivateAddress = ((websocketServer.host === 'localhost') || isPrivateIPv4(websocketServer.host))
+    return websocketServer.tls || hasPrivateAddress
+}
+
 const ENTRY_POINT_CONNECTION_ATTEMPTS = 5
 
-export class WebSocketConnector implements IWebSocketConnectorService {
-    private static readonly WEBSOCKET_CONNECTOR_SERVICE_ID = 'system/websocketconnector'
+interface WebSocketConnectorRpcLocalConfig {
+    transport: ITransport
+    canConnect: (peerDescriptor: PeerDescriptor, _ip: string, port: number) => boolean
+    onIncomingConnection: (connection: ManagedConnection) => boolean
+    portRange?: PortRange
+    maxMessageSize?: number
+    host?: string
+    entrypoints?: PeerDescriptor[]
+    tlsCertificate?: TlsCertificate
+}
+
+export class WebSocketConnectorRpcLocal implements IWebSocketConnectorRpc {
+
+    private static readonly WEBSOCKET_CONNECTOR_SERVICE_ID = 'system/websocket-connector'
     private readonly rpcCommunicator: ListeningRpcCommunicator
     private readonly canConnectFunction: (peerDescriptor: PeerDescriptor, _ip: string, port: number) => boolean
     private readonly webSocketServer?: WebSocketServer
     private connectivityChecker?: ConnectivityChecker
     private readonly ongoingConnectRequests: Map<PeerIDKey, ManagedConnection> = new Map()
-    private incomingConnectionCallback: (connection: ManagedConnection) => boolean
-    private portRange?: PortRange
+    private onIncomingConnection: (connection: ManagedConnection) => boolean
     private host?: string
-    private entrypoints?: PeerDescriptor[]
+    private readonly entrypoints?: PeerDescriptor[]
     private readonly tlsCertificate?: TlsCertificate
     private selectedPort?: number
-    private readonly protocolVersion: string
     private ownPeerDescriptor?: PeerDescriptor
     private connectingConnections: Map<PeerIDKey, ManagedConnection> = new Map()
     private destroyed = false
 
-    constructor(
-        protocolVersion: string,
-        rpcTransport: ITransport,
-        fnCanConnect: (peerDescriptor: PeerDescriptor, _ip: string, port: number) => boolean,
-        incomingConnectionCallback: (connection: ManagedConnection) => boolean,
-        portRange?: PortRange,
-        host?: string,
-        entrypoints?: PeerDescriptor[],
-        tlsCertificate?: TlsCertificate
-    ) {
-        this.protocolVersion = protocolVersion
-        this.webSocketServer = portRange ? new WebSocketServer() : undefined
-        this.incomingConnectionCallback = incomingConnectionCallback
-        this.portRange = portRange
-        this.host = host
-        this.entrypoints = entrypoints
-        this.tlsCertificate = tlsCertificate
-
-        this.canConnectFunction = fnCanConnect.bind(this)
-
-        this.rpcCommunicator = new ListeningRpcCommunicator(WebSocketConnector.WEBSOCKET_CONNECTOR_SERVICE_ID, rpcTransport, {
+    constructor(config: WebSocketConnectorRpcLocalConfig) {
+        this.webSocketServer = config.portRange ? new WebSocketServer({
+            portRange: config.portRange!,
+            tlsCertificate: config.tlsCertificate,
+            maxMessageSize: config.maxMessageSize
+        }) : undefined
+        this.onIncomingConnection = config.onIncomingConnection
+        this.host = config.host
+        this.entrypoints = config.entrypoints
+        this.tlsCertificate = config.tlsCertificate
+
+        this.canConnectFunction = config.canConnect.bind(this)
+
+        this.rpcCommunicator = new ListeningRpcCommunicator(WebSocketConnectorRpcLocal.WEBSOCKET_CONNECTOR_SERVICE_ID, config.transport, {
             rpcRequestTimeout: 15000
         })
 
@@ -80,12 +91,12 @@ export class WebSocketConnector implements IWebSocketConnectorService {
             WebSocketConnectionRequest,
             WebSocketConnectionResponse,
             'requestConnection',
-            (req: WebSocketConnectionRequest, context) => this.requestConnection(req, context)
+            (req: WebSocketConnectionRequest, context: ServerCallContext) => this.requestConnection(req, context)
         )
     }
 
     private attachHandshaker(connection: IConnection) {
-        const handshaker = new Handshaker(this.ownPeerDescriptor!, this.protocolVersion, connection)
+        const handshaker = new Handshaker(this.ownPeerDescriptor!, connection)
         handshaker.once('handshakeRequest', (peerDescriptor: PeerDescriptor) => {
             this.onServerSocketHandshakeRequest(peerDescriptor, connection)
         })
@@ -100,10 +111,10 @@ export class WebSocketConnector implements IWebSocketConnectorService {
                     serverSocket.resourceURL.query) {
                     const query = serverSocket.resourceURL.query as unknown as ParsedUrlQuery
                     if (query.connectivityRequest) {
-                        logger.trace('Received connectivity request connection')
+                        logger.trace('Received connectivity request connection from ' + serverSocket.getRemoteAddress())
                         this.connectivityChecker!.listenToIncomingConnectivityRequests(serverSocket)
                     } else if (query.connectivityProbe) {
-                        logger.trace('Received connectivity probe connection')
+                        logger.trace('Received connectivity probe connection from ' + serverSocket.getRemoteAddress())
                     } else {
                         this.attachHandshaker(connection)
                     }
@@ -111,13 +122,13 @@ export class WebSocketConnector implements IWebSocketConnectorService {
                     this.attachHandshaker(connection)
                 }
             })
-            const port = await this.webSocketServer.start(this.portRange!, this.tlsCertificate)
+            const port = await this.webSocketServer.start()
             this.selectedPort = port
             this.connectivityChecker = new ConnectivityChecker(this.selectedPort, this.tlsCertificate !== undefined, this.host)
         }
     }
 
-    public async checkConnectivity(reattempt = 0): Promise<ConnectivityResponse> {
+    public async checkConnectivity(): Promise<ConnectivityResponse> {
         // TODO: this could throw if the server is not running
         const noServerConnectivityResponse: ConnectivityResponse = {
             openInternet: false,
@@ -127,35 +138,47 @@ export class WebSocketConnector implements IWebSocketConnectorService {
         if (this.destroyed) {
             return noServerConnectivityResponse
         }
-        try {
-            if (!this.webSocketServer) {
-                // If no websocket server, return openInternet: false
-                return noServerConnectivityResponse
-            } else {
-                if (!this.entrypoints || this.entrypoints.length < 1) {
-                    // return connectivity info given in config
-                    const preconfiguredConnectivityResponse: ConnectivityResponse = {
-                        openInternet: true,
-                        host: this.host!,
-                        natType: NatType.OPEN_INTERNET,
-                        websocket: { host: this.host!, port: this.selectedPort!, tls: this.tlsCertificate !== undefined }
-                    }
-                    return preconfiguredConnectivityResponse
+        for (const reattempt of range(ENTRY_POINT_CONNECTION_ATTEMPTS)) {
+            const entryPoint = sample(this.entrypoints)!
+            try {
+                if (!this.webSocketServer) {
+                    // If no websocket server, return openInternet: false
+                    return noServerConnectivityResponse
                 } else {
-                    // Do real connectivity checking     
-                    return await this.connectivityChecker!.sendConnectivityRequest(sample(this.entrypoints)!)
+                    if (!this.entrypoints || this.entrypoints.length < 1) {
+                        // return connectivity info given in config
+                        const preconfiguredConnectivityResponse: ConnectivityResponse = {
+                            openInternet: true,
+                            host: this.host!,
+                            natType: NatType.OPEN_INTERNET,
+                            websocket: { host: this.host!, port: this.selectedPort!, tls: this.tlsCertificate !== undefined }
+                        }
+                        return preconfiguredConnectivityResponse
+                    } else {
+                        // Do real connectivity checking
+                        return await this.connectivityChecker!.sendConnectivityRequest(entryPoint)
+                    }
+                }
+            } catch (err) {
+                if (reattempt < ENTRY_POINT_CONNECTION_ATTEMPTS) {
+                    const error = `Failed to connect to entrypoint with id ${binaryToHex(entryPoint.kademliaId)} `
+                        + `and URL ${connectivityMethodToWebSocketUrl(entryPoint.websocket!)}`
+                    logger.error(error, { error: err })
+                    await wait(2000)
                 }
-            }
-        } catch (err) {
-            if (reattempt < ENTRY_POINT_CONNECTION_ATTEMPTS) {
-                logger.error('Failed to connect to the entrypoint', { error: err })
-                await wait(2000)
-                return this.checkConnectivity(reattempt + 1)
-            } else {
-                throw err
             }
         }
+        throw Error(`Failed to connect to the entrypoints after ${ENTRY_POINT_CONNECTION_ATTEMPTS} attempts`)
+    }
 
+    public isPossibleToFormConnection(targetPeerDescriptor: PeerDescriptor): boolean {
+        if (this.ownPeerDescriptor!.websocket !== undefined) {
+            return (targetPeerDescriptor.type !== NodeType.BROWSER) || canOpenConnectionFromBrowser(this.ownPeerDescriptor!.websocket)
+        } else if (targetPeerDescriptor.websocket !== undefined) {
+            return (this.ownPeerDescriptor!.type !== NodeType.BROWSER) || canOpenConnectionFromBrowser(targetPeerDescriptor.websocket)
+        } else {
+            return false
+        }
     }
 
     public connect(targetPeerDescriptor: PeerDescriptor): ManagedConnection {
@@ -172,8 +195,7 @@ export class WebSocketConnector implements IWebSocketConnectorService {
 
             const url = connectivityMethodToWebSocketUrl(targetPeerDescriptor.websocket!)
 
-            const managedConnection = new ManagedConnection(this.ownPeerDescriptor!, this.protocolVersion,
-                ConnectionType.WEBSOCKET_CLIENT, socket, undefined)
+            const managedConnection = new ManagedConnection(this.ownPeerDescriptor!, ConnectionType.WEBSOCKET_CLIENT, socket, undefined)
             managedConnection.setPeerDescriptor(targetPeerDescriptor)
 
             this.connectingConnections.set(keyFromPeerDescriptor(targetPeerDescriptor), managedConnection)
@@ -194,15 +216,16 @@ export class WebSocketConnector implements IWebSocketConnectorService {
         }
     }
 
-    public requestConnectionFromPeer(ownPeerDescriptor: PeerDescriptor, targetPeerDescriptor: PeerDescriptor): ManagedConnection {
+    private requestConnectionFromPeer(ownPeerDescriptor: PeerDescriptor, targetPeerDescriptor: PeerDescriptor): ManagedConnection {
         setImmediate(() => {
-            const remoteConnector = new RemoteWebSocketConnector(
+            const remoteConnector = new WebSocketConnectorRpcRemote(
+                ownPeerDescriptor,
                 targetPeerDescriptor,
-                toProtoRpcClient(new WebSocketConnectorServiceClient(this.rpcCommunicator.getRpcClientTransport()))
+                toProtoRpcClient(new WebSocketConnectorRpcClient(this.rpcCommunicator.getRpcClientTransport()))
             )
-            remoteConnector.requestConnection(ownPeerDescriptor, ownPeerDescriptor.websocket!.host, ownPeerDescriptor.websocket!.port)
+            remoteConnector.requestConnection(ownPeerDescriptor.websocket!.host, ownPeerDescriptor.websocket!.port)
         })
-        const managedConnection = new ManagedConnection(this.ownPeerDescriptor!, this.protocolVersion, ConnectionType.WEBSOCKET_SERVER)
+        const managedConnection = new ManagedConnection(this.ownPeerDescriptor!, ConnectionType.WEBSOCKET_SERVER)
         managedConnection.on('disconnected', () => this.ongoingConnectRequests.delete(keyFromPeerDescriptor(targetPeerDescriptor)))
         managedConnection.setPeerDescriptor(targetPeerDescriptor)
         this.ongoingConnectRequests.set(keyFromPeerDescriptor(targetPeerDescriptor), managedConnection)
@@ -215,16 +238,15 @@ export class WebSocketConnector implements IWebSocketConnectorService {
 
         if (this.ongoingConnectRequests.has(peerId.toKey())) {
             const ongoingConnectReguest = this.ongoingConnectRequests.get(peerId.toKey())!
-            ongoingConnectReguest.attachImplementation(serverWebSocket, peerDescriptor)
+            ongoingConnectReguest.attachImplementation(serverWebSocket)
             ongoingConnectReguest.acceptHandshake()
             this.ongoingConnectRequests.delete(peerId.toKey())
         } else {
-            const managedConnection = new ManagedConnection(this.ownPeerDescriptor!, this.protocolVersion,
-                ConnectionType.WEBSOCKET_SERVER, undefined, serverWebSocket)
+            const managedConnection = new ManagedConnection(this.ownPeerDescriptor!, ConnectionType.WEBSOCKET_SERVER, undefined, serverWebSocket)
 
             managedConnection.setPeerDescriptor(peerDescriptor)
 
-            if (this.incomingConnectionCallback(managedConnection)) {
+            if (this.onIncomingConnection(managedConnection)) {
                 managedConnection.acceptHandshake()
             } else {
                 managedConnection.rejectHandshake('Duplicate connection')
@@ -250,15 +272,16 @@ export class WebSocketConnector implements IWebSocketConnectorService {
         await this.webSocketServer?.stop()
     }
 
-    // IWebSocketConnectorService implementation
-    public async requestConnection(request: WebSocketConnectionRequest, _context: ServerCallContext): Promise<WebSocketConnectionResponse> {
-        if (!this.destroyed && this.canConnectFunction(request.requester!, request.ip, request.port)) {
+    // IWebSocketConnectorRpc implementation
+    public async requestConnection(request: WebSocketConnectionRequest, context: ServerCallContext): Promise<WebSocketConnectionResponse> {
+        const senderPeerDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
+        if (!this.destroyed && this.canConnectFunction(senderPeerDescriptor, request.ip, request.port)) {
             setImmediate(() => {
                 if (this.destroyed) {
                     return
                 }
-                const connection = this.connect(request.requester!)
-                this.incomingConnectionCallback(connection)
+                const connection = this.connect(senderPeerDescriptor)
+                this.onIncomingConnection(connection)
             })
             const res: WebSocketConnectionResponse = {
                 accepted: true

package/src/connection/WebSocket/WebSocketConnectorRpcRemote.ts

@@ -0,0 +1,45 @@
+import {
+    PeerDescriptor,
+    WebSocketConnectionRequest
+} from '../../proto/packages/dht/protos/DhtRpc'
+import { IWebSocketConnectorRpcClient } from '../../proto/packages/dht/protos/DhtRpc.client'
+import { Logger } from '@streamr/utils'
+import * as Err from '../../helpers/errors'
+import { ProtoRpcClient } from '@streamr/proto-rpc'
+import { keyFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
+import { Remote } from '../../dht/contact/Remote'
+
+const logger = new Logger(module)
+
+export class WebSocketConnectorRpcRemote extends Remote<IWebSocketConnectorRpcClient> {
+
+    constructor(
+        localPeerDescriptor: PeerDescriptor,
+        remotePeerDescriptor: PeerDescriptor,
+        client: ProtoRpcClient<IWebSocketConnectorRpcClient>
+    ) {
+        super(localPeerDescriptor, remotePeerDescriptor, 'DUMMY', client)
+    }
+
+    async requestConnection(ip: string, port: number): Promise<boolean> {
+        logger.trace(`Requesting WebSocket connection from ${keyFromPeerDescriptor(this.getLocalPeerDescriptor())}`)
+        const request: WebSocketConnectionRequest = {
+            ip,
+            port
+        }
+        const options = this.formDhtRpcOptions()
+        try {
+            const res = await this.getClient().requestConnection(request, options)
+            
+            if (res.reason) {
+                logger.debug('WebSocketConnectionRequest Rejected', {
+                    stack: new Err.WebSocketConnectionRequestRejected(res.reason).stack
+                })
+            }
+            return res.accepted
+        } catch (err) {
+            logger.debug(new Err.WebSocketConnectionRequestRejected('WebSocketConnectionRequest rejected', err).stack!)
+            return false
+        }
+    }
+}

package/src/connection/WebSocket/WebSocketServer.ts

@@ -22,17 +22,33 @@ const logger = new Logger(module)
 
 declare class NodeJsWsServer extends WsServer { }
 
+interface WebSocketServerConfig {
+    portRange: PortRange
+    tlsCertificate?: TlsCertificate
+    maxMessageSize?: number
+}
+
 export class WebSocketServer extends EventEmitter<ConnectionSourceEvents> {
 
     private httpServer?: HttpServer | HttpsServer
     private wsServer?: WsServer
     private readonly abortController = new AbortController()
+    private readonly portRange: PortRange
+    private readonly tlsCertificate?: TlsCertificate
+    private readonly maxMessageSize: number
+
+    constructor(config: WebSocketServerConfig) {
+        super()
+        this.portRange = config.portRange
+        this.tlsCertificate = config.tlsCertificate
+        this.maxMessageSize = config.maxMessageSize ?? 1048576
+    }
 
-    public async start(portRange: PortRange, tlsCertificate?: TlsCertificate): Promise<number> {
-        const ports = range(portRange.min, portRange.max + 1)
+    public async start(): Promise<number> {
+        const ports = range(this.portRange.min, this.portRange.max + 1)
         for (const port of ports) {
             try {
-                await asAbortable(this.startServer(port, tlsCertificate), this.abortController.signal)
+                await asAbortable(this.startServer(port, this.tlsCertificate), this.abortController.signal)
                 return port
             } catch (err) {
                 if (err.originalError?.code === 'EADDRINUSE') {
@@ -42,7 +58,7 @@ export class WebSocketServer extends EventEmitter<ConnectionSourceEvents> {
                 }
             }
         }
-        throw new WebSocketServerStartError(`Failed to start WebSocket server on any port in range: ${portRange.min}-${portRange.min}`)
+        throw new WebSocketServerStartError(`Failed to start WebSocket server on any port in range: ${this.portRange.min}-${this.portRange.min}`)
     }
 
     private startServer(port: number, tlsCertificate?: TlsCertificate): Promise<void> {
@@ -60,14 +76,14 @@ export class WebSocketServer extends EventEmitter<ConnectionSourceEvents> {
                 : 
                 createHttpServer(requestListener)
 
-            function originIsAllowed(_uorigin: string) {
+            function originIsAllowed() {
                 return true
             }
 
             this.wsServer = this.createWsServer(this.httpServer)
             
             this.wsServer.on('request', (request) => {
-                if (!originIsAllowed(request.origin)) {
+                if (!originIsAllowed()) {
                     // Make sure we only accept requests from an allowed origin
                     request.reject()
                     logger.trace('IConnection from origin ' + request.origin + ' rejected.')
@@ -115,12 +131,14 @@ export class WebSocketServer extends EventEmitter<ConnectionSourceEvents> {
         if (typeof NodeJsWsServer !== 'undefined') {
             return new NodeJsWsServer({
                 httpServer,
-                autoAcceptConnections: false
+                autoAcceptConnections: false,
+                maxReceivedMessageSize: this.maxMessageSize
             })
         } else {
             return this.wsServer = new WsServer({
                 httpServer,
-                autoAcceptConnections: false
+                autoAcceptConnections: false,
+                maxReceivedMessageSize: this.maxMessageSize
             })
         }
     }