@streamr/dht 100.0.0-pretestnet.4 → 100.0.0-rc.0

package/src/connection/ConnectivityChecker.ts

@@ -1,199 +0,0 @@
-import {
-    ConnectivityRequest, ConnectivityResponse,
-    Message, MessageType, PeerDescriptor
-} from '../proto/packages/dht/protos/DhtRpc'
-import { ConnectionEvents, IConnection } from './IConnection'
-import { Logger, runAndRaceEvents3, RunAndRaceEventsReturnType } from '@streamr/utils'
-import * as Err from '../helpers/errors'
-import { ClientWebsocket } from './websocket/ClientWebsocket'
-import { v4 } from 'uuid'
-import { NatType } from './ConnectionManager'
-import { ServerWebsocket } from './websocket/ServerWebsocket'
-import { connectivityMethodToWebsocketUrl } from './websocket/WebsocketConnector'
-
-const logger = new Logger(module)
-
-// Class for handling both client and server side of the connectivity
-// checks. This is attached to all ServerWebsockets to listen to
-// ConnectivityRequest messages. 
-
-export class ConnectivityChecker {
-
-    private static readonly CONNECTIVITY_CHECKER_SERVICE_ID = 'system/connectivity-checker'
-    private static readonly CONNECTIVITY_CHECKER_TIMEOUT = 5000
-    private destroyed = false
-    private readonly websocketPort: number
-    private readonly tls: boolean 
-    private host?: string
-
-    constructor(websocketPort: number, tls: boolean, host?: string) {
-        this.websocketPort = websocketPort
-        this.tls = tls
-        this.host = host
-    }
-
-    public async sendConnectivityRequest(entryPoint: PeerDescriptor, selfSigned: boolean): Promise<ConnectivityResponse> {
-        if (this.destroyed) {
-            throw new Err.ConnectionFailed('ConnectivityChecker is destroyed')
-        }
-        let outgoingConnection: IConnection
-        const wsServerInfo = {
-            host: entryPoint.websocket!.host, 
-            port: entryPoint.websocket!.port,
-            tls: entryPoint.websocket!.tls,
-        }
-        const url = connectivityMethodToWebsocketUrl(wsServerInfo, 'connectivityRequest')
-        try {
-            outgoingConnection = await this.connectAsync({
-                url,
-                selfSigned
-            })
-        } catch (e) {
-            throw new Err.ConnectionFailed(`Failed to connect to the entrypoint ${url}`, e)
-        }
-        // send connectivity request
-        const connectivityRequestMessage: ConnectivityRequest = { port: this.websocketPort, host: this.host, tls: this.tls, selfSigned }
-        const msg: Message = {
-            serviceId: ConnectivityChecker.CONNECTIVITY_CHECKER_SERVICE_ID,
-            messageType: MessageType.CONNECTIVITY_REQUEST, messageId: v4(),
-            body: {
-                oneofKind: 'connectivityRequest',
-                connectivityRequest: connectivityRequestMessage
-            }
-        }
-        const responseAwaiter = () => {
-            return new Promise((resolve: (res: ConnectivityResponse) => void, reject) => {
-                const timeoutId = setTimeout(() => {
-                    outgoingConnection.close(false)
-                    reject(new Err.ConnectivityResponseTimeout('timeout'))
-                }, ConnectivityChecker.CONNECTIVITY_CHECKER_TIMEOUT)
-                const listener = (bytes: Uint8Array) => {
-                    outgoingConnection.close(false)
-                    try {
-                        const message: Message = Message.fromBinary(bytes)
-                        if (message.body.oneofKind === 'connectivityResponse') {
-                            logger.trace('ConnectivityResponse received: ' + JSON.stringify(Message.toJson(message)))
-                            const connectivityResponseMessage = message.body.connectivityResponse
-                            outgoingConnection!.off('data', listener)
-                            clearTimeout(timeoutId)
-                            resolve(connectivityResponseMessage)
-                        } else {
-                            return
-                        }
-                    } catch (err) {
-                        logger.trace(`Could not parse message: ${err}`)
-                    }
-                }
-                outgoingConnection!.on('data', listener)
-            })
-        }
-        try {
-            const retPromise = responseAwaiter()
-            outgoingConnection.send(Message.toBinary(msg))
-            logger.trace('ConnectivityRequest sent: ' + JSON.stringify(Message.toJson(msg)))
-            return await retPromise
-        } catch (e) {
-            logger.error('error getting connectivityresponse')
-            throw e
-        }
-    }
-
-    public setHost(hostName: string): void {
-        this.host = hostName
-    }
-
-    public listenToIncomingConnectivityRequests(connectionToListenTo: ServerWebsocket): void {
-        connectionToListenTo.on('data', (data: Uint8Array) => {
-            logger.trace('server received data')
-            try {
-                const message = Message.fromBinary(data)
-                if (message.body.oneofKind === 'connectivityRequest') {
-                    logger.trace('ConnectivityRequest received: ' + JSON.stringify(Message.toJson(message)))
-                    this.handleIncomingConnectivityRequest(connectionToListenTo, message.body.connectivityRequest).then(() => {
-                        logger.trace('handleIncomingConnectivityRequest ok')
-                        return
-                    }).catch((e) => {
-                        logger.error('handleIncomingConnectivityRequest' + e)
-                    })
-                }
-            } catch (err) {
-                logger.trace(`Could not parse message: ${err}`)
-            }
-            
-        })
-    }
-
-    private async handleIncomingConnectivityRequest(
-        connection: ServerWebsocket,
-        connectivityRequest: ConnectivityRequest
-    ): Promise<void> {
-        if (this.destroyed) {
-            return
-        }
-        let outgoingConnection: IConnection | undefined
-        let connectivityResponseMessage: ConnectivityResponse | undefined
-        const host = connectivityRequest.host ?? connection.getRemoteAddress()
-        try {
-            const wsServerInfo = {
-                host,
-                port: connectivityRequest.port,
-                tls: connectivityRequest.tls
-            }
-            const url = connectivityMethodToWebsocketUrl(wsServerInfo, 'connectivityProbe')
-            logger.trace(`Attempting Connectivity Check to ${url}`)
-            outgoingConnection = await this.connectAsync({
-                url,
-                selfSigned: connectivityRequest.selfSigned
-            })
-        } catch (err) {
-            logger.debug('error', { err })
-            connectivityResponseMessage = {
-                host,
-                natType: NatType.UNKNOWN
-            }
-        }
-        if (outgoingConnection) {
-            outgoingConnection.close(false)
-            logger.trace('Connectivity test produced positive result, communicating reply to the requester ' + host + ':' + connectivityRequest.port)
-            connectivityResponseMessage = {
-                host,
-                natType: NatType.OPEN_INTERNET,
-                websocket: { host, port: connectivityRequest.port, tls: connectivityRequest.tls }
-            }
-        }
-        const msg: Message = {
-            serviceId: ConnectivityChecker.CONNECTIVITY_CHECKER_SERVICE_ID,
-            messageType: MessageType.CONNECTIVITY_RESPONSE, messageId: v4(),
-            body: {
-                oneofKind: 'connectivityResponse',
-                connectivityResponse: connectivityResponseMessage!
-            }
-        }
-        connection.send(Message.toBinary(msg))
-        logger.trace('ConnectivityResponse sent: ' + JSON.stringify(Message.toJson(msg)))
-    }
-
-    // eslint-disable-next-line class-methods-use-this
-    private async connectAsync({ url, selfSigned, timeoutMs = 1000, }:
-        { url: string, selfSigned: boolean, timeoutMs?: number }
-    ): Promise<IConnection> {
-        const socket = new ClientWebsocket()
-        let result: RunAndRaceEventsReturnType<ConnectionEvents>
-        try {
-            result = await runAndRaceEvents3<ConnectionEvents>([
-                () => { socket.connect(url, selfSigned) }],
-            socket, ['connected', 'error'],
-            timeoutMs)
-        } catch (e) {
-            throw (new Err.ConnectionFailed('WebSocket connection timed out'))
-        }
-        if (result.winnerName === 'error') {
-            throw (new Err.ConnectionFailed('Could not open WebSocket connection'))
-        }
-        return socket
-    }
-
-    public destroy(): void {
-        this.destroyed = true
-    }
-}

package/src/connection/websocket/ServerWebsocket.ts

@@ -1,114 +0,0 @@
-import EventEmitter from 'eventemitter3'
-import { IConnection, ConnectionID, ConnectionEvents, ConnectionType } from '../IConnection'
-import { connection as WsConnection } from 'websocket'
-import { Logger } from '@streamr/utils'
-import { Url } from 'url'
-import { GOING_AWAY } from './ClientWebsocket'
-
-const logger = new Logger(module)
-
-// NodeJsBuffer is global defined in preload.js of Karma
-// It is used to make Karma/Electron tests to use the NodeJS
-// implementation of Buffer instead of the browser polyfill
-
-declare let NodeJsBuffer: BufferConstructor
-
-enum MessageType {
-    UTF8 = 'utf8',
-    BINARY = 'binary'
-}
-
-export class ServerWebsocket extends EventEmitter<ConnectionEvents> implements IConnection {
-
-    public readonly connectionId: ConnectionID
-    public readonly connectionType = ConnectionType.WEBSOCKET_SERVER
-    public readonly resourceURL: Url
-    private socket?: WsConnection
-    private stopped = false
-
-    constructor(socket: WsConnection, resourceURL: Url) {
-        super()
-
-        this.resourceURL = resourceURL
-        this.connectionId = new ConnectionID()
-
-        socket.on('message', (message) => {
-            logger.trace('ServerWebsocket::onMessage')
-            if (message.type === MessageType.UTF8) {
-                logger.debug('Received string Message: ' + message.utf8Data)
-            } else if (message.type === MessageType.BINARY) {
-                logger.trace('Received Binary Message of ' + message.binaryData.length + ' bytes')
-                this.emit('data',
-                    new Uint8Array(message.binaryData.buffer, message.binaryData.byteOffset,
-                        message.binaryData.byteLength / Uint8Array.BYTES_PER_ELEMENT))
-            }
-        })
-        socket.on('close', (reasonCode, description) => {
-            logger.trace('Peer ' + socket.remoteAddress + ' disconnected.')            
-            this.doDisconnect(reasonCode, description)
-        })
-
-        socket.on('error', (error) => {
-            this.emit('error', error.name)
-        })
-
-        this.socket = socket
-    }
-
-    private doDisconnect(reasonCode: number, description: string): void {
-        this.stopped = true
-        this.socket?.removeAllListeners()
-        this.socket = undefined
-        const gracefulLeave = reasonCode === GOING_AWAY
-        this.emit('disconnected', gracefulLeave, reasonCode, description)
-    }
-
-    public send(data: Uint8Array): void {
-        // If in an Karma / Electron test, use the NodeJS implementation
-        // of Buffer instead of the browser polyfill
-
-        if (!this.stopped && this.socket) {
-            if (typeof NodeJsBuffer !== 'undefined') {
-                this.socket.sendBytes(NodeJsBuffer.from(data))
-            } else {
-                this.socket.sendBytes(Buffer.from(data))
-            }
-        } else {
-            logger.debug('Tried to call send() on a stopped socket')
-        }
-
-    }
-
-    public async close(gracefulLeave: boolean): Promise<void> {
-        this.emit('disconnected', gracefulLeave, undefined, 'close() called')
-        this.removeAllListeners()
-        if (!this.stopped) {
-            this.socket?.close(gracefulLeave === true ? GOING_AWAY : undefined)
-        } else {
-            logger.debug('Tried to close a stopped connection')
-        }
-    }
-
-    public destroy(): void {
-        if (!this.stopped) {
-            this.removeAllListeners()
-            if (this.socket) {
-                this.socket.removeAllListeners()
-                this.socket.close()
-                this.socket = undefined
-            }
-            this.stopped = true
-        } else {
-            logger.debug('Tried to destroy() a stopped connection')
-        }
-    }
-
-    public getRemoteAddress(): string {
-        if (!this.stopped && this.socket) {
-            return this.socket.remoteAddress
-        } else {
-            logger.error('Tried to get the remoteAddress of a stopped connection')
-            return ''
-        }
-    }
-}

package/src/dht/find/FindRpcLocal.ts

@@ -1,35 +0,0 @@
-import { Logger } from '@streamr/utils'
-import { PeerDescriptor, RouteMessageAck, RouteMessageWrapper } from '../../proto/packages/dht/protos/DhtRpc'
-import { IFindRpc } from '../../proto/packages/dht/protos/DhtRpc.server'
-import { getNodeIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
-import { createRouteMessageAck } from '../routing/RouterRpcLocal'
-import { getPreviousPeer } from '../routing/getPreviousPeer'
-
-const logger = new Logger(module)
-
-interface FindRpcLocalConfig {
-    doRouteFindRequest: (routedMessage: RouteMessageWrapper) => RouteMessageAck
-    addContact: (contact: PeerDescriptor, setActive?: boolean) => void
-    isMostLikelyDuplicate: (requestId: string) => boolean
-    addToDuplicateDetector: (requestId: string) => void
-}
-
-export class FindRpcLocal implements IFindRpc {
-
-    private readonly config: FindRpcLocalConfig
-
-    constructor(config: FindRpcLocalConfig) {
-        this.config = config
-    }
-
-    async routeFindRequest(routedMessage: RouteMessageWrapper): Promise<RouteMessageAck> {
-        if (this.config.isMostLikelyDuplicate(routedMessage.requestId)) {
-            return createRouteMessageAck(routedMessage, 'message given to routeFindRequest() service is likely a duplicate')
-        }
-        const senderId = getNodeIdFromPeerDescriptor(getPreviousPeer(routedMessage) ?? routedMessage.sourcePeer!)
-        logger.trace(`Received routeFindRequest call from ${senderId}`)
-        this.config.addContact(routedMessage.sourcePeer!, true)
-        this.config.addToDuplicateDetector(routedMessage.requestId)
-        return this.config.doRouteFindRequest(routedMessage)
-    }
-}

package/src/dht/find/FindSession.ts

@@ -1,178 +0,0 @@
-import EventEmitter from 'eventemitter3'
-import { PeerID, PeerIDKey } from '../../helpers/PeerID'
-import { DataEntry, PeerDescriptor, FindResponse } from '../../proto/packages/dht/protos/DhtRpc'
-import { ITransport } from '../../transport/ITransport'
-import { ListeningRpcCommunicator } from '../../transport/ListeningRpcCommunicator'
-import { Contact } from '../contact/Contact'
-import { SortedContactList } from '../contact/SortedContactList'
-import { FindResult } from './Finder'
-import { keyFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
-import { ServiceID } from '../../types/ServiceID'
-import { FindSessionRpcLocal } from './FindSessionRpcLocal'
-
-export interface FindSessionEvents {
-    findCompleted: (results: PeerDescriptor[]) => void
-}
-
-export interface FindSessionConfig {
-    serviceId: ServiceID
-    transport: ITransport
-    kademliaIdToFind: Uint8Array
-    localPeerId: PeerID
-    waitedRoutingPathCompletions: number
-    fetchData: boolean
-}
-
-export class FindSession extends EventEmitter<FindSessionEvents> {
-    private readonly serviceId: ServiceID
-    private readonly transport: ITransport
-    private readonly kademliaIdToFind: Uint8Array
-    private readonly localPeerId: PeerID
-    private readonly waitedRoutingPathCompletions: number
-    private readonly rpcCommunicator: ListeningRpcCommunicator
-    private readonly fetchData: boolean
-    private results: SortedContactList<Contact>
-    private foundData: Map<string, DataEntry> = new Map()
-    private allKnownHops: Set<PeerIDKey> = new Set()
-    private reportedHops: Set<PeerIDKey> = new Set()
-    private reportFindCompletedTimeout?: NodeJS.Timeout
-    private findCompletedEmitted = false
-    private noCloserNodesReceivedCounter = 0
-
-    constructor(config: FindSessionConfig) {
-        super()
-        this.serviceId = config.serviceId
-        this.transport = config.transport
-        this.kademliaIdToFind = config.kademliaIdToFind
-        this.localPeerId = config.localPeerId
-        this.waitedRoutingPathCompletions = config.waitedRoutingPathCompletions
-        this.results = new SortedContactList(PeerID.fromValue(this.kademliaIdToFind), 10, undefined, true)
-        this.fetchData = config.fetchData
-        this.rpcCommunicator = new ListeningRpcCommunicator(this.serviceId, this.transport, {
-            rpcRequestTimeout: 15000
-        })
-        this.registerLocalRpcMethods()
-    }
-
-    private registerLocalRpcMethods() {
-        const rpcLocal = new FindSessionRpcLocal({
-            doSendFindResponse: (routingPath: PeerDescriptor[], nodes: PeerDescriptor[], dataEntries: DataEntry[], noCloserNodesFound?: boolean) => {
-                this.doSendFindResponse(routingPath, nodes, dataEntries, noCloserNodesFound)
-            }
-        })
-        this.rpcCommunicator.registerRpcNotification(FindResponse, 'sendFindResponse',
-            (req: FindResponse) => rpcLocal.sendFindResponse(req))
-    }
-
-    private isFindCompleted(): boolean {
-        const unreportedHops: Set<PeerIDKey> = new Set(this.allKnownHops)
-        this.reportedHops.forEach((id) => {
-            unreportedHops.delete(id)
-        })
-        if (this.noCloserNodesReceivedCounter >= 1 && unreportedHops.size === 0) {
-            if (this.fetchData
-                && (this.hasNonStaleData() || this.noCloserNodesReceivedCounter >= this.waitedRoutingPathCompletions)) {
-                return true
-            } else if (this.fetchData) {
-                return false
-            }
-            return true
-        }
-        return false
-    }
-
-    private hasNonStaleData(): boolean {
-        return Array.from(this.foundData.values()).some((entry) => entry.stale === false)
-    }
-
-    public doSendFindResponse(
-        routingPath: PeerDescriptor[],
-        nodes: PeerDescriptor[],
-        dataEntries: DataEntry[],
-        noCloserNodesFound?: boolean
-    ): void {
-        this.addKnownHops(routingPath)
-        if (routingPath.length >= 1) {
-            this.setHopAsReported(routingPath[routingPath.length - 1])
-        }
-        nodes.forEach((descriptor: PeerDescriptor) => {
-            this.results.addContact(new Contact(descriptor))
-        })
-        this.processFoundData(dataEntries)
-        if (noCloserNodesFound) {
-            this.onNoCloserPeersFound()
-        }
-    }
-
-    private addKnownHops(routingPath: PeerDescriptor[]) {
-        routingPath.forEach((desc) => {
-            const newPeerId = PeerID.fromValue(desc.kademliaId)
-            if (!this.localPeerId.equals(newPeerId)) {
-                this.allKnownHops.add(newPeerId.toKey())
-            }
-        })
-    }
-
-    private setHopAsReported(desc: PeerDescriptor) {
-        const newPeerId = PeerID.fromValue(desc.kademliaId)
-        if (!this.localPeerId.equals(newPeerId)) {
-            this.reportedHops.add(newPeerId.toKey())
-        }
-        if (this.isFindCompleted()) {
-            if (!this.findCompletedEmitted && this.isFindCompleted()) {
-                if (this.reportFindCompletedTimeout) {
-                    clearTimeout(this.reportFindCompletedTimeout)
-                    this.reportFindCompletedTimeout = undefined
-                }
-                this.emit('findCompleted', this.results.getAllContacts().map((contact) => contact.getPeerDescriptor()))
-                this.findCompletedEmitted = true
-            }
-        }
-    }
-
-    private processFoundData(dataEntries: DataEntry[]): void {
-        dataEntries.forEach((entry) => {
-            const storerKey = keyFromPeerDescriptor(entry.storer!)
-            const existingEntry = this.foundData.get(storerKey)
-            if (!existingEntry || existingEntry.storerTime! < entry.storerTime! 
-                || (existingEntry.storerTime! <= entry.storerTime! && entry.deleted)) {
-                this.foundData.set(storerKey, entry)
-            }
-        })
-    }
-
-    private onNoCloserPeersFound(): void {
-        this.noCloserNodesReceivedCounter += 1
-        if (this.isFindCompleted()) {
-            this.emit('findCompleted', this.results.getAllContacts().map((contact) => contact.getPeerDescriptor()))
-            this.findCompletedEmitted = true
-            if (this.reportFindCompletedTimeout) {
-                clearTimeout(this.reportFindCompletedTimeout)
-                this.reportFindCompletedTimeout = undefined
-            }
-        } else {
-            if (!this.reportFindCompletedTimeout && !this.findCompletedEmitted) {
-                this.reportFindCompletedTimeout = setTimeout(() => {
-                    if (!this.findCompletedEmitted) {
-                        this.emit('findCompleted', this.results.getAllContacts().map((contact) => contact.getPeerDescriptor()))
-                        this.findCompletedEmitted = true
-                    }
-                }, 4000)
-            }
-        }
-    }
-
-    public getResults = (): FindResult => ({
-        closestNodes: this.results.getAllContacts().map((contact) => contact.getPeerDescriptor()),
-        dataEntries: (this.foundData && this.foundData.size > 0) ? Array.from(this.foundData.values()) : undefined
-    })
-
-    public stop(): void {
-        if (this.reportFindCompletedTimeout) {
-            clearTimeout(this.reportFindCompletedTimeout)
-            this.reportFindCompletedTimeout = undefined
-        }
-        this.rpcCommunicator.destroy()
-        this.emit('findCompleted', [])
-    }
-}

package/src/dht/find/FindSessionRpcLocal.ts

@@ -1,25 +0,0 @@
-import { IFindSessionRpc } from '../../proto/packages/dht/protos/DhtRpc.server'
-import { Empty } from '../../proto/google/protobuf/empty'
-import { DataEntry, FindResponse, PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
-import { Logger } from '@streamr/utils'
-
-const logger = new Logger(module)
-
-interface FindSessionRpcLocalConfig {
-    doSendFindResponse: (routingPath: PeerDescriptor[], nodes: PeerDescriptor[], dataEntries: DataEntry[], noCloserNodesFound?: boolean) => void
-}
-
-export class FindSessionRpcLocal implements IFindSessionRpc {
-
-    private readonly config: FindSessionRpcLocalConfig
-
-    constructor(config: FindSessionRpcLocalConfig) {
-        this.config = config
-    }
-    
-    async sendFindResponse(report: FindResponse): Promise<Empty> {
-        logger.trace('FindResponse arrived: ' + JSON.stringify(report))
-        this.config.doSendFindResponse(report.routingPath, report.closestConnectedPeers, report.dataEntries, report.noCloserNodesFound)
-        return {}
-    }
-}

package/src/dht/find/FindSessionRpcRemote.ts

@@ -1,30 +0,0 @@
-import {
-    DataEntry,
-    PeerDescriptor,
-    FindResponse
-} from '../../proto/packages/dht/protos/DhtRpc'
-import { IFindSessionRpcClient } from '../../proto/packages/dht/protos/DhtRpc.client'
-import { Logger } from '@streamr/utils'
-import { Remote } from '../contact/Remote'
-
-const logger = new Logger(module)
-
-export class FindSessionRpcRemote extends Remote<IFindSessionRpcClient> {
-
-    sendFindResponse(
-        routingPath: PeerDescriptor[],
-        closestNodes: PeerDescriptor[],
-        dataEntries: DataEntry[],
-        noCloserNodesFound: boolean
-    ): void {
-        const report: FindResponse = {
-            routingPath,
-            closestConnectedPeers: closestNodes,
-            dataEntries,
-            noCloserNodesFound
-        }
-        this.getClient().sendFindResponse(report, this.formDhtRpcOptions()).catch((_e) => {
-            logger.trace('Failed to send FindResponse')
-        })
-    }
-}