@streamr/dht 0.0.1-tatum.0

package/src/dht/routing/DuplicateDetector.ts

@@ -0,0 +1,53 @@
+import { Message } from '../../proto/packages/dht/protos/DhtRpc'
+
+type QueueEntry = [timeStamp: number, value: string, senderId: string, message?: Message]
+
+export class DuplicateDetector {
+
+    private values: Set<string> = new Set()
+    private queue: Array<QueueEntry> = []
+    private maxAge: number
+    private maxNumberOfValues: number
+
+    constructor(
+        maxNumberOfValues: number,
+        maxAgeInSeconds: number
+    ) {
+        this.maxNumberOfValues = maxNumberOfValues
+        this.maxAge = maxAgeInSeconds * 1000
+    }
+
+    public add(value: string, senderId: string, message?: Message): void {
+        this.values.add(value)
+        if (message) {
+            this.queue.push([Date.now(), value, senderId, message])
+        } else {
+            this.queue.push([Date.now(), value, senderId])
+        }
+        this.cleanUp()
+    }
+
+    public isMostLikelyDuplicate(value: string): boolean {
+        return this.values.has(value)
+    }
+
+    private cleanUp(): void {
+        const currentTime = Date.now()
+
+        while (this.queue.length > 0 && (this.queue.length > this.maxNumberOfValues ||
+            (currentTime - this.queue[0][0]) > this.maxAge)) {
+            const oldestEntry = this.queue.shift()
+            this.values.delete(oldestEntry![1])
+        }
+    }
+
+    public size(): number {
+        return this.values.size
+    }
+
+    public clear(): void {
+        this.values.clear()
+        this.queue = []
+    }
+
+}

package/src/dht/routing/RemoteRouter.ts

@@ -0,0 +1,115 @@
+import { PeerDescriptor, RouteMessageWrapper } from '../../proto/packages/dht/protos/DhtRpc'
+import { v4 } from 'uuid'
+import { DhtRpcOptions } from '../../rpc-protocol/DhtRpcOptions'
+import {
+    isSamePeerDescriptor,
+    keyFromPeerDescriptor,
+    peerIdFromPeerDescriptor
+} from '../../helpers/peerIdFromPeerDescriptor'
+import { IRoutingServiceClient } from '../../proto/packages/dht/protos/DhtRpc.client'
+import { Remote } from '../contact/Remote'
+import { Logger } from '@streamr/utils'
+
+const logger = new Logger(module)
+
+export class RemoteRouter extends Remote<IRoutingServiceClient> {
+
+    async routeMessage(params: RouteMessageWrapper): Promise<boolean> {
+        const message: RouteMessageWrapper = {
+            destinationPeer: params.destinationPeer,
+            sourcePeer: params.sourcePeer,
+            previousPeer: params.previousPeer,
+            message: params.message,
+            requestId: params.requestId || v4(),
+            reachableThrough: params.reachableThrough || [],
+            routingPath: params.routingPath
+        }
+        const options: DhtRpcOptions = {
+            sourceDescriptor: params.previousPeer as PeerDescriptor,
+            targetDescriptor: this.peerDescriptor as PeerDescriptor,
+            timeout: 10000
+        }
+        try {
+            logger.trace('calling dhtClient.routeMessage')
+            const ack = await this.client.routeMessage(message, options)
+            logger.trace('dhtClient.routeMessage returned')
+            // Success signal if sent to destination and error includes duplicate
+            if (
+                isSamePeerDescriptor(params.destinationPeer!, this.peerDescriptor)
+                && ack.error.includes('duplicate')
+            ) {
+                return true
+            } else if (ack.error!.length > 0) {
+                return false
+            }
+        } catch (err) {
+            const fromNode = params.previousPeer ?
+                peerIdFromPeerDescriptor(params.previousPeer) : keyFromPeerDescriptor(params.sourcePeer!)
+            logger.debug(`Failed to send routeMessage from ${fromNode} to ${this.peerId.toKey()} with: ${err}`)
+            return false
+        }
+        return true
+    }
+
+    async forwardMessage(params: RouteMessageWrapper): Promise<boolean> {
+        const message: RouteMessageWrapper = {
+            destinationPeer: params.destinationPeer,
+            sourcePeer: params.sourcePeer,
+            previousPeer: params.previousPeer,
+            message: params.message,
+            requestId: params.requestId || v4(),
+            reachableThrough: params.reachableThrough || [],
+            routingPath: params.routingPath
+        }
+        const options: DhtRpcOptions = {
+            sourceDescriptor: params.previousPeer as PeerDescriptor,
+            targetDescriptor: this.peerDescriptor as PeerDescriptor,
+            timeout: 10000
+        }
+        try {
+            const ack = await this.client.forwardMessage(message, options)
+            if (ack.error!.length > 0) {
+                return false
+            }
+        } catch (err) {
+            const fromNode = params.previousPeer ?
+                keyFromPeerDescriptor(params.previousPeer) : keyFromPeerDescriptor(params.sourcePeer!)
+
+            logger.debug(
+                `Failed to send forwardMessage from ${fromNode} to ${this.peerId.toKey()} with: ${err}`
+            )
+            return false
+        }
+        return true
+    }
+
+    async findRecursively(params: RouteMessageWrapper): Promise<boolean> {
+        const message: RouteMessageWrapper = {
+            destinationPeer: params.destinationPeer,
+            sourcePeer: params.sourcePeer,
+            previousPeer: params.previousPeer,
+            message: params.message,
+            requestId: params.requestId || v4(),
+            reachableThrough: params.reachableThrough || [],
+            routingPath: params.routingPath
+        }
+        const options: DhtRpcOptions = {
+            sourceDescriptor: params.previousPeer as PeerDescriptor,
+            targetDescriptor: this.peerDescriptor as PeerDescriptor,
+            timeout: 10000
+        }
+        try {
+            const ack = await this.client.findRecursively(message, options)
+            if (ack.error!.length > 0) {
+                logger.debug('Next hop responded with error ' + ack.error)
+                return false
+            }
+        } catch (err) {
+            const fromNode = params.previousPeer ? keyFromPeerDescriptor(params.previousPeer) : keyFromPeerDescriptor(params.sourcePeer!)
+            logger.debug(`Failed to send recursiveFind message from ${fromNode} to ${this.peerId.toKey()} with: ${err}`)
+            return false
+        }
+        return true
+    }
+
+}

package/src/dht/routing/Router.ts

@@ -0,0 +1,266 @@
+import { Message, PeerDescriptor, RouteMessageAck, RouteMessageWrapper } from '../../proto/packages/dht/protos/DhtRpc'
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import { keyFromPeerDescriptor, peerIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
+import { RoutingMode, RoutingSession, RoutingSessionEvents } from './RoutingSession'
+import { Logger, raceEvents3 } from '@streamr/utils'
+import { RoutingRpcCommunicator } from '../../transport/RoutingRpcCommunicator'
+import { PeerID, PeerIDKey } from '../../helpers/PeerID'
+import { DuplicateDetector } from './DuplicateDetector'
+import { ConnectionManager } from '../../connection/ConnectionManager'
+import { DhtPeer } from '../DhtPeer'
+import { v4 } from 'uuid'
+import { IRoutingService } from '../../proto/packages/dht/protos/DhtRpc.server'
+
+export const createRouteMessageAck = (routedMessage: RouteMessageWrapper, error?: string): RouteMessageAck => {
+    const ack: RouteMessageAck = {
+        requestId: routedMessage.requestId,
+        destinationPeer: routedMessage.sourcePeer,
+        sourcePeer: routedMessage.destinationPeer,
+        error: error ? error : ''
+    }
+    return ack
+}
+
+export enum RoutingErrors {
+    NO_CANDIDATES_FOUND = 'No routing candidates found',
+    STOPPED = 'DhtNode Stopped'
+}
+
+export interface RouterConfig {
+    rpcCommunicator: RoutingRpcCommunicator
+    ownPeerDescriptor: PeerDescriptor
+    ownPeerId: PeerID
+    connections: Map<PeerIDKey, DhtPeer>
+    addContact: (contact: PeerDescriptor, setActive?: boolean) => void
+    serviceId: string
+    connectionManager?: ConnectionManager
+}
+
+interface ForwardingTableEntry {
+    timeout: NodeJS.Timeout
+    peerDescriptors: PeerDescriptor[]
+}
+
+interface IRouterFunc {
+    doRouteMessage(routedMessage: RouteMessageWrapper, mode: RoutingMode, excludedPeer?: PeerDescriptor): RouteMessageAck
+    send(msg: Message, reachableThrough: PeerDescriptor[]): Promise<void>
+    checkDuplicate(messageId: string): boolean
+    addToDuplicateDetector(messageId: string, senderId: string, message?: Message): void
+    addRoutingSession(session: RoutingSession): void
+    removeRoutingSession(sessionId: string): void
+    stop(): void
+}
+
+export interface IRouter extends Omit<IRoutingService, 'findRecursively'>, IRouterFunc {}
+
+const logger = new Logger(module)
+
+export class Router implements IRouter {
+    private readonly rpcCommunicator: RoutingRpcCommunicator
+    private readonly ownPeerDescriptor: PeerDescriptor
+    private readonly ownPeerId: PeerID
+    private readonly connections: Map<PeerIDKey, DhtPeer>
+    private readonly addContact: (contact: PeerDescriptor, setActive?: boolean) => void
+    private readonly serviceId: string
+    private readonly connectionManager?: ConnectionManager
+    private readonly forwardingTable: Map<string, ForwardingTableEntry> = new Map()
+    private ongoingRoutingSessions: Map<string, RoutingSession> = new Map()
+    private readonly routerDuplicateDetector: DuplicateDetector = new DuplicateDetector(100000, 100)
+    private stopped = false
+
+    constructor(config: RouterConfig) {
+        this.rpcCommunicator = config.rpcCommunicator
+        this.ownPeerDescriptor = config.ownPeerDescriptor
+        this.ownPeerId = config.ownPeerId
+        this.connections = config.connections
+        this.addContact = config.addContact
+        this.serviceId = config.serviceId
+        this.connectionManager = config.connectionManager
+        this.rpcCommunicator.registerRpcMethod(RouteMessageWrapper, RouteMessageAck, 'forwardMessage',
+            (forwardMessage: RouteMessageWrapper, context) => this.forwardMessage(forwardMessage, context))
+        this.rpcCommunicator.registerRpcMethod(RouteMessageWrapper, RouteMessageAck, 'routeMessage',
+            (routedMessage: RouteMessageWrapper, context) => this.routeMessage(routedMessage, context))
+    }
+
+    public async send(msg: Message, reachableThrough: PeerDescriptor[]): Promise<void> {
+        msg.sourceDescriptor = this.ownPeerDescriptor
+        const targetPeerDescriptor = msg.targetDescriptor!
+        const forwardingEntry = this.forwardingTable.get(keyFromPeerDescriptor(targetPeerDescriptor))
+        if (forwardingEntry && forwardingEntry.peerDescriptors.length > 0) {
+            const forwardingPeer = forwardingEntry.peerDescriptors[0]
+            const forwardedMessage: RouteMessageWrapper = {
+                message: msg,
+                requestId: v4(),
+                destinationPeer: forwardingPeer,
+                sourcePeer: this.ownPeerDescriptor!,
+                reachableThrough,
+                routingPath: []
+            }
+            this.doRouteMessage(forwardedMessage, RoutingMode.FORWARD)
+        } else {
+            const routedMessage: RouteMessageWrapper = {
+                message: msg,
+                requestId: v4(),
+                destinationPeer: targetPeerDescriptor,
+                sourcePeer: this.ownPeerDescriptor!,
+                reachableThrough,
+                routingPath: []
+            }
+            this.doRouteMessage(routedMessage, RoutingMode.ROUTE)
+        }
+    }
+
+    public doRouteMessage(routedMessage: RouteMessageWrapper, mode = RoutingMode.ROUTE, excludedPeer?: PeerDescriptor): RouteMessageAck {
+        if (this.stopped) {
+            return createRouteMessageAck(routedMessage, RoutingErrors.STOPPED)
+        }
+        logger.trace(`Peer ${this.ownPeerId.value} routing message ${routedMessage.requestId} 
+            from ${routedMessage.sourcePeer?.kademliaId} to ${routedMessage.destinationPeer?.kademliaId}`)
+        routedMessage.routingPath.push(this.ownPeerDescriptor!)
+        const session = this.createRoutingSession(routedMessage, mode, excludedPeer)
+        this.addRoutingSession(session)
+        try {
+            // eslint-disable-next-line promise/catch-or-return
+            logger.trace('starting to raceEvents from routingSession: ' + session.sessionId)
+            raceEvents3<RoutingSessionEvents>(session, ['routingSucceeded', 'partialSuccess', 'routingFailed', 'stopped', 'noCandidatesFound'], 10000)
+                .then(() => {
+                    logger.trace('raceEvents ended from routingSession: ' + session.sessionId)
+                    this.removeRoutingSession(session.sessionId)
+                })
+                .catch(() => {
+                    logger.trace('raceEvents timed out for routingSession ' + session.sessionId) 
+                    this.removeRoutingSession(session.sessionId) 
+                })
+            session.start()
+        } catch (e) {
+            if (peerIdFromPeerDescriptor(routedMessage.sourcePeer!).equals(this.ownPeerId!)) {
+                logger.warn(
+                    `Failed to send (routeMessage: ${this.serviceId}) to ${keyFromPeerDescriptor(routedMessage.destinationPeer!)}: ${e}`
+                )
+            }
+            return createRouteMessageAck(routedMessage, RoutingErrors.NO_CANDIDATES_FOUND)
+        }
+        return createRouteMessageAck(routedMessage)
+    }
+
+    private createRoutingSession(routedMessage: RouteMessageWrapper, mode: RoutingMode, excludedPeer?: PeerDescriptor): RoutingSession {
+        const excludedPeers = routedMessage.routingPath.map((descriptor) => peerIdFromPeerDescriptor(descriptor))
+        if (excludedPeer) {
+            excludedPeers.push(peerIdFromPeerDescriptor(excludedPeer))
+        }
+        logger.trace(' routing session created with connections: ' + this.connections.size )
+        return new RoutingSession(
+            this.rpcCommunicator,
+            this.ownPeerDescriptor!,
+            routedMessage,
+            this.connections,
+            this.ownPeerId!.equals(peerIdFromPeerDescriptor(routedMessage.sourcePeer!)) ? 2 : 1,
+            mode,
+            undefined,
+            excludedPeers
+        )
+    }
+
+    public checkDuplicate(messageId: string): boolean {
+        return this.routerDuplicateDetector.isMostLikelyDuplicate(messageId)
+    }
+
+    public addToDuplicateDetector(messageId: string, senderId: string, message?: Message): void {
+        this.routerDuplicateDetector.add(messageId, senderId, message)
+    }
+
+    public addRoutingSession(session: RoutingSession): void {
+        this.ongoingRoutingSessions.set(session.sessionId, session)
+    }
+
+    public removeRoutingSession(sessionId: string): void {
+        this.ongoingRoutingSessions.delete(sessionId)
+    }
+
+    public stop(): void {
+        this.stopped = true
+        this.ongoingRoutingSessions.forEach((session, _id) => {
+            session.stop()
+        })
+        this.ongoingRoutingSessions.clear()
+        this.forwardingTable.forEach((entry) => {
+            clearTimeout(entry.timeout)
+        })
+        this.forwardingTable.clear()
+        this.routerDuplicateDetector.clear()
+    }
+    
+    // IRoutingService method
+    async routeMessage(routedMessage: RouteMessageWrapper, _context: ServerCallContext): Promise<RouteMessageAck> {
+        if (this.stopped) {
+            return createRouteMessageAck(routedMessage, 'routeMessage() service is not running')
+        } else if (this.routerDuplicateDetector.isMostLikelyDuplicate(routedMessage.requestId)) {
+            logger.trace(`Peer ${this.ownPeerId?.value} routing message ${routedMessage.requestId} 
+                from ${routedMessage.sourcePeer!.kademliaId} to ${routedMessage.destinationPeer!.kademliaId} is likely a duplicate`)
+            return createRouteMessageAck(routedMessage, 'message given to routeMessage() service is likely a duplicate')
+        }
+        logger.trace(`Processing received routeMessage ${routedMessage.requestId}`)
+        this.addContact(routedMessage.sourcePeer!, true)
+        this.addToDuplicateDetector(routedMessage.requestId, keyFromPeerDescriptor(routedMessage.sourcePeer!))
+        if (this.ownPeerId!.equals(peerIdFromPeerDescriptor(routedMessage.destinationPeer!))) {
+            logger.trace(`${this.ownPeerDescriptor.nodeName} routing message targeted to self ${routedMessage.requestId}`)
+            this.setForwardingEntries(routedMessage)
+            this.connectionManager?.handleMessage(routedMessage.message!)
+            return createRouteMessageAck(routedMessage)
+        } else {
+            return this.doRouteMessage(routedMessage)
+        }
+    }
+
+    private setForwardingEntries(routedMessage: RouteMessageWrapper): void {
+        const reachableThroughWithoutSelf = routedMessage.reachableThrough.filter((peer) => {
+            return !peerIdFromPeerDescriptor(peer).equals(this.ownPeerId!)
+        })
+        
+        if (reachableThroughWithoutSelf.length > 0) {
+            const sourceKey = keyFromPeerDescriptor(routedMessage.sourcePeer!)
+            if (this.forwardingTable.has(sourceKey)) {
+                const oldEntry = this.forwardingTable.get(sourceKey)
+                clearTimeout(oldEntry!.timeout)
+                this.forwardingTable.delete(sourceKey)
+            }
+            const forwardingEntry: ForwardingTableEntry = {
+                peerDescriptors: reachableThroughWithoutSelf,
+                timeout: setTimeout(() => {
+                    this.forwardingTable.delete(sourceKey)
+                }, 10000)
+            }
+            this.forwardingTable.set(sourceKey, forwardingEntry)
+        }
+    }
+
+    // IRoutingService method
+    async forwardMessage(forwardMessage: RouteMessageWrapper, _context: ServerCallContext): Promise<RouteMessageAck> {
+        if (this.stopped) {
+            return createRouteMessageAck(forwardMessage, 'forwardMessage() service is not running')
+        } else if (this.routerDuplicateDetector.isMostLikelyDuplicate(forwardMessage.requestId)) {
+            logger.trace(`Peer ${this.ownPeerId.value} forwarding message ${forwardMessage.requestId} 
+        from ${forwardMessage.sourcePeer?.kademliaId} to ${forwardMessage.destinationPeer?.kademliaId} is likely a duplicate`)
+            return createRouteMessageAck(forwardMessage, 'message given to forwardMessage() service is likely a duplicate')
+        }
+        logger.trace(`Processing received forward routeMessage ${forwardMessage.requestId}`)
+        this.addContact(forwardMessage.sourcePeer!, true)
+        this.addToDuplicateDetector(forwardMessage.requestId, keyFromPeerDescriptor(forwardMessage.sourcePeer!))
+        if (this.ownPeerId.equals(peerIdFromPeerDescriptor(forwardMessage.destinationPeer!))) {
+            return this.forwardToDestination(forwardMessage)
+        } else {
+            return this.doRouteMessage(forwardMessage, RoutingMode.FORWARD)
+        }
+    }
+
+    private forwardToDestination(routedMessage: RouteMessageWrapper): RouteMessageAck {
+        logger.trace(`Peer ${this.ownPeerId.value} forwarding found message targeted to self ${routedMessage.requestId}`)
+        const forwardedMessage = routedMessage.message!
+        if (this.ownPeerId!.equals(peerIdFromPeerDescriptor(forwardedMessage.targetDescriptor!))) {
+            this.connectionManager?.handleMessage(forwardedMessage!)
+            return createRouteMessageAck(routedMessage)
+        }
+        return this.doRouteMessage({ ...routedMessage, destinationPeer: forwardedMessage.targetDescriptor })
+    }
+
+}

package/src/dht/routing/RoutingSession.ts

@@ -0,0 +1,222 @@
+import { PeerDescriptor } from '../../exports'
+import { DhtPeer } from '../DhtPeer'
+import { SortedContactList } from '../contact/SortedContactList'
+import { PeerID, PeerIDKey } from '../../helpers/PeerID'
+import { Logger } from '@streamr/utils'
+import EventEmitter from 'eventemitter3'
+import { v4 } from 'uuid'
+import { RouteMessageWrapper } from '../../proto/packages/dht/protos/DhtRpc'
+import { RemoteRouter } from './RemoteRouter'
+import { RoutingRpcCommunicator } from '../../transport/RoutingRpcCommunicator'
+import { RoutingServiceClient } from '../../proto/packages/dht/protos/DhtRpc.client'
+import { toProtoRpcClient } from '@streamr/proto-rpc'
+
+const logger = new Logger(module)
+
+const MAX_FAILED_HOPS = 2
+
+export interface RoutingSessionEvents {
+    // This event is emitted when a peer responds with a success ack
+    // to routeMessage call
+    routingSucceeded: (sessionId: string) => void
+    partialSuccess: (sessionId: string) => void
+
+    // This event is emitted when all the candidates have been gone
+    // through, and none of them responds with a success ack
+    routingFailed: (sessionId: string) => void
+    stopped: (sessionId: string) => void
+    noCandidatesFound: (sessionId: string) => void
+}
+
+export enum RoutingMode { ROUTE, FORWARD, RECURSIVE_FIND }
+
+export class RoutingSession extends EventEmitter<RoutingSessionEvents> {
+
+    public readonly sessionId = v4()
+    private readonly rpcCommunicator: RoutingRpcCommunicator
+    private ongoingRequests: Set<PeerIDKey> = new Set()
+    private contactList: SortedContactList<RemoteRouter>
+    private readonly ownPeerDescriptor: PeerDescriptor
+    private readonly messageToRoute: RouteMessageWrapper
+    private connections: Map<PeerIDKey, DhtPeer>
+    private readonly parallelism: number
+    private failedHopCounter = 0
+    private successfulHopCounter = 0
+    private readonly mode: RoutingMode = RoutingMode.ROUTE
+    private stopped = false
+
+    constructor(
+        rpcCommunicator: RoutingRpcCommunicator,
+        ownPeerDescriptor: PeerDescriptor,
+        messageToRoute: RouteMessageWrapper,
+        connections: Map<PeerIDKey, DhtPeer>,
+        parallelism: number,
+        mode: RoutingMode = RoutingMode.ROUTE,
+        destinationId?: Uint8Array,
+        excludedPeerIDs?: PeerID[]
+    ) {
+        super()
+        this.rpcCommunicator = rpcCommunicator
+        this.ownPeerDescriptor = ownPeerDescriptor
+        this.messageToRoute = messageToRoute
+        this.connections = connections
+        this.parallelism = parallelism
+        this.mode = mode
+        const previousId = messageToRoute.previousPeer ? PeerID.fromValue(messageToRoute.previousPeer.kademliaId) : undefined
+        this.contactList = new SortedContactList(
+            destinationId ? PeerID.fromValue(destinationId) : PeerID.fromValue(this.messageToRoute!.destinationPeer!.kademliaId),
+            10000,
+            undefined,
+            true,
+            previousId,
+            excludedPeerIDs
+        )
+    }
+
+    private onRequestFailed = (peerId: PeerID) => {
+        logger.trace('onRequestFailed() sessionId: ' + this.sessionId)
+        if (this.stopped) {
+            return
+        }
+        if (this.ongoingRequests.has(peerId.toKey())) {
+            this.ongoingRequests.delete(peerId.toKey())
+        }
+        const contacts = this.findMoreContacts()
+        if (contacts.length < 1 && this.ongoingRequests.size < 1) {
+            logger.trace('routing failed, emitting routingFailed sessionId: ' + this.sessionId)
+            this.stopped = true
+            this.emitFailure()
+        } else {
+            this.failedHopCounter += 1
+            logger.trace('routing failed, retrying to route sessionId: ' + this.sessionId + ' failedHopCounter: ' + this.failedHopCounter)
+            this.sendMoreRequests(contacts)
+        }
+    }
+
+    private emitFailure = () => {
+        if (this.successfulHopCounter >= 1) {
+            this.emit('partialSuccess', this.sessionId)
+
+        } else {
+            this.emit('routingFailed', this.sessionId)
+        }
+    }
+
+    private onRequestSucceeded = (_peerId: PeerID) => {
+        logger.trace('onRequestSucceeded() sessionId: ' + this.sessionId)
+        if (this.stopped) {
+            return
+        }
+        this.successfulHopCounter += 1
+        const contacts = this.findMoreContacts()
+        if (this.successfulHopCounter >= this.parallelism || contacts.length < 1) {
+            this.stopped = true
+            this.emit('routingSucceeded', this.sessionId)
+        } else if (contacts.length > 0 && this.ongoingRequests.size < 1) {
+            this.sendMoreRequests(contacts)
+        }
+    }
+
+    private sendRouteMessageRequest = async (contact: RemoteRouter): Promise<boolean> => {
+        logger.trace('sendRouteMessageRequest() sessionId: ' + this.sessionId)
+        logger.trace(`Sending routeMessage request from ${this.ownPeerDescriptor.kademliaId} to contact: ${contact.getPeerId()}`)
+        this.contactList.setContacted(contact.getPeerId())
+        this.ongoingRequests.add(contact.getPeerId().toKey())
+        if (this.mode === RoutingMode.FORWARD) {
+            return contact.forwardMessage({
+                ...this.messageToRoute,
+                previousPeer: this.ownPeerDescriptor
+            })
+        } else if (this.mode === RoutingMode.RECURSIVE_FIND) {
+            return contact.findRecursively({
+                ...this.messageToRoute,
+                previousPeer: this.ownPeerDescriptor
+            })
+        } else {
+            return contact.routeMessage({
+                ...this.messageToRoute,
+                previousPeer: this.ownPeerDescriptor
+            })
+        }
+    }
+
+    private findMoreContacts = (): RemoteRouter[] => {
+        logger.trace('findMoreContacts() sessionId: ' + this.sessionId)
+        // the contents of the connections might have changed between the rounds
+        // addContacts() will only add new contacts that were not there yet
+        const contacts = Array.from(this.connections.values())
+            .map((contact) => {
+                return new RemoteRouter(
+                    this.ownPeerDescriptor,
+                    contact.getPeerDescriptor(),
+                    toProtoRpcClient(new RoutingServiceClient(this.rpcCommunicator!.getRpcClientTransport())),
+                    contact.getServiceId()
+                )  
+            })
+        this.contactList.addContacts(contacts)
+        return this.contactList.getUncontactedContacts(this.parallelism)
+    }
+
+    public getClosestContacts = (limit: number): PeerDescriptor[] => {
+        const contacts = this.contactList.getClosestContacts(limit)
+        return contacts.map((contact) => contact.getPeerDescriptor())
+    }
+
+    private sendMoreRequests = (uncontacted: RemoteRouter[]) => {
+        logger.trace('sendMoreRequests() sessionId: ' + this.sessionId)
+        if (this.stopped) {
+            return
+        }
+        if (uncontacted.length < 1) {
+            this.emitFailure()
+            return
+        }
+        if (this.failedHopCounter >= MAX_FAILED_HOPS) {
+            logger.trace(`Stopping routing after ${MAX_FAILED_HOPS} failed attempts for sessionId: ${this.sessionId}`)
+            this.emitFailure()
+            return
+        }
+        while (this.ongoingRequests.size < this.parallelism && uncontacted.length > 0) {
+            if (this.stopped) {
+                return
+            }
+            const nextPeer = uncontacted.shift()
+            logger.trace('sendRouteMessageRequest')
+            // eslint-disable-next-line promise/catch-or-return
+            this.sendRouteMessageRequest(nextPeer!)
+                .then((succeeded) => {
+                    if (succeeded) {
+                        this.onRequestSucceeded(nextPeer!.getPeerId())
+                    } else {
+                        this.onRequestFailed(nextPeer!.getPeerId())
+                    }
+                }).catch((e) => { 
+                    logger.error(e)
+                }).finally(() => {
+                    logger.trace('sendRouteMessageRequest returned')
+                })
+        }
+    }
+
+    public start(): void {
+        logger.trace('start() sessionId: ' + this.sessionId)
+        const contacts = this.findMoreContacts()
+        if (contacts.length < 1) {
+            logger.trace('start() throwing noCandidatesFound sessionId: ' + this.sessionId)
+            
+            this.stopped = true
+            this.emit('noCandidatesFound', this.sessionId)
+            throw new Error('noCandidatesFound ' + this.sessionId)
+        }
+        this.sendMoreRequests(contacts)
+    }
+
+    public stop(): void {
+        this.stopped = true
+        this.contactList.stop()
+
+        this.emit('stopped', this.sessionId)
+        this.removeAllListeners()
+    }
+
+}