@streamr/dht 100.0.0-pretestnet.6 → 100.0.0-rc.0

package/src/dht/discovery/PeerDiscovery.ts

@@ -1,48 +1,72 @@
 import { DiscoverySession } from './DiscoverySession'
 import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
-import { areEqualPeerDescriptors, getNodeIdFromPeerDescriptor, peerIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
 import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
 import { Logger, scheduleAtInterval, setAbortableTimeout } from '@streamr/utils'
-import KBucket from 'k-bucket'
-import { SortedContactList } from '../contact/SortedContactList'
 import { ConnectionManager } from '../../connection/ConnectionManager'
-import { RoutingRpcCommunicator } from '../../transport/RoutingRpcCommunicator'
-import { createRandomKademliaId } from '../../helpers/kademliaId'
+import { PeerManager } from '../PeerManager'
+import { DhtAddress, areEqualPeerDescriptors, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../identifiers'
 import { ServiceID } from '../../types/ServiceID'
+import { RingDiscoverySession } from './RingDiscoverySession'
+import { RingIdRaw, getRingIdRawFromPeerDescriptor } from '../contact/ringIdentifiers'
 
 interface PeerDiscoveryConfig {
-    rpcCommunicator: RoutingRpcCommunicator
     localPeerDescriptor: PeerDescriptor
-    bucket: KBucket<DhtNodeRpcRemote>
-    neighborList: SortedContactList<DhtNodeRpcRemote>
     joinNoProgressLimit: number
     peerDiscoveryQueryBatchSize: number
     serviceId: ServiceID
     parallelism: number
     joinTimeout: number
-    addContact: (contact: PeerDescriptor, setActive?: boolean) => void
     connectionManager?: ConnectionManager
-    rpcRequestTimeout?: number
+    peerManager: PeerManager
+}
+
+export const createDistantDhtAddress = (address: DhtAddress): DhtAddress => {
+    const raw = getRawFromDhtAddress(address)
+    const flipped = raw.map((val) => ~val)
+    return getDhtAddressFromRaw(flipped)
 }
 
 const logger = new Logger(module)
 
 export class PeerDiscovery {
 
-    private readonly config: PeerDiscoveryConfig
     private ongoingDiscoverySessions: Map<string, DiscoverySession> = new Map()
+    private ongoingRingDiscoverySessions: Map<string, RingDiscoverySession> = new Map()
+    
     private rejoinOngoing = false
     private joinCalled = false
-    private rejoinTimeoutRef?: NodeJS.Timeout
     private readonly abortController: AbortController
     private recoveryIntervalStarted = false
+    private readonly config: PeerDiscoveryConfig
 
     constructor(config: PeerDiscoveryConfig) {
         this.config = config
         this.abortController = new AbortController()
     }
 
-    async joinDht(entryPointDescriptor: PeerDescriptor, doAdditionalRandomPeerDiscovery = true, retry = true): Promise<void> {
+    async joinDht(
+        entryPoints: PeerDescriptor[],
+        doAdditionalDistantPeerDiscovery = true,
+        retry = true
+    ): Promise<void> {
+        const contactedPeers = new Set<DhtAddress>()
+        const distantJoinConfig = doAdditionalDistantPeerDiscovery 
+            ? { enabled: true, contactedPeers: new Set<DhtAddress>() } : { enabled: false } as const
+        await Promise.all(entryPoints.map((entryPoint) => this.joinThroughEntryPoint(
+            entryPoint,
+            contactedPeers,
+            distantJoinConfig,
+            retry
+        )))
+    }
+
+    async joinThroughEntryPoint(
+        entryPointDescriptor: PeerDescriptor,
+        // Note that this set is mutated by DiscoverySession
+        contactedPeers: Set<DhtAddress>,
+        additionalDistantJoin: { enabled: true, contactedPeers: Set<DhtAddress> } | { enabled: false },
+        retry = true
+    ): Promise<void> {
         if (this.isStopped()) {
             return
         }
@@ -55,53 +79,79 @@ export class PeerDiscovery {
             return
         }
         this.config.connectionManager?.lockConnection(entryPointDescriptor, `${this.config.serviceId}::joinDht`)
-        this.config.addContact(entryPointDescriptor)
-        const targetId = peerIdFromPeerDescriptor(this.config.localPeerDescriptor).value
-        const closest = this.config.bucket.closest(targetId, this.config.peerDiscoveryQueryBatchSize)
-        this.config.neighborList.addContacts(closest)
-        const sessions = [this.createSession(targetId)]
-        if (doAdditionalRandomPeerDiscovery) {
-            sessions.push(this.createSession(createRandomKademliaId()))
+        this.config.peerManager.addContact([entryPointDescriptor])
+        const targetId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
+        const sessions = [this.createSession(targetId, contactedPeers)]
+        if (additionalDistantJoin.enabled) {
+            sessions.push(this.createSession(createDistantDhtAddress(targetId), additionalDistantJoin.contactedPeers))
         }
         await this.runSessions(sessions, entryPointDescriptor, retry)
         this.config.connectionManager?.unlockConnection(entryPointDescriptor, `${this.config.serviceId}::joinDht`)
 
     }
 
-    private createSession(targetId: Uint8Array): DiscoverySession {
+    async joinRing(): Promise<void> {
+        const contactedPeers = new Set<DhtAddress>()
+        const sessions = [this.createRingSession(getRingIdRawFromPeerDescriptor(this.config.localPeerDescriptor), contactedPeers)]
+        await this.runRingSessions(sessions)
+    }
+
+    private createSession(targetId: DhtAddress, contactedPeers: Set<DhtAddress>): DiscoverySession {
         const sessionOptions = {
-            bucket: this.config.bucket,
-            neighborList: this.config.neighborList,
             targetId,
-            localPeerDescriptor: this.config.localPeerDescriptor,
-            serviceId: this.config.serviceId,
-            rpcCommunicator: this.config.rpcCommunicator,
             parallelism: this.config.parallelism,
             noProgressLimit: this.config.joinNoProgressLimit,
-            newContactListener: (newPeer: DhtNodeRpcRemote) => this.config.addContact(newPeer.getPeerDescriptor())
+            peerManager: this.config.peerManager,
+            contactedPeers
         }
         return new DiscoverySession(sessionOptions)
     }
 
+    private createRingSession(targetId: RingIdRaw, contactedPeers: Set<DhtAddress>): RingDiscoverySession {
+        const sessionOptions = {
+            targetId,
+            parallelism: this.config.parallelism,
+            noProgressLimit: this.config.joinNoProgressLimit,
+            peerManager: this.config.peerManager,
+            contactedPeers
+        }
+        return new RingDiscoverySession(sessionOptions)
+    }
+
     private async runSessions(sessions: DiscoverySession[], entryPointDescriptor: PeerDescriptor, retry: boolean): Promise<void> {
         try {
             for (const session of sessions) {
-                this.ongoingDiscoverySessions.set(session.sessionId, session)
+                this.ongoingDiscoverySessions.set(session.id, session)
                 await session.findClosestNodes(this.config.joinTimeout)
             }
         } catch (_e) {
             logger.debug(`DHT join on ${this.config.serviceId} timed out`)
         } finally {
             if (!this.isStopped()) {
-                if (this.config.bucket.count() === 0) {
+                if (this.config.peerManager.getNeighborCount() === 0) {
                     if (retry) {
+                        // TODO should we catch possible promise rejection?
+                        // TODO use config option or named constant?
                         setAbortableTimeout(() => this.rejoinDht(entryPointDescriptor), 1000, this.abortController.signal)
                     }
                 } else {
                     await this.ensureRecoveryIntervalIsRunning()
                 }
             }
-            sessions.forEach((session) => this.ongoingDiscoverySessions.delete(session.sessionId))
+            sessions.forEach((session) => this.ongoingDiscoverySessions.delete(session.id))
+        }
+    }
+
+    private async runRingSessions(sessions: RingDiscoverySession[]): Promise<void> {
+        try {
+            for (const session of sessions) {
+                this.ongoingRingDiscoverySessions.set(session.id, session)
+                await session.findClosestNodes(this.config.joinTimeout)
+            }
+        } catch (_e) {
+            logger.debug(`Ring join on ${this.config.serviceId} timed out`)
+        } finally {
+            sessions.forEach((session) => this.ongoingDiscoverySessions.delete(session.id))
         }
     }
 
@@ -112,12 +162,13 @@ export class PeerDiscovery {
         logger.debug(`Rejoining DHT ${this.config.serviceId}`)
         this.rejoinOngoing = true
         try {
-            this.config.neighborList.clear()
-            await this.joinDht(entryPoint)
+            await this.joinThroughEntryPoint(entryPoint, new Set(), { enabled: false })
             logger.debug(`Rejoined DHT successfully ${this.config.serviceId}!`)
         } catch (err) {
             logger.warn(`Rejoining DHT ${this.config.serviceId} failed`)
             if (!this.isStopped()) {
+                // TODO should we catch possible promise rejection?
+                // TODO use config option or named constant?
                 setAbortableTimeout(() => this.rejoinDht(entryPoint), 5000, this.abortController.signal)
             }
         } finally {
@@ -128,6 +179,7 @@ export class PeerDiscovery {
     private async ensureRecoveryIntervalIsRunning(): Promise<void> {
         if (!this.recoveryIntervalStarted) {
             this.recoveryIntervalStarted = true
+            // TODO use config option or named constant?
             await scheduleAtInterval(() => this.fetchClosestPeersFromBucket(), 60000, true, this.abortController.signal)
         }
     }
@@ -136,13 +188,17 @@ export class PeerDiscovery {
         if (this.isStopped()) {
             return
         }
-        const nodes = this.config.bucket.closest(peerIdFromPeerDescriptor(this.config.localPeerDescriptor).value, this.config.parallelism)
-        await Promise.allSettled(nodes.map(async (peer: DhtNodeRpcRemote) => {
-            const contacts = await peer.getClosestPeers(this.config.localPeerDescriptor.kademliaId)
-            contacts.forEach((contact) => {
-                this.config.addContact(contact)
+        const localNodeId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
+        const nodes = this.config.peerManager.getClosestNeighborsTo(
+            localNodeId,
+            this.config.parallelism
+        )
+        await Promise.allSettled(
+            nodes.map(async (peer: DhtNodeRpcRemote) => {
+                const contacts = await peer.getClosestPeers(localNodeId)
+                this.config.peerManager.addContact(contacts)
             })
-        }))
+        )
     }
 
     public isJoinOngoing(): boolean {
@@ -159,11 +215,10 @@ export class PeerDiscovery {
 
     public stop(): void {
         this.abortController.abort()
-        if (this.rejoinTimeoutRef) {
-            clearTimeout(this.rejoinTimeoutRef)
-            this.rejoinTimeoutRef = undefined
-        }
-        this.ongoingDiscoverySessions.forEach((session, _id) => {
+        this.ongoingDiscoverySessions.forEach((session) => {
+            session.stop()
+        })
+        this.ongoingRingDiscoverySessions.forEach((session) => {
             session.stop()
         })
     }

package/src/dht/discovery/RingDiscoverySession.ts

@@ -0,0 +1,160 @@
+import { Logger, runAndWaitForEvents3 } from '@streamr/utils'
+import EventEmitter from 'eventemitter3'
+import { v4 } from 'uuid'
+import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
+import { PeerManager } from '../PeerManager'
+import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
+import { DhtAddress, getNodeIdFromPeerDescriptor } from '../../identifiers'
+import { RingId, RingIdRaw, getLeftDistance, getRingIdFromPeerDescriptor, getRingIdFromRaw } from '../contact/ringIdentifiers'
+import { RingContacts } from '../contact/RingContactList'
+
+const logger = new Logger(module)
+
+interface RingDiscoverySessionEvents {
+    discoveryCompleted: () => void
+}
+
+interface RingDiscoverySessionConfig {
+    targetId: RingIdRaw
+    parallelism: number
+    noProgressLimit: number
+    peerManager: PeerManager
+    // Note that contacted peers will be mutated by the DiscoverySession or other parallel sessions
+    contactedPeers: Set<DhtAddress>
+}
+
+export class RingDiscoverySession {
+
+    public readonly id = v4()
+    private stopped = false
+    private emitter = new EventEmitter<RingDiscoverySessionEvents>()
+    private noProgressCounter = 0
+    private ongoingClosestPeersRequests: Set<DhtAddress> = new Set()
+    private readonly config: RingDiscoverySessionConfig
+    private numContactedPeers = 0
+    private targetIdAsRingId: RingId
+
+    constructor(config: RingDiscoverySessionConfig) {
+        this.config = config
+        this.targetIdAsRingId = getRingIdFromRaw(this.config.targetId)
+    }
+
+    private addContacts(contacts: PeerDescriptor[]): void {
+        if (this.stopped) {
+            return
+        }
+        this.config.peerManager.addContact(contacts)
+    }
+
+    private async getClosestPeersFromContact(contact: DhtNodeRpcRemote): Promise<RingContacts> {
+        if (this.stopped) {
+            return { left: [], right: [] }
+        }
+        logger.trace(`Getting closest ring peers from contact: ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
+        this.numContactedPeers++
+        this.config.contactedPeers.add(contact.getNodeId())
+        const returnedContacts = await contact.getClosestRingPeers(this.config.targetId)
+        this.config.peerManager.setContactActive(contact.getNodeId())
+        return returnedContacts
+    }
+
+    private onClosestPeersRequestSucceeded(nodeId: DhtAddress, contacts: RingContacts) {
+        if (!this.ongoingClosestPeersRequests.has(nodeId)) {
+            return
+        }
+        this.ongoingClosestPeersRequests.delete(nodeId)
+        const oldClosestContacts = this.config.peerManager.getClosestRingContactsTo(this.config.targetId, 1)
+        const oldClosestLeftDistance = getLeftDistance(
+            this.targetIdAsRingId,
+            getRingIdFromPeerDescriptor(oldClosestContacts.left[0].getPeerDescriptor())
+        )
+        const oldClosestRightDistance = getLeftDistance(
+            this.targetIdAsRingId,
+            getRingIdFromPeerDescriptor(oldClosestContacts.right[0].getPeerDescriptor())
+        )
+        this.addContacts(contacts.left.concat(contacts.right))
+        const newClosestContacts = this.config.peerManager.getClosestRingContactsTo(this.config.targetId, 1)
+        const newClosestLeftDistance = getLeftDistance(this.targetIdAsRingId,
+            getRingIdFromPeerDescriptor(newClosestContacts.left[0].getPeerDescriptor()))
+        const newClosestRightDistance = getLeftDistance(this.targetIdAsRingId,
+            getRingIdFromPeerDescriptor(newClosestContacts.right[0].getPeerDescriptor()))
+        if (newClosestLeftDistance >= oldClosestLeftDistance && newClosestRightDistance >= oldClosestRightDistance) {
+            this.noProgressCounter++
+        }
+    }
+
+    private onClosestPeersRequestFailed(peer: DhtNodeRpcRemote) {
+        if (!this.ongoingClosestPeersRequests.has(peer.getNodeId())) {
+            return
+        }
+        this.ongoingClosestPeersRequests.delete(peer.getNodeId())
+        this.config.peerManager.removeContact(peer.getNodeId())
+    }
+
+    private findMoreContacts(): void {
+        if (this.stopped) {
+            return
+        }
+        const uncontacted = this.config.peerManager.getClosestRingContactsTo(
+            this.config.targetId,
+            this.config.parallelism,
+            this.config.contactedPeers
+        )
+        if ((uncontacted.left.length === 0 && uncontacted.right.length === 0)
+            || this.noProgressCounter >= this.config.noProgressLimit) {
+            this.emitter.emit('discoveryCompleted')
+            this.stopped = true
+            return
+        }
+        // ask from both sides equally
+        const merged = []
+        const alreadyInMerged: Set<DhtAddress> = new Set()
+        const length = Math.max(uncontacted.left.length, uncontacted.right.length)
+        for (let i = 0; i < length; i++) {
+            if (i < uncontacted.left.length) {
+                if (!alreadyInMerged.has(uncontacted.left[i].getNodeId())) {
+                    merged.push(uncontacted.left[i])
+                    alreadyInMerged.add(uncontacted.left[i].getNodeId())
+                }
+            }
+            if (i < uncontacted.right.length) {
+                if (!alreadyInMerged.has(uncontacted.right[i].getNodeId())) {
+                    merged.push(uncontacted.right[i])
+                    alreadyInMerged.add(uncontacted.right[i].getNodeId())
+                }
+            }
+        }
+
+        for (const nextPeer of merged) {
+            if (this.ongoingClosestPeersRequests.size >= this.config.parallelism) {
+                break
+            }
+            this.ongoingClosestPeersRequests.add(nextPeer.getNodeId())
+            // eslint-disable-next-line promise/catch-or-return
+            this.getClosestPeersFromContact(nextPeer)
+                .then((contacts) => this.onClosestPeersRequestSucceeded(nextPeer.getNodeId(), contacts))
+                .catch(() => this.onClosestPeersRequestFailed(nextPeer))
+                .finally(() => {
+                    this.findMoreContacts()
+                })
+        }
+    }
+
+    public async findClosestNodes(timeout: number): Promise<void> {
+        if (this.config.peerManager.getContactCount(this.config.contactedPeers) === 0) {
+            return
+        }
+        // TODO add abortController and signal it in stop()
+        await runAndWaitForEvents3<RingDiscoverySessionEvents>(
+            [this.findMoreContacts.bind(this)],
+            [[this.emitter, 'discoveryCompleted']],
+            timeout
+        )
+    }
+
+    public stop(): void {
+        this.stopped = true
+        this.emitter.emit('discoveryCompleted')
+        this.emitter.removeAllListeners()
+    }
+}

package/src/dht/recursive-operation/RecursiveOperationManager.ts

@@ -0,0 +1,246 @@
+import {
+    DataEntry,
+    PeerDescriptor,
+    RecursiveOperation,
+    RecursiveOperationRequest,
+    RouteMessageAck,
+    RouteMessageError,
+    RouteMessageWrapper
+} from '../../proto/packages/dht/protos/DhtRpc'
+import { Router } from '../routing/Router'
+import { RoutingMode } from '../routing/RoutingSession'
+import { Logger, areEqualBinaries, runAndWaitForEvents3, wait } from '@streamr/utils'
+import { RoutingRpcCommunicator } from '../../transport/RoutingRpcCommunicator'
+import { RecursiveOperationSessionRpcRemote } from './RecursiveOperationSessionRpcRemote'
+import { RecursiveOperationSession, RecursiveOperationSessionEvents } from './RecursiveOperationSession'
+import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
+import { ITransport } from '../../transport/ITransport'
+import { LocalDataStore } from '../store/LocalDataStore'
+import { ListeningRpcCommunicator } from '../../transport/ListeningRpcCommunicator'
+import { RecursiveOperationSessionRpcClient } from '../../proto/packages/dht/protos/DhtRpc.client'
+import { SortedContactList } from '../contact/SortedContactList'
+import { getPreviousPeer } from '../routing/getPreviousPeer'
+import { createRouteMessageAck } from '../routing/RouterRpcLocal'
+import { ServiceID } from '../../types/ServiceID'
+import { RecursiveOperationRpcLocal } from './RecursiveOperationRpcLocal'
+import { DhtAddress, areEqualPeerDescriptors, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../identifiers'
+import { getDistance } from '../PeerManager'
+
+interface RecursiveOperationManagerConfig {
+    rpcCommunicator: RoutingRpcCommunicator
+    sessionTransport: ITransport
+    connections: Map<DhtAddress, DhtNodeRpcRemote>
+    router: Router
+    localPeerDescriptor: PeerDescriptor
+    serviceId: ServiceID
+    localDataStore: LocalDataStore
+    addContact: (contact: PeerDescriptor) => void
+}
+
+export interface RecursiveOperationResult { closestNodes: Array<PeerDescriptor>, dataEntries?: Array<DataEntry> }
+
+const logger = new Logger(module)
+
+export class RecursiveOperationManager {
+
+    private ongoingSessions: Map<string, RecursiveOperationSession> = new Map()
+    private stopped = false
+    private readonly config: RecursiveOperationManagerConfig
+
+    constructor(config: RecursiveOperationManagerConfig) {
+        this.config = config
+        this.registerLocalRpcMethods()
+    }
+
+    private registerLocalRpcMethods() {
+        const rpcLocal = new RecursiveOperationRpcLocal({
+            doRouteRequest: (routedMessage: RouteMessageWrapper) => this.doRouteRequest(routedMessage),
+            addContact: (contact: PeerDescriptor) => this.config.addContact(contact),
+            isMostLikelyDuplicate: (requestId: string) => this.config.router.isMostLikelyDuplicate(requestId),
+            addToDuplicateDetector: (requestId: string) => this.config.router.addToDuplicateDetector(requestId)
+        })
+        this.config.rpcCommunicator.registerRpcMethod(
+            RouteMessageWrapper,
+            RouteMessageAck,
+            'routeRequest',
+            async (routedMessage: RouteMessageWrapper) => {
+                if (this.stopped) {
+                    return createRouteMessageAck(routedMessage, RouteMessageError.STOPPED)
+                } else {
+                    return rpcLocal.routeRequest(routedMessage)
+                }
+            }
+        )
+    }
+
+    public async execute(
+        targetId: DhtAddress,
+        operation: RecursiveOperation,
+        excludedPeer?: DhtAddress,
+        waitForCompletion = true
+    ): Promise<RecursiveOperationResult> {
+        if (this.stopped) {
+            return { closestNodes: [] }
+        }
+        const session = new RecursiveOperationSession({
+            transport: this.config.sessionTransport,
+            targetId,
+            localPeerDescriptor: this.config.localPeerDescriptor,
+            // TODO use config option or named constant?
+            waitedRoutingPathCompletions: this.config.connections.size > 1 ? 2 : 1,
+            operation,
+            // TODO would it make sense to give excludedPeer as one of the fields RecursiveOperationSession?
+            doRouteRequest: (routedMessage: RouteMessageWrapper) => {
+                return this.doRouteRequest(routedMessage, excludedPeer)
+            }
+        })
+        if (this.config.connections.size === 0) {
+            const dataEntries = Array.from(this.config.localDataStore.values(targetId))
+            session.onResponseReceived(
+                getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor),
+                [this.config.localPeerDescriptor],
+                [this.config.localPeerDescriptor],
+                dataEntries,
+                true
+            )
+            return session.getResults()
+        }
+        this.ongoingSessions.set(session.getId(), session)
+        if (waitForCompletion === true) {
+            try {
+                await runAndWaitForEvents3<RecursiveOperationSessionEvents>(
+                    [() => session.start(this.config.serviceId)],
+                    [[session, 'completed']],
+                    // TODO use config option or named constant?
+                    15000
+                )
+            } catch (err) {
+                logger.debug(`start failed with error ${err}`)
+            }
+        } else {
+            session.start(this.config.serviceId)
+            // Wait for delete operation to be sent out by the router
+            // TODO: Add a feature to wait for the router to pass the message?
+            await wait(50)
+        }
+        if (operation === RecursiveOperation.FETCH_DATA) {
+            const dataEntries = Array.from(this.config.localDataStore.values(targetId))
+            if (dataEntries.length > 0) {
+                this.sendResponse([this.config.localPeerDescriptor], this.config.localPeerDescriptor, session.getId(), [], dataEntries, true)
+            }
+        } else if (operation === RecursiveOperation.DELETE_DATA) {
+            this.config.localDataStore.markAsDeleted(targetId, getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor))
+        }
+        this.ongoingSessions.delete(session.getId())
+        session.stop()
+        return session.getResults()
+    }
+
+    private sendResponse(
+        routingPath: PeerDescriptor[],
+        targetPeerDescriptor: PeerDescriptor,
+        serviceId: ServiceID,
+        closestNodes: PeerDescriptor[],
+        dataEntries: DataEntry[],
+        noCloserNodesFound: boolean = false
+    ): void {
+        const isOwnNode = areEqualPeerDescriptors(this.config.localPeerDescriptor, targetPeerDescriptor)
+        if (isOwnNode && this.ongoingSessions.has(serviceId)) {
+            this.ongoingSessions.get(serviceId)!
+                .onResponseReceived(
+                    getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor),
+                    routingPath,
+                    closestNodes,
+                    dataEntries,
+                    noCloserNodesFound
+                )
+        } else {
+            // TODO use config option or named constant?
+            const remoteCommunicator = new ListeningRpcCommunicator(serviceId, this.config.sessionTransport, { rpcRequestTimeout: 15000 })
+            const rpcRemote = new RecursiveOperationSessionRpcRemote(
+                this.config.localPeerDescriptor,
+                targetPeerDescriptor,
+                remoteCommunicator,
+                RecursiveOperationSessionRpcClient,
+                // TODO use config option or named constant?
+                10000
+            )
+            rpcRemote.sendResponse(routingPath, closestNodes, dataEntries, noCloserNodesFound)
+            remoteCommunicator.destroy()
+        }
+    }
+
+    private doRouteRequest(routedMessage: RouteMessageWrapper, excludedPeer?: DhtAddress): RouteMessageAck {
+        if (this.stopped) {
+            return createRouteMessageAck(routedMessage, RouteMessageError.STOPPED)
+        }
+        const targetId = getDhtAddressFromRaw(routedMessage.target)
+        const request = (routedMessage.message!.body as { recursiveOperationRequest: RecursiveOperationRequest }).recursiveOperationRequest
+        // TODO use config option or named constant?
+        const closestPeersToDestination = this.getClosestConnections(targetId, 5)
+        const dataEntries = (request.operation === RecursiveOperation.FETCH_DATA) 
+            ? Array.from(this.config.localDataStore.values(targetId))
+            : []
+        if (request.operation === RecursiveOperation.DELETE_DATA) {
+            this.config.localDataStore.markAsDeleted(targetId, getNodeIdFromPeerDescriptor(routedMessage.sourcePeer!))
+        }
+        if (areEqualBinaries(this.config.localPeerDescriptor.nodeId, routedMessage.target)) {
+            // TODO this is also very similar case to what we do at line 255, could simplify the code paths?
+            this.sendResponse(
+                routedMessage.routingPath,
+                routedMessage.sourcePeer!,
+                request.sessionId,
+                closestPeersToDestination,
+                dataEntries,
+                true
+            )
+            return createRouteMessageAck(routedMessage)
+        } else {
+            const ack = this.config.router.doRouteMessage(routedMessage, RoutingMode.RECURSIVE, excludedPeer)
+            if ((ack.error === undefined) || (ack.error === RouteMessageError.NO_TARGETS)) {
+                const noCloserContactsFound = (ack.error === RouteMessageError.NO_TARGETS) ||
+                    (
+                        closestPeersToDestination.length > 0 
+                        && getPreviousPeer(routedMessage) 
+                        && !this.isPeerCloserToIdThanSelf(closestPeersToDestination[0], targetId)
+                    )
+                this.sendResponse(
+                    routedMessage.routingPath,
+                    routedMessage.sourcePeer!,
+                    request.sessionId,
+                    closestPeersToDestination,
+                    dataEntries,
+                    noCloserContactsFound
+                )
+            }
+            return ack
+        }    
+    }
+
+    private getClosestConnections(referenceId: DhtAddress, limit: number): PeerDescriptor[] {
+        const connectedPeers = Array.from(this.config.connections.values())
+        const closestPeers = new SortedContactList<DhtNodeRpcRemote>({
+            referenceId,
+            maxSize: limit,
+            allowToContainReferenceId: true,
+            emitEvents: false
+        })
+        closestPeers.addContacts(connectedPeers)
+        return closestPeers.getClosestContacts(limit).map((peer) => peer.getPeerDescriptor())
+    }
+
+    private isPeerCloserToIdThanSelf(peer: PeerDescriptor, nodeIdOrDataKey: DhtAddress): boolean {
+        const nodeIdOrDataKeyRaw = getRawFromDhtAddress(nodeIdOrDataKey)
+        const distance1 = getDistance(peer.nodeId, nodeIdOrDataKeyRaw)
+        const distance2 = getDistance(this.config.localPeerDescriptor.nodeId, nodeIdOrDataKeyRaw)
+        return distance1 < distance2
+    }
+
+    public stop(): void {
+        this.stopped = true
+        this.ongoingSessions.forEach((session, _id) => {
+            session.stop()
+        })
+        this.ongoingSessions.clear()
+    }
+}