@streamr/dht 100.0.0-pretestnet.6 → 100.0.0-rc.0

package/src/dht/DhtNode.ts

@@ -1,70 +1,68 @@
-import { DhtNodeRpcRemote } from './DhtNodeRpcRemote'
-import KBucket from 'k-bucket'
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import {
+    Logger,
+    MetricsContext,
+    merge,
+    waitForCondition
+} from '@streamr/utils'
 import { EventEmitter } from 'eventemitter3'
-import { SortedContactList } from './contact/SortedContactList'
-import { RoutingRpcCommunicator } from '../transport/RoutingRpcCommunicator'
-import { PeerID, PeerIDKey } from '../helpers/PeerID'
+import { sample } from 'lodash'
+import { MarkRequired } from 'ts-essentials'
+import { ConnectionManager, PortRange, TlsCertificate } from '../connection/ConnectionManager'
+import { DefaultConnectorFacade, DefaultConnectorFacadeConfig } from '../connection/ConnectorFacade'
+import { IceServer } from '../connection/webrtc/WebrtcConnector'
+import { isBrowserEnvironment } from '../helpers/browser/isBrowserEnvironment'
+import { DhtAddress, KADEMLIA_ID_LENGTH_IN_BYTES, getNodeIdFromPeerDescriptor } from '../identifiers'
+import { Any } from '../proto/google/protobuf/any'
 import {
     ClosestPeersRequest,
     ClosestPeersResponse,
-    LeaveNotice,
+    ClosestRingPeersRequest,
+    ClosestRingPeersResponse,
     ConnectivityResponse,
+    DataEntry,
+    ExternalFetchDataRequest,
+    ExternalFetchDataResponse,
+    ExternalStoreDataRequest,
+    ExternalStoreDataResponse,
+    LeaveNotice,
     Message,
-    NodeType,
     PeerDescriptor,
     PingRequest,
     PingResponse,
-    DataEntry,
-    ExternalFindDataRequest,
-    ExternalFindDataResponse,
-    ExternalStoreDataRequest,
-    ExternalStoreDataResponse,
+    RecursiveOperation
 } from '../proto/packages/dht/protos/DhtRpc'
+import { ExternalApiRpcClient, StoreRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
 import { ITransport, TransportEvents } from '../transport/ITransport'
-import { ConnectionManager, PortRange, TlsCertificate } from '../connection/ConnectionManager'
-import { DhtNodeRpcClient, ExternalApiRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
-import {
-    Logger,
-    MetricsContext,
-    hexToBinary,
-    merge,
-    waitForCondition
-} from '@streamr/utils'
-import { toProtoRpcClient } from '@streamr/proto-rpc'
-import { RandomContactList } from './contact/RandomContactList'
-import { Any } from '../proto/google/protobuf/any'
-import {
-    areEqualPeerDescriptors,
-    getNodeIdFromPeerDescriptor,
-    keyFromPeerDescriptor,
-    peerIdFromPeerDescriptor
-} from '../helpers/peerIdFromPeerDescriptor'
-import { Router } from './routing/Router'
-import { Finder, FindResult } from './find/Finder'
-import { StoreRpcLocal } from './store/StoreRpcLocal'
-import { PeerDiscovery } from './discovery/PeerDiscovery'
-import { LocalDataStore } from './store/LocalDataStore'
-import { IceServer } from '../connection/webrtc/WebrtcConnector'
-import { ExternalApiRpcRemote } from './ExternalApiRpcRemote'
-import { UUID } from '../helpers/UUID'
-import { isBrowserEnvironment } from '../helpers/browser/isBrowserEnvironment'
-import { sample } from 'lodash'
-import { DefaultConnectorFacade, DefaultConnectorFacadeConfig } from '../connection/ConnectorFacade'
-import { MarkRequired } from 'ts-essentials'
+import { RoutingRpcCommunicator } from '../transport/RoutingRpcCommunicator'
+import { ServiceID } from '../types/ServiceID'
 import { DhtNodeRpcLocal } from './DhtNodeRpcLocal'
-import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import { DhtNodeRpcRemote } from './DhtNodeRpcRemote'
 import { ExternalApiRpcLocal } from './ExternalApiRpcLocal'
+import { ExternalApiRpcRemote } from './ExternalApiRpcRemote'
+import { PeerManager } from './PeerManager'
+import { PeerDiscovery } from './discovery/PeerDiscovery'
+import { RecursiveOperationManager } from './recursive-operation/RecursiveOperationManager'
+import { Router } from './routing/Router'
+import { LocalDataStore } from './store/LocalDataStore'
+import { StoreManager } from './store/StoreManager'
+import { StoreRpcRemote } from './store/StoreRpcRemote'
+import { createPeerDescriptor } from '../helpers/createPeerDescriptor'
+import { RingIdRaw } from './contact/ringIdentifiers'
+import { getLocalRegion } from '@streamr/cdn-location'
+import { RingContacts } from './contact/RingContactList'
 
 export interface DhtNodeEvents {
-    newContact: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
+    contactAdded: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
     contactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
-    joinCompleted: () => void
-    newRandomContact: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
+    randomContactAdded: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
     randomContactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
+    ringContactAdded: (peerDescriptor: PeerDescriptor, closestPeers: RingContacts) => void
+    ringContactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: RingContacts) => void
 }
 
 export interface DhtNodeOptions {
-    serviceId?: string
+    serviceId?: ServiceID
     joinParallelism?: number
     maxNeighborListSize?: number
     numberOfNodesPerKBucket?: number
@@ -76,6 +74,7 @@ export interface DhtNodeOptions {
     storeMaxTtl?: number
     networkConnectivityTimeout?: number
     storageRedundancyFactor?: number
+    region?: number
 
     transport?: ITransport
     peerDescriptor?: PeerDescriptor
@@ -83,7 +82,7 @@ export interface DhtNodeOptions {
     websocketHost?: string
     websocketPortRange?: PortRange
     websocketServerEnableTls?: boolean
-    peerId?: string
+    nodeId?: DhtAddress
 
     rpcRequestTimeout?: number
     iceServers?: IceServer[]
@@ -100,7 +99,7 @@ export interface DhtNodeOptions {
     autoCertifierConfigFile?: string
 }
 
-type StrictDhtNodeOptions = MarkRequired<DhtNodeOptions, 
+type StrictDhtNodeOptions = MarkRequired<DhtNodeOptions,
     'serviceId' |
     'joinParallelism' |
     'maxNeighborListSize' |
@@ -113,49 +112,28 @@ type StrictDhtNodeOptions = MarkRequired<DhtNodeOptions,
     'storeMaxTtl' |
     'networkConnectivityTimeout' |
     'storageRedundancyFactor' |
-    'metricsContext' |
-    'peerId'>
+    'metricsContext'>
 
 const logger = new Logger(module)
 
 export type Events = TransportEvents & DhtNodeEvents
 
-export const createPeerDescriptor = (msg?: ConnectivityResponse, peerId?: string): PeerDescriptor => {
-    let kademliaId: Uint8Array
-    if (msg) {
-        kademliaId = peerId ? hexToBinary(peerId) : PeerID.fromIp(msg.host).value
-    } else {
-        kademliaId = hexToBinary(peerId!)
-    }
-    const nodeType = isBrowserEnvironment() ? NodeType.BROWSER : NodeType.NODEJS 
-    const ret: PeerDescriptor = { kademliaId, type: nodeType }
-    if (msg && msg.websocket) {
-        ret.websocket = { host: msg.websocket.host, port: msg.websocket.port, tls: msg.websocket.tls }
-    }
-    return ret
-}
-
 export class DhtNode extends EventEmitter<Events> implements ITransport {
 
     private readonly config: StrictDhtNodeOptions
-    private bucket?: KBucket<DhtNodeRpcRemote>
-    private connections: Map<PeerIDKey, DhtNodeRpcRemote> = new Map()
-    private neighborList?: SortedContactList<DhtNodeRpcRemote>
-    private randomPeers?: RandomContactList<DhtNodeRpcRemote>
     private rpcCommunicator?: RoutingRpcCommunicator
     private transport?: ITransport
     private localPeerDescriptor?: PeerDescriptor
     public router?: Router
-    private storeRpcLocal?: StoreRpcLocal
-    private localDataStore = new LocalDataStore()
-    private finder?: Finder
+    private storeManager?: StoreManager
+    private localDataStore: LocalDataStore
+    private recursiveOperationManager?: RecursiveOperationManager
     private peerDiscovery?: PeerDiscovery
-
+    private peerManager?: PeerManager
     public connectionManager?: ConnectionManager
+    private region?: number
     private started = false
-    private stopped = false
-    private entryPointDisconnectTimeout?: NodeJS.Timeout
-
+    private abortController = new AbortController()
     constructor(conf: DhtNodeOptions) {
         super()
         this.config = merge({
@@ -163,7 +141,7 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
             joinParallelism: 3,
             maxNeighborListSize: 200,
             numberOfNodesPerKBucket: 8,
-            joinNoProgressLimit: 4,
+            joinNoProgressLimit: 5,
             dhtJoinTimeout: 60000,
             peerDiscoveryQueryBatchSize: 5,
             maxConnections: 80,
@@ -171,14 +149,31 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
             storeMaxTtl: 60000,
             networkConnectivityTimeout: 10000,
             storageRedundancyFactor: 5,
-            metricsContext: new MetricsContext(),
-            peerId: new UUID().toHex()
+            metricsContext: new MetricsContext()
         }, conf)
+        this.validateConfig()
+        this.localDataStore = new LocalDataStore(this.config.storeMaxTtl)
         this.send = this.send.bind(this)
     }
 
+    private validateConfig(): void {
+        const expectedNodeIdLength = KADEMLIA_ID_LENGTH_IN_BYTES * 2
+        if (this.config.nodeId !== undefined) {
+            if (!/^[0-9a-fA-F]+$/.test(this.config.nodeId)) {
+                throw new Error('Invalid nodeId, the nodeId should be a hex string')
+            } else if (this.config.nodeId.length !== expectedNodeIdLength) {
+                throw new Error(`Invalid nodeId, the length of the nodeId should be ${expectedNodeIdLength}`)
+            }
+        }
+        if (this.config.peerDescriptor !== undefined) {
+            if (this.config.peerDescriptor.nodeId.length !== KADEMLIA_ID_LENGTH_IN_BYTES) {
+                throw new Error(`Invalid peerDescriptor, the length of the nodeId should be ${KADEMLIA_ID_LENGTH_IN_BYTES} bytes`)
+            }
+        }
+    }
+
     public async start(): Promise<void> {
-        if (this.started || this.stopped) {
+        if (this.started || this.abortController.signal.aborted) {
             return
         }
         logger.trace(`Starting new Streamr Network DHT Node with serviceId ${this.config.serviceId}`)
@@ -190,6 +185,14 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
                 this.config.peerDescriptor.websocket = undefined
             }
         }
+        if (this.region !== undefined) {
+            this.region = this.config.region
+        } else if (this.config.peerDescriptor?.region !== undefined) {
+            this.region = this.config.peerDescriptor.region
+        } else {
+            this.region = await getLocalRegion()
+        }
+            
         // If transport is given, do not create a ConnectionManager
         if (this.config.transport) {
             this.transport = this.config.transport
@@ -218,12 +221,12 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
             // If own PeerDescriptor is given in config, create a ConnectionManager with ws server
             if (this.config.peerDescriptor?.websocket) {
                 connectorFacadeConfig.websocketHost = this.config.peerDescriptor.websocket.host
-                connectorFacadeConfig.websocketPortRange = { 
+                connectorFacadeConfig.websocketPortRange = {
                     min: this.config.peerDescriptor.websocket.port,
                     max: this.config.peerDescriptor.websocket.port
                 }
-            // If websocketPortRange is given, create ws server using it, websocketHost can be undefined
-            } else if (this.config.websocketPortRange) { 
+                // If websocketPortRange is given, create ws server using it, websocketHost can be undefined
+            } else if (this.config.websocketPortRange) {
                 connectorFacadeConfig.websocketHost = this.config.websocketHost
                 connectorFacadeConfig.websocketPortRange = this.config.websocketPortRange
             }
@@ -244,426 +247,296 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
             { rpcRequestTimeout: this.config.rpcRequestTimeout }
         )
 
-        this.transport.on('message', (message: Message) => this.handleMessage(message))
+        this.transport.on('message', (message: Message) => this.handleMessageFromTransport(message))
+
+        this.initPeerManager()
 
-        this.initKBuckets(peerIdFromPeerDescriptor(this.localPeerDescriptor!))
         this.peerDiscovery = new PeerDiscovery({
-            rpcCommunicator: this.rpcCommunicator,
             localPeerDescriptor: this.localPeerDescriptor!,
-            bucket: this.bucket!,
-            neighborList: this.neighborList!,
             joinNoProgressLimit: this.config.joinNoProgressLimit,
             peerDiscoveryQueryBatchSize: this.config.peerDiscoveryQueryBatchSize,
             joinTimeout: this.config.dhtJoinTimeout,
             serviceId: this.config.serviceId,
             parallelism: this.config.joinParallelism,
-            addContact: this.addNewContact.bind(this),
             connectionManager: this.connectionManager,
-            rpcRequestTimeout: this.config.rpcRequestTimeout
+            peerManager: this.peerManager!
         })
         this.router = new Router({
             rpcCommunicator: this.rpcCommunicator,
-            connections: this.connections,
+            connections: this.peerManager!.connections,
             localPeerDescriptor: this.localPeerDescriptor!,
-            addContact: this.addNewContact.bind(this),
-            serviceId: this.config.serviceId,
-            connectionManager: this.connectionManager
+            addContact: (contact: PeerDescriptor, setActive?: boolean) => this.peerManager!.addContact([contact], setActive),
+            handleMessage: (message: Message) => this.handleMessageFromRouter(message),
         })
-        this.finder = new Finder({
+        this.recursiveOperationManager = new RecursiveOperationManager({
             rpcCommunicator: this.rpcCommunicator,
             router: this.router,
             sessionTransport: this,
-            connections: this.connections,
+            connections: this.peerManager!.connections,
             localPeerDescriptor: this.localPeerDescriptor!,
             serviceId: this.config.serviceId,
-            addContact: this.addNewContact.bind(this),
-            isPeerCloserToIdThanSelf: this.isPeerCloserToIdThanSelf.bind(this),
+            addContact: (contact: PeerDescriptor) => this.peerManager!.addContact([contact]),
             localDataStore: this.localDataStore
         })
-        this.storeRpcLocal = new StoreRpcLocal({
+        this.storeManager = new StoreManager({
             rpcCommunicator: this.rpcCommunicator,
-            finder: this.finder,
+            recursiveOperationManager: this.recursiveOperationManager,
             localPeerDescriptor: this.localPeerDescriptor!,
             serviceId: this.config.serviceId,
             highestTtl: this.config.storeHighestTtl,
-            maxTtl: this.config.storeMaxTtl,
             redundancyFactor: this.config.storageRedundancyFactor,
             localDataStore: this.localDataStore,
-            dhtNodeEmitter: this,
-            getNodesClosestToIdFromBucket: (id: Uint8Array, n?: number) => {
-                return this.bucket!.closest(id, n)
+            getClosestNeighborsTo: (key: DhtAddress, n?: number) => {
+                return this.peerManager!.getClosestNeighborsTo(key, n).map((n) => n.getPeerDescriptor())
             },
-            rpcRequestTimeout: this.config.rpcRequestTimeout
+            createRpcRemote: (contact: PeerDescriptor) => {
+                return new StoreRpcRemote(
+                    this.localPeerDescriptor!,
+                    contact,
+                    this.rpcCommunicator!,
+                    StoreRpcClient,
+                    this.config.rpcRequestTimeout
+                )
+            }
+        })
+        this.on('contactAdded', (peerDescriptor: PeerDescriptor) => {
+            this.storeManager!.onContactAdded(peerDescriptor)
         })
         this.bindRpcLocalMethods()
-        if (this.connectionManager! && this.config.entryPoints && this.config.entryPoints.length > 0 
-            && !areEqualPeerDescriptors(this.config.entryPoints[0], this.localPeerDescriptor!)) {
-            this.connectToEntryPoint(this.config.entryPoints[0])
-        }
     }
 
-    private initKBuckets(selfId: PeerID) {
-        this.bucket = new KBucket<DhtNodeRpcRemote>({
-            localNodeId: selfId.value,
+    private initPeerManager() {
+        this.peerManager = new PeerManager({
             numberOfNodesPerKBucket: this.config.numberOfNodesPerKBucket,
-            numberOfNodesToPing: this.config.numberOfNodesPerKBucket
+            maxContactListSize: this.config.maxNeighborListSize,
+            localNodeId: this.getNodeId(),
+            localPeerDescriptor: this.localPeerDescriptor!,
+            connectionManager: this.connectionManager!,
+            peerDiscoveryQueryBatchSize: this.config.peerDiscoveryQueryBatchSize,
+            isLayer0: (this.connectionManager !== undefined),
+            createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => this.createDhtNodeRpcRemote(peerDescriptor),
+            lockId: this.config.serviceId
         })
-        this.bucket.on('ping', (oldContacts: DhtNodeRpcRemote[], newContact: DhtNodeRpcRemote) => this.onKBucketPing(oldContacts, newContact))
-        this.bucket.on('removed', (contact: DhtNodeRpcRemote) => this.onKBucketRemoved(contact))
-        this.bucket.on('added', (contact: DhtNodeRpcRemote) => this.onKBucketAdded(contact))
-        this.bucket.on('updated', () => {
-            // TODO: Update contact info to the connection manager and reconnect
+        this.peerManager.on('contactRemoved', (peerDescriptor: PeerDescriptor, activeContacts: PeerDescriptor[]) => {
+            this.emit('contactRemoved', peerDescriptor, activeContacts)
         })
-        this.neighborList = new SortedContactList(selfId, this.config.maxNeighborListSize)
-        this.neighborList.on('contactRemoved', (removedContact: DhtNodeRpcRemote, activeContacts: DhtNodeRpcRemote[]) => {
-            if (this.stopped) {
-                return
+        this.peerManager.on('contactAdded', (peerDescriptor: PeerDescriptor, activeContacts: PeerDescriptor[]) =>
+            this.emit('contactAdded', peerDescriptor, activeContacts)
+        )
+        this.peerManager.on('randomContactRemoved', (peerDescriptor: PeerDescriptor, activeContacts: PeerDescriptor[]) =>
+            this.emit('randomContactRemoved', peerDescriptor, activeContacts)
+        )
+        this.peerManager.on('randomContactAdded', (peerDescriptor: PeerDescriptor, activeContacts: PeerDescriptor[]) =>
+            this.emit('randomContactAdded', peerDescriptor, activeContacts)
+        )
+        this.peerManager.on('ringContactRemoved', (peerDescriptor: PeerDescriptor, activeContacts: RingContacts) => {
+            this.emit('ringContactRemoved', peerDescriptor, activeContacts)
+        })
+        this.peerManager.on('ringContactAdded', (peerDescriptor: PeerDescriptor, activeContacts: RingContacts) => {
+            this.emit('ringContactAdded', peerDescriptor, activeContacts)
+        })
+        this.peerManager.on('kBucketEmpty', () => {
+            if (!this.peerDiscovery!.isJoinOngoing()
+                && this.config.entryPoints
+                && this.config.entryPoints.length > 0
+            ) {
+                setImmediate(async () => {
+                    // TODO should we catch possible promise rejection?
+                    await Promise.all(this.config.entryPoints!.map((entryPoint) =>
+                        this.peerDiscovery!.rejoinDht(entryPoint)
+                    ))
+                })
             }
-            this.emit('contactRemoved', removedContact.getPeerDescriptor(), activeContacts.map((c) => c.getPeerDescriptor()))
-            this.randomPeers!.addContact(
-                new DhtNodeRpcRemote(
-                    this.localPeerDescriptor!,
-                    removedContact.getPeerDescriptor(),
-                    toProtoRpcClient(new DhtNodeRpcClient(this.rpcCommunicator!.getRpcClientTransport())),
-                    this.config.serviceId,
-                    this.config.rpcRequestTimeout
-                )
-            )
         })
-        this.neighborList.on('newContact', (newContact: DhtNodeRpcRemote, activeContacts: DhtNodeRpcRemote[]) =>
-            this.emit('newContact', newContact.getPeerDescriptor(), activeContacts.map((c) => c.getPeerDescriptor()))
-        )
-        this.transport!.on('connected', (peerDescriptor: PeerDescriptor) => this.onTransportConnected(peerDescriptor))
-
+        this.transport!.on('connected', (peerDescriptor: PeerDescriptor) => {
+            this.peerManager!.onContactConnected(peerDescriptor)
+            this.router!.onNodeConnected(peerDescriptor)
+            this.emit('connected', peerDescriptor)
+        })
         this.transport!.on('disconnected', (peerDescriptor: PeerDescriptor, gracefulLeave: boolean) => {
-            this.onTransportDisconnected(peerDescriptor, gracefulLeave)
+            this.peerManager!.onContactDisconnected(getNodeIdFromPeerDescriptor(peerDescriptor), gracefulLeave)
+            this.router!.onNodeDisconnected(peerDescriptor)
+            this.emit('disconnected', peerDescriptor, gracefulLeave)
         })
-
-        this.transport!.getAllConnectionPeerDescriptors().forEach((peer) => {
-            const rpcRemote = new DhtNodeRpcRemote(
-                this.localPeerDescriptor!,
-                peer,
-                toProtoRpcClient(new DhtNodeRpcClient(this.rpcCommunicator!.getRpcClientTransport())),
-                this.config.serviceId,
-                this.config.rpcRequestTimeout
-            )
-            if (areEqualPeerDescriptors(peer, this.localPeerDescriptor!)) {
-                logger.error('own peerdescriptor added to connections in initKBucket')
-            }
-            this.connections.set(keyFromPeerDescriptor(peer), rpcRemote)
+        this.transport!.getConnections().forEach((peer) => {
+            this.peerManager!.onContactConnected(peer)
         })
-        this.randomPeers = new RandomContactList(selfId, this.config.maxNeighborListSize)
-        this.randomPeers.on('contactRemoved', (removedContact: DhtNodeRpcRemote, activeContacts: DhtNodeRpcRemote[]) =>
-            this.emit('randomContactRemoved', removedContact.getPeerDescriptor(), activeContacts.map((c) => c.getPeerDescriptor()))
-        )
-        this.randomPeers.on('newContact', (newContact: DhtNodeRpcRemote, activeContacts: DhtNodeRpcRemote[]) =>
-            this.emit('newRandomContact', newContact.getPeerDescriptor(), activeContacts.map((c) => c.getPeerDescriptor()))
-        )
-    }
-
-    private onTransportConnected(peerDescriptor: PeerDescriptor): void {
-
-        if (areEqualPeerDescriptors(this.localPeerDescriptor!, peerDescriptor)) {
-            logger.error('onTransportConnected() to self')
-        }
-
-        const rpcRemote = new DhtNodeRpcRemote(
-            this.localPeerDescriptor!,
-            peerDescriptor,
-            toProtoRpcClient(new DhtNodeRpcClient(this.rpcCommunicator!.getRpcClientTransport())),
-            this.config.serviceId,
-            this.config.rpcRequestTimeout
-        )
-        if (!this.connections.has(PeerID.fromValue(rpcRemote.id).toKey())) {
-            this.connections.set(PeerID.fromValue(rpcRemote.id).toKey(), rpcRemote)
-            logger.trace('connectionschange add ' + this.connections.size)
-        } else {
-            logger.trace('new connection not set to connections, there is already a connection with the peer ID')
-        }
-        logger.trace('connected: ' + getNodeIdFromPeerDescriptor(peerDescriptor) + ' ' + this.connections.size)
-        this.emit('connected', peerDescriptor)
-    }
-
-    private onTransportDisconnected(peerDescriptor: PeerDescriptor, gracefulLeave: boolean): void {
-        logger.trace('disconnected: ' + getNodeIdFromPeerDescriptor(peerDescriptor))
-        this.connections.delete(keyFromPeerDescriptor(peerDescriptor))
-        // only remove from bucket if we are on layer 0
-        if (this.connectionManager) {
-            this.bucket!.remove(peerDescriptor.kademliaId)
-
-            if (gracefulLeave === true) {
-                logger.trace(getNodeIdFromPeerDescriptor(peerDescriptor) + ' ' + 'onTransportDisconnected with gracefulLeave ' + gracefulLeave)
-                this.removeContact(peerDescriptor)
-            } else {
-                logger.trace(getNodeIdFromPeerDescriptor(peerDescriptor) + ' ' + 'onTransportDisconnected with gracefulLeave ' + gracefulLeave)
-            }
-        }
-
-        this.emit('disconnected', peerDescriptor, gracefulLeave)
     }
 
     private bindRpcLocalMethods(): void {
-        if (!this.started || this.stopped) {
+        if (!this.started || this.abortController.signal.aborted) {
             return
         }
         const dhtNodeRpcLocal = new DhtNodeRpcLocal({
-            bucket: this.bucket!,
-            serviceId: this.config.serviceId,
             peerDiscoveryQueryBatchSize: this.config.peerDiscoveryQueryBatchSize,
-            addNewContact: (contact: PeerDescriptor) => this.addNewContact(contact),
-            removeContact: (contact: PeerDescriptor) => this.removeContact(contact)
+            getClosestPeersTo: (nodeId: DhtAddress, limit: number) => {
+                return this.peerManager!.getClosestNeighborsTo(nodeId, limit)
+                    .map((dhtPeer: DhtNodeRpcRemote) => dhtPeer.getPeerDescriptor())
+            },
+            getClosestRingPeersTo: (ringIdRaw: RingIdRaw, limit: number) => {
+                return this.getClosestRingContactsTo(ringIdRaw, limit)
+            },
+            addContact: (contact: PeerDescriptor) => this.peerManager!.addContact([contact]),
+            removeContact: (nodeId: DhtAddress) => this.removeContact(nodeId)
         })
         this.rpcCommunicator!.registerRpcMethod(ClosestPeersRequest, ClosestPeersResponse, 'getClosestPeers',
             (req: ClosestPeersRequest, context) => dhtNodeRpcLocal.getClosestPeers(req, context))
+        this.rpcCommunicator!.registerRpcMethod(ClosestRingPeersRequest, ClosestRingPeersResponse, 'getClosestRingPeers',
+            (req: ClosestRingPeersRequest, context) => dhtNodeRpcLocal.getClosestRingPeers(req, context))
         this.rpcCommunicator!.registerRpcMethod(PingRequest, PingResponse, 'ping',
             (req: PingRequest, context) => dhtNodeRpcLocal.ping(req, context))
         this.rpcCommunicator!.registerRpcNotification(LeaveNotice, 'leaveNotice',
-            (req: LeaveNotice, context) => dhtNodeRpcLocal.leaveNotice(req, context))
+            (_req: LeaveNotice, context) => dhtNodeRpcLocal.leaveNotice(context))
         const externalApiRpcLocal = new ExternalApiRpcLocal({
-            startFind: (idToFind: Uint8Array, fetchData: boolean, excludedPeer: PeerDescriptor) => {
-                return this.startFind(idToFind, fetchData, excludedPeer)
+            executeRecursiveOperation: (key: DhtAddress, operation: RecursiveOperation, excludedPeer: DhtAddress) => {
+                return this.recursiveOperationManager!.execute(key, operation, excludedPeer)
             },
-            storeDataToDht: (key: Uint8Array, data: Any) => this.storeDataToDht(key, data)
+            storeDataToDht: (key: DhtAddress, data: Any, creator?: DhtAddress) => this.storeDataToDht(key, data, creator)
         })
         this.rpcCommunicator!.registerRpcMethod(
-            ExternalFindDataRequest,
-            ExternalFindDataResponse,
-            'externalFindData', 
-            (req: ExternalFindDataRequest, context: ServerCallContext) => externalApiRpcLocal.externalFindData(req, context),
-            { timeout: 10000 }
+            ExternalFetchDataRequest,
+            ExternalFetchDataResponse,
+            'externalFetchData',
+            (req: ExternalFetchDataRequest, context: ServerCallContext) => externalApiRpcLocal.externalFetchData(req, context),
+            { timeout: 10000 }  // TODO use config option or named constant?
         )
         this.rpcCommunicator!.registerRpcMethod(
             ExternalStoreDataRequest,
             ExternalStoreDataResponse,
             'externalStoreData',
-            (req: ExternalStoreDataRequest) => externalApiRpcLocal.externalStoreData(req),
-            { timeout: 10000 }
+            (req: ExternalStoreDataRequest, context: ServerCallContext) => externalApiRpcLocal.externalStoreData(req, context),
+            { timeout: 10000 }  // TODO use config option or named constant?
         )
     }
 
-    private isPeerCloserToIdThanSelf(peer1: PeerDescriptor, compareToId: PeerID): boolean {
-        const distance1 = this.bucket!.distance(peer1.kademliaId, compareToId.value)
-        const distance2 = this.bucket!.distance(this.localPeerDescriptor!.kademliaId, compareToId.value)
-        return distance1 < distance2
+    private handleMessageFromTransport(message: Message): void {
+        if (message.serviceId === this.config.serviceId) {
+            this.rpcCommunicator?.handleMessageFromPeer(message)
+        } 
     }
-
-    private handleMessage(message: Message): void {
+    
+    private handleMessageFromRouter(message: Message): void {
         if (message.serviceId === this.config.serviceId) {
-            logger.trace('callig this.handleMessageFromPeer ' + getNodeIdFromPeerDescriptor(message.sourceDescriptor!)
-                + ' ' + message.serviceId + ' ' + message.messageId)
             this.rpcCommunicator?.handleMessageFromPeer(message)
+        } else if (this.connectionManager?.handleIncomingMessage(message)) {
+            // message was handled by connectionManager
         } else {
-            logger.trace('emit "message" ' + getNodeIdFromPeerDescriptor(message.sourceDescriptor!)
-                + ' ' + message.serviceId + ' ' + message.messageId)
             this.emit('message', message)
         }
     }
 
     private generatePeerDescriptorCallBack(connectivityResponse: ConnectivityResponse) {
-        if (this.config.peerDescriptor) {
+        if (this.config.peerDescriptor !== undefined) {
             this.localPeerDescriptor = this.config.peerDescriptor
         } else {
-            this.localPeerDescriptor = createPeerDescriptor(connectivityResponse, this.config.peerId)
+            this.localPeerDescriptor = createPeerDescriptor(connectivityResponse, this.region!, this.config.nodeId)
         }
         return this.localPeerDescriptor
     }
 
-    private onKBucketPing(oldContacts: DhtNodeRpcRemote[], newContact: DhtNodeRpcRemote): void {
-        if (this.stopped) {
-            return
-        }
-        const sortingList: SortedContactList<DhtNodeRpcRemote> = new SortedContactList(this.getNodeId(), 100)
-        sortingList.addContacts(oldContacts)
-        const sortedContacts = sortingList.getAllContacts()
-        this.connectionManager?.weakUnlockConnection(sortedContacts[sortedContacts.length - 1].getPeerDescriptor())
-        this.bucket?.remove(sortedContacts[sortedContacts.length - 1].getPeerId().value)
-        this.bucket!.add(newContact)
-    }
-
-    private onKBucketRemoved(contact: DhtNodeRpcRemote): void {
-        if (this.stopped) {
-            return
-        }
-        this.connectionManager?.weakUnlockConnection(contact.getPeerDescriptor())
-        logger.trace(`Removed contact ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
-        if (this.bucket!.count() === 0
-            && !this.peerDiscovery!.isJoinOngoing()
-            && this.config.entryPoints
-            && this.config.entryPoints.length > 0
-        ) {
-            setImmediate(async () => {
-                await Promise.all(this.config.entryPoints!.map((entryPoint) => 
-                    this.peerDiscovery!.rejoinDht(entryPoint)
-                )) 
-            })
-        }
-    }
-
-    private onKBucketAdded(contact: DhtNodeRpcRemote): void {
-        if (this.stopped) {
-            return
-        }
-        if (!contact.getPeerId().equals(this.getNodeId())) {
-            // Important to lock here, before the ping result is known
-            this.connectionManager?.weakLockConnection(contact.getPeerDescriptor())
-            if (this.connections.has(contact.getPeerId().toKey())) {
-                logger.trace(`Added new contact ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
-            } else {    // open connection by pinging
-                logger.trace('starting ping ' + getNodeIdFromPeerDescriptor(contact.getPeerDescriptor()))
-                contact.ping().then((result) => {
-                    if (result) {
-                        logger.trace(`Added new contact ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
-                    } else {
-                        logger.trace('ping failed ' + getNodeIdFromPeerDescriptor(contact.getPeerDescriptor()))
-                        this.connectionManager?.weakUnlockConnection(contact.getPeerDescriptor())
-                        this.removeContact(contact.getPeerDescriptor())
-                        this.addClosestContactToBucket()
-                    }
-                    return
-                }).catch((_e) => {
-                    this.connectionManager?.weakUnlockConnection(contact.getPeerDescriptor())
-                    this.removeContact(contact.getPeerDescriptor())
-                    this.addClosestContactToBucket()
-                })
-            }
-        }
-    }
-
-    private addClosestContactToBucket(): void {
-        if (!this.started || this.stopped) {
-            return
-        }
-        const closest = this.getClosestActiveContactNotInBucket()
-        if (closest) {
-            this.addNewContact(closest.getPeerDescriptor())
-        }
+    public getClosestContacts(limit?: number): PeerDescriptor[] {
+        return this.peerManager!.getClosestContactsTo(
+            this.getNodeId(),
+            limit).map((peer) => peer.getPeerDescriptor())
     }
 
-    private getClosestActiveContactNotInBucket(): DhtNodeRpcRemote | undefined {
-        for (const contactId of this.neighborList!.getContactIds()) {
-            if (!this.bucket!.get(contactId.value) && this.neighborList!.isActive(contactId)) {
-                return this.neighborList!.getContact(contactId).contact
-            }
+    public getClosestRingContactsTo(ringIdRaw: RingIdRaw, limit?: number): RingContacts {
+        const closest = this.peerManager!.getClosestRingContactsTo(ringIdRaw, limit)
+        return {
+            left: closest.left.map((dhtPeer: DhtNodeRpcRemote) => dhtPeer.getPeerDescriptor()),
+            right: closest.right.map((dhtPeer: DhtNodeRpcRemote) => dhtPeer.getPeerDescriptor())
         }
-        return undefined
-    }
-
-    public getClosestContacts(maxCount?: number): PeerDescriptor[] {
-        return this.neighborList!.getClosestContacts(maxCount).map((c) => c.getPeerDescriptor())
     }
 
-    public getNodeId(): PeerID {
-        return peerIdFromPeerDescriptor(this.localPeerDescriptor!)
+    public getNodeId(): DhtAddress {
+        return getNodeIdFromPeerDescriptor(this.localPeerDescriptor!)
     }
 
-    public getBucketSize(): number {
-        return this.bucket!.count()
+    public getNeighborCount(): number {
+        return this.peerManager!.getNeighborCount()
     }
 
-    private addNewContact(contact: PeerDescriptor, setActive = false): void {
-        if (!this.started || this.stopped) {
+    public removeContact(nodeId: DhtAddress): void {
+        if (!this.started) {  // the stopped state is checked in PeerManager
             return
         }
-        if (!areEqualPeerDescriptors(contact, this.localPeerDescriptor!)) {
-            logger.trace(`Adding new contact ${getNodeIdFromPeerDescriptor(contact)}`)
-            const rpcRemote = new DhtNodeRpcRemote(
-                this.localPeerDescriptor!,
-                contact,
-                toProtoRpcClient(new DhtNodeRpcClient(this.rpcCommunicator!.getRpcClientTransport())),
-                this.config.serviceId,
-                this.config.rpcRequestTimeout
-            )
-            if (!this.bucket!.get(contact.kademliaId) && !this.neighborList!.getContact(peerIdFromPeerDescriptor(contact))) {
-                this.neighborList!.addContact(rpcRemote)
-                if (setActive) {
-                    const peerId = peerIdFromPeerDescriptor(contact)
-                    this.neighborList!.setActive(peerId)
-                }
-                this.bucket!.add(rpcRemote)
-            } else {
-                this.randomPeers!.addContact(rpcRemote)
-            }
-        }
-    }
-
-    private connectToEntryPoint(entryPoint: PeerDescriptor): void {
-        this.connectionManager!.lockConnection(entryPoint, 'temporary-layer0-connection')
-        this.entryPointDisconnectTimeout = setTimeout(() => {
-            this.connectionManager!.unlockConnection(entryPoint, 'temporary-layer0-connection')
-        }, 10 * 1000)
+        this.peerManager!.removeContact(nodeId)
     }
 
-    public removeContact(contact: PeerDescriptor): void {
-        if (!this.started || this.stopped) {
+    public async send(msg: Message): Promise<void> {
+        if (!this.started || this.abortController.signal.aborted) {
             return
         }
-        logger.trace(`Removing contact ${getNodeIdFromPeerDescriptor(contact)}`)
-        const peerId = peerIdFromPeerDescriptor(contact)
-        this.bucket!.remove(peerId.value)
-        this.neighborList!.removeContact(peerId)
-        this.randomPeers!.removeContact(peerId)
+        const reachableThrough = this.peerDiscovery!.isJoinOngoing() ? this.getConnectedEntryPoints() : []
+        this.router!.send(msg, reachableThrough)
     }
 
-    public async send(msg: Message): Promise<void> {
-        if (!this.started || this.stopped) {
-            return
-        }
-        const reachableThrough = this.peerDiscovery!.isJoinOngoing() ? this.config.entryPoints ?? [] : []
-        await this.router!.send(msg, reachableThrough)
+    private getConnectedEntryPoints(): PeerDescriptor[] {
+        return this.config.entryPoints !== undefined ? this.config.entryPoints.filter((entryPoint) =>
+            this.peerManager!.connections.has(getNodeIdFromPeerDescriptor(entryPoint))
+        ) : []
     }
 
-    public async joinDht(entryPointDescriptors: PeerDescriptor[], doAdditionalRandomPeerDiscovery?: boolean, retry?: boolean): Promise<void> {
+    public async joinDht(entryPointDescriptors: PeerDescriptor[], doAdditionalDistantPeerDiscovery?: boolean, retry?: boolean): Promise<void> {
         if (!this.started) {
             throw new Error('Cannot join DHT before calling start() on DhtNode')
         }
-        await Promise.all(entryPointDescriptors.map((entryPoint) => 
-            this.peerDiscovery!.joinDht(entryPoint, doAdditionalRandomPeerDiscovery, retry)
-        ))
+        await this.peerDiscovery!.joinDht(entryPointDescriptors, doAdditionalDistantPeerDiscovery, retry)
     }
 
-    public async startFind(idToFind: Uint8Array, fetchData?: boolean, excludedPeer?: PeerDescriptor): Promise<FindResult> {
-        return this.finder!.startFind(idToFind, fetchData, excludedPeer)
+    public async joinRing(): Promise<void> {
+        if (!this.started) {
+            throw new Error('Cannot join ring before calling start() on DhtNode')
+        }
+        await this.peerDiscovery!.joinRing()
     }
 
-    public async storeDataToDht(key: Uint8Array, data: Any): Promise<PeerDescriptor[]> {
-        if (this.peerDiscovery!.isJoinOngoing() && this.config.entryPoints && this.config.entryPoints.length > 0) {
-            return this.storeDataViaPeer(key, data, sample(this.config.entryPoints)!)
+    public async storeDataToDht(key: DhtAddress, data: Any, creator?: DhtAddress): Promise<PeerDescriptor[]> {
+        const connectedEntryPoints = this.getConnectedEntryPoints()
+        if (this.peerDiscovery!.isJoinOngoing() && connectedEntryPoints.length > 0) {
+            return this.storeDataToDhtViaPeer(key, data, sample(connectedEntryPoints)!)
         }
-        return this.storeRpcLocal!.storeDataToDht(key, data)
+        return this.storeManager!.storeDataToDht(key, data, creator ?? this.getNodeId())
     }
 
-    public async storeDataViaPeer(key: Uint8Array, data: Any, peer: PeerDescriptor): Promise<PeerDescriptor[]> {
+    public async storeDataToDhtViaPeer(key: DhtAddress, data: Any, peer: PeerDescriptor): Promise<PeerDescriptor[]> {
         const rpcRemote = new ExternalApiRpcRemote(
             this.localPeerDescriptor!,
             peer,
-            this.config.serviceId,
-            toProtoRpcClient(new ExternalApiRpcClient(this.rpcCommunicator!.getRpcClientTransport()))
+            this.rpcCommunicator!,
+            ExternalApiRpcClient
         )
         return await rpcRemote.storeData(key, data)
     }
 
-    public async getDataFromDht(idToFind: Uint8Array): Promise<DataEntry[]> {
-        if (this.peerDiscovery!.isJoinOngoing() && this.config.entryPoints && this.config.entryPoints.length > 0) {
-            return this.findDataViaPeer(idToFind, sample(this.config.entryPoints)!)
+    public async fetchDataFromDht(key: DhtAddress): Promise<DataEntry[]> {
+        const connectedEntryPoints = this.getConnectedEntryPoints()
+        if (this.peerDiscovery!.isJoinOngoing() && connectedEntryPoints.length > 0) {
+            return this.fetchDataFromDhtViaPeer(key, sample(connectedEntryPoints)!)
         }
-        const result = await this.finder!.startFind(idToFind, true)
-        return result.dataEntries ?? []
+        const result = await this.recursiveOperationManager!.execute(key, RecursiveOperation.FETCH_DATA)
+        return result.dataEntries ?? []  // TODO is this fallback needed?
     }
 
-    public async deleteDataFromDht(idToDelete: Uint8Array): Promise<void> {
-        if (!this.stopped) {
-            return this.storeRpcLocal!.deleteDataFromDht(idToDelete)
-        }
-    }
-
-    public async findDataViaPeer(idToFind: Uint8Array, peer: PeerDescriptor): Promise<DataEntry[]> {
+    public async fetchDataFromDhtViaPeer(key: DhtAddress, peer: PeerDescriptor): Promise<DataEntry[]> {
         const rpcRemote = new ExternalApiRpcRemote(
             this.localPeerDescriptor!,
             peer,
-            this.config.serviceId,
-            toProtoRpcClient(new ExternalApiRpcClient(this.rpcCommunicator!.getRpcClientTransport()))
+            this.rpcCommunicator!,
+            ExternalApiRpcClient
         )
-        return await rpcRemote.externalFindData(idToFind)
+        return await rpcRemote.externalFetchData(key)
+    }
+
+    public async deleteDataFromDht(key: DhtAddress, waitForCompletion: boolean): Promise<void> {
+        if (!this.abortController.signal.aborted) {
+            await this.recursiveOperationManager!.execute(key, RecursiveOperation.DELETE_DATA, undefined, waitForCompletion)
+        }
+    }
+
+    async findClosestNodesFromDht(key: DhtAddress): Promise<PeerDescriptor[]> {
+        const result = await this.recursiveOperationManager!.execute(key, RecursiveOperation.FIND_CLOSEST_NODES)
+        return result.closestNodes
     }
 
     public getTransport(): ITransport {
@@ -674,32 +547,38 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
         return this.localPeerDescriptor!
     }
 
-    public getAllConnectionPeerDescriptors(): PeerDescriptor[] {
-        return Array.from(this.connections.values()).map((peer) => peer.getPeerDescriptor())
+    public getConnections(): PeerDescriptor[] {
+        return Array.from(this.peerManager!.connections.values()).map((peer) => peer.getPeerDescriptor())
     }
 
-    public getKBucketPeers(): PeerDescriptor[] {
-        return this.bucket!.toArray().map((rpcRemote: DhtNodeRpcRemote) => rpcRemote.getPeerDescriptor())
+    public getNeighbors(): PeerDescriptor[] {
+        return this.started ? this.peerManager!.getNeighbors() : []
     }
 
-    public getNumberOfConnections(): number {
-        return this.connections.size
+    public getConnectionCount(): number {
+        return this.peerManager!.getConnectionCount()
     }
 
-    public getNumberOfLocalLockedConnections(): number {
-        return this.connectionManager!.getNumberOfLocalLockedConnections()
+    public getLocalLockedConnectionCount(): number {
+        return this.connectionManager!.getLocalLockedConnectionCount()
     }
 
-    public getNumberOfRemoteLockedConnections(): number {
-        return this.connectionManager!.getNumberOfRemoteLockedConnections()
+    public getRemoteLockedConnectionCount(): number {
+        return this.connectionManager!.getRemoteLockedConnectionCount()
     }
 
-    public getNumberOfWeakLockedConnections(): number {
-        return this.connectionManager!.getNumberOfWeakLockedConnections()
+    public getWeakLockedConnectionCount(): number {
+        return this.connectionManager!.getWeakLockedConnectionCount()
     }
 
     public async waitForNetworkConnectivity(): Promise<void> {
-        await waitForCondition(() => this.connections.size > 0, this.config.networkConnectivityTimeout)
+        await waitForCondition(() => {
+            if (!this.peerManager) {
+                return false
+            } else {
+                return (this.peerManager.getConnectionCount() > 0)
+            }
+        }, this.config.networkConnectivityTimeout, 100, this.abortController.signal)
     }
 
     public hasJoined(): boolean {
@@ -707,35 +586,35 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
     }
 
     public async stop(): Promise<void> {
-        if (this.stopped || !this.started) {
+        if (this.abortController.signal.aborted || !this.started) {
             return
         }
         logger.trace('stop()')
-        this.stopped = true
-
-        if (this.entryPointDisconnectTimeout) {
-            clearTimeout(this.entryPointDisconnectTimeout)
-        }
-        this.bucket!.toArray().forEach((rpcRemote: DhtNodeRpcRemote) => { 
-            rpcRemote.leaveNotice()
-            this.bucket!.remove(rpcRemote.id)
-        })
-        this.bucket!.removeAllListeners()
+        this.abortController.abort()
+        await this.storeManager!.destroy()
         this.localDataStore.clear()
-        this.neighborList!.stop()
-        this.randomPeers!.stop()
+        this.peerManager?.stop()
         this.rpcCommunicator!.stop()
         this.router!.stop()
-        this.finder!.stop()
+        this.recursiveOperationManager!.stop()
         this.peerDiscovery!.stop()
         if (this.config.transport === undefined) {
-            // if the transport was not given in config, the instance was created in start() and 
+            // if the transport was not given in config, the instance was created in start() and
             // this component is responsible for stopping it
             await this.transport!.stop()
         }
         this.transport = undefined
         this.connectionManager = undefined
-        this.connections.clear()
         this.removeAllListeners()
     }
+
+    private createDhtNodeRpcRemote(peerDescriptor: PeerDescriptor) {
+        return new DhtNodeRpcRemote(
+            this.localPeerDescriptor!,
+            peerDescriptor,
+            this.config.serviceId,
+            this.rpcCommunicator!,
+            this.config.rpcRequestTimeout
+        )
+    }
 }