@streamr/dht 100.0.0-pretestnet.4 → 100.0.0-rc.0

package/src/dht/DhtNodeRpcLocal.ts

@@ -1,26 +1,27 @@
 import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
 import { Logger } from '@streamr/utils'
-import KBucket from 'k-bucket'
-import { getNodeIdFromPeerDescriptor } from '../helpers/peerIdFromPeerDescriptor'
 import { Empty } from '../proto/google/protobuf/empty'
 import {
     ClosestPeersRequest,
     ClosestPeersResponse,
-    LeaveNotice,
+    ClosestRingPeersRequest,
+    ClosestRingPeersResponse,
     PeerDescriptor,
     PingRequest,
     PingResponse
 } from '../proto/packages/dht/protos/DhtRpc'
 import { IDhtNodeRpc } from '../proto/packages/dht/protos/DhtRpc.server'
 import { DhtCallContext } from '../rpc-protocol/DhtCallContext'
-import { DhtNodeRpcRemote } from './DhtNodeRpcRemote'
+import { DhtAddress, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor } from '../identifiers'
+import { RingIdRaw } from './contact/ringIdentifiers'
+import { RingContacts } from './contact/RingContactList'
 
 interface DhtNodeRpcLocalConfig {
-    bucket: KBucket<DhtNodeRpcRemote>
-    serviceId: string
     peerDiscoveryQueryBatchSize: number
-    addNewContact: (contact: PeerDescriptor) => void
-    removeContact: (contact: PeerDescriptor) => void
+    getClosestPeersTo: (nodeId: DhtAddress, limit: number) => PeerDescriptor[]
+    getClosestRingPeersTo: (id: RingIdRaw, limit: number) => RingContacts
+    addContact: (contact: PeerDescriptor) => void
+    removeContact: (nodeId: DhtAddress) => void
 }
 
 const logger = new Logger(module)
@@ -34,23 +35,29 @@ export class DhtNodeRpcLocal implements IDhtNodeRpc {
     }
 
     async getClosestPeers(request: ClosestPeersRequest, context: ServerCallContext): Promise<ClosestPeersResponse> {
-        this.config.addNewContact((context as DhtCallContext).incomingSourceDescriptor!)
+        this.config.addContact((context as DhtCallContext).incomingSourceDescriptor!)
         const response = {
-            peers: this.getClosestPeerDescriptors(request.kademliaId, this.config.peerDiscoveryQueryBatchSize),
+            peers: this.config.getClosestPeersTo(getDhtAddressFromRaw(request.nodeId), this.config.peerDiscoveryQueryBatchSize),
             requestId: request.requestId
         }
         return response
     }
 
-    private getClosestPeerDescriptors(kademliaId: Uint8Array, limit: number): PeerDescriptor[] {
-        const closestPeers = this.config.bucket.closest(kademliaId, limit)
-        return closestPeers.map((rpcRemote: DhtNodeRpcRemote) => rpcRemote.getPeerDescriptor())
+    async getClosestRingPeers(request: ClosestRingPeersRequest, context: ServerCallContext): Promise<ClosestRingPeersResponse> {
+        this.config.addContact((context as DhtCallContext).incomingSourceDescriptor!)
+        const closestPeers = this.config.getClosestRingPeersTo(request.ringId as RingIdRaw, this.config.peerDiscoveryQueryBatchSize)
+        const response = {
+            leftPeers: closestPeers.left,
+            rightPeers: closestPeers.right,
+            requestId: request.requestId
+        }
+        return response
     }
 
     async ping(request: PingRequest, context: ServerCallContext): Promise<PingResponse> {
         logger.trace('received ping request: ' + getNodeIdFromPeerDescriptor((context as DhtCallContext).incomingSourceDescriptor!))
         setImmediate(() => {
-            this.config.addNewContact((context as DhtCallContext).incomingSourceDescriptor!)
+            this.config.addContact((context as DhtCallContext).incomingSourceDescriptor!)
         })
         const response: PingResponse = {
             requestId: request.requestId
@@ -58,11 +65,12 @@ export class DhtNodeRpcLocal implements IDhtNodeRpc {
         return response
     }
 
-    async leaveNotice(_request: LeaveNotice, context: ServerCallContext): Promise<Empty> {
+    async leaveNotice(context: ServerCallContext): Promise<Empty> {
         // TODO check signature??
         const sender = (context as DhtCallContext).incomingSourceDescriptor!
-        logger.trace('received leave notice: ' + getNodeIdFromPeerDescriptor(sender))
-        this.config.removeContact(sender)
+        const senderNodeId = getNodeIdFromPeerDescriptor(sender)
+        logger.trace('received leave notice: ' + senderNodeId)
+        this.config.removeContact(senderNodeId)
         return {}
     }
 }

package/src/dht/DhtNodeRpcRemote.ts

@@ -1,61 +1,80 @@
-import { IDhtNodeRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
+import { RpcCommunicator } from '@streamr/proto-rpc'
+import { Logger } from '@streamr/utils'
+import { v4 } from 'uuid'
+import { DhtAddress, DhtAddressRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../identifiers'
 import {
     ClosestPeersRequest,
-    LeaveNotice,
+    ClosestRingPeersRequest,
     PeerDescriptor,
     PingRequest
 } from '../proto/packages/dht/protos/DhtRpc'
-import { v4 } from 'uuid'
-import { Logger } from '@streamr/utils'
-import { ProtoRpcClient } from '@streamr/proto-rpc'
-import { Remote } from './contact/Remote'
-import { PeerID } from '../helpers/PeerID'
-import { getNodeIdFromPeerDescriptor, peerIdFromPeerDescriptor } from '../helpers/peerIdFromPeerDescriptor'
+import { DhtNodeRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
 import { ServiceID } from '../types/ServiceID'
+import { RpcRemote } from './contact/RpcRemote'
+import { DhtCallContext } from '../rpc-protocol/DhtCallContext'
+import { RingIdRaw } from './contact/ringIdentifiers'
+import { RingContacts } from './contact/RingContactList'
 
 const logger = new Logger(module)
 
 // Fields required by objects stored in the k-bucket library
 export interface KBucketContact {
-    id: Uint8Array
+    id: DhtAddressRaw
     vectorClock: number
 }
 
-export class DhtNodeRpcRemote extends Remote<IDhtNodeRpcClient> implements KBucketContact {
+export class DhtNodeRpcRemote extends RpcRemote<DhtNodeRpcClient> implements KBucketContact {
 
     private static counter = 0
     public vectorClock: number
-    public readonly id: Uint8Array
+    public readonly id: DhtAddressRaw
+    private readonly serviceId: ServiceID
 
     constructor(
         localPeerDescriptor: PeerDescriptor,
         peerDescriptor: PeerDescriptor,
-        client: ProtoRpcClient<IDhtNodeRpcClient>,
         serviceId: ServiceID,
+        rpcCommunicator: RpcCommunicator<DhtCallContext>,
         rpcRequestTimeout?: number
     ) {
-        super(localPeerDescriptor, peerDescriptor, serviceId, client, rpcRequestTimeout)
-        this.id = this.getPeerId().value
+        super(localPeerDescriptor, peerDescriptor, rpcCommunicator, DhtNodeRpcClient, rpcRequestTimeout)
+        this.id = this.getPeerDescriptor().nodeId
         this.vectorClock = DhtNodeRpcRemote.counter++
+        this.serviceId = serviceId
     }
 
-    async getClosestPeers(kademliaId: Uint8Array): Promise<PeerDescriptor[]> {
-        logger.trace(`Requesting getClosestPeers on ${this.getServiceId()} from ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())}`)
+    async getClosestPeers(nodeId: DhtAddress): Promise<PeerDescriptor[]> {
+        logger.trace(`Requesting getClosestPeers on ${this.serviceId} from ${this.getNodeId()}`)
         const request: ClosestPeersRequest = {
-            kademliaId,
+            nodeId: getRawFromDhtAddress(nodeId),
             requestId: v4()
         }
         try {
             const peers = await this.getClient().getClosestPeers(request, this.formDhtRpcOptions())
             return peers.peers
         } catch (err) {
-            logger.trace(`getClosestPeers error ${this.getServiceId()}`, { err })
+            logger.trace(`getClosestPeers error ${this.serviceId}`, { err })
+            throw err
+        }
+    }
+
+    async getClosestRingPeers(ringIdRaw: RingIdRaw): Promise<RingContacts> {
+        logger.trace(`Requesting getClosestRingPeers on ${this.serviceId} from ${this.getNodeId()}`)
+        const request: ClosestRingPeersRequest = {
+            ringId: ringIdRaw,
+            requestId: v4()
+        }
+        try {
+            const response = await this.getClient().getClosestRingPeers(request, this.formDhtRpcOptions())
+            return { left: response.leftPeers ?? [], right: response.rightPeers ?? [] }
+        } catch (err) {
+            logger.trace(`getClosestRingPeers error ${this.serviceId}`, { err })
             throw err
         }
     }
 
     async ping(): Promise<boolean> {
-        logger.trace(`Requesting ping on ${this.getServiceId()} from ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())}`)
+        logger.trace(`Requesting ping on ${this.serviceId} from ${this.getNodeId()}`)
         const request: PingRequest = {
             requestId: v4()
         }
@@ -66,25 +85,22 @@ export class DhtNodeRpcRemote extends Remote<IDhtNodeRpcClient> implements KBuck
                 return true
             }
         } catch (err) {
-            logger.trace(`ping failed on ${this.getServiceId()} to ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())}: ${err}`)
+            logger.trace(`ping failed on ${this.serviceId} to ${this.getNodeId()}: ${err}`)
         }
         return false
     }
 
     leaveNotice(): void {
-        logger.trace(`Sending leaveNotice on ${this.getServiceId()} from ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())}`)
-        const request: LeaveNotice = {
-            serviceId: this.getServiceId()
-        }
+        logger.trace(`Sending leaveNotice on ${this.serviceId} from ${this.getNodeId()}`)
         const options = this.formDhtRpcOptions({
             notification: true
         })
-        this.getClient().leaveNotice(request, options).catch((e) => {
+        this.getClient().leaveNotice({}, options).catch((e) => {
             logger.trace('Failed to send leaveNotice' + e)
         })
     }
 
-    getPeerId(): PeerID {
-        return peerIdFromPeerDescriptor(this.getPeerDescriptor())
+    getNodeId(): DhtAddress {
+        return getNodeIdFromPeerDescriptor(this.getPeerDescriptor())
     }
 }

package/src/dht/ExternalApiRpcLocal.ts

@@ -1,19 +1,30 @@
 import { IExternalApiRpc } from '../proto/packages/dht/protos/DhtRpc.server'
 import {
-    ExternalFindDataRequest,
-    ExternalFindDataResponse,
+    ExternalFetchDataRequest,
+    ExternalFetchDataResponse,
     ExternalStoreDataRequest,
     ExternalStoreDataResponse,
+    RecursiveOperation,
     PeerDescriptor
 } from '../proto/packages/dht/protos/DhtRpc'
 import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
 import { DhtCallContext } from '../rpc-protocol/DhtCallContext'
-import { FindResult } from './find/Finder'
+import { RecursiveOperationResult } from './recursive-operation/RecursiveOperationManager'
 import { Any } from '../proto/google/protobuf/any'
+import { DhtAddress, getNodeIdFromPeerDescriptor } from '../identifiers'
+import { getDhtAddressFromRaw } from '../identifiers'
 
 interface ExternalApiRpcLocalConfig {
-    startFind: (idToFind: Uint8Array, fetchData: boolean, excludedPeer: PeerDescriptor) => Promise<FindResult>
-    storeDataToDht: (key: Uint8Array, data: Any) => Promise<PeerDescriptor[]>
+    executeRecursiveOperation: (
+        targetId: DhtAddress,
+        operation: RecursiveOperation,
+        excludedPeer: DhtAddress
+    ) => Promise<RecursiveOperationResult>
+    storeDataToDht: (
+        key: DhtAddress,
+        data: Any,
+        creator: DhtAddress
+    ) => Promise<PeerDescriptor[]>
 }
 
 export class ExternalApiRpcLocal implements IExternalApiRpc {
@@ -24,21 +35,23 @@ export class ExternalApiRpcLocal implements IExternalApiRpc {
         this.config = config
     }
 
-    async externalFindData(findDataRequest: ExternalFindDataRequest, context: ServerCallContext): Promise<ExternalFindDataResponse> {
+    async externalFetchData(request: ExternalFetchDataRequest, context: ServerCallContext): Promise<ExternalFetchDataResponse> {
         const senderPeerDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
-        const result = await this.config.startFind(findDataRequest.kademliaId, true, senderPeerDescriptor)
-        if (result.dataEntries) {
-            return ExternalFindDataResponse.create({ dataEntries: result.dataEntries })
-        } else {
-            return ExternalFindDataResponse.create({ 
-                dataEntries: [],
-                error: 'Could not find data with the given key' 
-            })
-        }
+        const result = await this.config.executeRecursiveOperation(
+            getDhtAddressFromRaw(request.key),
+            RecursiveOperation.FETCH_DATA,
+            getNodeIdFromPeerDescriptor(senderPeerDescriptor)
+        )
+        return ExternalFetchDataResponse.create({ entries: result.dataEntries ?? [] })
     }
 
-    async externalStoreData(request: ExternalStoreDataRequest): Promise<ExternalStoreDataResponse> {
-        const result = await this.config.storeDataToDht(request.key, request.data!)
+    async externalStoreData(request: ExternalStoreDataRequest, context: ServerCallContext): Promise<ExternalStoreDataResponse> {
+        const senderPeerDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
+        const result = await this.config.storeDataToDht(
+            getDhtAddressFromRaw(request.key),
+            request.data!,
+            getNodeIdFromPeerDescriptor(senderPeerDescriptor)
+        )
         return ExternalStoreDataResponse.create({
             storers: result
         })

package/src/dht/ExternalApiRpcRemote.ts

@@ -1,31 +1,34 @@
+import { DhtAddress, getRawFromDhtAddress } from '../identifiers'
 import { Any } from '../proto/google/protobuf/any'
-import { DataEntry, ExternalStoreDataRequest, ExternalFindDataRequest, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
-import { IExternalApiRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
-import { Remote } from './contact/Remote'
+import { DataEntry, ExternalFetchDataRequest, ExternalStoreDataRequest, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
+import { ExternalApiRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
+import { RpcRemote } from './contact/RpcRemote'
 
-export class ExternalApiRpcRemote extends Remote<IExternalApiRpcClient> {
+export class ExternalApiRpcRemote extends RpcRemote<ExternalApiRpcClient> {
 
-    async externalFindData(idToFind: Uint8Array): Promise<DataEntry[]> {
-        const request: ExternalFindDataRequest = {
-            kademliaId: idToFind
+    async externalFetchData(key: DhtAddress): Promise<DataEntry[]> {
+        const request: ExternalFetchDataRequest = {
+            key: getRawFromDhtAddress(key)
         }
         const options = this.formDhtRpcOptions({
+            // TODO use config option or named constant?
             timeout: 10000
         })
         try {
-            const data = await this.getClient().externalFindData(request, options)
-            return data.dataEntries
+            const data = await this.getClient().externalFetchData(request, options)
+            return data.entries
         } catch (err) {
             return []
         }
     }
 
-    async storeData(key: Uint8Array, data: Any): Promise<PeerDescriptor[]> {
+    async storeData(key: DhtAddress, data: Any): Promise<PeerDescriptor[]> {
         const request: ExternalStoreDataRequest = {
-            key,
+            key: getRawFromDhtAddress(key),
             data
         }
         const options = this.formDhtRpcOptions({
+            // TODO use config option or named constant?
             timeout: 10000
         })
         try {