@streamr/dht 0.0.1-tatum.0

package/src/dht/DhtPeer.ts

@@ -0,0 +1,96 @@
+import { IDhtRpcServiceClient } from '../proto/packages/dht/protos/DhtRpc.client'
+import {
+    ClosestPeersRequest,
+    LeaveNotice,
+    PeerDescriptor,
+    PingRequest
+} from '../proto/packages/dht/protos/DhtRpc'
+import { v4 } from 'uuid'
+import { DhtRpcOptions } from '../rpc-protocol/DhtRpcOptions'
+import { Logger } from '@streamr/utils'
+import { ProtoRpcClient } from '@streamr/proto-rpc'
+import { Remote } from './contact/Remote'
+
+const logger = new Logger(module)
+
+// Fields required by objects stored in the k-bucket library
+export interface KBucketContact {
+    id: Uint8Array
+    vectorClock: number
+}
+
+export class DhtPeer extends Remote<IDhtRpcServiceClient> implements KBucketContact {
+    private static counter = 0
+    public vectorClock: number
+    public readonly id: Uint8Array
+    constructor(
+        ownPeerDescriptor: PeerDescriptor,
+        peerDescriptor: PeerDescriptor,
+        client: ProtoRpcClient<IDhtRpcServiceClient>,
+        serviceId: string
+    ) {
+        super(ownPeerDescriptor, peerDescriptor, client, serviceId)
+        this.id = this.peerId.value
+        this.vectorClock = DhtPeer.counter++
+    }
+
+    async getClosestPeers(kademliaId: Uint8Array): Promise<PeerDescriptor[]> {
+        logger.trace(`Requesting getClosestPeers on ${this.serviceId} from ${this.peerId.toKey()}`)
+        const request: ClosestPeersRequest = {
+            kademliaId: kademliaId,
+            requestId: v4()
+        }
+        const options: DhtRpcOptions = {
+            sourceDescriptor: this.ownPeerDescriptor,
+            targetDescriptor: this.peerDescriptor
+        }
+        try {
+            const peers = await this.client.getClosestPeers(request, options)
+            return peers.peers
+        } catch (err) {
+            logger.trace(`getClosestPeers error ${this.serviceId}`, { err })
+            throw err
+        }
+    }
+
+    async ping(): Promise<boolean> {
+        logger.trace(`Requesting ping on ${this.serviceId} from ${this.peerId.toKey()}`)
+        const request: PingRequest = {
+            requestId: v4()
+        }
+        const options: DhtRpcOptions = {
+            sourceDescriptor: this.ownPeerDescriptor,
+            targetDescriptor: this.peerDescriptor,
+            timeout: 10000
+        }
+        try {
+            const pong = await this.client.ping(request, options)
+            if (pong.requestId === request.requestId) {
+                return true
+            }
+        } catch (err) {
+            logger.debug(`ping failed on ${this.serviceId} to ${this.peerId.toKey()}: ${err}`)
+        }
+        return false
+    }
+
+    leaveNotice(): void {
+        logger.trace(`Sending leaveNotice on ${this.serviceId} from ${this.peerId.toKey()}`)
+        const request: LeaveNotice = {
+            serviceId: this.serviceId
+        }
+        const options: DhtRpcOptions = {
+            sourceDescriptor: this.ownPeerDescriptor,
+            targetDescriptor: this.peerDescriptor,
+            notification: true
+        }
+        this.client.leaveNotice(request, options).catch((e) => {
+            logger.trace('Failed to send leaveNotice' + e)
+        })
+    }
+
+    getPeerDescriptor(): PeerDescriptor {
+        return this.peerDescriptor
+    }
+
+}

package/src/dht/ExternalApi.ts

@@ -0,0 +1,29 @@
+import { DhtNode } from '../exports'
+import { FindDataRequest, FindDataResponse, FindMode } from '../proto/packages/dht/protos/DhtRpc'
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+
+export class ExternalApi {
+
+    private readonly dhtNode: DhtNode
+
+    constructor(dhtNode: DhtNode) {
+        this.dhtNode = dhtNode
+        const rpcCommunicator = this.dhtNode.getRpcCommunicator!()
+        rpcCommunicator.registerRpcMethod(FindDataRequest, FindDataResponse, 'findData', 
+            (req: FindDataRequest, context) => this.findData(req, context), { timeout: 15000 })
+    }
+
+    // IDHTRpcService method for external findRecursive calls
+    async findData(findDataRequest: FindDataRequest, _context: ServerCallContext): Promise<FindDataResponse> {
+        const result = await this.dhtNode.startRecursiveFind(findDataRequest.kademliaId, FindMode.DATA, findDataRequest.requestor)
+        if (result.dataEntries) {
+            return FindDataResponse.create({ dataEntries: result.dataEntries })
+        } else {
+            return FindDataResponse.create({ 
+                dataEntries: [],
+                error: 'Could not find data with the given key' 
+            })
+        }
+    }
+
+}

package/src/dht/RemoteExternalApi.ts

@@ -0,0 +1,25 @@
+import { DhtRpcOptions } from '../exports'
+import { DataEntry, FindDataRequest } from '../proto/packages/dht/protos/DhtRpc'
+import { IExternalApiServiceClient } from '../proto/packages/dht/protos/DhtRpc.client'
+import { Remote } from './contact/Remote'
+
+export class RemoteExternalApi extends Remote<IExternalApiServiceClient> {
+
+    async findData(idToFind: Uint8Array): Promise<DataEntry[]> {
+        const request: FindDataRequest = {
+            kademliaId: idToFind,
+            requestor: this.ownPeerDescriptor,
+        }
+        const options: DhtRpcOptions = {
+            sourceDescriptor: this.ownPeerDescriptor,
+            targetDescriptor: this.peerDescriptor,
+            timeout: 10000
+        }
+        try {
+            const data = await this.client.findData(request, options)
+            return data.dataEntries
+        } catch (err) {
+            return []
+        }
+    }
+}

package/src/dht/contact/Contact.ts

@@ -0,0 +1,36 @@
+import { PeerID } from '../../helpers/PeerID'
+import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
+import { peerIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
+
+export class ContactState<TContact> {
+    public contacted = false
+    public active = false
+    public contact: TContact
+
+    constructor(contact: TContact) {
+        this.contact = contact
+    }
+}
+
+export interface IContact { getPeerId: () => PeerID, getPeerDescriptor: () => PeerDescriptor }
+
+export interface Events {
+    contactRemoved: (removedDescriptor: PeerDescriptor, closestDescriptors: PeerDescriptor[]) => void
+    newContact: (newDescriptor: PeerDescriptor, closestDescriptors: PeerDescriptor[]) => void
+}
+
+export class Contact implements IContact {
+    private peerDescriptor: PeerDescriptor
+
+    constructor(peerDescriptor: PeerDescriptor) {
+        this.peerDescriptor = peerDescriptor
+    }
+
+    public getPeerDescriptor(): PeerDescriptor {
+        return this.peerDescriptor
+    }
+
+    public getPeerId(): PeerID {
+        return peerIdFromPeerDescriptor(this.peerDescriptor)
+    }
+}

package/src/dht/contact/RandomContactList.ts

@@ -0,0 +1,92 @@
+import { Events, IContact, ContactState } from './Contact'
+
+import EventEmitter from 'eventemitter3'
+import { PeerID, PeerIDKey } from '../../helpers/PeerID'
+export class RandomContactList<Contact extends IContact> extends EventEmitter<Events> {
+    private contactsById: Map<PeerIDKey, ContactState<Contact>> = new Map()
+    private contactIds: PeerID[] = []
+    private ownId: PeerID
+    private maxSize: number
+    private randomness = 0.20
+    private getContactsLimit = 20
+
+    constructor(
+        ownId: PeerID,
+        maxSize: number,
+        randomness = 0.20,
+        getContactsLimit = 20
+    ) {
+        super()
+        this.ownId = ownId
+        this.maxSize = maxSize
+        this.randomness = randomness
+        this.getContactsLimit = getContactsLimit
+    }
+
+    addContact(contact: Contact): void {
+        if (this.ownId.equals(contact.getPeerId())) {
+            return
+        }
+        if (!this.contactsById.has(contact.getPeerId().toKey())) {
+            const roll = Math.random()
+            if (roll < this.randomness) {
+                if (this.getSize() === this.maxSize && this.getSize() > 0) {
+                    const toRemove = this.contactIds[0]
+                    this.removeContact(toRemove)
+                }
+                this.contactIds.push(contact.getPeerId())
+                this.contactsById.set(contact.getPeerId().toKey(), new ContactState(contact))
+                this.emit(
+                    'newContact',
+                    contact.getPeerDescriptor(),
+                    this.getContacts().map((contact: Contact) => contact.getPeerDescriptor())
+                )
+            }
+        }
+    }
+
+    addContacts(contacts: Contact[]): void {
+        contacts.forEach((contact) => this.addContact(contact))
+    }
+
+    removeContact(id: PeerID): boolean {
+        if (this.contactsById.has(id.toKey())) {
+            const removedDescriptor = this.contactsById.get(id.toKey())!.contact.getPeerDescriptor()
+            const index = this.contactIds.indexOf(id)
+            this.contactIds.splice(index, 1)
+            this.contactsById.delete(id.toKey())
+            this.emit('contactRemoved', removedDescriptor, this.getContacts().map((contact: Contact) => contact.getPeerDescriptor()))
+            return true
+        }
+        return false
+    }
+
+    public getSize(): number {
+        return this.contactIds.length
+    }
+
+    public getContact(id: PeerID): ContactState<Contact> {
+        return this.contactsById.get(id.toKey())!
+    }
+
+    public getContacts(limit = this.getContactsLimit): Contact[] {
+        const ret: Contact[] = []
+        this.contactIds.forEach((contactId) => {
+            const contact = this.contactsById.get(contactId.toKey())
+            if (contact) {
+                ret.push(contact.contact)
+            }
+        })
+        return ret.splice(0, limit)
+    }
+
+    public clear(): void {
+        this.contactsById.clear()
+        this.contactIds = []
+    }
+
+    public stop(): void {
+        this.removeAllListeners()
+        this.clear()
+    }
+}

package/src/dht/contact/Remote.ts

@@ -0,0 +1,40 @@
+import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
+import { ProtoRpcClient } from '@streamr/proto-rpc'
+import { peerIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
+import { PeerID } from '../../helpers/PeerID'
+import { IContact } from './Contact'
+
+export abstract class Remote<T> implements IContact {
+
+    protected readonly peerId: PeerID
+    protected readonly peerDescriptor: PeerDescriptor
+    protected readonly client: ProtoRpcClient<T>
+    protected readonly serviceId: string
+    protected readonly ownPeerDescriptor: PeerDescriptor
+
+    constructor(
+        ownPeerDescriptor: PeerDescriptor,
+        peerDescriptor: PeerDescriptor,
+        client: ProtoRpcClient<T>,
+        serviceId: string
+    ) {
+        this.ownPeerDescriptor = ownPeerDescriptor
+        this.peerId = peerIdFromPeerDescriptor(peerDescriptor)
+        this.peerDescriptor = peerDescriptor
+        this.client = client
+        this.serviceId = serviceId
+    }
+
+    getPeerId(): PeerID {
+        return this.peerId
+    }
+
+    getPeerDescriptor(): PeerDescriptor {
+        return this.peerDescriptor
+    }
+
+    getServiceId(): string {
+        return this.serviceId
+    }
+
+}

package/src/dht/contact/SortedContactList.ts

@@ -0,0 +1,196 @@
+import KBucket from 'k-bucket'
+import { PeerID, PeerIDKey } from '../../helpers/PeerID'
+import EventEmitter from 'eventemitter3'
+import { Events, IContact, ContactState } from './Contact'
+
+export class SortedContactList<Contact extends IContact> extends EventEmitter<Events> {
+    private contactsById: Map<PeerIDKey, ContactState<Contact>> = new Map()
+    private contactIds: PeerID[] = []
+    private ownId: PeerID
+    private maxSize: number
+    private getContactsLimit: number
+    private allowOwnPeerId: boolean
+    private peerIdDistanceLimit?: PeerID
+    private excludedPeerIDs?: PeerID[]
+
+    constructor(
+        ownId: PeerID,
+        maxSize: number,
+        getContactsLimit = 20,
+        allowOwnPeerId = false,
+        peerIdDistanceLimit?: PeerID,
+        excludedPeerIDs?: PeerID[]
+    ) {
+        super()
+        this.compareIds = this.compareIds.bind(this)
+        this.ownId = ownId
+        this.maxSize = maxSize
+        this.getContactsLimit = getContactsLimit
+        this.allowOwnPeerId = allowOwnPeerId
+        this.peerIdDistanceLimit = peerIdDistanceLimit
+        this.excludedPeerIDs = excludedPeerIDs
+    }
+
+    public getClosestContactId(): PeerID {
+        return this.contactIds[0]
+    }
+
+    public getContactIds(): PeerID[] {
+        return this.contactIds
+    }
+
+    public addContact(contact: Contact): void {
+        if (this.excludedPeerIDs
+            && this.excludedPeerIDs.some((peerId) => contact.getPeerId().equals(peerId))) {
+            return
+        }
+        
+        if ((!this.allowOwnPeerId && this.ownId.equals(contact.getPeerId())) ||
+            (this.peerIdDistanceLimit !== undefined && this.compareIds(this.peerIdDistanceLimit, contact.getPeerId()) < 0)) {
+            return
+        }
+        if (!this.contactsById.has(contact.getPeerId().toKey())) {
+            if (this.contactIds.length < this.maxSize) {
+                this.contactsById.set(contact.getPeerId().toKey(), new ContactState(contact))
+                this.contactIds.push(contact.getPeerId())
+                this.contactIds.sort(this.compareIds)
+            } else if (this.compareIds(this.contactIds[this.maxSize - 1], contact.getPeerId()) > 0) {
+                const removed = this.contactIds.pop()
+                this.contactsById.delete(removed!.toKey())
+                this.contactsById.set(contact.getPeerId().toKey(), new ContactState(contact))
+                this.contactIds.push(contact.getPeerId())
+                this.contactIds.sort(this.compareIds)
+                this.emit(
+                    'contactRemoved',
+                    contact.getPeerDescriptor(),
+                    this.getClosestContacts().map((contact: Contact) => contact.getPeerDescriptor())
+                )
+            }
+            this.emit(
+                'newContact',
+                contact.getPeerDescriptor(),
+                this.getClosestContacts().map((contact: Contact) => contact.getPeerDescriptor())
+            )
+        }
+    }
+
+    public addContacts(contacts: Contact[]): void {
+        contacts.forEach((contact) => this.addContact(contact))
+    }
+
+    public setContacted(contactId: PeerID): void {
+        if (this.contactsById.has(contactId.toKey())) {
+            this.contactsById.get(contactId.toKey())!.contacted = true
+        }
+    }
+
+    public setActive(contactId: PeerID): void {
+        if (this.contactsById.has(contactId.toKey())) {
+            this.contactsById.get(contactId.toKey())!.active = true
+        }
+    }
+
+    public getClosestContacts(limit = this.getContactsLimit): Contact[] {
+        const ret: Contact[] = []
+        this.contactIds.forEach((contactId) => {
+            const contact = this.contactsById.get(contactId.toKey())
+            if (contact) {
+                ret.push(contact.contact)
+            }
+        })
+        return ret.splice(0, limit)
+    }
+
+    public getUncontactedContacts(num: number): Contact[] {
+        const ret: Contact[] = []
+        for (const contactId of this.contactIds) {
+            const contact = this.contactsById.get(contactId.toKey())
+            if (contact && !contact.contacted) {
+                ret.push(contact.contact)
+                if (ret.length >= num) {
+                    return ret
+                }
+            }
+        }
+        return ret
+    }
+
+    public getActiveContacts(limit?: number): Contact[] {
+        const ret: Contact[] = []
+        this.contactIds.forEach((contactId) => {
+            const contact = this.contactsById.get(contactId.toKey())
+            if (contact && contact.active) {
+                ret.push(contact.contact)
+            }
+        })
+        if (limit !== undefined) {
+            return ret.splice(0, limit)
+        } else {
+            return ret
+        }
+    }
+
+    public compareIds(id1: PeerID, id2: PeerID): number {
+        const distance1 = KBucket.distance(this.ownId.value, id1.value)
+        const distance2 = KBucket.distance(this.ownId.value, id2.value)
+        return distance1 - distance2
+    }
+
+    public getStringIds(): string[] {
+        return this.contactIds.map((peerId) => peerId.toKey())
+    }
+
+    public getSize(): number {
+        return this.contactIds.length
+    }
+
+    public getContact(id: PeerID): ContactState<Contact> {
+        return this.contactsById.get(id.toKey())!
+    }
+
+    public removeContact(id: PeerID): boolean {
+        if (this.contactsById.has(id.toKey())) {
+            const removedDescriptor = this.contactsById.get(id.toKey())!.contact.getPeerDescriptor()
+            const index = this.contactIds.indexOf(id)
+            this.contactIds.splice(index, 1)
+            this.contactsById.delete(id.toKey())
+            this.emit(
+                'contactRemoved',
+                removedDescriptor,
+                this.getClosestContacts().map((contact: Contact) => contact.getPeerDescriptor())
+            )
+            return true
+        }
+        return false
+    }
+
+    public hasContact(id: PeerID): boolean {
+        return this.contactsById.has(id.toKey())
+    }
+
+    public isActive(id: PeerID): boolean {
+        return this.contactsById.has(id.toKey()) ? this.contactsById.get(id.toKey())!.active : false
+    }
+
+    public getAllContacts(): Contact[] {
+        return this.contactIds.map((peerId) => this.contactsById.get(peerId.toKey())!.contact)
+    }
+
+    public getMaxSize(): number {
+        return this.maxSize
+    }
+
+    public setAllAsUncontacted(): void {
+        this.contactsById.forEach((contact) => contact.contacted = false)
+    }
+
+    public clear(): void {
+        this.contactsById.clear()
+        this.contactIds = []
+    }
+
+    public stop(): void {
+        this.removeAllListeners()
+        this.clear()
+    }
+}

package/src/dht/discovery/DiscoverySession.ts

@@ -0,0 +1,150 @@
+import { RpcCommunicator, toProtoRpcClient } from '@streamr/proto-rpc'
+import { Logger, runAndWaitForEvents3 } from '@streamr/utils'
+import EventEmitter from 'eventemitter3'
+import KBucket from 'k-bucket'
+import { v4 } from 'uuid'
+import { PeerID } from '../../helpers/PeerID'
+import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
+import { DhtRpcServiceClient } from '../../proto/packages/dht/protos/DhtRpc.client'
+import { SortedContactList } from '../contact/SortedContactList'
+import { DhtPeer } from '../DhtPeer'
+import { peerIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
+
+const logger = new Logger(module)
+
+interface DiscoverySessionEvents {
+    discoveryCompleted: () => void
+}
+
+interface DiscoverySessionConfig {
+    bucket: KBucket<DhtPeer>
+    neighborList: SortedContactList<DhtPeer>
+    targetId: Uint8Array
+    ownPeerDescriptor: PeerDescriptor
+    serviceId: string
+    rpcCommunicator: RpcCommunicator
+    parallelism: number
+    noProgressLimit: number
+    newContactListener?: (dhtPeer: DhtPeer) => void
+    nodeName?: string
+}
+
+export class DiscoverySession {
+    public readonly sessionId = v4()
+
+    private stopped = false
+    private emitter = new EventEmitter<DiscoverySessionEvents>()
+    private outgoingClosestPeersRequestsCounter = 0
+    private noProgressCounter = 0
+    private ongoingClosestPeersRequests: Set<string> = new Set()
+    private readonly config: DiscoverySessionConfig
+    private readonly ownPeerId: PeerID
+
+    constructor(config: DiscoverySessionConfig) {
+        this.config = config
+        this.ownPeerId = peerIdFromPeerDescriptor(config.ownPeerDescriptor)
+    }
+
+    private addNewContacts(contacts: PeerDescriptor[]): void {
+        if (this.stopped) {
+            return
+        }
+        contacts.forEach((contact) => {
+            const dhtPeer = new DhtPeer(
+                this.config.ownPeerDescriptor,
+                contact,
+                toProtoRpcClient(new DhtRpcServiceClient(this.config.rpcCommunicator!.getRpcClientTransport())),
+                this.config.serviceId
+            )
+            if (!dhtPeer.getPeerId().equals(this.ownPeerId!)) {
+                if (this.config.newContactListener) {
+                    this.config.newContactListener(dhtPeer)
+                }
+                if (!this.config.neighborList.getContact(dhtPeer.getPeerId())) {
+                    this.config.neighborList!.addContact(dhtPeer)
+                }
+            }
+        })
+    }
+
+    private async getClosestPeersFromContact(contact: DhtPeer): Promise<PeerDescriptor[]> {
+        if (this.stopped) {
+            return []
+        }
+        logger.trace(`Getting closest peers from contact: ${contact.getPeerId().toKey()}`)
+        this.outgoingClosestPeersRequestsCounter++
+        this.config.neighborList!.setContacted(contact.getPeerId())
+        const returnedContacts = await contact.getClosestPeers(this.config.targetId)
+        this.config.neighborList!.setActive(contact.getPeerId())
+        return returnedContacts
+    }
+
+    private onClosestPeersRequestSucceeded(peerId: PeerID, contacts: PeerDescriptor[]) {
+        if (!this.ongoingClosestPeersRequests.has(peerId.toKey())) {
+            return
+        }
+        this.ongoingClosestPeersRequests.delete(peerId.toKey())
+        const oldClosestContact = this.config.neighborList!.getClosestContactId()
+        this.addNewContacts(contacts)
+        if (this.config.neighborList!.getClosestContactId().equals(oldClosestContact)) {
+            this.noProgressCounter++
+        } else {
+            this.noProgressCounter = 0
+        }
+    }
+
+    private onClosestPeersRequestFailed(peer: DhtPeer, _exception: Error) {
+        if (!this.ongoingClosestPeersRequests.has(peer.getPeerId().toKey())) {
+            return
+        }
+        this.ongoingClosestPeersRequests.delete(peer.getPeerId().toKey())
+        this.config.bucket.remove(peer.getPeerId().value)
+        this.config.neighborList!.removeContact(peer.getPeerId())
+    }
+
+    private findMoreContacts(): void {
+        if (this.stopped) {
+            return
+        }
+        const uncontacted = this.config.neighborList!.getUncontactedContacts(this.config.parallelism)
+        if (uncontacted.length < 1 || this.noProgressCounter >= this.config.noProgressLimit) {
+            this.emitter.emit('discoveryCompleted')
+            this.stopped = true
+            return
+        }
+        for (const nextPeer of uncontacted) {
+            if (this.ongoingClosestPeersRequests.size >= this.config.parallelism) {
+                break
+            }
+            this.ongoingClosestPeersRequests.add(nextPeer!.getPeerId().toKey())
+            // eslint-disable-next-line promise/catch-or-return
+            this.getClosestPeersFromContact(nextPeer!)
+                .then((contacts) => this.onClosestPeersRequestSucceeded(nextPeer!.getPeerId(), contacts))
+                .catch((err) => this.onClosestPeersRequestFailed(nextPeer!, err))
+                .finally(() => {
+                    this.outgoingClosestPeersRequestsCounter--
+                    this.findMoreContacts()
+                })
+        }
+    }
+
+    public async findClosestNodes(timeout: number): Promise<SortedContactList<DhtPeer>> {
+        if (this.config.neighborList!.getUncontactedContacts(this.config.parallelism).length < 1) {
+            logger.trace('getUncontactedContacts length was 0 in beginning of discovery, this.neighborList.size: '
+                + this.config.neighborList.getSize())
+            return this.config.neighborList
+        }
+        await runAndWaitForEvents3<DiscoverySessionEvents>(
+            [this.findMoreContacts.bind(this)],
+            [[this.emitter, 'discoveryCompleted']],
+            timeout
+        )
+        return this.config.neighborList
+    }
+
+    public stop(): void {
+        this.stopped = true
+        this.emitter.emit('discoveryCompleted')
+        this.emitter.removeAllListeners()
+    }
+}