@streamr/dht 100.0.0-testnet-one.4 → 100.0.0-testnet-two.0

package/src/dht/contact/SortedContactList.ts

@@ -1,27 +1,27 @@
-import { PeerID, PeerIDKey } from '../../helpers/PeerID'
 import { ContactState, Events } from './ContactList'
 import { sortedIndexBy } from 'lodash'
 import EventEmitter from 'eventemitter3'
 import { getDistance } from '../PeerManager'
+import { NodeID, areEqualNodeIds } from '../../helpers/nodeId'
 
 export interface SortedContactListConfig {
-    referenceId: PeerID  // all contacts in this list are in sorted by the distance to this ID
+    referenceId: NodeID  // all contacts in this list are in sorted by the distance to this ID
     allowToContainReferenceId: boolean
     // TODO could maybe optimize this by removing the flag and then we'd check whether we have 
     // any listeners before we emit the event
     emitEvents: boolean
     maxSize?: number
     // if set, the list can't contain any contacts which are futher away than this limit
-    peerIdDistanceLimit?: PeerID
+    nodeIdDistanceLimit?: NodeID
     // if set, the list can't contain contacts with these ids
-    excludedPeerIDs?: PeerID[]
+    excludedNodeIds?: Set<NodeID>
 }
 
-export class SortedContactList<C extends { getPeerId: () => PeerID }> extends EventEmitter<Events<C>> {
+export class SortedContactList<C extends { getNodeId: () => NodeID }> extends EventEmitter<Events<C>> {
 
     private config: SortedContactListConfig
-    private contactsById: Map<PeerIDKey, ContactState<C>> = new Map()
-    private contactIds: PeerID[] = []
+    private contactsById: Map<NodeID, ContactState<C>> = new Map()
+    private contactIds: NodeID[] = []
 
     constructor(
         config: SortedContactListConfig
@@ -31,38 +31,39 @@ export class SortedContactList<C extends { getPeerId: () => PeerID }> extends Ev
         this.compareIds = this.compareIds.bind(this)
     }
 
-    public getClosestContactId(): PeerID {
+    public getClosestContactId(): NodeID {
         return this.contactIds[0]
     }
 
-    public getContactIds(): PeerID[] {
+    public getContactIds(): NodeID[] {
         return this.contactIds
     }
 
     public addContact(contact: C): void {
-        if (this.config.excludedPeerIDs !== undefined
-            && this.config.excludedPeerIDs.some((peerId) => contact.getPeerId().equals(peerId))) {
+        if (this.config.excludedNodeIds !== undefined && this.config.excludedNodeIds.has(contact.getNodeId())) {
             return
         }
 
-        if ((!this.config.allowToContainReferenceId && this.config.referenceId.equals(contact.getPeerId())) ||
-            (this.config.peerIdDistanceLimit !== undefined && this.compareIds(this.config.peerIdDistanceLimit, contact.getPeerId()) < 0)) {
+        if ((!this.config.allowToContainReferenceId && areEqualNodeIds(this.config.referenceId, contact.getNodeId())) ||
+            (this.config.nodeIdDistanceLimit !== undefined && this.compareIds(this.config.nodeIdDistanceLimit, contact.getNodeId()) < 0)) {
             return
         }
-        if (!this.contactsById.has(contact.getPeerId().toKey())) {
+        if (!this.contactsById.has(contact.getNodeId())) {
             if ((this.config.maxSize === undefined) || (this.contactIds.length < this.config.maxSize)) {
-                this.contactsById.set(contact.getPeerId().toKey(), new ContactState(contact))
+                this.contactsById.set(contact.getNodeId(), new ContactState(contact))
 
-                const index = sortedIndexBy(this.contactIds, contact.getPeerId(), (id: PeerID) => { return this.distanceToReferenceId(id) })
-                this.contactIds.splice(index, 0, contact.getPeerId())
-            } else if (this.compareIds(this.contactIds[this.config.maxSize - 1], contact.getPeerId()) > 0) {
+                // eslint-disable-next-line max-len
+                const index = sortedIndexBy(this.contactIds, contact.getNodeId(), (id: NodeID) => { return this.distanceToReferenceId(id) })
+                this.contactIds.splice(index, 0, contact.getNodeId())
+            } else if (this.compareIds(this.contactIds[this.config.maxSize - 1], contact.getNodeId()) > 0) {
                 const removedId = this.contactIds.pop()
-                const removedContact = this.contactsById.get(removedId!.toKey())!.contact
-                this.contactsById.delete(removedId!.toKey())
-                this.contactsById.set(contact.getPeerId().toKey(), new ContactState(contact))
+                const removedContact = this.contactsById.get(removedId!)!.contact
+                this.contactsById.delete(removedId!)
+                this.contactsById.set(contact.getNodeId(), new ContactState(contact))
 
-                const index = sortedIndexBy(this.contactIds, contact.getPeerId(), (id: PeerID) => { return this.distanceToReferenceId(id) })
-                this.contactIds.splice(index, 0, contact.getPeerId())
+                // eslint-disable-next-line max-len
+                const index = sortedIndexBy(this.contactIds, contact.getNodeId(), (id: NodeID) => { return this.distanceToReferenceId(id) })
+                this.contactIds.splice(index, 0, contact.getNodeId())
                 if (this.config.emitEvents) {
                     this.emit(
                         'contactRemoved',
@@ -85,26 +86,26 @@ export class SortedContactList<C extends { getPeerId: () => PeerID }> extends Ev
         contacts.forEach((contact) => this.addContact(contact))
     }
 
-    public getContact(id: PeerID): ContactState<C> | undefined {
-        return this.contactsById.get(id.toKey())
+    public getContact(id: NodeID): ContactState<C> | undefined {
+        return this.contactsById.get(id)
     }
 
-    public setContacted(contactId: PeerID): void {
-        if (this.contactsById.has(contactId.toKey())) {
-            this.contactsById.get(contactId.toKey())!.contacted = true
+    public setContacted(contactId: NodeID): void {
+        if (this.contactsById.has(contactId)) {
+            this.contactsById.get(contactId)!.contacted = true
         }
     }
 
-    public setActive(contactId: PeerID): void {
-        if (this.contactsById.has(contactId.toKey())) {
-            this.contactsById.get(contactId.toKey())!.active = true
+    public setActive(contactId: NodeID): void {
+        if (this.contactsById.has(contactId)) {
+            this.contactsById.get(contactId)!.active = true
         }
     }
 
     public getClosestContacts(limit?: number): C[] {
         const ret: C[] = []
         this.contactIds.forEach((contactId) => {
-            const contact = this.contactsById.get(contactId.toKey())
+            const contact = this.contactsById.get(contactId)
             if (contact) {
                 ret.push(contact.contact)
             }
@@ -119,7 +120,7 @@ export class SortedContactList<C extends { getPeerId: () => PeerID }> extends Ev
     public getUncontactedContacts(num: number): C[] {
         const ret: C[] = []
         for (const contactId of this.contactIds) {
-            const contact = this.contactsById.get(contactId.toKey())
+            const contact = this.contactsById.get(contactId)
             if (contact && !contact.contacted) {
                 ret.push(contact.contact)
                 if (ret.length >= num) {
@@ -133,7 +134,7 @@ export class SortedContactList<C extends { getPeerId: () => PeerID }> extends Ev
     public getActiveContacts(limit?: number): C[] {
         const ret: C[] = []
         this.contactIds.forEach((contactId) => {
-            const contact = this.contactsById.get(contactId.toKey())
+            const contact = this.contactsById.get(contactId)
             if (contact && contact.active) {
                 ret.push(contact.contact)
             }
@@ -145,23 +146,25 @@ export class SortedContactList<C extends { getPeerId: () => PeerID }> extends Ev
         }
     }
 
-    public compareIds(id1: PeerID, id2: PeerID): number {
+    public compareIds(id1: NodeID, id2: NodeID): number {
         const distance1 = this.distanceToReferenceId(id1)
         const distance2 = this.distanceToReferenceId(id2)
         return distance1 - distance2
     }
 
     // TODO inline this method?
-    private distanceToReferenceId(id: PeerID): number {
-        return getDistance(this.config.referenceId.value, id.value)
+    private distanceToReferenceId(id: NodeID): number {
+        // TODO maybe this class should store the referenceId also as UInt8Array so that we don't need to convert it here?
+        return getDistance(this.config.referenceId, id)
     }
 
-    public removeContact(id: PeerID): boolean {
-        if (this.contactsById.has(id.toKey())) {
-            const removed = this.contactsById.get(id.toKey())!.contact
-            const index = this.contactIds.findIndex((element) => element.equals(id))
+    public removeContact(id: NodeID): boolean {
+        if (this.contactsById.has(id)) {
+            const removed = this.contactsById.get(id)!.contact
+            // TODO use sortedIndexBy?
+            const index = this.contactIds.findIndex((nodeId) => areEqualNodeIds(nodeId, id))
             this.contactIds.splice(index, 1)
-            this.contactsById.delete(id.toKey())
+            this.contactsById.delete(id)
             if (this.config.emitEvents) {
                 this.emit(
                     'contactRemoved',
@@ -174,12 +177,12 @@ export class SortedContactList<C extends { getPeerId: () => PeerID }> extends Ev
         return false
     }
 
-    public isActive(id: PeerID): boolean {
-        return this.contactsById.has(id.toKey()) ? this.contactsById.get(id.toKey())!.active : false
+    public isActive(id: NodeID): boolean {
+        return this.contactsById.has(id) ? this.contactsById.get(id)!.active : false
     }
 
     public getAllContacts(): C[] {
-        return this.contactIds.map((peerId) => this.contactsById.get(peerId.toKey())!.contact)
+        return this.contactIds.map((nodeId) => this.contactsById.get(nodeId)!.contact)
     }
 
     public getSize(): number {

package/src/dht/discovery/DiscoverySession.ts

@@ -1,11 +1,11 @@
 import { Logger, runAndWaitForEvents3 } from '@streamr/utils'
 import EventEmitter from 'eventemitter3'
 import { v4 } from 'uuid'
-import { PeerID, PeerIDKey } from '../../helpers/PeerID'
 import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
 import { PeerManager, getDistance } from '../PeerManager'
 import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
 import { getNodeIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
+import { NodeID, getNodeIdFromBinary } from '../../helpers/nodeId'
 
 const logger = new Logger(module)
 
@@ -18,18 +18,16 @@ interface DiscoverySessionConfig {
     parallelism: number
     noProgressLimit: number
     peerManager: PeerManager
-    // Note that contacted peers will be mutated by the DiscoverySession or other parallel sessions
-    contactedPeers: Set<PeerIDKey>
 }
 
 export class DiscoverySession {
-    public readonly sessionId = v4()
-
+    
+    public readonly id = v4()
     private stopped = false
     private emitter = new EventEmitter<DiscoverySessionEvents>()
-    private outgoingClosestPeersRequestsCounter = 0
     private noProgressCounter = 0
-    private ongoingClosestPeersRequests: Set<string> = new Set()
+    private ongoingClosestPeersRequests: Set<NodeID> = new Set()
+    private contactedPeers: Set<NodeID> = new Set()
     private readonly config: DiscoverySessionConfig
 
     constructor(config: DiscoverySessionConfig) {
@@ -48,23 +46,22 @@ export class DiscoverySession {
             return []
         }
         logger.trace(`Getting closest peers from contact: ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
-        this.outgoingClosestPeersRequestsCounter++
-        this.config.contactedPeers.add(contact.getPeerId().toKey())
+        this.contactedPeers.add(contact.getNodeId())
         const returnedContacts = await contact.getClosestPeers(this.config.targetId)
-        this.config.peerManager.handlePeerActive(contact.getPeerId())
+        this.config.peerManager.handlePeerActive(contact.getNodeId())
         return returnedContacts
     }
 
-    private onClosestPeersRequestSucceeded(peerId: PeerID, contacts: PeerDescriptor[]) {
-        if (!this.ongoingClosestPeersRequests.has(peerId.toKey())) {
+    private onClosestPeersRequestSucceeded(nodeId: NodeID, contacts: PeerDescriptor[]) {
+        if (!this.ongoingClosestPeersRequests.has(nodeId)) {
             return
         }
-        this.ongoingClosestPeersRequests.delete(peerId.toKey())
-        const oldClosestNeighbor = this.config.peerManager.getClosestNeighborsTo(this.config.targetId, 1)[0]
-        const oldClosestDistance = getDistance(this.config.targetId, oldClosestNeighbor.getPeerId().value)
+        this.ongoingClosestPeersRequests.delete(nodeId)
+        const oldClosestNeighbor = this.config.peerManager.getClosestNeighborsTo(getNodeIdFromBinary(this.config.targetId), 1)[0]
+        const oldClosestDistance = getDistance(getNodeIdFromBinary(this.config.targetId), oldClosestNeighbor.getNodeId())
         this.addNewContacts(contacts)
-        const newClosestNeighbor = this.config.peerManager.getClosestNeighborsTo(this.config.targetId, 1)[0]
-        const newClosestDistance = getDistance(this.config.targetId, newClosestNeighbor.getPeerId().value)
+        const newClosestNeighbor = this.config.peerManager.getClosestNeighborsTo(getNodeIdFromBinary(this.config.targetId), 1)[0]
+        const newClosestDistance = getDistance(getNodeIdFromBinary(this.config.targetId), newClosestNeighbor.getNodeId())
         if (newClosestDistance >= oldClosestDistance) {
             this.noProgressCounter++
         } else {
@@ -73,18 +70,22 @@ export class DiscoverySession {
     }
 
     private onClosestPeersRequestFailed(peer: DhtNodeRpcRemote) {
-        if (!this.ongoingClosestPeersRequests.has(peer.getPeerId().toKey())) {
+        if (!this.ongoingClosestPeersRequests.has(peer.getNodeId())) {
             return
         }
-        this.ongoingClosestPeersRequests.delete(peer.getPeerId().toKey())
-        this.config.peerManager.handlePeerUnresponsive(peer.getPeerId())
+        this.ongoingClosestPeersRequests.delete(peer.getNodeId())
+        this.config.peerManager.handlePeerUnresponsive(peer.getNodeId())
     }
 
     private findMoreContacts(): void {
         if (this.stopped) {
             return
         }
-        const uncontacted = this.config.peerManager.getClosestContactsTo(this.config.targetId, this.config.parallelism, this.config.contactedPeers)
+        const uncontacted = this.config.peerManager.getClosestContactsTo(
+            getNodeIdFromBinary(this.config.targetId),
+            this.config.parallelism,
+            this.contactedPeers
+        )
         if (uncontacted.length === 0 || this.noProgressCounter >= this.config.noProgressLimit) {
             this.emitter.emit('discoveryCompleted')
             this.stopped = true
@@ -94,20 +95,19 @@ export class DiscoverySession {
             if (this.ongoingClosestPeersRequests.size >= this.config.parallelism) {
                 break
             }
-            this.ongoingClosestPeersRequests.add(nextPeer.getPeerId().toKey())
+            this.ongoingClosestPeersRequests.add(nextPeer.getNodeId())
             // eslint-disable-next-line promise/catch-or-return
             this.getClosestPeersFromContact(nextPeer)
-                .then((contacts) => this.onClosestPeersRequestSucceeded(nextPeer.getPeerId(), contacts))
+                .then((contacts) => this.onClosestPeersRequestSucceeded(nextPeer.getNodeId(), contacts))
                 .catch(() => this.onClosestPeersRequestFailed(nextPeer))
                 .finally(() => {
-                    this.outgoingClosestPeersRequestsCounter--
                     this.findMoreContacts()
                 })
         }
     }
 
     public async findClosestNodes(timeout: number): Promise<void> {
-        if (this.config.peerManager.getNumberOfContacts(this.config.contactedPeers) === 0) {
+        if (this.config.peerManager.getNumberOfContacts(this.contactedPeers) === 0) {
             return
         }
         // TODO add abortController and signal it in stop()

package/src/dht/discovery/PeerDiscovery.ts

@@ -7,7 +7,6 @@ import { ConnectionManager } from '../../connection/ConnectionManager'
 import { PeerManager } from '../PeerManager'
 import { createRandomNodeId } from '../../helpers/nodeId'
 import { ServiceID } from '../../types/ServiceID'
-import { PeerIDKey } from '../../helpers/PeerID'
 
 interface PeerDiscoveryConfig {
     localPeerDescriptor: PeerDescriptor
@@ -24,40 +23,19 @@ const logger = new Logger(module)
 
 export class PeerDiscovery {
 
-    private readonly config: PeerDiscoveryConfig
     private ongoingDiscoverySessions: Map<string, DiscoverySession> = new Map()
     private rejoinOngoing = false
     private joinCalled = false
-    private rejoinTimeoutRef?: NodeJS.Timeout
     private readonly abortController: AbortController
     private recoveryIntervalStarted = false
+    private readonly config: PeerDiscoveryConfig
 
     constructor(config: PeerDiscoveryConfig) {
         this.config = config
         this.abortController = new AbortController()
     }
 
-    async joinDht(
-        entryPoints: PeerDescriptor[],
-        doAdditionalRandomPeerDiscovery = true,
-        retry = true
-    ): Promise<void> {
-        const contactedPeers = new Set<PeerIDKey>()
-        await Promise.all(entryPoints.map((entryPoint) => this.joinThroughEntryPoint(
-            entryPoint,
-            contactedPeers,
-            doAdditionalRandomPeerDiscovery,
-            retry
-        )))
-    }
-
-    async joinThroughEntryPoint(
-        entryPointDescriptor: PeerDescriptor,
-        // Note that this set is mutated by DiscoverySession
-        contactedPeers: Set<PeerIDKey>,
-        doAdditionalRandomPeerDiscovery = true,
-        retry = true
-    ): Promise<void> {
+    async joinDht(entryPointDescriptor: PeerDescriptor, doAdditionalRandomPeerDiscovery = true, retry = true): Promise<void> {
         if (this.isStopped()) {
             return
         }
@@ -72,22 +50,21 @@ export class PeerDiscovery {
         this.config.connectionManager?.lockConnection(entryPointDescriptor, `${this.config.serviceId}::joinDht`)
         this.config.peerManager.handleNewPeers([entryPointDescriptor])
         const targetId = peerIdFromPeerDescriptor(this.config.localPeerDescriptor).value
-        const sessions = [this.createSession(targetId, contactedPeers)]
+        const sessions = [this.createSession(targetId)]
         if (doAdditionalRandomPeerDiscovery) {
-            sessions.push(this.createSession(createRandomNodeId(), contactedPeers))
+            sessions.push(this.createSession(createRandomNodeId()))
         }
         await this.runSessions(sessions, entryPointDescriptor, retry)
         this.config.connectionManager?.unlockConnection(entryPointDescriptor, `${this.config.serviceId}::joinDht`)
 
     }
 
-    private createSession(targetId: Uint8Array, contactedPeers: Set<PeerIDKey>): DiscoverySession {
+    private createSession(targetId: Uint8Array): DiscoverySession {
         const sessionOptions = {
             targetId,
             parallelism: this.config.parallelism,
             noProgressLimit: this.config.joinNoProgressLimit,
-            peerManager: this.config.peerManager,
-            contactedPeers
+            peerManager: this.config.peerManager
         }
         return new DiscoverySession(sessionOptions)
     }
@@ -95,7 +72,7 @@ export class PeerDiscovery {
     private async runSessions(sessions: DiscoverySession[], entryPointDescriptor: PeerDescriptor, retry: boolean): Promise<void> {
         try {
             for (const session of sessions) {
-                this.ongoingDiscoverySessions.set(session.sessionId, session)
+                this.ongoingDiscoverySessions.set(session.id, session)
                 await session.findClosestNodes(this.config.joinTimeout)
             }
         } catch (_e) {
@@ -105,13 +82,14 @@ export class PeerDiscovery {
                 if (this.config.peerManager.getNumberOfNeighbors() === 0) {
                     if (retry) {
                         // TODO should we catch possible promise rejection?
+                        // TODO use config option or named constant?
                         setAbortableTimeout(() => this.rejoinDht(entryPointDescriptor), 1000, this.abortController.signal)
                     }
                 } else {
                     await this.ensureRecoveryIntervalIsRunning()
                 }
             }
-            sessions.forEach((session) => this.ongoingDiscoverySessions.delete(session.sessionId))
+            sessions.forEach((session) => this.ongoingDiscoverySessions.delete(session.id))
         }
     }
 
@@ -122,12 +100,13 @@ export class PeerDiscovery {
         logger.debug(`Rejoining DHT ${this.config.serviceId}`)
         this.rejoinOngoing = true
         try {
-            await this.joinThroughEntryPoint(entryPoint, new Set())
+            await this.joinDht(entryPoint)
             logger.debug(`Rejoined DHT successfully ${this.config.serviceId}!`)
         } catch (err) {
             logger.warn(`Rejoining DHT ${this.config.serviceId} failed`)
             if (!this.isStopped()) {
                 // TODO should we catch possible promise rejection?
+                // TODO use config option or named constant?
                 setAbortableTimeout(() => this.rejoinDht(entryPoint), 5000, this.abortController.signal)
             }
         } finally {
@@ -138,6 +117,7 @@ export class PeerDiscovery {
     private async ensureRecoveryIntervalIsRunning(): Promise<void> {
         if (!this.recoveryIntervalStarted) {
             this.recoveryIntervalStarted = true
+            // TODO use config option or named constant?
             await scheduleAtInterval(() => this.fetchClosestPeersFromBucket(), 60000, true, this.abortController.signal)
         }
     }
@@ -146,7 +126,10 @@ export class PeerDiscovery {
         if (this.isStopped()) {
             return
         }
-        const nodes = this.config.peerManager.getClosestNeighborsTo(this.config.localPeerDescriptor.nodeId, this.config.parallelism)
+        const nodes = this.config.peerManager.getClosestNeighborsTo(
+            getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor),
+            this.config.parallelism
+        )
         await Promise.allSettled(
             nodes.map(async (peer: DhtNodeRpcRemote) => {
                 const contacts = await peer.getClosestPeers(this.config.localPeerDescriptor.nodeId!)
@@ -169,10 +152,6 @@ export class PeerDiscovery {
 
     public stop(): void {
         this.abortController.abort()
-        if (this.rejoinTimeoutRef) {
-            clearTimeout(this.rejoinTimeoutRef)
-            this.rejoinTimeoutRef = undefined
-        }
         this.ongoingDiscoverySessions.forEach((session) => {
             session.stop()
         })