@streamr/dht 0.0.1-tatum.4 → 0.0.1-tatum.5

package/src/connection/ConnectivityChecker.ts

@@ -69,14 +69,18 @@ export class ConnectivityChecker {
                 }, ConnectivityChecker.CONNECTIVITY_CHECKER_TIMEOUT)
                 const listener = (bytes: Uint8Array) => {
                     outgoingConnection.close('OTHER')
-                    const message: Message = Message.fromBinary(bytes)
-                    if (message.body.oneofKind === 'connectivityResponse') {
-                        const connectivityResponseMessage = message.body.connectivityResponse
-                        outgoingConnection!.off('data', listener)
-                        clearTimeout(timeoutId)
-                        resolve(connectivityResponseMessage)
-                    } else {
-                        return
+                    try {
+                        const message: Message = Message.fromBinary(bytes)
+                        if (message.body.oneofKind === 'connectivityResponse') {
+                            const connectivityResponseMessage = message.body.connectivityResponse
+                            outgoingConnection!.off('data', listener)
+                            clearTimeout(timeoutId)
+                            resolve(connectivityResponseMessage)
+                        } else {
+                            return
+                        }
+                    } catch (err) {
+                        logger.trace(`Could not parse message: ${err}`)
                     }
                 }
                 outgoingConnection!.on('data', listener)
@@ -97,16 +101,21 @@ export class ConnectivityChecker {
     public listenToIncomingConnectivityRequests(connectionToListenTo: ServerWebSocket): void {
         connectionToListenTo.on('data', (data: Uint8Array) => {
             logger.trace('server received data')
-            const message = Message.fromBinary(data)
-            if (message.body.oneofKind === 'connectivityRequest') {
-                logger.trace('received connectivity request')
-                this.handleIncomingConnectivityRequest(connectionToListenTo, message.body.connectivityRequest).then(() => {
-                    logger.trace('handleIncomingConnectivityRequest ok')
-                    return
-                }).catch((e) => {
-                    logger.error('handleIncomingConnectivityRequest' + e)
-                })
+            try {
+                const message = Message.fromBinary(data)
+                if (message.body.oneofKind === 'connectivityRequest') {
+                    logger.trace('received connectivity request')
+                    this.handleIncomingConnectivityRequest(connectionToListenTo, message.body.connectivityRequest).then(() => {
+                        logger.trace('handleIncomingConnectivityRequest ok')
+                        return
+                    }).catch((e) => {
+                        logger.error('handleIncomingConnectivityRequest' + e)
+                    })
+                }
+            } catch (err) {
+                logger.trace(`Could not parse message: ${err}`)
             }
+            
         })
     }
 

package/src/connection/Handshaker.ts

@@ -32,21 +32,26 @@ export class Handshaker extends EventEmitter<HandshakerEvents> {
     }
 
     private onData = (data: Uint8Array) => {
-        const message = Message.fromBinary(data)
-        if (message.body.oneofKind === 'handshakeRequest') {
-            logger.trace('handshake request received')
-            const handshake = message.body.handshakeRequest
-            this.emit('handshakeRequest', handshake.peerDescriptor!)
-        }
-        if (message.body.oneofKind === 'handshakeResponse') {
-            logger.trace('handshake response received')
-            const handshake = message.body.handshakeResponse
-            if (handshake.responseError) {
-                this.emit('handshakeFailed', handshake.responseError)
-            } else {
-                this.emit('handshakeCompleted', handshake.peerDescriptor!)
+        try {
+            const message = Message.fromBinary(data)
+            if (message.body.oneofKind === 'handshakeRequest') {
+                logger.trace('handshake request received')
+                const handshake = message.body.handshakeRequest
+                this.emit('handshakeRequest', handshake.peerDescriptor!)
+            }
+            if (message.body.oneofKind === 'handshakeResponse') {
+                logger.trace('handshake response received')
+                const handshake = message.body.handshakeResponse
+                if (handshake.responseError) {
+                    this.emit('handshakeFailed', handshake.responseError)
+                } else {
+                    this.emit('handshakeCompleted', handshake.peerDescriptor!)
+                }
             }
+        } catch (err) {
+            logger.error('error while parsing handshake message', err)
         }
+        
     }
 
     public sendHandshakeRequest(): void {

package/src/connection/WebSocket/ServerWebSocket.ts

@@ -67,7 +67,6 @@ export class ServerWebSocket extends EventEmitter<ConnectionEvents> implements I
         this.socket = undefined
 
         this.emit('disconnected', disconnectionType, reasonCode, description)
-        this.removeAllListeners()
     }
 
     public send(data: Uint8Array): void {

package/src/dht/DhtNode.ts

@@ -18,7 +18,6 @@ import {
     FindMode,
     DataEntry,
 } from '../proto/packages/dht/protos/DhtRpc'
-import * as Err from '../helpers/errors'
 import { DisconnectionType, ITransport, TransportEvents } from '../transport/ITransport'
 import { ConnectionManager, ConnectionManagerConfig, PortRange, TlsCertificate } from '../connection/ConnectionManager'
 import { DhtRpcServiceClient, ExternalApiServiceClient } from '../proto/packages/dht/protos/DhtRpc.client'
@@ -40,10 +39,11 @@ import { DataStore } from './store/DataStore'
 import { PeerDiscovery } from './discovery/PeerDiscovery'
 import { LocalDataStore } from './store/LocalDataStore'
 import { IceServer } from '../connection/WebRTC/WebRtcConnector'
-import { registerExternalApiRpcMethod } from './registerExternalApiRpcMethod'
+import { registerExternalApiRpcMethods } from './registerExternalApiRpcMethods'
 import { RemoteExternalApi } from './RemoteExternalApi'
 import { UUID } from '../exports'
 import { isNodeJS } from '../helpers/browser/isNodeJS'
+import { sample } from 'lodash'
 
 export interface DhtNodeEvents {
     newContact: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
@@ -299,7 +299,7 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
                 return this.bucket!.closest(id, n)
             }
         })
-        registerExternalApiRpcMethod(this)
+        registerExternalApiRpcMethods(this)
         if (this.connectionManager! && this.config.entryPoints && this.config.entryPoints.length > 0 
             && !isSamePeerDescriptor(this.config.entryPoints[0], this.ownPeerDescriptor!)) {
             this.connectToEntryPoint(this.config.entryPoints[0])
@@ -640,9 +640,22 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
     }
 
     public async storeDataToDht(key: Uint8Array, data: Any): Promise<PeerDescriptor[]> {
+        if (this.isJoinOngoing() && this.config.entryPoints && this.config.entryPoints.length > 0) {
+            return this.storeDataViaPeer(key, data, sample(this.config.entryPoints)!)
+        }
         return this.dataStore!.storeDataToDht(key, data)
     }
 
+    public async storeDataViaPeer(key: Uint8Array, data: Any, peer: PeerDescriptor): Promise<PeerDescriptor[]> {
+        const target = new RemoteExternalApi(
+            this.ownPeerDescriptor!,
+            peer,
+            this.config.serviceId,
+            toProtoRpcClient(new ExternalApiServiceClient(this.rpcCommunicator!.getRpcClientTransport()))
+        )
+        return await target.storeData(key, data)
+    }
+
     public async getDataFromDht(idToFind: Uint8Array): Promise<RecursiveFindResult> {
         return this.recursiveFinder!.startRecursiveFind(idToFind, FindMode.DATA)
     }
@@ -657,8 +670,8 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
         const target = new RemoteExternalApi(
             this.ownPeerDescriptor!,
             peer,
-            toProtoRpcClient(new ExternalApiServiceClient(this.rpcCommunicator!.getRpcClientTransport())),
-            this.config.serviceId
+            this.config.serviceId,
+            toProtoRpcClient(new ExternalApiServiceClient(this.rpcCommunicator!.getRpcClientTransport()))
         )
         return await target.findData(idToFind)
     }
@@ -720,13 +733,10 @@ export class DhtNode extends EventEmitter<Events> implements ITransport {
     }
 
     public async stop(): Promise<void> {
-        if (this.stopped) {
+        if (this.stopped || !this.started) {
             return
         }
         logger.trace('stop()')
-        if (!this.started) {
-            throw new Err.CouldNotStop('Cannot not stop() before start()')
-        }
         this.stopped = true
 
         if (this.entryPointDisconnectTimeout) {

package/src/dht/DhtPeer.ts

@@ -6,7 +6,6 @@ import {
     PingRequest
 } from '../proto/packages/dht/protos/DhtRpc'
 import { v4 } from 'uuid'
-import { DhtRpcOptions } from '../rpc-protocol/DhtRpcOptions'
 import { Logger } from '@streamr/utils'
 import { ProtoRpcClient } from '@streamr/proto-rpc'
 import { Remote } from './contact/Remote'
@@ -20,77 +19,66 @@ export interface KBucketContact {
 }
 
 export class DhtPeer extends Remote<IDhtRpcServiceClient> implements KBucketContact {
+
     private static counter = 0
     public vectorClock: number
     public readonly id: Uint8Array
+
     constructor(
         ownPeerDescriptor: PeerDescriptor,
         peerDescriptor: PeerDescriptor,
         client: ProtoRpcClient<IDhtRpcServiceClient>,
         serviceId: string
     ) {
-        super(ownPeerDescriptor, peerDescriptor, client, serviceId)
-        this.id = this.peerId.value
+        super(ownPeerDescriptor, peerDescriptor, serviceId, client)
+        this.id = this.getPeerId().value
         this.vectorClock = DhtPeer.counter++
     }
 
     async getClosestPeers(kademliaId: Uint8Array): Promise<PeerDescriptor[]> {
-        logger.trace(`Requesting getClosestPeers on ${this.serviceId} from ${this.peerId.toKey()}`)
+        logger.trace(`Requesting getClosestPeers on ${this.getServiceId()} from ${this.getPeerId().toKey()}`)
         const request: ClosestPeersRequest = {
             kademliaId,
             requestId: v4()
         }
-        const options: DhtRpcOptions = {
-            sourceDescriptor: this.ownPeerDescriptor,
-            targetDescriptor: this.peerDescriptor
-        }
         try {
-            const peers = await this.client.getClosestPeers(request, options)
+            const peers = await this.getClient().getClosestPeers(request, this.formDhtRpcOptions())
             return peers.peers
         } catch (err) {
-            logger.trace(`getClosestPeers error ${this.serviceId}`, { err })
+            logger.trace(`getClosestPeers error ${this.getServiceId()}`, { err })
             throw err
         }
     }
 
     async ping(): Promise<boolean> {
-        logger.trace(`Requesting ping on ${this.serviceId} from ${this.peerId.toKey()}`)
+        logger.trace(`Requesting ping on ${this.getServiceId()} from ${this.getPeerId().toKey()}`)
         const request: PingRequest = {
             requestId: v4()
         }
-        const options: DhtRpcOptions = {
-            sourceDescriptor: this.ownPeerDescriptor,
-            targetDescriptor: this.peerDescriptor,
+        const options = this.formDhtRpcOptions({
             timeout: 10000
-        }
+        })
         try {
-            const pong = await this.client.ping(request, options)
+            const pong = await this.getClient().ping(request, options)
             if (pong.requestId === request.requestId) {
                 return true
             }
         } catch (err) {
-            logger.trace(`ping failed on ${this.serviceId} to ${this.peerId.toKey()}: ${err}`)
+            logger.trace(`ping failed on ${this.getServiceId()} to ${this.getPeerId().toKey()}: ${err}`)
         }
         return false
     }
 
     leaveNotice(): void {
-        logger.trace(`Sending leaveNotice on ${this.serviceId} from ${this.peerId.toKey()}`)
+        logger.trace(`Sending leaveNotice on ${this.getServiceId()} from ${this.getPeerId().toKey()}`)
         const request: LeaveNotice = {
-            serviceId: this.serviceId
+            serviceId: this.getServiceId()
         }
-        const options: DhtRpcOptions = {
-            sourceDescriptor: this.ownPeerDescriptor,
-            targetDescriptor: this.peerDescriptor,
+        const options = this.formDhtRpcOptions({
             notification: true
-        }
-        this.client.leaveNotice(request, options).catch((e) => {
+        })
+        this.getClient().leaveNotice(request, options).catch((e) => {
             logger.trace('Failed to send leaveNotice' + e)
         })
     }
-
-    getPeerDescriptor(): PeerDescriptor {
-        return this.peerDescriptor
-    }
-
 }

package/src/dht/RemoteExternalApi.ts

@@ -1,5 +1,5 @@
-import { DhtRpcOptions } from '../exports'
-import { DataEntry, FindDataRequest } from '../proto/packages/dht/protos/DhtRpc'
+import { Any } from '../proto/google/protobuf/any'
+import { DataEntry, ExternalStoreDataRequest, FindDataRequest, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
 import { IExternalApiServiceClient } from '../proto/packages/dht/protos/DhtRpc.client'
 import { Remote } from './contact/Remote'
 
@@ -8,18 +8,32 @@ export class RemoteExternalApi extends Remote<IExternalApiServiceClient> {
     async findData(idToFind: Uint8Array): Promise<DataEntry[]> {
         const request: FindDataRequest = {
             kademliaId: idToFind,
-            requestor: this.ownPeerDescriptor,
+            requestor: this.getLocalPeerDescriptor(),
         }
-        const options: DhtRpcOptions = {
-            sourceDescriptor: this.ownPeerDescriptor,
-            targetDescriptor: this.peerDescriptor,
+        const options = this.formDhtRpcOptions({
             timeout: 10000
-        }
+        })
         try {
-            const data = await this.client.findData(request, options)
+            const data = await this.getClient().findData(request, options)
             return data.dataEntries
         } catch (err) {
             return []
         }
     }
+
+    async storeData(key: Uint8Array, data: Any): Promise<PeerDescriptor[]> {
+        const request: ExternalStoreDataRequest = {
+            key,
+            data
+        }
+        const options = this.formDhtRpcOptions({
+            timeout: 10000
+        })
+        try {
+            const response = await this.getClient().externalStoreData(request, options)
+            return response.storers
+        } catch (err) {
+            return []
+        }
+    }
 }

package/src/dht/contact/ContactList.ts

@@ -0,0 +1,45 @@
+import { PeerID, PeerIDKey } from '../../helpers/PeerID'
+import EventEmitter from 'eventemitter3'
+import { Events, IContact, ContactState } from './Contact'
+
+export class ContactList<C extends IContact> extends EventEmitter<Events> {
+
+    protected contactsById: Map<PeerIDKey, ContactState<C>> = new Map()
+    protected contactIds: PeerID[] = []
+    protected ownId: PeerID
+    protected maxSize: number
+    protected defaultContactQueryLimit
+
+    constructor(
+        ownId: PeerID,
+        maxSize: number,
+        defaultContactQueryLimit = 20
+    ) {
+        super()
+        this.ownId = ownId
+        this.maxSize = maxSize
+        this.defaultContactQueryLimit = defaultContactQueryLimit
+    }
+
+    public getContact(id: PeerID): ContactState<C> {
+        return this.contactsById.get(id.toKey())!
+    }
+
+    public hasContact(id: PeerID): boolean {
+        return this.contactsById.has(id.toKey())
+    }
+
+    public getSize(): number {
+        return this.contactIds.length
+    }
+
+    public clear(): void {
+        this.contactsById.clear()
+        this.contactIds = []
+    }
+
+    public stop(): void {
+        this.removeAllListeners()
+        this.clear()
+    }
+}

package/src/dht/contact/RandomContactList.ts

@@ -1,29 +1,22 @@
-import { Events, IContact, ContactState } from './Contact'
+import { PeerID } from '../../helpers/PeerID'
+import { ContactState, IContact } from './Contact'
+import { ContactList } from './ContactList'
 
-import EventEmitter from 'eventemitter3'
-import { PeerID, PeerIDKey } from '../../helpers/PeerID'
-export class RandomContactList<Contact extends IContact> extends EventEmitter<Events> {
-    private contactsById: Map<PeerIDKey, ContactState<Contact>> = new Map()
-    private contactIds: PeerID[] = []
-    private ownId: PeerID
-    private maxSize: number
-    private randomness = 0.20
-    private getContactsLimit = 20
+export class RandomContactList<C extends IContact> extends ContactList<C> {
+
+    private randomness: number
 
     constructor(
         ownId: PeerID,
         maxSize: number,
         randomness = 0.20,
-        getContactsLimit = 20
+        defaultContactQueryLimit?: number
     ) {
-        super()
-        this.ownId = ownId
-        this.maxSize = maxSize
+        super(ownId, maxSize, defaultContactQueryLimit)
         this.randomness = randomness
-        this.getContactsLimit = getContactsLimit
     }
 
-    addContact(contact: Contact): void {
+    addContact(contact: C): void {
         if (this.ownId.equals(contact.getPeerId())) {
             return
         }
@@ -39,54 +32,36 @@ export class RandomContactList<Contact extends IContact> extends EventEmitter<Ev
                 this.emit(
                     'newContact',
                     contact.getPeerDescriptor(),
-                    this.getContacts().map((contact: Contact) => contact.getPeerDescriptor())
+                    this.getContacts().map((contact: C) => contact.getPeerDescriptor())
                 )
             }
         }
     }
 
-    addContacts(contacts: Contact[]): void {
+    addContacts(contacts: C[]): void {
         contacts.forEach((contact) => this.addContact(contact))
     }
 
     removeContact(id: PeerID): boolean {
         if (this.contactsById.has(id.toKey())) {
             const removedDescriptor = this.contactsById.get(id.toKey())!.contact.getPeerDescriptor()
-            const index = this.contactIds.indexOf(id)
+            const index = this.contactIds.findIndex((element) => element.equals(id))
             this.contactIds.splice(index, 1)
             this.contactsById.delete(id.toKey())
-            this.emit('contactRemoved', removedDescriptor, this.getContacts().map((contact: Contact) => contact.getPeerDescriptor()))
+            this.emit('contactRemoved', removedDescriptor, this.getContacts().map((contact: C) => contact.getPeerDescriptor()))
             return true
         }
         return false
     }
 
-    public getSize(): number {
-        return this.contactIds.length
-    }
-
-    public getContact(id: PeerID): ContactState<Contact> {
-        return this.contactsById.get(id.toKey())!
-    }
-
-    public getContacts(limit = this.getContactsLimit): Contact[] {
-        const ret: Contact[] = []
+    public getContacts(limit = this.defaultContactQueryLimit): C[] {
+        const ret: C[] = []
         this.contactIds.forEach((contactId) => {
             const contact = this.contactsById.get(contactId.toKey())
             if (contact) {
                 ret.push(contact.contact)
             }
         })
-        return ret.splice(0, limit)
-    }
-
-    public clear(): void {
-        this.contactsById.clear()
-        this.contactIds = []
-    }
-
-    public stop(): void {
-        this.removeAllListeners()
-        this.clear()
+        return ret.slice(0, limit)
     }
 }

package/src/dht/contact/Remote.ts

@@ -3,38 +3,54 @@ import { ProtoRpcClient } from '@streamr/proto-rpc'
 import { peerIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
 import { PeerID } from '../../helpers/PeerID'
 import { IContact } from './Contact'
+import { DhtRpcOptions } from '../../rpc-protocol/DhtRpcOptions'
 
 export abstract class Remote<T> implements IContact {
 
-    protected readonly peerId: PeerID
-    protected readonly peerDescriptor: PeerDescriptor
-    protected readonly client: ProtoRpcClient<T>
-    protected readonly serviceId: string
-    protected readonly ownPeerDescriptor: PeerDescriptor
+    private readonly localPeerDescriptor: PeerDescriptor
+    private readonly remotePeerId: PeerID
+    private readonly remotePeerDescriptor: PeerDescriptor
+    private readonly serviceId: string
+    private readonly client: ProtoRpcClient<T>
 
     constructor(
-        ownPeerDescriptor: PeerDescriptor,
-        peerDescriptor: PeerDescriptor,
-        client: ProtoRpcClient<T>,
-        serviceId: string
+        localPeerDescriptor: PeerDescriptor,
+        remotePeerDescriptor: PeerDescriptor,
+        serviceId: string,
+        client: ProtoRpcClient<T>
     ) {
-        this.ownPeerDescriptor = ownPeerDescriptor
-        this.peerId = peerIdFromPeerDescriptor(peerDescriptor)
-        this.peerDescriptor = peerDescriptor
+        this.localPeerDescriptor = localPeerDescriptor
+        this.remotePeerId = peerIdFromPeerDescriptor(remotePeerDescriptor)
+        this.remotePeerDescriptor = remotePeerDescriptor
         this.client = client
         this.serviceId = serviceId
     }
 
     getPeerId(): PeerID {
-        return this.peerId
+        return this.remotePeerId
     }
 
     getPeerDescriptor(): PeerDescriptor {
-        return this.peerDescriptor
+        return this.remotePeerDescriptor
+    }
+
+    getLocalPeerDescriptor(): PeerDescriptor {
+        return this.localPeerDescriptor
     }
 
     getServiceId(): string {
         return this.serviceId
     }
 
+    getClient(): ProtoRpcClient<T> {
+        return this.client
+    }
+
+    formDhtRpcOptions(opts?: Omit<DhtRpcOptions, 'sourceDescriptor' | 'targetDescriptor'>): DhtRpcOptions {
+        return {
+            sourceDescriptor: this.localPeerDescriptor,
+            targetDescriptor: this.remotePeerDescriptor,
+            ...opts
+        }
+    }
 }