@streamr/dht 100.0.0-testnet-one.3 → 100.0.0-testnet-two.0

package/src/dht/find/FindSession.ts

@@ -1,183 +0,0 @@
-import EventEmitter from 'eventemitter3'
-import { PeerID, PeerIDKey } from '../../helpers/PeerID'
-import { DataEntry, PeerDescriptor, FindResponse, FindAction } from '../../proto/packages/dht/protos/DhtRpc'
-import { ITransport } from '../../transport/ITransport'
-import { ListeningRpcCommunicator } from '../../transport/ListeningRpcCommunicator'
-import { Contact } from '../contact/Contact'
-import { SortedContactList } from '../contact/SortedContactList'
-import { FindResult } from './Finder'
-import { keyFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
-import { ServiceID } from '../../types/ServiceID'
-import { FindSessionRpcLocal } from './FindSessionRpcLocal'
-
-export interface FindSessionEvents {
-    findCompleted: (results: PeerDescriptor[]) => void
-}
-
-export interface FindSessionConfig {
-    serviceId: ServiceID
-    transport: ITransport
-    nodeIdToFind: Uint8Array
-    localPeerId: PeerID
-    waitedRoutingPathCompletions: number
-    action: FindAction
-}
-
-export class FindSession extends EventEmitter<FindSessionEvents> {
-    private readonly serviceId: ServiceID
-    private readonly transport: ITransport
-    private readonly nodeIdToFind: Uint8Array
-    private readonly localPeerId: PeerID
-    private readonly waitedRoutingPathCompletions: number
-    private readonly rpcCommunicator: ListeningRpcCommunicator
-    private readonly action: FindAction
-    private results: SortedContactList<Contact>
-    private foundData: Map<string, DataEntry> = new Map()
-    private allKnownHops: Set<PeerIDKey> = new Set()
-    private reportedHops: Set<PeerIDKey> = new Set()
-    private reportFindCompletedTimeout?: NodeJS.Timeout
-    private findCompletedEmitted = false
-    private noCloserNodesReceivedCounter = 0
-
-    constructor(config: FindSessionConfig) {
-        super()
-        this.serviceId = config.serviceId
-        this.transport = config.transport
-        this.nodeIdToFind = config.nodeIdToFind
-        this.localPeerId = config.localPeerId
-        this.waitedRoutingPathCompletions = config.waitedRoutingPathCompletions
-        this.results = new SortedContactList({
-            referenceId: PeerID.fromValue(this.nodeIdToFind), 
-            maxSize: 10,  // TODO use config option or named constant?
-            allowToContainReferenceId: true,
-            emitEvents: false
-        })
-        this.action = config.action
-        this.rpcCommunicator = new ListeningRpcCommunicator(this.serviceId, this.transport, {
-            rpcRequestTimeout: 15000
-        })
-        this.registerLocalRpcMethods()
-    }
-
-    private registerLocalRpcMethods() {
-        const rpcLocal = new FindSessionRpcLocal({
-            doSendFindResponse: (routingPath: PeerDescriptor[], nodes: PeerDescriptor[], dataEntries: DataEntry[], noCloserNodesFound: boolean) => {
-                this.doSendFindResponse(routingPath, nodes, dataEntries, noCloserNodesFound)
-            }
-        })
-        this.rpcCommunicator.registerRpcNotification(FindResponse, 'sendFindResponse',
-            (req: FindResponse) => rpcLocal.sendFindResponse(req))
-    }
-
-    private isFindCompleted(): boolean {
-        const unreportedHops: Set<PeerIDKey> = new Set(this.allKnownHops)
-        this.reportedHops.forEach((id) => {
-            unreportedHops.delete(id)
-        })
-        if (this.noCloserNodesReceivedCounter >= 1 && unreportedHops.size === 0) {
-            if (this.action === FindAction.FETCH_DATA
-                && (this.hasNonStaleData() || this.noCloserNodesReceivedCounter >= this.waitedRoutingPathCompletions)) {
-                return true
-            } else if (this.action === FindAction.FETCH_DATA) {
-                return false
-            }
-            return true
-        }
-        return false
-    }
-
-    private hasNonStaleData(): boolean {
-        return Array.from(this.foundData.values()).some((entry) => entry.stale === false)
-    }
-
-    public doSendFindResponse(
-        routingPath: PeerDescriptor[],
-        nodes: PeerDescriptor[],
-        dataEntries: DataEntry[],
-        noCloserNodesFound: boolean
-    ): void {
-        this.addKnownHops(routingPath)
-        if (routingPath.length >= 1) {
-            this.setHopAsReported(routingPath[routingPath.length - 1])
-        }
-        nodes.forEach((descriptor: PeerDescriptor) => {
-            this.results.addContact(new Contact(descriptor))
-        })
-        this.processFoundData(dataEntries)
-        if (noCloserNodesFound) {
-            this.onNoCloserPeersFound()
-        }
-    }
-
-    private addKnownHops(routingPath: PeerDescriptor[]) {
-        routingPath.forEach((desc) => {
-            const newPeerId = PeerID.fromValue(desc.nodeId)
-            if (!this.localPeerId.equals(newPeerId)) {
-                this.allKnownHops.add(newPeerId.toKey())
-            }
-        })
-    }
-
-    private setHopAsReported(desc: PeerDescriptor) {
-        const newPeerId = PeerID.fromValue(desc.nodeId)
-        if (!this.localPeerId.equals(newPeerId)) {
-            this.reportedHops.add(newPeerId.toKey())
-        }
-        if (this.isFindCompleted()) {
-            if (!this.findCompletedEmitted && this.isFindCompleted()) {
-                if (this.reportFindCompletedTimeout) {
-                    clearTimeout(this.reportFindCompletedTimeout)
-                    this.reportFindCompletedTimeout = undefined
-                }
-                this.emit('findCompleted', this.results.getAllContacts().map((contact) => contact.getPeerDescriptor()))
-                this.findCompletedEmitted = true
-            }
-        }
-    }
-
-    private processFoundData(dataEntries: DataEntry[]): void {
-        dataEntries.forEach((entry) => {
-            const creatorKey = keyFromPeerDescriptor(entry.creator!)
-            const existingEntry = this.foundData.get(creatorKey)
-            if (!existingEntry || existingEntry.createdAt! < entry.createdAt! 
-                || (existingEntry.createdAt! <= entry.createdAt! && entry.deleted)) {
-                this.foundData.set(creatorKey, entry)
-            }
-        })
-    }
-
-    private onNoCloserPeersFound(): void {
-        this.noCloserNodesReceivedCounter += 1
-        if (this.isFindCompleted()) {
-            this.emit('findCompleted', this.results.getAllContacts().map((contact) => contact.getPeerDescriptor()))
-            this.findCompletedEmitted = true
-            if (this.reportFindCompletedTimeout) {
-                clearTimeout(this.reportFindCompletedTimeout)
-                this.reportFindCompletedTimeout = undefined
-            }
-        } else {
-            if (!this.reportFindCompletedTimeout && !this.findCompletedEmitted) {
-                this.reportFindCompletedTimeout = setTimeout(() => {
-                    if (!this.findCompletedEmitted) {
-                        this.emit('findCompleted', this.results.getAllContacts().map((contact) => contact.getPeerDescriptor()))
-                        this.findCompletedEmitted = true
-                    }
-                }, 4000)
-            }
-        }
-    }
-
-    public getResults = (): FindResult => ({
-        closestNodes: this.results.getAllContacts().map((contact) => contact.getPeerDescriptor()),
-        dataEntries: Array.from(this.foundData.values())
-    })
-
-    public stop(): void {
-        if (this.reportFindCompletedTimeout) {
-            clearTimeout(this.reportFindCompletedTimeout)
-            this.reportFindCompletedTimeout = undefined
-        }
-        this.rpcCommunicator.destroy()
-        this.emit('findCompleted', [])
-    }
-}

package/src/dht/find/FindSessionRpcLocal.ts

@@ -1,25 +0,0 @@
-import { IFindSessionRpc } from '../../proto/packages/dht/protos/DhtRpc.server'
-import { Empty } from '../../proto/google/protobuf/empty'
-import { DataEntry, FindResponse, PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
-import { Logger } from '@streamr/utils'
-
-const logger = new Logger(module)
-
-interface FindSessionRpcLocalConfig {
-    doSendFindResponse: (routingPath: PeerDescriptor[], nodes: PeerDescriptor[], dataEntries: DataEntry[], noCloserNodesFound: boolean) => void
-}
-
-export class FindSessionRpcLocal implements IFindSessionRpc {
-
-    private readonly config: FindSessionRpcLocalConfig
-
-    constructor(config: FindSessionRpcLocalConfig) {
-        this.config = config
-    }
-    
-    async sendFindResponse(report: FindResponse): Promise<Empty> {
-        logger.trace('FindResponse arrived: ' + JSON.stringify(report))
-        this.config.doSendFindResponse(report.routingPath, report.closestConnectedPeers, report.dataEntries, report.noCloserNodesFound)
-        return {}
-    }
-}

package/src/dht/find/Finder.ts

@@ -1,295 +0,0 @@
-import {
-    DataEntry,
-    Message,
-    MessageType,
-    NodeType,
-    PeerDescriptor,
-    FindRequest,
-    RouteMessageAck,
-    RouteMessageWrapper,
-    RouteMessageError,
-    FindAction
-} from '../../proto/packages/dht/protos/DhtRpc'
-import { PeerID, PeerIDKey } from '../../helpers/PeerID'
-import { IRouter } from '../routing/Router'
-import { RoutingMode } from '../routing/RoutingSession'
-import { areEqualPeerDescriptors, peerIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
-import { Logger, runAndWaitForEvents3, wait } from '@streamr/utils'
-import { RoutingRpcCommunicator } from '../../transport/RoutingRpcCommunicator'
-import { FindSessionRpcRemote } from './FindSessionRpcRemote'
-import { v4 } from 'uuid'
-import { FindSession, FindSessionEvents } from './FindSession'
-import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
-import { ITransport } from '../../transport/ITransport'
-import { LocalDataStore } from '../store/LocalDataStore'
-import { ListeningRpcCommunicator } from '../../transport/ListeningRpcCommunicator'
-import { FindSessionRpcClient } from '../../proto/packages/dht/protos/DhtRpc.client'
-import { toProtoRpcClient } from '@streamr/proto-rpc'
-import { SortedContactList } from '../contact/SortedContactList'
-import { getPreviousPeer } from '../routing/getPreviousPeer'
-import { createRouteMessageAck } from '../routing/RouterRpcLocal'
-import { ServiceID } from '../../types/ServiceID'
-import { FindRpcLocal } from './FindRpcLocal'
-
-interface FinderConfig {
-    rpcCommunicator: RoutingRpcCommunicator
-    sessionTransport: ITransport
-    connections: Map<PeerIDKey, DhtNodeRpcRemote>
-    router: IRouter
-    localPeerDescriptor: PeerDescriptor
-    serviceId: ServiceID
-    localDataStore: LocalDataStore
-    addContact: (contact: PeerDescriptor) => void
-    isPeerCloserToIdThanSelf: (peer1: PeerDescriptor, compareToId: PeerID) => boolean
-}
-
-export interface IFinder {
-    startFind(idToFind: Uint8Array, action?: FindAction): Promise<FindResult>
-}
-
-export interface FindResult { closestNodes: Array<PeerDescriptor>, dataEntries?: Array<DataEntry> }
-
-const logger = new Logger(module)
-
-export class Finder implements IFinder {
-
-    private readonly rpcCommunicator: RoutingRpcCommunicator
-    private readonly sessionTransport: ITransport
-    private readonly connections: Map<PeerIDKey, DhtNodeRpcRemote>
-    private readonly router: IRouter
-    private readonly localPeerDescriptor: PeerDescriptor
-    private readonly serviceId: ServiceID
-    private readonly localDataStore: LocalDataStore
-    private readonly isPeerCloserToIdThanSelf: (peer1: PeerDescriptor, compareToId: PeerID) => boolean
-    private ongoingSessions: Map<string, FindSession> = new Map()
-    private stopped = false
-
-    constructor(config: FinderConfig) {
-        this.rpcCommunicator = config.rpcCommunicator
-        this.sessionTransport = config.sessionTransport
-        this.connections = config.connections
-        this.router = config.router
-        this.localPeerDescriptor = config.localPeerDescriptor
-        this.serviceId = config.serviceId
-        this.localDataStore = config.localDataStore
-        this.isPeerCloserToIdThanSelf = config.isPeerCloserToIdThanSelf
-        this.registerLocalRpcMethods(config)
-    }
-
-    private registerLocalRpcMethods(config: FinderConfig) {
-        const rpcLocal = new FindRpcLocal({
-            doRouteFindRequest: (routedMessage: RouteMessageWrapper) => this.doRouteFindRequest(routedMessage),
-            addContact: (contact: PeerDescriptor) => config.addContact(contact),
-            isMostLikelyDuplicate: (requestId: string) => this.router.isMostLikelyDuplicate(requestId),
-            addToDuplicateDetector: (requestId: string) => this.router.addToDuplicateDetector(requestId)
-        })
-        this.rpcCommunicator.registerRpcMethod(
-            RouteMessageWrapper,
-            RouteMessageAck,
-            'routeFindRequest',
-            async (routedMessage: RouteMessageWrapper) => {
-                if (this.stopped) {
-                    return createRouteMessageAck(routedMessage, RouteMessageError.STOPPED)
-                } else {
-                    return rpcLocal.routeFindRequest(routedMessage)
-                }
-            }
-        )
-    }
-
-    public async startFind(
-        idToFind: Uint8Array,
-        action: FindAction = FindAction.NODE,
-        excludedPeer?: PeerDescriptor,
-        waitForCompletion = true
-    ): Promise<FindResult> {
-        if (this.stopped) {
-            return { closestNodes: [] }
-        }
-        const sessionId = v4()
-        const session = new FindSession({
-            serviceId: sessionId,
-            transport: this.sessionTransport,
-            nodeIdToFind: idToFind,
-            localPeerId: peerIdFromPeerDescriptor(this.localPeerDescriptor),
-            waitedRoutingPathCompletions: this.connections.size > 1 ? 2 : 1,
-            action
-        })
-        if (this.connections.size === 0) {
-            const data = this.localDataStore.getEntry(PeerID.fromValue(idToFind))
-            session.doSendFindResponse(
-                [this.localPeerDescriptor],
-                [this.localPeerDescriptor],
-                Array.from(data.values()),
-                true
-            )
-            return session.getResults()
-        }
-        const routeMessage = this.wrapFindRequest(idToFind, sessionId, action)
-        this.ongoingSessions.set(sessionId, session)
-        if (waitForCompletion === true) {
-            try {
-                await runAndWaitForEvents3<FindSessionEvents>(
-                    [() => this.doRouteFindRequest(routeMessage, excludedPeer)],
-                    [[session, 'findCompleted']],
-                    15000
-                )
-            } catch (err) {
-                logger.debug(`doRouteFindRequest failed with error ${err}`)
-            }
-        } else {
-            this.doRouteFindRequest(routeMessage, excludedPeer)
-            // Wait for delete operation to be sent out by the router
-            // TODO: Add a feature to wait for the router to pass the message?
-            await wait(50)
-        }
-        if (action === FindAction.FETCH_DATA) {
-            this.findAndReportLocalData(idToFind, [], this.localPeerDescriptor, sessionId)
-        } else if (action === FindAction.DELETE_DATA) {
-            this.localDataStore.markAsDeleted(idToFind, peerIdFromPeerDescriptor(this.localPeerDescriptor))
-        }
-        this.ongoingSessions.delete(sessionId)
-        session.stop()
-        return session.getResults()
-    }
-
-    private wrapFindRequest(idToFind: Uint8Array, sessionId: string, action: FindAction): RouteMessageWrapper {
-        const targetDescriptor: PeerDescriptor = {
-            nodeId: idToFind,
-            type: NodeType.VIRTUAL
-        }
-        const request: FindRequest = {
-            sessionId,
-            action
-        }
-        const msg: Message = {
-            messageType: MessageType.FIND_REQUEST,
-            messageId: v4(),
-            serviceId: this.serviceId,
-            body: {
-                oneofKind: 'findRequest',
-                findRequest: request
-            }
-        }
-        const routeMessage: RouteMessageWrapper = {
-            message: msg,
-            requestId: v4(),
-            destinationPeer: targetDescriptor,
-            sourcePeer: this.localPeerDescriptor,
-            reachableThrough: [],
-            routingPath: []
-        }
-        return routeMessage
-    }
-
-    private findAndReportLocalData(
-        idToFind: Uint8Array,
-        routingPath: PeerDescriptor[],
-        sourcePeer: PeerDescriptor,
-        sessionId: string
-    ): void {
-        const data = this.localDataStore.getEntry(PeerID.fromValue(idToFind))
-        if (data.size > 0) {
-            this.sendFindResponse(routingPath, sourcePeer, sessionId, [], data, true)
-        }
-    }
-
-    private findLocalData(idToFind: Uint8Array, fetchData: boolean): Map<PeerIDKey, DataEntry> | undefined {
-        if (fetchData) {
-            return this.localDataStore.getEntry(PeerID.fromValue(idToFind))
-        }
-        return undefined
-    }
-
-    private sendFindResponse(
-        routingPath: PeerDescriptor[],
-        targetPeerDescriptor: PeerDescriptor,
-        serviceId: ServiceID,
-        closestNodes: PeerDescriptor[],
-        data: Map<PeerIDKey, DataEntry> | undefined,
-        noCloserNodesFound: boolean = false
-    ): void {
-        const dataEntries = data ? Array.from(data.values(), DataEntry.create.bind(DataEntry)) : []
-        const isOwnNode = areEqualPeerDescriptors(this.localPeerDescriptor, targetPeerDescriptor)
-        if (isOwnNode && this.ongoingSessions.has(serviceId)) {
-            this.ongoingSessions.get(serviceId)!
-                .doSendFindResponse(routingPath, closestNodes, dataEntries, noCloserNodesFound)
-        } else {
-            const remoteCommunicator = new ListeningRpcCommunicator(serviceId, this.sessionTransport, { rpcRequestTimeout: 15000 })
-            const rpcRemote = new FindSessionRpcRemote(
-                this.localPeerDescriptor,
-                targetPeerDescriptor,
-                serviceId,
-                toProtoRpcClient(new FindSessionRpcClient(remoteCommunicator.getRpcClientTransport())),
-                10000
-            )
-            rpcRemote.sendFindResponse(routingPath, closestNodes, dataEntries, noCloserNodesFound)
-            remoteCommunicator.destroy()
-        }
-    }
-
-    private doRouteFindRequest(routedMessage: RouteMessageWrapper, excludedPeer?: PeerDescriptor): RouteMessageAck {
-        if (this.stopped) {
-            return createRouteMessageAck(routedMessage, RouteMessageError.STOPPED)
-        }
-        const idToFind = peerIdFromPeerDescriptor(routedMessage.destinationPeer!)
-        const msg = routedMessage.message
-        const findRequest = msg?.body.oneofKind === 'findRequest' ? msg.body.findRequest : undefined
-        const closestPeersToDestination = this.getClosestConnections(routedMessage.destinationPeer!.nodeId, 5)
-        const data = this.findLocalData(idToFind.value, findRequest!.action === FindAction.FETCH_DATA)
-        if (findRequest!.action === FindAction.DELETE_DATA) {
-            this.localDataStore.markAsDeleted(idToFind.value, peerIdFromPeerDescriptor(routedMessage.sourcePeer!))
-        }
-        if (areEqualPeerDescriptors(this.localPeerDescriptor, routedMessage.destinationPeer!)) {
-            // TODO this is also very similar case to what we do at line 255, could simplify the code paths?
-            this.sendFindResponse(
-                routedMessage.routingPath,
-                routedMessage.sourcePeer!,
-                findRequest!.sessionId,
-                closestPeersToDestination,
-                data,
-                true
-            )
-            return createRouteMessageAck(routedMessage)
-        } else {
-            const ack = this.router.doRouteMessage(routedMessage, RoutingMode.FIND, excludedPeer)
-            if ((ack.error === undefined) || (ack.error === RouteMessageError.NO_TARGETS)) {
-                const noCloserContactsFound = (ack.error === RouteMessageError.NO_TARGETS) ||
-                    (
-                        closestPeersToDestination.length > 0 
-                        && getPreviousPeer(routedMessage) 
-                        && !this.isPeerCloserToIdThanSelf(closestPeersToDestination[0], idToFind)
-                    )
-                this.sendFindResponse(
-                    routedMessage.routingPath,
-                    routedMessage.sourcePeer!,
-                    findRequest!.sessionId,
-                    closestPeersToDestination,
-                    data,
-                    noCloserContactsFound
-                )
-            }
-            return ack
-        }    
-    }
-
-    private getClosestConnections(nodeId: Uint8Array, limit: number): PeerDescriptor[] {
-        const connectedPeers = Array.from(this.connections.values())
-        const closestPeers = new SortedContactList<DhtNodeRpcRemote>({
-            referenceId: PeerID.fromValue(nodeId),
-            maxSize: limit,
-            allowToContainReferenceId: true,
-            emitEvents: false
-        })
-        closestPeers.addContacts(connectedPeers)
-        return closestPeers.getClosestContacts(limit).map((peer) => peer.getPeerDescriptor())
-    }
-
-    public stop(): void {
-        this.stopped = true
-        this.ongoingSessions.forEach((session, _id) => {
-            session.stop()
-        })
-        this.ongoingSessions.clear()
-    }
-}