@streamr/dht 100.0.0-pretestnet.4 → 100.0.0-rc.0

package/src/dht/recursive-operation/RecursiveOperationRpcLocal.ts

@@ -0,0 +1,34 @@
+import { Logger } from '@streamr/utils'
+import { PeerDescriptor, RouteMessageAck, RouteMessageError, RouteMessageWrapper } from '../../proto/packages/dht/protos/DhtRpc'
+import { IRecursiveOperationRpc } from '../../proto/packages/dht/protos/DhtRpc.server'
+import { createRouteMessageAck } from '../routing/RouterRpcLocal'
+import { getPreviousPeer } from '../routing/getPreviousPeer'
+import { getNodeIdFromPeerDescriptor } from '../../identifiers'
+
+const logger = new Logger(module)
+
+interface RecursiveOperationRpcLocalConfig {
+    doRouteRequest: (routedMessage: RouteMessageWrapper) => RouteMessageAck
+    addContact: (contact: PeerDescriptor, setActive?: boolean) => void
+    isMostLikelyDuplicate: (requestId: string) => boolean
+    addToDuplicateDetector: (requestId: string) => void
+}
+
+export class RecursiveOperationRpcLocal implements IRecursiveOperationRpc {
+
+    private readonly config: RecursiveOperationRpcLocalConfig
+
+    constructor(config: RecursiveOperationRpcLocalConfig) {
+        this.config = config
+    }
+
+    async routeRequest(routedMessage: RouteMessageWrapper): Promise<RouteMessageAck> {
+        if (this.config.isMostLikelyDuplicate(routedMessage.requestId)) {
+            return createRouteMessageAck(routedMessage, RouteMessageError.DUPLICATE)
+        }
+        const senderId = getNodeIdFromPeerDescriptor(getPreviousPeer(routedMessage) ?? routedMessage.sourcePeer!)
+        logger.trace(`Received routeRequest call from ${senderId}`)
+        this.config.addToDuplicateDetector(routedMessage.requestId)
+        return this.config.doRouteRequest(routedMessage)
+    }
+}

package/src/dht/recursive-operation/RecursiveOperationRpcRemote.ts

@@ -0,0 +1,43 @@
+import { Logger } from '@streamr/utils'
+import { v4 } from 'uuid'
+import { RouteMessageWrapper } from '../../proto/packages/dht/protos/DhtRpc'
+import { RecursiveOperationRpcClient } from '../../proto/packages/dht/protos/DhtRpc.client'
+import { RpcRemote } from '../contact/RpcRemote'
+import { getPreviousPeer } from '../routing/getPreviousPeer'
+import { getNodeIdFromPeerDescriptor } from '../../identifiers'
+
+const logger = new Logger(module)
+
+export class RecursiveOperationRpcRemote extends RpcRemote<RecursiveOperationRpcClient> {
+
+    async routeRequest(params: RouteMessageWrapper): Promise<boolean> {
+        const message: RouteMessageWrapper = {
+            target: params.target,
+            sourcePeer: params.sourcePeer,
+            message: params.message,
+            requestId: params.requestId ?? v4(),
+            reachableThrough: params.reachableThrough ?? [],
+            routingPath: params.routingPath,
+            parallelRootNodeIds: params.parallelRootNodeIds
+        }
+        const options = this.formDhtRpcOptions({
+            connect: false
+        })
+        try {
+            const ack = await this.getClient().routeRequest(message, options)
+            if (ack.error !== undefined) {
+                logger.trace('Next hop responded with error ' + ack.error)
+                return false
+            }
+        } catch (err) {
+            const previousPeer = getPreviousPeer(params)
+            const fromNode = previousPeer
+                ? getNodeIdFromPeerDescriptor(previousPeer)
+                : getNodeIdFromPeerDescriptor(params.sourcePeer!)
+            const toNode = getNodeIdFromPeerDescriptor(this.getPeerDescriptor())
+            logger.debug(`Failed to send routeRequest message from ${fromNode} to ${toNode} with: ${err}`)
+            return false
+        }
+        return true
+    }
+}

package/src/dht/recursive-operation/RecursiveOperationSession.ts

@@ -0,0 +1,232 @@
+import EventEmitter from 'eventemitter3'
+import { v4 } from 'uuid'
+import { 
+    DataEntry,
+    PeerDescriptor,
+    RecursiveOperationResponse,
+    RecursiveOperation,
+    RouteMessageWrapper,
+    RouteMessageAck,
+    RecursiveOperationRequest,
+    Message
+} from '../../proto/packages/dht/protos/DhtRpc'
+import { ITransport } from '../../transport/ITransport'
+import { ListeningRpcCommunicator } from '../../transport/ListeningRpcCommunicator'
+import { Contact } from '../contact/Contact'
+import { SortedContactList } from '../contact/SortedContactList'
+import { RecursiveOperationResult } from './RecursiveOperationManager'
+import { ServiceID } from '../../types/ServiceID'
+import { RecursiveOperationSessionRpcLocal } from './RecursiveOperationSessionRpcLocal'
+import { DhtAddress, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../identifiers'
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+
+export interface RecursiveOperationSessionEvents {
+    completed: () => void
+}
+
+export interface RecursiveOperationSessionConfig {
+    transport: ITransport
+    targetId: DhtAddress
+    localPeerDescriptor: PeerDescriptor
+    waitedRoutingPathCompletions: number
+    operation: RecursiveOperation
+    doRouteRequest: (routedMessage: RouteMessageWrapper) => RouteMessageAck
+}
+
+export class RecursiveOperationSession extends EventEmitter<RecursiveOperationSessionEvents> {
+
+    private readonly id = v4()
+    private readonly rpcCommunicator: ListeningRpcCommunicator
+    private results: SortedContactList<Contact>
+    private foundData: Map<DhtAddress, DataEntry> = new Map()
+    private allKnownHops: Set<DhtAddress> = new Set()
+    private reportedHops: Set<DhtAddress> = new Set()
+    private timeoutTask?: NodeJS.Timeout 
+    private completionEventEmitted = false
+    private noCloserNodesReceivedCounter = 0
+    private readonly noCloserNodesReceivedFrom: Set<DhtAddress> = new Set()
+    private readonly config: RecursiveOperationSessionConfig
+
+    constructor(config: RecursiveOperationSessionConfig) {
+        super()
+        this.config = config
+        this.results = new SortedContactList({
+            referenceId: config.targetId, 
+            maxSize: 10,  // TODO use config option or named constant?
+            allowToContainReferenceId: true,
+            emitEvents: false
+        })
+        this.rpcCommunicator = new ListeningRpcCommunicator(this.id, config.transport, {
+            rpcRequestTimeout: 15000  // TODO use config option or named constant?
+        })
+        this.registerLocalRpcMethods()
+    }
+
+    private registerLocalRpcMethods() {
+        const rpcLocal = new RecursiveOperationSessionRpcLocal({
+            onResponseReceived: (
+                sourceId: DhtAddress,
+                routingPath: PeerDescriptor[],
+                nodes: PeerDescriptor[],
+                dataEntries: DataEntry[],
+                noCloserNodesFound: boolean
+            ) => {
+                this.onResponseReceived(sourceId, routingPath, nodes, dataEntries, noCloserNodesFound)
+            }
+        })
+        this.rpcCommunicator.registerRpcNotification(RecursiveOperationResponse, 'sendResponse',
+            (req: RecursiveOperationResponse, context: ServerCallContext) => rpcLocal.sendResponse(req, context))
+    }
+
+    public start(serviceId: ServiceID): void {
+        const routeMessage = this.wrapRequest(serviceId)
+        this.config.doRouteRequest(routeMessage)
+    }
+
+    private wrapRequest(serviceId: ServiceID): RouteMessageWrapper {
+        const request: RecursiveOperationRequest = {
+            sessionId: this.getId(),
+            operation: this.config.operation
+        }
+        const msg: Message = {
+            messageId: v4(),
+            serviceId,
+            body: {
+                oneofKind: 'recursiveOperationRequest',
+                recursiveOperationRequest: request
+            }
+        }
+        const routeMessage: RouteMessageWrapper = {
+            message: msg,
+            requestId: v4(),
+            target: getRawFromDhtAddress(this.config.targetId),
+            sourcePeer: this.config.localPeerDescriptor,
+            reachableThrough: [],
+            routingPath: [],
+            parallelRootNodeIds: []
+        }
+        return routeMessage
+    }
+
+    private isCompleted(): boolean {
+        const unreportedHops: Set<DhtAddress> = new Set(this.allKnownHops)
+        this.reportedHops.forEach((id) => {
+            unreportedHops.delete(id)
+        })
+        if (this.noCloserNodesReceivedCounter >= 1 && unreportedHops.size === 0) {
+            if (this.config.operation === RecursiveOperation.FETCH_DATA
+                && (this.hasNonStaleData() || this.noCloserNodesReceivedCounter >= this.config.waitedRoutingPathCompletions)) {
+                return true
+            } else if (this.config.operation === RecursiveOperation.FETCH_DATA) {
+                return false
+            }
+            return true
+        }
+        return false
+    }
+
+    private hasNonStaleData(): boolean {
+        return Array.from(this.foundData.values()).some((entry) => entry.stale === false)
+    }
+
+    public onResponseReceived(
+        sourceId: DhtAddress,
+        routingPath: PeerDescriptor[],
+        nodes: PeerDescriptor[],
+        dataEntries: DataEntry[],
+        noCloserNodesFound: boolean
+    ): void {
+        this.addKnownHops(routingPath)
+        if (routingPath.length >= 1) {
+            this.setHopAsReported(routingPath[routingPath.length - 1])
+        }
+        nodes.forEach((descriptor: PeerDescriptor) => {
+            this.results.addContact(new Contact(descriptor))
+        })
+        this.processFoundData(dataEntries)
+        if (noCloserNodesFound || this.noCloserNodesReceivedFrom.has(sourceId)) {
+            this.onNoCloserPeersFound(sourceId)
+        }
+    }
+
+    private addKnownHops(routingPath: PeerDescriptor[]) {
+        const localNodeId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
+        routingPath.forEach((desc) => {
+            const newNodeId = getNodeIdFromPeerDescriptor(desc)
+            if (localNodeId !== newNodeId) {
+                this.allKnownHops.add(newNodeId)
+            }
+        })
+    }
+
+    private setHopAsReported(desc: PeerDescriptor) {
+        const localNodeId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
+        const newNodeId = getNodeIdFromPeerDescriptor(desc)
+        if (localNodeId !== newNodeId) {
+            this.reportedHops.add(newNodeId)
+        }
+        if (this.isCompleted()) {
+            if (!this.completionEventEmitted && this.isCompleted()) {
+                if (this.timeoutTask) {
+                    clearTimeout(this.timeoutTask)
+                    this.timeoutTask = undefined
+                }
+                this.emit('completed')
+                this.completionEventEmitted = true
+            }
+        }
+    }
+
+    private processFoundData(dataEntries: DataEntry[]): void {
+        dataEntries.forEach((entry) => {
+            const creatorNodeId = getDhtAddressFromRaw(entry.creator)
+            const existingEntry = this.foundData.get(creatorNodeId)
+            if (!existingEntry || existingEntry.createdAt! < entry.createdAt! 
+                || (existingEntry.createdAt! <= entry.createdAt! && entry.deleted)) {
+                this.foundData.set(creatorNodeId, entry)
+            }
+        })
+    }
+
+    private onNoCloserPeersFound(sourceId: DhtAddress): void {
+        this.noCloserNodesReceivedCounter += 1
+        this.noCloserNodesReceivedFrom.add(sourceId)
+        if (this.isCompleted()) {
+            this.emit('completed')
+            this.completionEventEmitted = true
+            if (this.timeoutTask) {
+                clearTimeout(this.timeoutTask)
+                this.timeoutTask = undefined
+            }
+        } else {
+            if (!this.timeoutTask && !this.completionEventEmitted) {
+                this.timeoutTask = setTimeout(() => {
+                    if (!this.completionEventEmitted) {
+                        this.emit('completed')
+                        this.completionEventEmitted = true
+                    }
+                }, 4000)  // TODO use config option or named constant?
+            }
+        }
+    }
+
+    public getResults(): RecursiveOperationResult {
+        return {
+            closestNodes: this.results.getAllContacts().map((contact) => contact.getPeerDescriptor()),
+            dataEntries: Array.from(this.foundData.values())
+        }
+    }
+
+    public getId(): string {
+        return this.id
+    }
+
+    public stop(): void {
+        if (this.timeoutTask) {
+            clearTimeout(this.timeoutTask)
+            this.timeoutTask = undefined
+        }
+        this.rpcCommunicator.destroy()
+        this.emit('completed')
+    }
+}

package/src/dht/recursive-operation/RecursiveOperationSessionRpcLocal.ts

@@ -0,0 +1,35 @@
+import { IRecursiveOperationSessionRpc } from '../../proto/packages/dht/protos/DhtRpc.server'
+import { Empty } from '../../proto/google/protobuf/empty'
+import { DataEntry, RecursiveOperationResponse, PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
+import { Logger } from '@streamr/utils'
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import { DhtCallContext } from '../../rpc-protocol/DhtCallContext'
+import { DhtAddress, getNodeIdFromPeerDescriptor } from '../../identifiers'
+
+const logger = new Logger(module)
+
+interface RecursiveOperationSessionRpcLocalConfig {
+    onResponseReceived: (
+        sourceId: DhtAddress,
+        routingPath: PeerDescriptor[],
+        nodes: PeerDescriptor[],
+        dataEntries: DataEntry[],
+        noCloserNodesFound: boolean
+    ) => void
+}
+
+export class RecursiveOperationSessionRpcLocal implements IRecursiveOperationSessionRpc {
+
+    private readonly config: RecursiveOperationSessionRpcLocalConfig
+
+    constructor(config: RecursiveOperationSessionRpcLocalConfig) {
+        this.config = config
+    }
+    
+    async sendResponse(report: RecursiveOperationResponse, context: ServerCallContext): Promise<Empty> {
+        const sourceId = getNodeIdFromPeerDescriptor((context as DhtCallContext).incomingSourceDescriptor!)
+        logger.trace('RecursiveOperationResponse arrived: ' + JSON.stringify(report))
+        this.config.onResponseReceived(sourceId, report.routingPath, report.closestConnectedPeers, report.dataEntries, report.noCloserNodesFound)
+        return {}
+    }
+}

package/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.ts

@@ -0,0 +1,30 @@
+import { Logger } from '@streamr/utils'
+import {
+    DataEntry,
+    PeerDescriptor,
+    RecursiveOperationResponse
+} from '../../proto/packages/dht/protos/DhtRpc'
+import { RecursiveOperationSessionRpcClient } from '../../proto/packages/dht/protos/DhtRpc.client'
+import { RpcRemote } from '../contact/RpcRemote'
+
+const logger = new Logger(module)
+
+export class RecursiveOperationSessionRpcRemote extends RpcRemote<RecursiveOperationSessionRpcClient> {
+
+    sendResponse(
+        routingPath: PeerDescriptor[],
+        closestNodes: PeerDescriptor[],
+        dataEntries: DataEntry[],
+        noCloserNodesFound: boolean
+    ): void {
+        const report: RecursiveOperationResponse = {
+            routingPath,
+            closestConnectedPeers: closestNodes,
+            dataEntries,
+            noCloserNodesFound
+        }
+        this.getClient().sendResponse(report, this.formDhtRpcOptions()).catch((_e) => {
+            logger.trace('Failed to send RecursiveOperationResponse')
+        })
+    }
+}

package/src/dht/routing/DuplicateDetector.ts

@@ -1,40 +1,28 @@
-type QueueEntry = [timestamp: number, value: string]
-
 export class DuplicateDetector {
 
     private values: Set<string> = new Set()
-    private queue: Array<QueueEntry> = []
-    private maxAge: number
-    private maxNumberOfValues: number
+    private queue: Array<string> = []
+    private maxItemCount: number
 
     constructor(
-        maxNumberOfValues: number,
-        maxAgeInSeconds: number
+        maxItemCount: number,
     ) {
-        this.maxNumberOfValues = maxNumberOfValues
-        this.maxAge = maxAgeInSeconds * 1000
+        this.maxItemCount = maxItemCount
     }
 
     public add(value: string): void {
         this.values.add(value)
-        this.queue.push([Date.now(), value])
-        this.cleanUp()
+        this.queue.push(value)
+        if (this.queue.length > this.maxItemCount) {
+            const removed = this.queue.shift()!
+            this.values.delete(removed)
+        }
     }
 
     public isMostLikelyDuplicate(value: string): boolean {
         return this.values.has(value)
     }
 
-    private cleanUp(): void {
-        const currentTime = Date.now()
-
-        while (this.queue.length > 0 && (this.queue.length > this.maxNumberOfValues ||
-            (currentTime - this.queue[0][0]) > this.maxAge)) {
-            const oldestEntry = this.queue.shift()
-            this.values.delete(oldestEntry![1])
-        }
-    }
-
     public size(): number {
         return this.values.size
     }