@streamr/trackerless-network 100.0.0-testnet-one.4 → 100.0.0-testnet-two.1

package/src/logic/EntryPointDiscovery.ts

@@ -51,7 +51,7 @@ const exponentialRunOff = async (
 
 const logger = new Logger(module)
 
-const ENTRYPOINT_STORE_LIMIT = 8
+export const ENTRYPOINT_STORE_LIMIT = 8
 export const NETWORK_SPLIT_AVOIDANCE_LIMIT = 4
 
 interface EntryPointDiscoveryConfig {
@@ -69,16 +69,14 @@ export class EntryPointDiscovery {
     private readonly config: EntryPointDiscoveryConfig
     private readonly storeInterval: number
     private readonly networkSplitAvoidedNodes: Set<NodeID> = new Set()
-
+    private isLocalNodeStoredAsEntryPoint = false
     constructor(config: EntryPointDiscoveryConfig) {
         this.config = config
         this.abortController = new AbortController()
         this.storeInterval = this.config.storeInterval ?? 60000
     }
 
-    async discoverEntryPointsFromDht(
-        knownEntryPointCount: number
-    ): Promise<FindEntryPointsResult> {
+    async discoverEntryPointsFromDht(knownEntryPointCount: number): Promise<FindEntryPointsResult> {
         if (knownEntryPointCount > 0) {
             return {
                 entryPointsFromDht: false,
@@ -124,6 +122,7 @@ export class EntryPointDiscovery {
         }
         const possibleNetworkSplitDetected = this.config.layer1Node.getNumberOfNeighbors() < NETWORK_SPLIT_AVOIDANCE_LIMIT
         if ((currentEntrypointCount < ENTRYPOINT_STORE_LIMIT) || possibleNetworkSplitDetected) {
+            this.isLocalNodeStoredAsEntryPoint = true
             await this.storeSelfAsEntryPoint()
             await this.keepSelfAsEntryPoint()
         }
@@ -165,7 +164,8 @@ export class EntryPointDiscovery {
             if (this.config.layer1Node.getNumberOfNeighbors() < NETWORK_SPLIT_AVOIDANCE_LIMIT) {
                 // Filter out nodes that are not neighbors as those nodes are assumed to be offline
                 const nodesToAvoid = rediscoveredEntrypoints
-                    .filter((peer) => !this.config.layer1Node.getAllNeighborPeerDescriptors().includes(peer))
+                    .filter((peer) => !this.config.layer1Node.getAllNeighborPeerDescriptors()
+                        .some((neighbor) => areEqualPeerDescriptors(neighbor, peer)))
                     .map((peer) => getNodeIdFromPeerDescriptor(peer))
                 nodesToAvoid.forEach((node) => this.networkSplitAvoidedNodes.add(node))
                 throw new Error(`Network split is still possible`)
@@ -175,6 +175,10 @@ export class EntryPointDiscovery {
         logger.trace(`Network split avoided`)
     }
 
+    public isLocalNodeEntryPoint(): boolean {
+        return this.isLocalNodeStoredAsEntryPoint
+    }
+
     async destroy(): Promise<void> {
         this.abortController.abort()
         await this.config.deleteEntryPointData(streamPartIdToDataKey(this.config.streamPartId))

package/src/logic/RandomGraphNode.ts

@@ -19,23 +19,24 @@ import { DeliveryRpcRemote } from './DeliveryRpcRemote'
 import { IDeliveryRpc } from '../proto/packages/trackerless-network/protos/NetworkRpc.server'
 import { DuplicateMessageDetector } from './DuplicateMessageDetector'
 import { Logger, addManagedEventListener } from '@streamr/utils'
-import { toProtoRpcClient } from '@streamr/proto-rpc'
-import { IHandshaker } from './neighbor-discovery/Handshaker'
+import { Handshaker } from './neighbor-discovery/Handshaker'
 import { Propagation } from './propagation/Propagation'
-import { INeighborFinder } from './neighbor-discovery/NeighborFinder'
-import { INeighborUpdateManager } from './neighbor-discovery/NeighborUpdateManager'
+import { NeighborFinder } from './neighbor-discovery/NeighborFinder'
+import { NeighborUpdateManager } from './neighbor-discovery/NeighborUpdateManager'
 import { DeliveryRpcLocal } from './DeliveryRpcLocal'
 import { ProxyConnectionRpcLocal } from './proxy/ProxyConnectionRpcLocal'
-import { IInspector } from './inspect/Inspector'
+import { Inspector } from './inspect/Inspector'
 import { TemporaryConnectionRpcLocal } from './temporary-connection/TemporaryConnectionRpcLocal'
 import { markAndCheckDuplicate } from './utils'
 import { NodeID, getNodeIdFromPeerDescriptor } from '../identifiers'
 import { Layer1Node } from './Layer1Node'
 import { StreamPartID } from '@streamr/protocol'
+import { uniqBy } from 'lodash'
 
 export interface Events {
     message: (message: StreamMessage) => void
     targetNeighborConnected: (nodeId: NodeID) => void
+    entryPointLeaveDetected: () => void
 }
 
 export interface StrictRandomGraphNodeConfig {
@@ -48,14 +49,16 @@ export interface StrictRandomGraphNodeConfig {
     nearbyNodeView: NodeList
     randomNodeView: NodeList
     targetNeighbors: NodeList
-    handshaker: IHandshaker
-    neighborFinder: INeighborFinder
-    neighborUpdateManager: INeighborUpdateManager
+    handshaker: Handshaker
+    neighborFinder: NeighborFinder
+    neighborUpdateManager: NeighborUpdateManager
     propagation: Propagation
     rpcCommunicator: ListeningRpcCommunicator
     numOfTargetNeighbors: number
-    inspector: IInspector
+    inspector: Inspector
     temporaryConnectionRpcLocal: TemporaryConnectionRpcLocal
+    isLocalNodeEntryPoint: () => boolean
+
     proxyConnectionRpcLocal?: ProxyConnectionRpcLocal
     rpcRequestTimeout?: number
 }
@@ -80,19 +83,25 @@ export class RandomGraphNode extends EventEmitter<Events> {
             rpcCommunicator: this.config.rpcCommunicator,
             markAndCheckDuplicate: (msg: MessageID, prev?: MessageRef) => markAndCheckDuplicate(this.duplicateDetectors, msg, prev),
             broadcast: (message: StreamMessage, previousNode?: NodeID) => this.broadcast(message, previousNode),
-            onLeaveNotice: (senderId: NodeID) => {
-                const contact = this.config.nearbyNodeView.get(senderId)
-                || this.config.randomNodeView.get(senderId)
-                || this.config.targetNeighbors.get(senderId)
-                || this.config.proxyConnectionRpcLocal?.getConnection(senderId )?.remote
+            onLeaveNotice: (sourceId: NodeID, sourceIsStreamEntryPoint: boolean) => {
+                if (this.abortController.signal.aborted) {
+                    return
+                }
+                const contact = this.config.nearbyNodeView.get(sourceId)
+                || this.config.randomNodeView.get(sourceId)
+                || this.config.targetNeighbors.get(sourceId)
+                || this.config.proxyConnectionRpcLocal?.getConnection(sourceId )?.remote
                 // TODO: check integrity of notifier?
                 if (contact) {
                     this.config.layer1Node.removeContact(contact.getPeerDescriptor())
                     this.config.targetNeighbors.remove(contact.getPeerDescriptor())
                     this.config.nearbyNodeView.remove(contact.getPeerDescriptor())
                     this.config.connectionLocker.unlockConnection(contact.getPeerDescriptor(), this.config.streamPartId)
-                    this.config.neighborFinder.start([senderId])
-                    this.config.proxyConnectionRpcLocal?.removeConnection(senderId)
+                    this.config.neighborFinder.start([sourceId])
+                    this.config.proxyConnectionRpcLocal?.removeConnection(sourceId)
+                }
+                if (sourceIsStreamEntryPoint) {
+                    this.emit('entryPointLeaveDetected')
                 }
             },
             markForInspection: (senderId: NodeID, messageId: MessageID) => this.config.inspector.markMessage(senderId, messageId)
@@ -191,7 +200,8 @@ export class RandomGraphNode extends EventEmitter<Events> {
                 this.config.localPeerDescriptor,
                 descriptor,
                 this.config.streamPartId,
-                toProtoRpcClient(new DeliveryRpcClient(this.config.rpcCommunicator.getRpcClientTransport())),
+                this.config.rpcCommunicator,
+                DeliveryRpcClient,
                 this.config.rpcRequestTimeout
             )
         ))
@@ -204,7 +214,8 @@ export class RandomGraphNode extends EventEmitter<Events> {
                     this.config.localPeerDescriptor,
                     descriptor,
                     this.config.streamPartId,
-                    toProtoRpcClient(new DeliveryRpcClient(this.config.rpcCommunicator.getRpcClientTransport())),
+                    this.config.rpcCommunicator,
+                    DeliveryRpcClient,
                     this.config.rpcRequestTimeout
 
                 )
@@ -221,7 +232,8 @@ export class RandomGraphNode extends EventEmitter<Events> {
                 this.config.localPeerDescriptor,
                 descriptor,
                 this.config.streamPartId,
-                toProtoRpcClient(new DeliveryRpcClient(this.config.rpcCommunicator.getRpcClientTransport())),
+                this.config.rpcCommunicator,
+                DeliveryRpcClient,
                 this.config.rpcRequestTimeout
             )
         ))
@@ -240,7 +252,8 @@ export class RandomGraphNode extends EventEmitter<Events> {
                 this.config.localPeerDescriptor,
                 descriptor,
                 this.config.streamPartId,
-                toProtoRpcClient(new DeliveryRpcClient(this.config.rpcCommunicator.getRpcClientTransport())),
+                this.config.rpcCommunicator,
+                DeliveryRpcClient,
                 this.config.rpcRequestTimeout
             )
         ))
@@ -256,14 +269,14 @@ export class RandomGraphNode extends EventEmitter<Events> {
     }
 
     private getNeighborCandidatesFromLayer1(): PeerDescriptor[] {
-        const uniqueNodes = new Set<PeerDescriptor>()
+        const nodes: PeerDescriptor[] = []
         this.config.layer1Node.getClosestContacts(this.config.nodeViewSize).forEach((peer: PeerDescriptor) => {
-            uniqueNodes.add(peer)
+            nodes.push(peer)
         })
         this.config.layer1Node.getAllNeighborPeerDescriptors().forEach((peer: PeerDescriptor) => {
-            uniqueNodes.add(peer)
+            nodes.push(peer)
         })
-        return Array.from(uniqueNodes)
+        return uniqBy(nodes, (p) => getNodeIdFromPeerDescriptor(p))
     }
 
     hasProxyConnection(nodeId: NodeID): boolean {
@@ -279,7 +292,7 @@ export class RandomGraphNode extends EventEmitter<Events> {
         }
         this.abortController.abort()
         this.config.proxyConnectionRpcLocal?.stop()
-        this.config.targetNeighbors.getAll().map((remote) => remote.leaveStreamPartNotice())
+        this.config.targetNeighbors.getAll().map((remote) => remote.leaveStreamPartNotice(this.config.isLocalNodeEntryPoint()))
         this.config.rpcCommunicator.destroy()
         this.removeAllListeners()
         this.config.nearbyNodeView.stop()

package/src/logic/StreamrNode.ts

@@ -130,7 +130,6 @@ export class StreamrNode extends EventEmitter<Events> {
             return
         }
         const layer1Node = this.createLayer1Node(streamPartId, this.knownStreamPartEntryPoints.get(streamPartId) ?? [])
-        const node = this.createRandomGraphNode(streamPartId, layer1Node)
         const entryPointDiscovery = new EntryPointDiscovery({
             streamPartId,
             localPeerDescriptor: this.getPeerDescriptor(),
@@ -139,6 +138,11 @@ export class StreamrNode extends EventEmitter<Events> {
             storeEntryPointData: (key, data) => this.layer0Node!.storeDataToDht(key, data),
             deleteEntryPointData: async (key: Uint8Array) => this.layer0Node!.deleteDataFromDht(key, false)
         })
+        const node = this.createRandomGraphNode(
+            streamPartId,
+            layer1Node, 
+            () => entryPointDiscovery.isLocalNodeEntryPoint()
+        )
         streamPart = {
             proxied: false,
             layer1Node,
@@ -155,6 +159,14 @@ export class StreamrNode extends EventEmitter<Events> {
         node.on('message', (message: StreamMessage) => {
             this.emit('newMessage', message)
         })
+        const handleEntryPointLeave = async () => {
+            if (this.destroyed || entryPointDiscovery.isLocalNodeEntryPoint() || this.knownStreamPartEntryPoints.has(streamPartId)) {
+                return
+            }
+            const entryPoints = await entryPointDiscovery.discoverEntryPointsFromDht(0)
+            await entryPointDiscovery.storeSelfAsEntryPointIfNecessary(entryPoints.discoveredEntryPoints.length)
+        }
+        node.on('entryPointLeaveDetected', () => handleEntryPointLeave())
         setImmediate(async () => {
             try {
                 await this.startLayersAndJoinDht(streamPartId, entryPointDiscovery)
@@ -190,13 +202,17 @@ export class StreamrNode extends EventEmitter<Events> {
             serviceId: 'layer1::' + streamPartId,
             peerDescriptor: this.layer0Node!.getLocalPeerDescriptor(),
             entryPoints,
-            numberOfNodesPerKBucket: 4,
+            numberOfNodesPerKBucket: 4,  // TODO use config option or named constant?
             rpcRequestTimeout: EXISTING_CONNECTION_TIMEOUT,
-            dhtJoinTimeout: 20000
+            dhtJoinTimeout: 20000  // TODO use config option or named constant?
         })
     }
 
-    private createRandomGraphNode(streamPartId: StreamPartID, layer1Node: Layer1Node) {
+    private createRandomGraphNode(
+        streamPartId: StreamPartID,
+        layer1Node: Layer1Node,
+        isLocalNodeEntryPoint: () => boolean
+    ) {
         return createRandomGraphNode({
             streamPartId,
             transport: this.transport!,
@@ -206,7 +222,8 @@ export class StreamrNode extends EventEmitter<Events> {
             minPropagationTargets: this.config.streamPartitionMinPropagationTargets,
             numOfTargetNeighbors: this.config.streamPartitionNumOfNeighbors,
             acceptProxyConnections: this.config.acceptProxyConnections,
-            rpcRequestTimeout: this.config.rpcRequestTimeout
+            rpcRequestTimeout: this.config.rpcRequestTimeout,
+            isLocalNodeEntryPoint
         })
     }
 

package/src/logic/formStreamPartDeliveryServiceId.ts

@@ -1,5 +1,6 @@
+import { ServiceID } from '@streamr/dht'
 import { StreamPartID } from '@streamr/protocol'
 
-export const formStreamPartDeliveryServiceId = (streamPartId: StreamPartID): string => {
+export const formStreamPartDeliveryServiceId = (streamPartId: StreamPartID): ServiceID => {
     return `stream-part-delivery-${streamPartId}`
 }

package/src/logic/inspect/Inspector.ts

@@ -1,8 +1,8 @@
-import { PeerDescriptor, ConnectionLocker } from '@streamr/dht'
+import { PeerDescriptor, ConnectionLocker, LockID } from '@streamr/dht'
 import { MessageID } from '../../proto/packages/trackerless-network/protos/NetworkRpc'
 import { InspectSession, Events as InspectSessionEvents } from './InspectSession'
 import { TemporaryConnectionRpcClient } from '../../proto/packages/trackerless-network/protos/NetworkRpc.client'
-import { ProtoRpcClient, RpcCommunicator, toProtoRpcClient } from '@streamr/proto-rpc'
+import { RpcCommunicator } from '@streamr/proto-rpc'
 import { Logger, waitForEvent3 } from '@streamr/utils'
 import { TemporaryConnectionRpcRemote } from '../temporary-connection/TemporaryConnectionRpcRemote'
 import { NodeID, getNodeIdFromPeerDescriptor } from '../../identifiers'
@@ -14,40 +14,39 @@ interface InspectorConfig {
     rpcCommunicator: RpcCommunicator
     connectionLocker: ConnectionLocker
     inspectionTimeout?: number
-    openInspectConnection?: (peerDescriptor: PeerDescriptor, lockId: string) => Promise<void>
-}
-
-export interface IInspector {
-    inspect(peerDescriptor: PeerDescriptor): Promise<boolean>
-    markMessage(sender: NodeID, messageId: MessageID): void
-    isInspected(nodeId: NodeID): boolean
-    stop(): void
+    openInspectConnection?: (peerDescriptor: PeerDescriptor, lockId: LockID) => Promise<void>
 }
 
 const logger = new Logger(module)
 const DEFAULT_TIMEOUT = 60 * 1000
 
-export class Inspector implements IInspector {
+export class Inspector {
 
     private readonly sessions: Map<NodeID, InspectSession> = new Map()
     private readonly streamPartId: StreamPartID
-    private readonly client: ProtoRpcClient<TemporaryConnectionRpcClient>
     private readonly localPeerDescriptor: PeerDescriptor
+    private readonly rpcCommunicator: RpcCommunicator
     private readonly connectionLocker: ConnectionLocker
     private readonly inspectionTimeout: number
-    private readonly openInspectConnection: (peerDescriptor: PeerDescriptor, lockId: string) => Promise<void>
+    private readonly openInspectConnection: (peerDescriptor: PeerDescriptor, lockId: LockID) => Promise<void>
 
     constructor(config: InspectorConfig) {
         this.streamPartId = config.streamPartId
         this.localPeerDescriptor = config.localPeerDescriptor
-        this.client = toProtoRpcClient(new TemporaryConnectionRpcClient(config.rpcCommunicator.getRpcClientTransport()))
+        this.rpcCommunicator = config.rpcCommunicator
         this.connectionLocker = config.connectionLocker
         this.inspectionTimeout = config.inspectionTimeout ?? DEFAULT_TIMEOUT
         this.openInspectConnection = config.openInspectConnection ?? this.defaultOpenInspectConnection
     }
 
-    async defaultOpenInspectConnection(peerDescriptor: PeerDescriptor, lockId: string): Promise<void> {
-        const rpcRemote = new TemporaryConnectionRpcRemote(this.localPeerDescriptor, peerDescriptor, this.streamPartId, this.client)
+    async defaultOpenInspectConnection(peerDescriptor: PeerDescriptor, lockId: LockID): Promise<void> {
+        const rpcRemote = new TemporaryConnectionRpcRemote(
+            this.localPeerDescriptor,
+            peerDescriptor,
+            this.streamPartId,
+            this.rpcCommunicator,
+            TemporaryConnectionRpcClient
+        )
         await rpcRemote.openConnection()
         this.connectionLocker.lockConnection(peerDescriptor, lockId)
     }

package/src/logic/neighbor-discovery/HandshakeRpcLocal.ts

@@ -1,5 +1,9 @@
-import { Empty } from '../../proto/google/protobuf/empty'
-import { InterleaveNotice, StreamPartHandshakeRequest, StreamPartHandshakeResponse } from '../../proto/packages/trackerless-network/protos/NetworkRpc'
+import { 
+    InterleaveRequest,
+    InterleaveResponse,
+    StreamPartHandshakeRequest,
+    StreamPartHandshakeResponse
+} from '../../proto/packages/trackerless-network/protos/NetworkRpc'
 import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
 import { NodeList } from '../NodeList'
 import { ConnectionLocker, DhtCallContext, PeerDescriptor } from '@streamr/dht'
@@ -7,7 +11,7 @@ import { IHandshakeRpc } from '../../proto/packages/trackerless-network/protos/N
 import { HandshakeRpcRemote } from './HandshakeRpcRemote'
 import { DeliveryRpcRemote } from '../DeliveryRpcRemote'
 import { NodeID, getNodeIdFromPeerDescriptor } from '../../identifiers'
-import { binaryToHex } from '@streamr/utils'
+import { Logger, binaryToHex } from '@streamr/utils'
 import { StreamPartID } from '@streamr/protocol'
 
 interface HandshakeRpcLocalConfig {
@@ -15,12 +19,15 @@ interface HandshakeRpcLocalConfig {
     targetNeighbors: NodeList
     connectionLocker: ConnectionLocker
     ongoingHandshakes: Set<NodeID>
+    ongoingInterleaves: Set<NodeID>
     maxNeighborCount: number
     createRpcRemote: (target: PeerDescriptor) => HandshakeRpcRemote
     createDeliveryRpcRemote: (peerDescriptor: PeerDescriptor) => DeliveryRpcRemote
     handshakeWithInterleaving: (target: PeerDescriptor, senderId: NodeID) => Promise<boolean>
 }
 
+const logger = new Logger(module)
+
 export class HandshakeRpcLocal implements IHandshakeRpc {
 
     private readonly config: HandshakeRpcLocalConfig
@@ -36,13 +43,17 @@ export class HandshakeRpcLocal implements IHandshakeRpc {
     private handleRequest(request: StreamPartHandshakeRequest, context: ServerCallContext): StreamPartHandshakeResponse {
         const senderDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
         const getInterleaveSourceIds = () => (request.interleaveSourceId !== undefined) ? [binaryToHex(request.interleaveSourceId) as NodeID] : []
-        if (this.config.targetNeighbors.hasNode(senderDescriptor)
+        if (this.config.ongoingInterleaves.has(getNodeIdFromPeerDescriptor(senderDescriptor))) {
+            return this.rejectHandshake(request)
+        } else if (this.config.targetNeighbors.hasNode(senderDescriptor)
             || this.config.ongoingHandshakes.has(getNodeIdFromPeerDescriptor(senderDescriptor))
         ) {
             return this.acceptHandshake(request, senderDescriptor)
         } else if (this.config.targetNeighbors.size() + this.config.ongoingHandshakes.size < this.config.maxNeighborCount) {
             return this.acceptHandshake(request, senderDescriptor)
-        } else if (this.config.targetNeighbors.size(getInterleaveSourceIds()) >= 2) {
+        } else if (this.config.targetNeighbors.size(getInterleaveSourceIds()) - this.config.ongoingInterleaves.size >= 2) {
+            // Do not accept the handshakes requests if the target neighbor count can potentially drop below 2 
+            // due to interleaving. This ensures that a stable number of connections is kept during high churn.
             return this.acceptHandshakeWithInterleaving(request, senderDescriptor)
         } else {
             return this.rejectHandshake(request)
@@ -69,7 +80,9 @@ export class HandshakeRpcLocal implements IHandshakeRpc {
     }
 
     private acceptHandshakeWithInterleaving(request: StreamPartHandshakeRequest, requester: PeerDescriptor): StreamPartHandshakeResponse {
-        const exclude = request.neighborIds.map((id: Uint8Array) => binaryToHex(id) as NodeID)
+        const exclude: NodeID[] = []
+        request.neighborIds.forEach((id: Uint8Array) => exclude.push(binaryToHex(id) as NodeID))
+        this.config.ongoingInterleaves.forEach((id) => exclude.push(id))
         exclude.push(getNodeIdFromPeerDescriptor(requester))
         if (request.interleaveSourceId !== undefined) {
             exclude.push(binaryToHex(request.interleaveSourceId) as NodeID)
@@ -77,10 +90,25 @@ export class HandshakeRpcLocal implements IHandshakeRpc {
         const furthest = this.config.targetNeighbors.getFurthest(exclude)
         const furthestPeerDescriptor = furthest ? furthest.getPeerDescriptor() : undefined
         if (furthest) {
+            const nodeId = getNodeIdFromPeerDescriptor(furthest.getPeerDescriptor())
             const remote = this.config.createRpcRemote(furthest.getPeerDescriptor())
-            remote.interleaveNotice(requester)
-            this.config.targetNeighbors.remove(furthest.getPeerDescriptor())
-            this.config.connectionLocker.unlockConnection(furthestPeerDescriptor!, this.config.streamPartId)
+            this.config.ongoingInterleaves.add(nodeId)
+            // Run this with then catch instead of setImmediate to avoid changes in state
+            // eslint-disable-next-line promise/catch-or-return
+            remote.interleaveRequest(requester).then((response) => {
+                // If response is accepted, remove the furthest node from the target neighbors
+                // and unlock the connection
+                // If response is not accepted, keep the furthest node as a neighbor
+                if (response.accepted) {
+                    this.config.targetNeighbors.remove(furthest.getPeerDescriptor())
+                    this.config.connectionLocker.unlockConnection(furthestPeerDescriptor!, this.config.streamPartId)
+                }
+                return
+            }).catch(() => {
+                // no-op: InterleaveRequest cannot reject
+            }).finally(() => {
+                this.config.ongoingInterleaves.delete(nodeId)
+            })
         }
         this.config.targetNeighbors.add(this.config.createDeliveryRpcRemote(requester))
         this.config.connectionLocker.lockConnection(requester, this.config.streamPartId)
@@ -91,16 +119,19 @@ export class HandshakeRpcLocal implements IHandshakeRpc {
         }
     }
 
-    async interleaveNotice(message: InterleaveNotice, context: ServerCallContext): Promise<Empty> {
-        if (message.streamPartId === this.config.streamPartId) {
-            const senderPeerDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
-            const senderId = getNodeIdFromPeerDescriptor(senderPeerDescriptor)
+    async interleaveRequest(message: InterleaveRequest, context: ServerCallContext): Promise<InterleaveResponse> {
+        const senderPeerDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
+        const senderId = getNodeIdFromPeerDescriptor(senderPeerDescriptor)
+        try {
+            await this.config.handshakeWithInterleaving(message.interleaveTargetDescriptor!, senderId)
             if (this.config.targetNeighbors.hasNodeById(senderId)) {
                 this.config.connectionLocker.unlockConnection(senderPeerDescriptor, this.config.streamPartId)
                 this.config.targetNeighbors.remove(senderPeerDescriptor)
             }
-            this.config.handshakeWithInterleaving(message.interleaveTargetDescriptor!, senderId).catch((_e) => {})
+            return { accepted: true }
+        } catch (err) {
+            logger.debug(`interleaveRequest to ${getNodeIdFromPeerDescriptor(message.interleaveTargetDescriptor!)} failed: ${err}`)
+            return { accepted: false }
         }
-        return Empty
     }
 }

package/src/logic/neighbor-discovery/HandshakeRpcRemote.ts

@@ -2,8 +2,8 @@ import { PeerDescriptor, RpcRemote } from '@streamr/dht'
 import { Logger, hexToBinary } from '@streamr/utils'
 import { v4 } from 'uuid'
 import { NodeID, getNodeIdFromPeerDescriptor } from '../../identifiers'
-import { InterleaveNotice, StreamPartHandshakeRequest } from '../../proto/packages/trackerless-network/protos/NetworkRpc'
-import { IHandshakeRpcClient } from '../../proto/packages/trackerless-network/protos/NetworkRpc.client'
+import { InterleaveRequest, InterleaveResponse, StreamPartHandshakeRequest } from '../../proto/packages/trackerless-network/protos/NetworkRpc'
+import { HandshakeRpcClient } from '../../proto/packages/trackerless-network/protos/NetworkRpc.client'
 
 const logger = new Logger(module)
 
@@ -12,7 +12,9 @@ interface HandshakeResponse {
     interleaveTargetDescriptor?: PeerDescriptor
 }
 
-export class HandshakeRpcRemote extends RpcRemote<IHandshakeRpcClient> {
+export const INTERLEAVE_REQUEST_TIMEOUT = 15000
+
+export class HandshakeRpcRemote extends RpcRemote<HandshakeRpcClient> {
 
     async handshake(
         neighborIds: NodeID[],
@@ -40,16 +42,25 @@ export class HandshakeRpcRemote extends RpcRemote<IHandshakeRpcClient> {
         }
     }
 
-    interleaveNotice(originatorDescriptor: PeerDescriptor): void {
-        const notification: InterleaveNotice = {
-            streamPartId: this.getServiceId(),
+    async interleaveRequest(originatorDescriptor: PeerDescriptor): Promise<InterleaveResponse> {
+        const request: InterleaveRequest = {
             interleaveTargetDescriptor: originatorDescriptor
         }
         const options = this.formDhtRpcOptions({
-            notification: true
-        })
-        this.getClient().interleaveNotice(notification, options).catch(() => {
-            logger.debug('Failed to send interleaveNotice')
+            connect: false,
+            timeout: INTERLEAVE_REQUEST_TIMEOUT
         })
+        try {
+            const res = await this.getClient().interleaveRequest(request, options)
+            return {
+                accepted: res.accepted
+            }
+        } catch (err) {
+            logger.debug(`interleaveRequest to ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())} failed: ${err}`)
+            return {
+                accepted: false
+            }
+        }
+        
     }
 }

package/src/logic/neighbor-discovery/Handshaker.ts

@@ -1,19 +1,19 @@
 import { ConnectionLocker, PeerDescriptor } from '@streamr/dht'
 import { NodeList } from '../NodeList'
 import { DeliveryRpcRemote } from '../DeliveryRpcRemote'
-import { ProtoRpcClient, RpcCommunicator, toProtoRpcClient } from '@streamr/proto-rpc'
+import { RpcCommunicator } from '@streamr/proto-rpc'
 import {
-    HandshakeRpcClient,
-    IHandshakeRpcClient, DeliveryRpcClient
+    DeliveryRpcClient, HandshakeRpcClient
 } from '../../proto/packages/trackerless-network/protos/NetworkRpc.client'
 import {
-    InterleaveNotice,
+    InterleaveRequest,
+    InterleaveResponse,
     StreamPartHandshakeRequest,
     StreamPartHandshakeResponse
 } from '../../proto/packages/trackerless-network/protos/NetworkRpc'
 import { Logger } from '@streamr/utils'
 import { IHandshakeRpc } from '../../proto/packages/trackerless-network/protos/NetworkRpc.server'
-import { HandshakeRpcRemote } from './HandshakeRpcRemote'
+import { HandshakeRpcRemote, INTERLEAVE_REQUEST_TIMEOUT } from './HandshakeRpcRemote'
 import { HandshakeRpcLocal } from './HandshakeRpcLocal'
 import { NodeID, getNodeIdFromPeerDescriptor } from '../../identifiers'
 import { StreamPartID } from '@streamr/protocol'
@@ -34,38 +34,33 @@ const logger = new Logger(module)
 
 const PARALLEL_HANDSHAKE_COUNT = 2
 
-export interface IHandshaker {
-    attemptHandshakesOnContacts(excludedIds: NodeID[]): Promise<NodeID[]>
-    getOngoingHandshakes(): Set<NodeID>
-}
-
-export class Handshaker implements IHandshaker {
+export class Handshaker {
 
     private readonly ongoingHandshakes: Set<NodeID> = new Set()
     private config: HandshakerConfig
-    private readonly client: ProtoRpcClient<IHandshakeRpcClient>
     private readonly rpcLocal: IHandshakeRpc
 
     constructor(config: HandshakerConfig) {
         this.config = config
-        this.client = toProtoRpcClient(new HandshakeRpcClient(this.config.rpcCommunicator.getRpcClientTransport()))
         this.rpcLocal = new HandshakeRpcLocal({
             streamPartId: this.config.streamPartId,
             targetNeighbors: this.config.targetNeighbors,
             connectionLocker: this.config.connectionLocker,
             ongoingHandshakes: this.ongoingHandshakes,
+            ongoingInterleaves: new Set(),
             maxNeighborCount: this.config.maxNeighborCount,
             handshakeWithInterleaving: (target: PeerDescriptor, senderId: NodeID) => this.handshakeWithInterleaving(target, senderId),
             createRpcRemote: (target: PeerDescriptor) => this.createRpcRemote(target),
             createDeliveryRpcRemote: (target: PeerDescriptor) => this.createDeliveryRpcRemote(target)
         })
-        this.config.rpcCommunicator.registerRpcNotification(InterleaveNotice, 'interleaveNotice',
-            (req: InterleaveNotice, context) => this.rpcLocal.interleaveNotice(req, context))
+        this.config.rpcCommunicator.registerRpcMethod(InterleaveRequest, InterleaveResponse, 'interleaveRequest',
+            (req: InterleaveRequest, context) => this.rpcLocal.interleaveRequest(req, context), { timeout: INTERLEAVE_REQUEST_TIMEOUT })
         this.config.rpcCommunicator.registerRpcMethod(StreamPartHandshakeRequest, StreamPartHandshakeResponse, 'handshake',
             (req: StreamPartHandshakeRequest, context) => this.rpcLocal.handshake(req, context))
     }
 
     async attemptHandshakesOnContacts(excludedIds: NodeID[]): Promise<NodeID[]> {
+        // TODO use config option or named constant? or why the value 2?
         if (this.config.targetNeighbors.size() + this.ongoingHandshakes.size < this.config.maxNeighborCount - 2) {
             logger.trace(`Attempting parallel handshakes with ${PARALLEL_HANDSHAKE_COUNT} targets`)
             return this.selectParallelTargetsAndHandshake(excludedIds)
@@ -142,13 +137,7 @@ export class Handshaker implements IHandshaker {
     }
 
     private async handshakeWithInterleaving(target: PeerDescriptor, interleaveSourceId: NodeID): Promise<boolean> {
-        const targetNeighbor = new HandshakeRpcRemote(
-            this.config.localPeerDescriptor,
-            target,
-            this.config.streamPartId,
-            this.client,
-            this.config.rpcRequestTimeout
-        )
+        const targetNeighbor = this.createRpcRemote(target)
         const targetNodeId = getNodeIdFromPeerDescriptor(targetNeighbor.getPeerDescriptor())
         this.ongoingHandshakes.add(targetNodeId)
         const result = await targetNeighbor.handshake(
@@ -169,7 +158,8 @@ export class Handshaker implements IHandshaker {
             this.config.localPeerDescriptor,
             targetPeerDescriptor,
             this.config.streamPartId,
-            this.client,
+            this.config.rpcCommunicator,
+            HandshakeRpcClient,
             this.config.rpcRequestTimeout
         )
     }
@@ -179,7 +169,8 @@ export class Handshaker implements IHandshaker {
             this.config.localPeerDescriptor,
             targetPeerDescriptor,
             this.config.streamPartId,
-            toProtoRpcClient(new DeliveryRpcClient(this.config.rpcCommunicator.getRpcClientTransport())),
+            this.config.rpcCommunicator,
+            DeliveryRpcClient,
             this.config.rpcRequestTimeout
         )
     }

package/src/logic/neighbor-discovery/NeighborFinder.ts

@@ -12,13 +12,7 @@ interface FindNeighborsSessionConfig {
 const INITIAL_WAIT = 100
 const INTERVAL = 250
 
-export interface INeighborFinder {
-    start(excluded?: NodeID[]): void
-    stop(): void
-    isRunning(): boolean
-}
-
-export class NeighborFinder implements INeighborFinder {
+export class NeighborFinder {
     private readonly abortController: AbortController
     private readonly config: FindNeighborsSessionConfig
     private running = false