@streamr/dht 0.0.1-tatum.0

package/test/integration/RemoteRouter.test.ts

@@ -0,0 +1,83 @@
+import { RpcCommunicator, toProtoRpcClient } from '@streamr/proto-rpc'
+import { RemoteRouter } from '../../src/dht/routing/RemoteRouter'
+import { Message, MessageType, PeerDescriptor, RouteMessageAck, RouteMessageWrapper } from '../../src/proto/packages/dht/protos/DhtRpc'
+import { RoutingServiceClient } from '../../src/proto/packages/dht/protos/DhtRpc.client'
+import { RpcMessage } from '../../src/proto/packages/proto-rpc/protos/ProtoRpc'
+import { DhtCallContext } from '../../src/rpc-protocol/DhtCallContext'
+import { createWrappedClosestPeersRequest, generateId, MockRoutingService } from '../utils/utils'
+
+describe('RemoteRouter', () => {
+
+    let remoteRouter: RemoteRouter
+    let clientRpcCommunicator: RpcCommunicator
+    let serverRpcCommunicator: RpcCommunicator
+    const serviceId = 'test'
+    const clientPeerDescriptor: PeerDescriptor = {
+        kademliaId: generateId('dhtPeer'),
+        type: 0
+    }
+    const serverPeerDescriptor: PeerDescriptor = {
+        kademliaId: generateId('server'),
+        type: 0
+    }
+
+    beforeEach(() => {
+        clientRpcCommunicator = new RpcCommunicator()
+        serverRpcCommunicator = new RpcCommunicator()
+        serverRpcCommunicator.registerRpcMethod(RouteMessageWrapper, RouteMessageAck, 'routeMessage', MockRoutingService.routeMessage)
+        clientRpcCommunicator.on('outgoingMessage', (message: RpcMessage, _requestId: string, _ucallContext?: DhtCallContext) => {
+            serverRpcCommunicator.handleIncomingMessage(message)
+        })
+        serverRpcCommunicator.on('outgoingMessage', (message: RpcMessage, _requestId: string, _ucallContext?: DhtCallContext) => {
+            clientRpcCommunicator.handleIncomingMessage(message)
+        })
+        const client = toProtoRpcClient(new RoutingServiceClient(clientRpcCommunicator.getRpcClientTransport()))
+        remoteRouter = new RemoteRouter(clientPeerDescriptor, serverPeerDescriptor, client, serviceId)
+    })
+
+    it('routeMessage happy path', async () => {
+        const rpcWrapper = createWrappedClosestPeersRequest(clientPeerDescriptor, serverPeerDescriptor)
+        const routed: Message = {
+            serviceId: serviceId,
+            messageId: 'routed',
+            messageType: MessageType.RPC,
+            body: {
+                oneofKind: 'rpcMessage',
+                rpcMessage: rpcWrapper
+            }
+        }
+        const routable = await remoteRouter.routeMessage({
+            requestId: 'routed',
+            message: routed,
+            sourcePeer: clientPeerDescriptor,
+            destinationPeer: serverPeerDescriptor,
+            reachableThrough: [],
+            routingPath: []
+        })
+        expect(routable).toEqual(true)
+    })
+
+    it('routeMessage error path', async () => {
+        serverRpcCommunicator.registerRpcMethod(RouteMessageWrapper, RouteMessageAck, 'routeMessage', MockRoutingService.throwRouteMessageError)
+        const rpcWrapper = createWrappedClosestPeersRequest(clientPeerDescriptor, serverPeerDescriptor)
+        const routed: Message = {
+            serviceId: serviceId,
+            messageId: 'routed',
+            messageType: MessageType.RPC,
+            body: {
+                oneofKind: 'rpcMessage',
+                rpcMessage: rpcWrapper
+            }
+        }
+        const routable = await remoteRouter.routeMessage({
+            requestId: 'routed',
+            message: routed,
+            sourcePeer: clientPeerDescriptor,
+            destinationPeer: serverPeerDescriptor,
+            reachableThrough: [],
+            routingPath: []
+        })
+        expect(routable).toEqual(false)
+    })
+
+})

package/test/integration/RemoteStore.test.ts

@@ -0,0 +1,66 @@
+import { RpcCommunicator, toProtoRpcClient } from '@streamr/proto-rpc'
+import {
+    PeerDescriptor,
+    StoreDataRequest,
+    StoreDataResponse
+} from '../../src/proto/packages/dht/protos/DhtRpc'
+import { generateId, MockStoreService } from '../utils/utils'
+import { RpcMessage } from '../../src/proto/packages/proto-rpc/protos/ProtoRpc'
+import { DhtCallContext } from '../../src/rpc-protocol/DhtCallContext'
+import { StoreServiceClient } from '../../src/proto/packages/dht/protos/DhtRpc.client'
+import { RemoteStore } from '../../src/dht/store/RemoteStore'
+import { Any } from '../../src/proto/google/protobuf/any'
+
+describe('RemoteStore', () => {
+
+    let remoteStore: RemoteStore
+    let clientRpcCommunicator: RpcCommunicator
+    let serverRpcCommunicator: RpcCommunicator
+    const serviceId = 'test'
+    const clientPeerDescriptor: PeerDescriptor = {
+        kademliaId: generateId('dhtPeer'),
+        type: 0
+    }
+    const serverPeerDescriptor: PeerDescriptor = {
+        kademliaId: generateId('server'),
+        type: 0
+    }
+    const data = Any.pack(clientPeerDescriptor, PeerDescriptor)
+    const request: StoreDataRequest = {
+        kademliaId: clientPeerDescriptor.kademliaId,
+        data,
+        ttl: 10
+    }
+
+    beforeEach(() => {
+        clientRpcCommunicator = new RpcCommunicator()
+        serverRpcCommunicator = new RpcCommunicator()
+        serverRpcCommunicator.registerRpcMethod(StoreDataRequest, StoreDataResponse, 'storeData', MockStoreService.storeData)
+        clientRpcCommunicator.on('outgoingMessage', (message: RpcMessage, _requestId: string, _ucallContext?: DhtCallContext) => {
+            serverRpcCommunicator.handleIncomingMessage(message)
+        })
+        serverRpcCommunicator.on('outgoingMessage', (message: RpcMessage, _requestId: string, _ucallContext?: DhtCallContext) => {
+            clientRpcCommunicator.handleIncomingMessage(message)
+        })
+        const client = toProtoRpcClient(new StoreServiceClient(clientRpcCommunicator.getRpcClientTransport()))
+        remoteStore = new RemoteStore(clientPeerDescriptor, serverPeerDescriptor, client, serviceId)
+    })
+
+    it('storeData happy path', async () => {
+        const response = await remoteStore.storeData(request)
+        expect(response.error).toBeEmpty()
+    })
+
+    it('storeData rejects', async () => {
+        serverRpcCommunicator.registerRpcMethod(StoreDataRequest, StoreDataResponse, 'storeData', MockStoreService.throwStoreDataError)
+        await expect(remoteStore.storeData(request))
+            .rejects.toThrowError('Could not store data to 736572766572 from 64687450656572 Error: Mock')
+    })
+
+    it('storeData response error', async () => {
+        serverRpcCommunicator.registerRpcMethod(StoreDataRequest, StoreDataResponse, 'storeData', MockStoreService.storeDataErrorString)
+        const response = await remoteStore.storeData(request)
+        expect(response.error).toEqual('Mock')
+    })
+
+})

package/test/integration/RouteMessage.test.ts

@@ -0,0 +1,254 @@
+import { DhtNode, Events as DhtNodeEvents } from '../../src/dht/DhtNode'
+import { Message, MessageType, PeerDescriptor, RouteMessageWrapper } from '../../src/proto/packages/dht/protos/DhtRpc'
+import { RpcMessage } from '../../src/proto/packages/proto-rpc/protos/ProtoRpc'
+import { Logger, runAndWaitForEvents3, waitForCondition } from '@streamr/utils'
+import { createMockConnectionDhtNode, createWrappedClosestPeersRequest } from '../utils/utils'
+import { PeerID } from '../../src/helpers/PeerID'
+import { Simulator } from '../../src/connection/Simulator/Simulator'
+import { v4 } from 'uuid'
+import { UUID } from '../../src/helpers/UUID'
+import { Any } from '../../src/proto/google/protobuf/any'
+import { RoutingMode } from '../../src/dht/routing/RoutingSession'
+
+const logger = new Logger(module)
+
+describe('Route Message With Mock Connections', () => {
+    let entryPoint: DhtNode
+    let sourceNode: DhtNode
+    let destinationNode: DhtNode
+    let routerNodes: DhtNode[]
+    let simulator: Simulator
+    let entryPointDescriptor: PeerDescriptor
+
+    const entryPointId = '0'
+    const sourceId = 'eeeeeeeee'
+    const destinationId = '000000000'
+    const NUM_NODES = 30
+
+    const receiveMatrix: Array<Array<number>> = []
+
+    beforeEach(async () => {
+        routerNodes = []
+        simulator = new Simulator()
+        entryPoint = await createMockConnectionDhtNode(entryPointId, simulator)
+
+        entryPointDescriptor = {
+            kademliaId: entryPoint.getNodeId().value,
+            nodeName: 'entrypoint',
+            type: 0
+        }
+
+        sourceNode = await createMockConnectionDhtNode(sourceId, simulator)
+        destinationNode = await createMockConnectionDhtNode(destinationId, simulator)
+
+        for (let i = 1; i < NUM_NODES; i++) {
+            const nodeId = `${i}`
+            const node = await createMockConnectionDhtNode(nodeId, simulator)
+            routerNodes.push(node)
+        }
+
+        await destinationNode.joinDht([entryPointDescriptor])
+        await sourceNode.joinDht([entryPointDescriptor])
+        await Promise.all(routerNodes.map((node) => node.joinDht([entryPointDescriptor])))
+        await entryPoint.joinDht([entryPointDescriptor])
+    }, 15000)
+
+    afterEach(async () => {
+        await Promise.allSettled(routerNodes.map((node) => node.stop()))
+        await Promise.allSettled([
+            entryPoint.stop(),
+            destinationNode.stop(),
+            sourceNode.stop()
+        ])
+
+        logger.info('calling simulator stop')
+        simulator.stop()
+        logger.info('simulator stop called')
+    }, 10000)
+
+    it('Happy path', async () => {
+        const rpcWrapper = createWrappedClosestPeersRequest(sourceNode.getPeerDescriptor(), destinationNode.getPeerDescriptor())
+        const message: Message = {
+            serviceId: 'unknown',
+            messageId: v4(),
+            messageType: MessageType.RPC,
+            body: {
+                oneofKind: 'rpcMessage',
+                rpcMessage: rpcWrapper
+            },
+            sourceDescriptor: sourceNode.getPeerDescriptor(),
+            targetDescriptor: destinationNode.getPeerDescriptor()
+        }
+
+        await runAndWaitForEvents3<DhtNodeEvents>([() => {
+            sourceNode.router!.doRouteMessage({
+                message: message,
+                destinationPeer: destinationNode.getPeerDescriptor(),
+                requestId: v4(),
+                sourcePeer: sourceNode.getPeerDescriptor(),
+                reachableThrough: [],
+                routingPath: []
+
+            })
+        }], [[destinationNode, 'message']], 20000)
+    }, 30000)
+
+    it('Receives multiple messages', async () => {
+        const numOfMessages = 20
+        let receivedMessages = 0
+        destinationNode.on('message', (_message: Message) => {
+            receivedMessages += 1
+        })
+        const rpcWrapper = createWrappedClosestPeersRequest(sourceNode.getPeerDescriptor(), destinationNode.getPeerDescriptor())
+
+        for (let i = 0; i < numOfMessages; i++) {
+            const message: Message = {
+                serviceId: 'unknown',
+                messageId: v4(),
+                messageType: MessageType.RPC,
+                body: {
+                    oneofKind: 'rpcMessage',
+                    rpcMessage: rpcWrapper
+                },
+                sourceDescriptor: sourceNode.getPeerDescriptor(),
+                targetDescriptor: destinationNode.getPeerDescriptor()
+            }
+            await sourceNode.router!.doRouteMessage({
+                message: message,
+                destinationPeer: destinationNode.getPeerDescriptor(),
+                requestId: v4(),
+                sourcePeer: sourceNode.getPeerDescriptor(),
+                reachableThrough: [],
+                routingPath: []
+            })
+        }
+        await waitForCondition(() => receivedMessages === numOfMessages)
+    })
+
+    it('From all to all', async () => {
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        for (const i in routerNodes) {
+            const arr: Array<number> = []
+            // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            for (const j in routerNodes) {
+                arr.push(0)
+            }
+            receiveMatrix.push(arr)
+        }
+
+        const numsOfReceivedMessages: Record<string, number> = {}
+        routerNodes.map((node) => {
+            numsOfReceivedMessages[node.getNodeId().toKey()] = 0
+            node.on('message', (msg: Message) => {
+                numsOfReceivedMessages[node.getNodeId().toKey()] = numsOfReceivedMessages[node.getNodeId().toKey()] + 1
+                try {
+                    const target = receiveMatrix[parseInt(node.getNodeId().toString()) - 1]
+                    target[parseInt(PeerID.fromValue(msg.sourceDescriptor!.kademliaId!).toString()) - 1]++
+                } catch (e) {
+                    console.error(e)
+                }
+                if (parseInt(node.getNodeId().toString()) > routerNodes.length || parseInt(node.getNodeId().toString()) < 1) {
+                    console.error(node.getNodeId().toString())
+                }
+            })
+        }
+        )
+        await Promise.all(
+            routerNodes.map(async (node) =>
+                Promise.all(routerNodes.map(async (receiver) => {
+                    if (!node.getNodeId().equals(receiver.getNodeId())) {
+                        const rpcWrapper = createWrappedClosestPeersRequest(sourceNode.getPeerDescriptor(), destinationNode.getPeerDescriptor())
+                        const message: Message = {
+                            serviceId: 'nonexisting_service',
+                            messageId: v4(),
+                            messageType: MessageType.RPC,
+                            body: {
+                                oneofKind: 'rpcMessage',
+                                rpcMessage: rpcWrapper
+                            },
+                            sourceDescriptor: node.getPeerDescriptor(),
+                            targetDescriptor: destinationNode.getPeerDescriptor()
+                        }
+                        await node.router!.doRouteMessage({
+                            message: message,
+                            destinationPeer: receiver.getPeerDescriptor(),
+                            sourcePeer: node.getPeerDescriptor(),
+                            requestId: v4(),
+                            reachableThrough: [],
+                            routingPath: []
+                        })
+                    }
+                }))
+            )
+        )
+        await waitForCondition(() => numsOfReceivedMessages[PeerID.fromString('1').toKey()] >= routerNodes.length - 1
+            , 30000)
+        await Promise.all(
+            Object.keys(numsOfReceivedMessages).map(async (key) =>
+                waitForCondition(() => numsOfReceivedMessages[key] >= routerNodes.length - 1, 30000)
+            )
+        )
+
+    }, 90000)
+
+    it('Destination receives forwarded message', async () => {
+        const closestPeersRequest = createWrappedClosestPeersRequest(sourceNode.getPeerDescriptor(), destinationNode.getPeerDescriptor())
+        const closestPeersRequestMessage: Message = {
+            serviceId: 'unknown',
+            messageId: v4(),
+            messageType: MessageType.RPC,
+            body: {
+                oneofKind: 'rpcMessage',
+                rpcMessage: closestPeersRequest
+            },
+            sourceDescriptor: sourceNode.getPeerDescriptor()!,
+            targetDescriptor: destinationNode.getPeerDescriptor()!
+        }
+
+        const routeMessageWrapper: RouteMessageWrapper = {
+            message: closestPeersRequestMessage,
+            destinationPeer: destinationNode.getPeerDescriptor(),
+            requestId: new UUID().toString(),
+            sourcePeer: sourceNode.getPeerDescriptor(),
+            reachableThrough: [entryPointDescriptor],
+            routingPath: []
+        }
+
+        const rpcMessage: RpcMessage = {
+            body: Any.pack(routeMessageWrapper, RouteMessageWrapper),
+            header: {
+                method: 'routeMessage',
+                request: 'request'
+            },
+            requestId: v4()
+        }
+
+        const requestMessage: Message = {
+            serviceId: 'layer0',
+            messageId: v4(),
+            messageType: MessageType.RPC,
+            body: {
+                oneofKind: 'rpcMessage',
+                rpcMessage: rpcMessage
+            },
+            sourceDescriptor: sourceNode.getPeerDescriptor()!,
+            targetDescriptor: entryPoint.getPeerDescriptor()!
+        }
+
+        const forwardedMessage: RouteMessageWrapper = {
+            message: requestMessage,
+            requestId: v4(),
+            sourcePeer: sourceNode.getPeerDescriptor(),
+            destinationPeer: entryPoint.getPeerDescriptor()!,
+            reachableThrough: [],
+            routingPath: []
+        }
+
+        await runAndWaitForEvents3<DhtNodeEvents>([() => {
+            sourceNode.router!.doRouteMessage(forwardedMessage, RoutingMode.FORWARD)
+        }], [[destinationNode, 'message']])
+
+    })
+
+})
+

package/test/integration/RpcErrors.test.ts

@@ -0,0 +1,153 @@
+import { ConnectionManager } from '../../src/connection/ConnectionManager'
+import { LatencyType, Simulator } from '../../src/connection/Simulator/Simulator'
+import { PeerID } from '../../src/helpers/PeerID'
+import { ITransport } from '../../src/transport/ITransport'
+import { v4 } from 'uuid'
+import { DhtRpcOptions, ListeningRpcCommunicator, SimulatorTransport } from '../../src/exports'
+import { ProtoRpcClient, toProtoRpcClient } from '@streamr/proto-rpc'
+import { DhtRpcServiceClient } from '../../src/proto/packages/dht/protos/DhtRpc.client'
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import { NodeType, PeerDescriptor, PingRequest, PingResponse } from '../../src/proto/packages/dht/protos/DhtRpc'
+
+describe('RPC errors', () => {
+
+    let manager1: ConnectionManager
+    let manager2: ConnectionManager
+
+    let rpcCommunicator1: ListeningRpcCommunicator
+    let rpcCommunicator2: ListeningRpcCommunicator
+
+    let client1: ProtoRpcClient<DhtRpcServiceClient>
+    //let client2: ProtoRpcClient<DhtRpcServiceClient>
+
+    let simulator: Simulator
+
+    const peerDescriptor1: PeerDescriptor = {
+        kademliaId: PeerID.fromString('peer1').value,
+        type: NodeType.NODEJS,
+    }
+
+    const peerDescriptor2: PeerDescriptor = {
+        kademliaId: PeerID.fromString('peer2').value,
+        type: NodeType.NODEJS,
+    }
+
+    let connectorTransport1: ITransport
+    let connectorTransport2: ITransport
+
+    const serviceId = 'test'
+
+    beforeEach(async () => {
+
+        simulator = new Simulator(LatencyType.FIXED, 50)
+        connectorTransport1 = new SimulatorTransport(peerDescriptor1, simulator)
+        manager1 = new ConnectionManager({ transportLayer: connectorTransport1 })
+        rpcCommunicator1 = new ListeningRpcCommunicator(serviceId, manager1)
+        client1 = toProtoRpcClient(new DhtRpcServiceClient(rpcCommunicator1.getRpcClientTransport()))
+
+        connectorTransport2 = new SimulatorTransport(peerDescriptor2, simulator)
+        manager2 = new ConnectionManager({ transportLayer: connectorTransport2 })
+        rpcCommunicator2 = new ListeningRpcCommunicator(serviceId, manager2)
+        //client2 = toProtoRpcClient(new DhtRpcServiceClient(rpcCommunicator2.getRpcClientTransport()))
+
+        await manager1.start((_msg) => peerDescriptor1)
+        await manager2.start((_msg) => peerDescriptor2)
+
+    })
+
+    afterEach(async () => {
+        await connectorTransport1.stop()
+        await connectorTransport2.stop()
+        await manager1.stop()
+        await manager2.stop()
+        simulator.stop()
+    })
+
+    it('Can make a RPC call over WebRTC', async () => {
+        const ping = async (request: PingRequest, _context: ServerCallContext):
+            Promise<PingResponse> => {
+            const response: PingResponse = {
+                requestId: request.requestId
+            }
+            return response
+        }
+
+        rpcCommunicator2.registerRpcMethod(PingRequest, PingResponse, 'ping', ping)
+
+        const request: PingRequest = {
+            requestId: v4()
+        }
+        const options: DhtRpcOptions = {
+            sourceDescriptor: peerDescriptor1,
+            targetDescriptor: peerDescriptor2
+        }
+
+        const response = await client1.ping(request, options)
+
+        expect(response.requestId).toEqual(request.requestId)
+    }, 60000)
+
+    it('Throws an exception if RPC method is not defined', async () => {
+
+        const request: PingRequest = {
+            requestId: v4()
+        }
+        const options: DhtRpcOptions = {
+            sourceDescriptor: peerDescriptor1,
+            targetDescriptor: peerDescriptor2
+        }
+
+        await expect(client1.ping(request, options))
+            .rejects.toThrow('Server does not implement method ping')
+    }, 60000)
+
+    /*
+    it.only('Throws a client-side exception if WebRTC connection fails', async () => {
+
+        const request: PingRequest = {
+            requestId: v4()
+        }
+        const options: DhtRpcOptions = {
+            sourceDescriptor: peerDescriptor1,
+            targetDescriptor: peerDescriptor2
+        }
+        await connectorTransport1.stop()
+        await manager2.stop()
+        
+        const result = await client1.ping(request, options)
+        
+    }, 60000)
+
+    it('Disconnects WebRtcConnection while being connected', async () => {
+        
+        const rpcMessage: RpcMessage = {
+            header: {},
+            body: new Uint8Array(10),
+            requestId: v4()
+        }
+
+        const msg: Message = {
+            serviceId: serviceId,
+            messageType: MessageType.RPC,
+            messageId: '1',
+            body: RpcMessage.toBinary(rpcMessage)
+        }
+
+        const disconnectedPromise1 = new Promise<void>((resolve, _reject) => {
+            manager1.on('disconnected', (_peerDescriptor: PeerDescriptor) => {
+                //expect(message.messageType).toBe(MessageType.RPC)
+                resolve()
+            })
+        })
+
+        msg.targetDescriptor = peerDescriptor2
+        manager1.send(msg).catch((e) => {
+            expect(e.code).toEqual('CONNECTION_FAILED')
+        })
+        
+        manager1.disconnect(peerDescriptor2!, undefined, 100)
+        await disconnectedPromise1
+
+    }, 20000)
+    */
+})

package/test/integration/ScaleDownDht.test.ts

@@ -0,0 +1,66 @@
+import { LatencyType, Simulator } from '../../src/connection/Simulator/Simulator'
+import { DhtNode } from '../../src/dht/DhtNode'
+import { NodeType, PeerDescriptor } from '../../src/proto/packages/dht/protos/DhtRpc'
+import { createMockConnectionDhtNode } from '../utils/utils'
+import { isSamePeerDescriptor } from '../../src/helpers/peerIdFromPeerDescriptor'
+import { Logger } from '@streamr/utils'
+
+const logger = new Logger(module)
+
+describe('Scaling down a Dht network', () => {
+    let entryPoint: DhtNode
+    let nodes: DhtNode[]
+    let entrypointDescriptor: PeerDescriptor
+    const simulator = new Simulator(LatencyType.RANDOM)
+    const NUM_NODES = 80
+    const MAX_CONNECTIONS = 15
+    const K = 2
+
+    beforeEach(async () => {
+        nodes = []
+        const entryPointId = '0'
+        entryPoint = await createMockConnectionDhtNode(entryPointId, simulator,
+            undefined, K, entryPointId, MAX_CONNECTIONS)
+        nodes.push(entryPoint)
+
+        entrypointDescriptor = {
+            kademliaId: entryPoint.getNodeId().value,
+            type: NodeType.NODEJS,
+            nodeName: entryPointId
+        }
+
+        for (let i = 1; i < NUM_NODES; i++) {
+            const nodeId = `${i}`
+            const node = await createMockConnectionDhtNode(nodeId, simulator, undefined, K, nodeId, MAX_CONNECTIONS)
+            nodes.push(node)
+        }
+        await Promise.all(nodes.map((node) => node.joinDht([entrypointDescriptor])))
+    }, 60000)
+
+    afterEach(async () => {
+        await Promise.all(nodes.map((node) => node.stop()))
+    }, 10000)
+
+    it('Remaining nodes cleaned up stopped nodes from states', async () => {
+        const randomIndices = []
+        for (let i = 1; i < nodes.length; i++) {
+            randomIndices.push(i)
+        }
+        while (randomIndices.length > 1) {
+            const index = Math.floor(Math.random() * randomIndices.length)
+            const nodeIndex = randomIndices[index]
+            randomIndices.splice(index, 1)
+            const stoppingPeerDescriptor = nodes[nodeIndex].getPeerDescriptor()
+            await nodes[nodeIndex].stop()
+            const nodeIsCleaned = nodes.every((node) =>
+                node.getAllConnectionPeerDescriptors().every((peer) => {
+                    if (isSamePeerDescriptor(peer, stoppingPeerDescriptor)) {
+                        logger.error(' ' + node.getNodeName() + ', ' + stoppingPeerDescriptor.nodeName + ' cleaning up failed')
+                    }
+                    return !isSamePeerDescriptor(peer, stoppingPeerDescriptor)
+                })
+            )
+            expect(nodeIsCleaned).toEqual(true)
+        }
+    }, 180000)
+})