@streamr/dht 102.0.0-beta.1 → 102.0.0-beta.2

package/test/integration/MultipleEntryPointJoining.test.ts

@@ -1,105 +0,0 @@
-import { LatencyType, Simulator } from '../../src/connection/simulator/Simulator'
-import { DhtNode } from '../../src/dht/DhtNode'
-import { PeerDescriptor } from '../../generated/packages/dht/protos/DhtRpc'
-import { createMockConnectionDhtNode } from '../utils/utils'
-
-describe('multiple entry point joining', () => {
-
-    describe('all nodes are entry points', () => {
-
-        let simulator: Simulator
-        let node1: DhtNode
-        let node2: DhtNode
-        let node3: DhtNode
-        let entryPoints: PeerDescriptor[]
-        
-        beforeEach(async () => {
-            simulator = new Simulator(LatencyType.REAL)
-
-            node1 = await createMockConnectionDhtNode(simulator)
-            node2 = await createMockConnectionDhtNode(simulator)
-            node3 = await createMockConnectionDhtNode(simulator)
-
-            entryPoints = [
-                node1.getLocalPeerDescriptor(),
-                node2.getLocalPeerDescriptor(),
-                node3.getLocalPeerDescriptor()
-            ]
-        })
-
-        afterEach(async () => {
-            await Promise.all([
-                node1.stop(),
-                node2.stop(),
-                node3.stop()
-            ])
-            simulator.stop()
-        })
-
-        it('can join simultaneously', async () => {
-            await Promise.all([
-                node1.joinDht(entryPoints),
-                node2.joinDht(entryPoints),
-                node3.joinDht(entryPoints)
-            ])
-            expect(node1.getNeighborCount()).toEqual(2)
-            expect(node2.getNeighborCount()).toEqual(2)
-            expect(node3.getNeighborCount()).toEqual(2)
-        })
-
-        it('can join even if a node is offline', async () => {
-            await node3.stop()
-            await Promise.all([
-                node1.joinDht(entryPoints),
-                node2.joinDht(entryPoints)
-            ])
-            expect(node1.getNeighborCount()).toEqual(1)
-            expect(node2.getNeighborCount()).toEqual(1)
-        }, 10000)
-    })
-
-    describe('non entry point nodes can join via multiple entry points', () => {
-        let simulator: Simulator
-        let entryPoint1: DhtNode
-        let entryPoint2: DhtNode
-        let node1: DhtNode
-        let node2: DhtNode
-        let entryPoints: PeerDescriptor[]
-        
-        beforeEach(async () => {
-            simulator = new Simulator(LatencyType.REAL)
-            
-            entryPoint1 = await createMockConnectionDhtNode(simulator)
-            entryPoint2 = await createMockConnectionDhtNode(simulator)
-            
-            node1 = await createMockConnectionDhtNode(simulator)
-            node2 = await createMockConnectionDhtNode(simulator)
-
-            entryPoints = [
-                entryPoint1.getLocalPeerDescriptor(),
-                entryPoint2.getLocalPeerDescriptor(),
-            ]
-
-            await entryPoint1.joinDht(entryPoints)
-            await entryPoint2.joinDht(entryPoints)
-        })
-
-        afterEach(async () => {
-            await Promise.all([
-                entryPoint1.stop(),
-                entryPoint2.stop(),
-                node1.stop(),
-                node2.stop()
-            ])
-            simulator.stop()
-        })
-
-        it('non-entry point nodes can join', async () => {
-            await node1.joinDht(entryPoints)
-            expect(node1.getNeighborCount()).toEqual(2)
-            await node2.joinDht(entryPoints)
-            expect(node2.getNeighborCount()).toEqual(3)
-        })
-
-    })
-})

package/test/integration/ReplicateData.test.ts

@@ -1,104 +0,0 @@
-import { LatencyType, Simulator } from '../../src/connection/simulator/Simulator'
-import { DhtNode } from '../../src/dht/DhtNode'
-import { createMockConnectionDhtNode, waitForStableTopology } from '../utils/utils'
-import { SortedContactList } from '../../src/dht/contact/SortedContactList'
-import { createMockDataEntry, expectEqualData } from '../utils/mock/mockDataEntry'
-import { DhtAddress, randomDhtAddress, toDhtAddress, toNodeId } from '../../src/identifiers'
-import { sample } from 'lodash'
-import { DataEntry, PeerDescriptor } from '../../generated/packages/dht/protos/DhtRpc'
-
-const DATA = createMockDataEntry()
-const NUM_NODES = 100
-const MAX_CONNECTIONS = 80
-const K = 8
-const ENTRY_POINT_INDEX = 0
-
-const getDataEntries = (node: DhtNode): DataEntry[] => {
-    // @ts-expect-error private field
-    const store = node.localDataStore
-    return Array.from(store.values(toDhtAddress(DATA.key)))
-}
-
-describe('Replicate data from node to node in DHT', () => {
-
-    let nodes: DhtNode[]
-    let entryPointDescriptor: PeerDescriptor
-    const simulator = new Simulator(LatencyType.FIXED, 20)
-
-    beforeEach(async () => {
-        const entryPoint = await createMockConnectionDhtNode(simulator, randomDhtAddress(), K, MAX_CONNECTIONS)
-        entryPointDescriptor = entryPoint.getLocalPeerDescriptor()
-        await entryPoint.joinDht([entryPointDescriptor])
-        nodes = []
-        nodes.push(entryPoint)
-        for (let i = 1; i < NUM_NODES; i++) {
-            const node = await createMockConnectionDhtNode(
-                simulator,
-                randomDhtAddress(),
-                K,
-                MAX_CONNECTIONS
-            )
-            nodes.push(node)
-        }
-    }, 60000)
-
-    afterEach(async () => {
-        await Promise.all(nodes.map(async (node) => await node.stop()))
-    }, 60000)
-
-    afterAll(async () => {
-        simulator.stop()
-    })
-
-    it('Data replicates to the closest node no matter where it is stored', async () => {
-        // calculate offline which node is closest to the data
-        const sortedList = new SortedContactList<DhtNode>({ 
-            referenceId: toDhtAddress(DATA.key),
-            maxSize: 10000, 
-            allowToContainReferenceId: true
-        })
-        nodes.forEach((node) => sortedList.addContact(node))
-
-        const closest = sortedList.getClosestContacts()
-        const successfulStorers = await nodes[0].storeDataToDht(toDhtAddress(DATA.key), DATA.data!)
-        expect(successfulStorers.length).toBe(1)
-
-        await Promise.all(
-            nodes.map(async (node, i) => {
-                if (i !== ENTRY_POINT_INDEX) {
-                    await node.joinDht([entryPointDescriptor])
-                }
-            })
-        )
-        await waitForStableTopology(nodes)
-
-        const data = getDataEntries(closest[0])
-        expect(data).toHaveLength(1)
-        expectEqualData(data[0], DATA)
-    }, 180000)
-
-    it('Data replicates to the other nodes when storers are stopped', async () => {
-        await Promise.all(
-            nodes.map(async (node, i) => {
-                if (i !== ENTRY_POINT_INDEX) {
-                    await node.joinDht([entryPointDescriptor])
-                }
-            })
-        )
-        await waitForStableTopology(nodes)
-
-        const randomIndex = Math.floor(Math.random() * nodes.length)
-        const storerDescriptors = await nodes[randomIndex].storeDataToDht(toDhtAddress(DATA.key), DATA.data!)
-        const stoppedNodeIds: DhtAddress[] = []
-        await Promise.all(storerDescriptors.map(async (storerDescriptor) => {
-            const storer = nodes.find((n) => n.getNodeId() === toNodeId(storerDescriptor))!
-            await storer.stop()
-            stoppedNodeIds.push(storer.getNodeId())
-        }))
-
-        const randomNonStoppedNode = sample(nodes.filter((n) => !stoppedNodeIds.includes(n.getNodeId())))!
-        const data = await randomNonStoppedNode.fetchDataFromDht(toDhtAddress(DATA.key))
-        expect(data).toHaveLength(1)
-        expectEqualData(data[0], DATA)
-    }, 180000)
-})

package/test/integration/RouteMessage.test.ts

@@ -1,230 +0,0 @@
-import { DhtNode, Events as DhtNodeEvents } from '../../src/dht/DhtNode'
-import { Message, NodeType, PeerDescriptor, RouteMessageWrapper } from '../../generated/packages/dht/protos/DhtRpc'
-import { RpcMessage } from '../../generated/packages/proto-rpc/protos/ProtoRpc'
-import { Logger, runAndWaitForEvents3, until } from '@streamr/utils'
-import { createMockConnectionDhtNode, createWrappedClosestPeersRequest } from '../utils/utils'
-import { Simulator } from '../../src/connection/simulator/Simulator'
-import { v4 } from 'uuid'
-import { Any } from '../../generated/google/protobuf/any'
-import { RoutingMode } from '../../src/dht/routing/RoutingSession'
-import { DhtAddress, randomDhtAddress, toDhtAddressRaw } from '../../src/identifiers'
-
-const logger = new Logger(module)
-
-const NUM_NODES = 30
-
-describe('Route Message With Mock Connections', () => {
-
-    let entryPoint: DhtNode
-    let sourceNode: DhtNode
-    let destinationNode: DhtNode
-    let routerNodes: DhtNode[]
-    let simulator: Simulator
-    let entryPointDescriptor: PeerDescriptor
-
-    beforeEach(async () => {
-        routerNodes = []
-        simulator = new Simulator()
-        entryPoint = await createMockConnectionDhtNode(simulator, randomDhtAddress())
-
-        entryPointDescriptor = {
-            nodeId: toDhtAddressRaw(entryPoint.getNodeId()),
-            type: NodeType.NODEJS
-        }
-
-        sourceNode = await createMockConnectionDhtNode(simulator, randomDhtAddress())
-        destinationNode = await createMockConnectionDhtNode(simulator, randomDhtAddress())
-
-        for (let i = 1; i < NUM_NODES; i++) {
-            const node = await createMockConnectionDhtNode(simulator, randomDhtAddress())
-            routerNodes.push(node)
-        }
-
-        await destinationNode.joinDht([entryPointDescriptor])
-        await sourceNode.joinDht([entryPointDescriptor])
-        await Promise.all(routerNodes.map((node) => node.joinDht([entryPointDescriptor])))
-        await entryPoint.joinDht([entryPointDescriptor])
-    }, 15000)
-
-    afterEach(async () => {
-        await Promise.allSettled(routerNodes.map((node) => node.stop()))
-        await Promise.allSettled([
-            entryPoint.stop(),
-            destinationNode.stop(),
-            sourceNode.stop()
-        ])
-
-        logger.info('calling simulator stop')
-        simulator.stop()
-        logger.info('simulator stop called')
-    }, 10000)
-
-    it('Happy path', async () => {
-        const rpcWrapper = createWrappedClosestPeersRequest(sourceNode.getLocalPeerDescriptor())
-        const message: Message = {
-            serviceId: 'unknown',
-            messageId: v4(),
-            body: {
-                oneofKind: 'rpcMessage',
-                rpcMessage: rpcWrapper
-            },
-            sourceDescriptor: sourceNode.getLocalPeerDescriptor(),
-            targetDescriptor: destinationNode.getLocalPeerDescriptor()
-        }
-
-        await runAndWaitForEvents3<DhtNodeEvents>([() => {
-            // @ts-expect-error private
-            sourceNode.router!.doRouteMessage({
-                message,
-                target: destinationNode.getLocalPeerDescriptor().nodeId,
-                requestId: v4(),
-                sourcePeer: sourceNode.getLocalPeerDescriptor(),
-                reachableThrough: [],
-                routingPath: [],
-                parallelRootNodeIds: []
-            })
-        }], [[destinationNode, 'message']], 20000)
-    }, 30000)
-
-    it('Receives multiple messages', async () => {
-        const messageCount = 20
-        let receivedMessages = 0
-        destinationNode.on('message', () => {
-            receivedMessages += 1
-        })
-        const rpcWrapper = createWrappedClosestPeersRequest(sourceNode.getLocalPeerDescriptor())
-
-        for (let i = 0; i < messageCount; i++) {
-            const message: Message = {
-                serviceId: 'unknown',
-                messageId: v4(),
-                body: {
-                    oneofKind: 'rpcMessage',
-                    rpcMessage: rpcWrapper
-                },
-                sourceDescriptor: sourceNode.getLocalPeerDescriptor(),
-                targetDescriptor: destinationNode.getLocalPeerDescriptor()
-            }
-            // @ts-expect-error private
-            sourceNode.router!.doRouteMessage({
-                message,
-                target: destinationNode.getLocalPeerDescriptor().nodeId,
-                requestId: v4(),
-                sourcePeer: sourceNode.getLocalPeerDescriptor(),
-                reachableThrough: [],
-                routingPath: [],
-                parallelRootNodeIds: []
-            })
-        }
-        await until(() => receivedMessages === messageCount)
-    })
-
-    it('From all to all', async () => {
-        const receivedMessageCounts: Record<DhtAddress, number> = {}
-        routerNodes.forEach((node) => {
-            const key = node.getNodeId()
-            receivedMessageCounts[key] = 0
-            node.on('message', () => {
-                receivedMessageCounts[key] = receivedMessageCounts[key] + 1
-            })
-        })
-        await Promise.all(
-            routerNodes.map(async (node) =>
-                Promise.all(routerNodes.map(async (receiver) => {
-                    if (node.getNodeId() !== receiver.getNodeId()) {
-                        const rpcWrapper = createWrappedClosestPeersRequest(sourceNode.getLocalPeerDescriptor())
-                        const message: Message = {
-                            serviceId: 'nonexisting_service',
-                            messageId: v4(),
-                            body: {
-                                oneofKind: 'rpcMessage',
-                                rpcMessage: rpcWrapper
-                            },
-                            sourceDescriptor: node.getLocalPeerDescriptor(),
-                            targetDescriptor: destinationNode.getLocalPeerDescriptor()
-                        }
-                        // @ts-expect-error private
-                        node.router!.doRouteMessage({
-                            message,
-                            target: receiver.getLocalPeerDescriptor().nodeId,
-                            sourcePeer: node.getLocalPeerDescriptor(),
-                            requestId: v4(),
-                            reachableThrough: [],
-                            routingPath: [],
-                            parallelRootNodeIds: []
-                        })
-                    }
-                }))
-            )
-        )
-        await until(() => receivedMessageCounts[routerNodes[0].getNodeId()] >= routerNodes.length - 1, 30000)
-        await Promise.all(
-            Object.keys(receivedMessageCounts).map(async (key) =>
-                until(() => receivedMessageCounts[key as DhtAddress] >= routerNodes.length - 1, 30000)
-            )
-        )
-
-    }, 90000)
-
-    it('Destination receives forwarded message', async () => {
-        const closestPeersRequest = createWrappedClosestPeersRequest(sourceNode.getLocalPeerDescriptor())
-        const closestPeersRequestMessage: Message = {
-            serviceId: 'unknown',
-            messageId: v4(),
-            body: {
-                oneofKind: 'rpcMessage',
-                rpcMessage: closestPeersRequest
-            },
-            sourceDescriptor: sourceNode.getLocalPeerDescriptor(),
-            targetDescriptor: destinationNode.getLocalPeerDescriptor()
-        }
-
-        const routeMessageWrapper: RouteMessageWrapper = {
-            message: closestPeersRequestMessage,
-            target: destinationNode.getLocalPeerDescriptor().nodeId,
-            requestId: v4(),
-            sourcePeer: sourceNode.getLocalPeerDescriptor(),
-            reachableThrough: [entryPointDescriptor],
-            routingPath: [],
-            parallelRootNodeIds: []
-        }
-
-        const rpcMessage: RpcMessage = {
-            body: Any.pack(routeMessageWrapper, RouteMessageWrapper),
-            header: {
-                method: 'routeMessage',
-                request: 'request'
-            },
-            requestId: v4()
-        }
-
-        const requestMessage: Message = {
-            serviceId: 'layer0',
-            messageId: v4(),
-            body: {
-                oneofKind: 'rpcMessage',
-                rpcMessage
-            },
-            sourceDescriptor: sourceNode.getLocalPeerDescriptor(),
-            targetDescriptor: entryPoint.getLocalPeerDescriptor()
-        }
-
-        const forwardedMessage: RouteMessageWrapper = {
-            message: requestMessage,
-            requestId: v4(),
-            sourcePeer: sourceNode.getLocalPeerDescriptor(),
-            target: entryPoint.getLocalPeerDescriptor().nodeId,
-            reachableThrough: [],
-            routingPath: [],
-            parallelRootNodeIds: []
-        }
-
-        await runAndWaitForEvents3<DhtNodeEvents>([() => {
-            // @ts-expect-error private
-            sourceNode.router!.doRouteMessage(forwardedMessage, RoutingMode.FORWARD)
-        }], [[destinationNode, 'message']])
-
-    })
-
-})
-

package/test/integration/RouterRpcRemote.test.ts

@@ -1,77 +0,0 @@
-import { RpcCommunicator } from '@streamr/proto-rpc'
-import { RouterRpcRemote } from '../../src/dht/routing/RouterRpcRemote'
-import { Message, RouteMessageAck, RouteMessageWrapper } from '../../generated/packages/dht/protos/DhtRpc'
-import { RouterRpcClient } from '../../generated/packages/dht/protos/DhtRpc.client'
-import { RpcMessage } from '../../generated/packages/proto-rpc/protos/ProtoRpc'
-import { createMockPeerDescriptor, createWrappedClosestPeersRequest, mockRouterRpc } from '../utils/utils'
-import { DhtCallContext } from '../../src/rpc-protocol/DhtCallContext'
-
-const SERVICE_ID = 'test'
-
-describe('RemoteRouter', () => {
-
-    let remoteRouter: RouterRpcRemote
-    let clientRpcCommunicator: RpcCommunicator<DhtCallContext>
-    let serverRpcCommunicator: RpcCommunicator<DhtCallContext>
-    const clientPeerDescriptor = createMockPeerDescriptor()
-    const serverPeerDescriptor = createMockPeerDescriptor()
-
-    beforeEach(() => {
-        clientRpcCommunicator = new RpcCommunicator()
-        serverRpcCommunicator = new RpcCommunicator()
-        serverRpcCommunicator.registerRpcMethod(RouteMessageWrapper, RouteMessageAck, 'routeMessage', mockRouterRpc.routeMessage)
-        clientRpcCommunicator.setOutgoingMessageListener(async (message: RpcMessage) => {
-            serverRpcCommunicator.handleIncomingMessage(message, new DhtCallContext())
-        })
-        serverRpcCommunicator.setOutgoingMessageListener(async (message: RpcMessage) => {
-            clientRpcCommunicator.handleIncomingMessage(message, new DhtCallContext())
-        })
-        remoteRouter = new RouterRpcRemote(clientPeerDescriptor, serverPeerDescriptor, clientRpcCommunicator, RouterRpcClient)
-    })
-
-    it('routeMessage happy path', async () => {
-        const rpcWrapper = createWrappedClosestPeersRequest(clientPeerDescriptor)
-        const routed: Message = {
-            serviceId: SERVICE_ID,
-            messageId: 'routed',
-            body: {
-                oneofKind: 'rpcMessage',
-                rpcMessage: rpcWrapper
-            }
-        }
-        const routable = await remoteRouter.routeMessage({
-            requestId: 'routed',
-            message: routed,
-            sourcePeer: clientPeerDescriptor,
-            target: serverPeerDescriptor.nodeId,
-            reachableThrough: [],
-            routingPath: [],
-            parallelRootNodeIds: []
-        })
-        expect(routable).toEqual(true)
-    })
-
-    it('routeMessage error path', async () => {
-        serverRpcCommunicator.registerRpcMethod(RouteMessageWrapper, RouteMessageAck, 'routeMessage', mockRouterRpc.throwRouteMessageError)
-        const rpcWrapper = createWrappedClosestPeersRequest(clientPeerDescriptor)
-        const routed: Message = {
-            serviceId: SERVICE_ID,
-            messageId: 'routed',
-            body: {
-                oneofKind: 'rpcMessage',
-                rpcMessage: rpcWrapper
-            }
-        }
-        const routable = await remoteRouter.routeMessage({
-            requestId: 'routed',
-            message: routed,
-            sourcePeer: clientPeerDescriptor,
-            target: serverPeerDescriptor.nodeId,
-            reachableThrough: [],
-            routingPath: [],
-            parallelRootNodeIds: []
-        })
-        expect(routable).toEqual(false)
-    })
-
-})