@streamr/trackerless-network 0.0.1-tatum.0

package/test/integration/RandomGraphNode-Layer1Node.test.ts

@@ -0,0 +1,226 @@
+/* eslint-disable @typescript-eslint/parameter-properties */
+
+import { DhtNode, Simulator, PeerDescriptor, PeerID, ConnectionManager, getRandomRegion } from '@streamr/dht'
+import { RandomGraphNode } from '../../src/logic/RandomGraphNode'
+import { range } from 'lodash'
+import { wait, waitForCondition, waitForEvent3 } from '@streamr/utils'
+import { Logger } from '@streamr/utils'
+import { createRandomGraphNode } from '../../src/logic/createRandomGraphNode'
+import { EventEmitter } from 'eventemitter3'
+
+const logger = new Logger(module)
+
+interface SuccessEvents {
+    success: () => void
+}
+
+class SuccessListener extends EventEmitter<SuccessEvents> {
+
+    private numNeighbors = 0
+    private numNearby = 0
+
+    constructor(private node: RandomGraphNode,
+        private wantedNumNeighbors: number,
+        private wantedNumNearby: number) {
+
+        super()
+        node.on('targetNeighborConnected', this.onTargetNeighborConnected)
+        node.on('nearbyContactPoolIdAdded', this.onNearbyContactPoolIdAdded)
+    }
+
+    private onTargetNeighborConnected = (_stringId: string) => {
+        this.numNeighbors++
+
+        if (this.numNeighbors >= this.wantedNumNeighbors
+            && this.numNearby >= this.wantedNumNearby) {
+            this.node.off('targetNeighborConnected', this.onTargetNeighborConnected)
+            this.node.off('nearbyContactPoolIdAdded', this.onNearbyContactPoolIdAdded)
+            this.emit('success')
+        }
+    }
+
+    private onNearbyContactPoolIdAdded = () => {
+        this.numNearby++
+
+        if (this.numNeighbors >= this.wantedNumNeighbors
+            && this.numNearby >= this.wantedNumNearby) {
+            this.node.off('targetNeighborConnected', this.onTargetNeighborConnected)
+            this.node.off('nearbyContactPoolIdAdded', this.onNearbyContactPoolIdAdded)
+            this.emit('success')
+        }
+    }
+
+    public async waitForSuccess(timeout: number): Promise<void> {
+        if (this.numNeighbors >= this.wantedNumNeighbors
+            && this.numNearby >= this.wantedNumNearby) {
+            return
+        } else {
+            await waitForEvent3<SuccessEvents>(this, 'success', timeout)
+        }
+    }
+}
+
+describe('RandomGraphNode-DhtNode', () => {
+    const numOfNodes = 64
+    let dhtNodes: DhtNode[]
+    let dhtEntryPoint: DhtNode
+    let entryPointRandomGraphNode: RandomGraphNode
+    let graphNodes: RandomGraphNode[]
+
+    const streamId = 'Stream1'
+    const entrypointDescriptor: PeerDescriptor = {
+        kademliaId: PeerID.fromString('entrypoint').value,
+        nodeName: 'entrypoint',
+        type: 0,
+        region: getRandomRegion()
+    }
+
+    const peerDescriptors: PeerDescriptor[] = range(numOfNodes).map((i) => {
+        return {
+            kademliaId: PeerID.fromString(`${i}`).value,
+            nodeName: `node${i}`,
+            type: 0,
+            region: getRandomRegion()
+        }
+    })
+    beforeEach(async () => {
+
+        Simulator.useFakeTimers()
+        const simulator = new Simulator()
+        const entrypointCm = new ConnectionManager({
+            ownPeerDescriptor: entrypointDescriptor,
+            nodeName: entrypointDescriptor.nodeName, simulator: simulator
+        })
+
+        const cms: ConnectionManager[] = range(numOfNodes).map((i) =>
+            new ConnectionManager({
+                ownPeerDescriptor: peerDescriptors[i],
+                nodeName: peerDescriptors[i].nodeName,
+                simulator: simulator
+            })
+        )
+
+        dhtEntryPoint = new DhtNode({
+            transportLayer: entrypointCm,
+            peerDescriptor: entrypointDescriptor,
+            serviceId: streamId
+        })
+
+        dhtNodes = range(numOfNodes).map((i) => new DhtNode({
+            transportLayer: cms[i],
+            peerDescriptor: peerDescriptors[i],
+            serviceId: streamId
+        }))
+
+        graphNodes = range(numOfNodes).map((i) => createRandomGraphNode({
+            randomGraphId: streamId,
+            layer1: dhtNodes[i],
+            P2PTransport: cms[i],
+            connectionLocker: cms[i],
+            ownPeerDescriptor: peerDescriptors[i],
+            neighborUpdateInterval: 2000
+        }))
+
+        entryPointRandomGraphNode = createRandomGraphNode({
+            randomGraphId: streamId,
+            layer1: dhtEntryPoint,
+            P2PTransport: entrypointCm,
+            connectionLocker: entrypointCm,
+            ownPeerDescriptor: entrypointDescriptor,
+            neighborUpdateInterval: 2000
+        })
+
+        await dhtEntryPoint.start()
+        await dhtEntryPoint.joinDht([entrypointDescriptor])
+        await Promise.all(dhtNodes.map((node) => node.start()))
+    })
+
+    afterEach(async () => {
+        await dhtEntryPoint.stop()
+        entryPointRandomGraphNode.stop()
+        await Promise.all(dhtNodes.map((node) => node.stop()))
+        await Promise.all(graphNodes.map((node) => node.stop()))
+        Simulator.useFakeTimers(false)
+    })
+
+    it('happy path single peer ', async () => {
+
+        const successListener = new SuccessListener(graphNodes[0], 1, 1)
+        await entryPointRandomGraphNode.start()
+        await dhtNodes[0].joinDht([entrypointDescriptor])
+
+        await graphNodes[0].start()
+
+        await successListener.waitForSuccess(15006)
+        expect(graphNodes[0].getNearbyContactPoolIds().length).toEqual(1)
+        expect(graphNodes[0].getTargetNeighborStringIds().length).toEqual(1)
+
+    })
+
+    it('happy path 4 peers', async () => {
+        const promise = Promise.all(range(4).map((i) => {
+            const successListener = new SuccessListener(graphNodes[i], 4, 4)
+            return waitForEvent3<SuccessEvents>(successListener, 'success', 15009)
+        }))
+
+        entryPointRandomGraphNode.start()
+        range(4).map((i) => graphNodes[i].start())
+        await Promise.all(range(4).map(async (i) => {
+            await dhtNodes[i].joinDht([entrypointDescriptor])
+        }))
+
+        await promise
+
+        range(4).map((i) => {
+            expect(graphNodes[i].getNearbyContactPoolIds().length).toBeGreaterThanOrEqual(4)
+            expect(graphNodes[i].getTargetNeighborStringIds().length).toBeGreaterThanOrEqual(4)
+        })
+
+        // Check bidirectionality
+        const allNodes = graphNodes
+        allNodes.push(entryPointRandomGraphNode)
+        range(5).map((i) => {
+            allNodes[i].getNearbyContactPoolIds().forEach((stringId) => {
+                const neighbor = allNodes.find((peer) => {
+                    return peer.getOwnStringId() === stringId
+                })
+                expect(neighbor!.getTargetNeighborStringIds().includes(allNodes[i].getOwnStringId())).toEqual(true)
+            })
+        })
+    }, 10000)
+
+    it('happy path 64 peers', async () => {
+        await Promise.all(range(numOfNodes).map((i) => graphNodes[i].start()))
+        await Promise.all(range(numOfNodes).map((i) => {
+            dhtNodes[i].joinDht([entrypointDescriptor])
+        }))
+        await Promise.all(graphNodes.map((node) =>
+            waitForCondition(() => node.getTargetNeighborStringIds().length >= 4, 10000)
+        ))
+
+        const avg = graphNodes.reduce((acc, curr) => {
+            return acc + curr.getTargetNeighborStringIds().length
+        }, 0) / numOfNodes
+
+        logger.info(`AVG Number of neighbors: ${avg}`)
+        await Promise.all(graphNodes.map((node) =>
+            waitForCondition(() => node.getNumberOfOutgoingHandshakes() === 0)
+        ))
+        await wait(10000)
+        let mismatchCounter = 0
+        graphNodes.forEach((node) => {
+            const nodeId = node.getOwnStringId()
+            node.getTargetNeighborStringIds().forEach((neighborId) => {
+                if (neighborId !== entryPointRandomGraphNode.getOwnStringId()) {
+                    const neighbor = graphNodes.find((n) => n.getOwnStringId() === neighborId)
+                    if (!neighbor!.getTargetNeighborStringIds().includes(nodeId)) {
+                        logger.info('mismatching ids length: ' + nodeId + ' ' + neighbor!.getTargetNeighborStringIds().length)
+                        mismatchCounter += 1
+                    }
+                }
+        
+            })
+        })
+        expect(mismatchCounter).toBeLessThanOrEqual(2)
+    }, 95000)
+})

package/test/integration/RemoteHandshaker.test.ts

@@ -0,0 +1,78 @@
+import {
+    StreamHandshakeRequest,
+    StreamHandshakeResponse
+} from '../../src/proto/packages/trackerless-network/protos/NetworkRpc'
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import {
+    ListeningRpcCommunicator,
+    PeerDescriptor,
+    Simulator,
+    SimulatorTransport
+} from '@streamr/dht'
+import { toProtoRpcClient } from '@streamr/proto-rpc'
+import {
+    HandshakeRpcClient,
+} from '../../src/proto/packages/trackerless-network/protos/NetworkRpc.client'
+import { RemoteHandshaker } from '../../src/logic/neighbor-discovery/RemoteHandshaker'
+
+describe('RemoteHandshaker', () => {
+    let mockServerRpc: ListeningRpcCommunicator
+    let clientRpc: ListeningRpcCommunicator
+    let remoteHandshaker: RemoteHandshaker
+
+    const clientPeer: PeerDescriptor = {
+        kademliaId: new Uint8Array([1, 1, 1]),
+        type: 1
+    }
+    const serverPeer: PeerDescriptor = {
+        kademliaId: new Uint8Array([2, 2, 2]),
+        type: 1
+    }
+
+    let simulator: Simulator
+    let mockConnectionManager1: SimulatorTransport
+    let mockConnectionManager2: SimulatorTransport
+
+    beforeEach(() => {
+        Simulator.useFakeTimers()
+        simulator = new Simulator()
+        mockConnectionManager1 = new SimulatorTransport(serverPeer, simulator)
+        mockConnectionManager2 = new SimulatorTransport(clientPeer, simulator)
+
+        mockServerRpc = new ListeningRpcCommunicator('test', mockConnectionManager1)
+        clientRpc = new ListeningRpcCommunicator('test', mockConnectionManager2)
+
+        mockServerRpc.registerRpcMethod(
+            StreamHandshakeRequest,
+            StreamHandshakeResponse,
+            'handshake',
+            async (msg: StreamHandshakeRequest, _context: ServerCallContext): Promise<StreamHandshakeResponse> => {
+                const res: StreamHandshakeResponse = {
+                    requestId: msg.requestId,
+                    accepted: true
+                }
+                return res
+            }
+        )
+
+        remoteHandshaker = new RemoteHandshaker(
+            serverPeer,
+            'test-stream',
+            toProtoRpcClient(new HandshakeRpcClient(clientRpc.getRpcClientTransport()))
+        )
+    })
+
+    afterEach(async () => {
+        clientRpc.stop()
+        mockServerRpc.stop()
+        await mockConnectionManager1.stop()
+        await mockConnectionManager2.stop()
+        simulator.stop()
+        Simulator.useFakeTimers(false)
+    })
+
+    it('handshake', async () => {
+        const result = await remoteHandshaker.handshake(clientPeer, [])
+        expect(result.accepted).toEqual(true)
+    })
+})

package/test/integration/RemoteNeighborUpdateManager.test.ts

@@ -0,0 +1,82 @@
+import { NeighborUpdate } from '../../src/proto/packages/trackerless-network/protos/NetworkRpc'
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import {
+    keyFromPeerDescriptor,
+    ListeningRpcCommunicator,
+    PeerDescriptor,
+    Simulator,
+    SimulatorTransport
+} from '@streamr/dht'
+import { toProtoRpcClient } from '@streamr/proto-rpc'
+import {
+    NeighborUpdateRpcClient,
+} from '../../src/proto/packages/trackerless-network/protos/NetworkRpc.client'
+import { RemoteNeighborUpdateManager } from '../../src/logic/neighbor-discovery/RemoteNeighborUpdateManager'
+
+describe('RemoteNeighborUpdateManager', () => {
+    let mockServerRpc: ListeningRpcCommunicator
+    let clientRpc: ListeningRpcCommunicator
+    let neighborUpdateRpcClient: RemoteNeighborUpdateManager
+
+    const clientPeer: PeerDescriptor = {
+        kademliaId: new Uint8Array([1, 1, 1]),
+        type: 1
+    }
+    const serverPeer: PeerDescriptor = {
+        kademliaId: new Uint8Array([2, 2, 2]),
+        type: 1
+    }
+
+    let simulator: Simulator
+    let mockConnectionManager1: SimulatorTransport
+    let mockConnectionManager2: SimulatorTransport
+
+    beforeEach(() => {
+        simulator = new Simulator()
+        mockConnectionManager1 = new SimulatorTransport(serverPeer, simulator)
+        mockConnectionManager2 = new SimulatorTransport(clientPeer, simulator)
+
+        mockServerRpc = new ListeningRpcCommunicator('test', mockConnectionManager1)
+        clientRpc = new ListeningRpcCommunicator('test', mockConnectionManager2)
+
+        mockServerRpc.registerRpcMethod(
+            NeighborUpdate,
+            NeighborUpdate,
+            'neighborUpdate',
+            async (_msg: NeighborUpdate, _context: ServerCallContext): Promise<NeighborUpdate> => {
+                const peer: PeerDescriptor = {
+                    kademliaId: new Uint8Array([4, 2, 4]),
+                    type: 0
+                }
+
+                const update: NeighborUpdate = {
+                    senderId: keyFromPeerDescriptor(peer),
+                    randomGraphId: 'testStream',
+                    neighborDescriptors: [
+                        peer
+                    ],
+                    removeMe: false
+                }
+                return update
+            }
+        )
+        neighborUpdateRpcClient = new RemoteNeighborUpdateManager(
+            serverPeer,
+            'test-stream',
+            toProtoRpcClient(new NeighborUpdateRpcClient(clientRpc.getRpcClientTransport()))
+        )
+    })
+
+    afterEach(async () => {
+        clientRpc.stop()
+        mockServerRpc.stop()
+        await mockConnectionManager1.stop()
+        await mockConnectionManager2.stop()
+        simulator.stop()
+    })
+
+    it('updateNeighbors', async () => {
+        const res = await neighborUpdateRpcClient.updateNeighbors(clientPeer, [])
+        expect(res.peers.length).toEqual(1)
+    })
+})

package/test/integration/RemoteRandomGraphNode.test.ts

@@ -0,0 +1,102 @@
+import {
+    ListeningRpcCommunicator,
+    Simulator,
+    PeerDescriptor,
+    SimulatorTransport,
+    peerIdFromPeerDescriptor
+} from '@streamr/dht'
+import { RemoteRandomGraphNode } from '../../src/logic/RemoteRandomGraphNode'
+import { NetworkRpcClient } from '../../src/proto/packages/trackerless-network/protos/NetworkRpc.client'
+import {
+    ContentMessage,
+    LeaveStreamNotice,
+    StreamMessage
+} from '../../src/proto/packages/trackerless-network/protos/NetworkRpc'
+import { Empty } from '../../src/proto/google/protobuf/empty'
+import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
+import { waitForCondition } from '@streamr/utils'
+import { toProtoRpcClient } from '@streamr/proto-rpc'
+import { createStreamMessage } from '../utils/utils'
+
+describe('RemoteRandomGraphNode', () => {
+    let mockServerRpc: ListeningRpcCommunicator
+    let clientRpc: ListeningRpcCommunicator
+    let remoteRandomGraphNode: RemoteRandomGraphNode
+
+    const clientPeer: PeerDescriptor = {
+        kademliaId: new Uint8Array([1, 1, 1]),
+        type: 1
+    }
+    const serverPeer: PeerDescriptor = {
+        kademliaId: new Uint8Array([2, 2, 2]),
+        type: 1
+    }
+
+    let recvCounter: number
+
+    let simulator: Simulator
+    let mockConnectionManager1: SimulatorTransport
+    let mockConnectionManager2: SimulatorTransport
+
+    beforeEach(() => {
+        recvCounter = 0
+        simulator = new Simulator()
+        mockConnectionManager1 = new SimulatorTransport(serverPeer, simulator)
+        mockConnectionManager2 = new SimulatorTransport(clientPeer, simulator)
+        
+        mockServerRpc = new ListeningRpcCommunicator('test', mockConnectionManager1)
+        clientRpc = new ListeningRpcCommunicator('test', mockConnectionManager2)
+
+        mockServerRpc.registerRpcNotification(
+            StreamMessage,
+            'sendData',
+            async (_msg: StreamMessage, _context: ServerCallContext): Promise<Empty> => {
+                recvCounter += 1
+                return Empty
+            }
+        )
+
+        mockServerRpc.registerRpcNotification(
+            LeaveStreamNotice,
+            'leaveStreamNotice',
+            async (_msg: LeaveStreamNotice, _context: ServerCallContext): Promise<Empty> => {
+                recvCounter += 1
+                return Empty
+            }
+        )
+
+        remoteRandomGraphNode = new RemoteRandomGraphNode(
+            serverPeer,
+            'test-stream',
+            toProtoRpcClient(new NetworkRpcClient(clientRpc.getRpcClientTransport()))
+        )
+    })
+
+    afterEach(async () => {
+        clientRpc.stop()
+        mockServerRpc.stop()
+        await mockConnectionManager1.stop()
+        await mockConnectionManager2.stop()
+        simulator.stop()
+    })
+
+    it('sendData', async () => {
+        const content: ContentMessage = {
+            body: JSON.stringify({ hello: 'WORLD' })
+        }
+        const msg = createStreamMessage(
+            content,
+            'test-stream',
+            peerIdFromPeerDescriptor(clientPeer).toString()
+        )
+
+        await remoteRandomGraphNode.sendData(clientPeer, msg)
+        await waitForCondition(() => recvCounter === 1)
+    })
+
+    it('leaveNotice', async () => {
+        await remoteRandomGraphNode.leaveStreamNotice(clientPeer)
+        await waitForCondition(() => recvCounter === 1)
+    })
+
+})

package/test/integration/StreamrNode.test.ts

@@ -0,0 +1,145 @@
+import {
+    DhtNode,
+    PeerDescriptor,
+    Simulator,
+    SimulatorTransport,
+    NodeType,
+    keyFromPeerDescriptor
+} from '@streamr/dht'
+import { StreamrNode, Events } from '../../src/logic/StreamrNode'
+import { ContentMessage } from '../../src/proto/packages/trackerless-network/protos/NetworkRpc'
+import { waitForEvent3, waitForCondition } from '@streamr/utils'
+import { createStreamMessage } from '../utils/utils'
+
+describe('StreamrNode', () => {
+
+    let layer01: DhtNode
+    let layer02: DhtNode
+    let transport1: SimulatorTransport
+    let transport2: SimulatorTransport
+    let node1: StreamrNode
+    let node2: StreamrNode
+
+    const peer1: PeerDescriptor = {
+        kademliaId: new Uint8Array([1, 2, 3]),
+        type: NodeType.NODEJS
+    }
+    const peer2: PeerDescriptor = {
+        kademliaId: new Uint8Array([1, 1, 1]),
+        type: NodeType.NODEJS
+    }
+    const STREAM_ID = 'test'
+
+    const content: ContentMessage = {
+        body: JSON.stringify({ hello: 'WORLD' })
+    }
+    const msg = createStreamMessage(
+        content,
+        STREAM_ID,
+        keyFromPeerDescriptor(peer2)
+    )
+
+    afterEach(async () => {
+        await Promise.all([
+            node1.destroy(),
+            node2.destroy()
+        ])
+    })
+
+    beforeEach(async () => {
+        const simulator = new Simulator()
+        transport1 = new SimulatorTransport(peer1, simulator)
+        transport2 = new SimulatorTransport(peer2, simulator)
+        layer01 = new DhtNode({
+            transportLayer: transport1,
+            peerDescriptor: peer1,
+            entryPoints: [peer1]
+        })
+        layer02 = new DhtNode({
+            transportLayer: transport2,
+            peerDescriptor: peer2,
+            entryPoints: [peer1]
+        })
+        await Promise.all([
+            layer01.start(),
+            layer02.start()
+        ])
+        await Promise.all([
+            layer01.joinDht([peer1]),
+            layer02.joinDht([peer1])
+        ])
+
+        node1 = new StreamrNode({})
+        node2 = new StreamrNode({})
+        await node1.start(layer01, transport1, transport1)
+        node1.setStreamPartEntryPoints(STREAM_ID, [peer1])
+        await node2.start(layer02, transport2, transport2)
+        node2.setStreamPartEntryPoints(STREAM_ID, [peer1])
+    })
+
+    it('starts', async () => {
+        expect(node1.getPeerDescriptor()).toEqual(peer1)
+        expect(node2.getPeerDescriptor()).toEqual(peer2)
+    })
+
+    it('Joining stream', async () => {
+        await node1.joinStream(STREAM_ID)
+        await node2.joinStream(STREAM_ID)
+        await waitForCondition(() => node1.getStream(STREAM_ID)!.layer2.getTargetNeighborStringIds().length === 1)
+        await waitForCondition(() => node2.getStream(STREAM_ID)!.layer2.getTargetNeighborStringIds().length === 1)
+        expect(node1.getStream(STREAM_ID)!.layer2.getTargetNeighborStringIds().length).toEqual(1)
+        expect(node2.getStream(STREAM_ID)!.layer2.getTargetNeighborStringIds().length).toEqual(1)
+    })
+
+    it('Publishing after joining and waiting for neighbors', async () => {
+        node1.subscribeToStream(STREAM_ID)
+        await node2.joinStream(STREAM_ID)
+        await waitForCondition(() => node1.getStream(STREAM_ID)!.layer2.getTargetNeighborStringIds().length === 1)
+        await waitForCondition(() => node2.getStream(STREAM_ID)!.layer2.getTargetNeighborStringIds().length === 1)
+        await Promise.all([
+            waitForEvent3<Events>(node1, 'newMessage'),
+            node2.publishToStream(STREAM_ID, msg)
+        ])
+    })
+
+    it('multi-stream pub/sub', async () => {
+        const stream2 = 'test2'
+        node1.setStreamPartEntryPoints(stream2, [peer1])
+        node2.setStreamPartEntryPoints(stream2, [peer1])
+        await node1.joinStream(STREAM_ID)
+        await node1.joinStream(stream2)
+        await node2.joinStream(STREAM_ID)
+        await node2.joinStream(stream2)
+        node1.subscribeToStream(STREAM_ID)
+        node2.subscribeToStream(stream2)
+        await Promise.all([
+            waitForCondition(() => node1.getStream(STREAM_ID)!.layer2.getTargetNeighborStringIds().length === 1),
+            waitForCondition(() => node2.getStream(STREAM_ID)!.layer2.getTargetNeighborStringIds().length === 1),
+            waitForCondition(() => node1.getStream(stream2)!.layer2.getTargetNeighborStringIds().length === 1),
+            waitForCondition(() => node2.getStream(stream2)!.layer2.getTargetNeighborStringIds().length === 1)
+        ])
+        const msg2 = createStreamMessage(
+            content,
+            stream2,
+            keyFromPeerDescriptor(peer1)
+        )
+        await Promise.all([
+            waitForEvent3<Events>(node1, 'newMessage'),
+            waitForEvent3<Events>(node2, 'newMessage'),
+            node1.publishToStream(stream2, msg2),
+            node2.publishToStream(STREAM_ID, msg)
+        ])
+    })
+
+    it('leaving streams', async () => {
+        await node1.joinStream(STREAM_ID)
+        await node2.joinStream(STREAM_ID)
+        await Promise.all([
+            waitForCondition(() => node1.getStream(STREAM_ID)!.layer2.getTargetNeighborStringIds().length === 1),
+            waitForCondition(() => node2.getStream(STREAM_ID)!.layer2.getTargetNeighborStringIds().length === 1)
+        ])
+        node2.leaveStream(STREAM_ID)
+        await waitForCondition(() => node1.getStream(STREAM_ID)!.layer2.getTargetNeighborStringIds().length === 0)
+    })
+
+})