@streamr/dht 100.0.0-testnet-two.4 → 100.0.0-testnet-three.0

package/test/integration/ReplicateData.test.ts

@@ -1,215 +1,106 @@
 /* eslint-disable no-console */
 import { LatencyType, Simulator } from '../../src/connection/simulator/Simulator'
 import { DhtNode } from '../../src/dht/DhtNode'
-import { NodeType, PeerDescriptor } from '../../src/proto/packages/dht/protos/DhtRpc'
 import { createMockConnectionDhtNode, waitNodesReadyForTesting } from '../utils/utils'
-import { execSync } from 'child_process'
-import fs from 'fs'
-import { Logger } from '@streamr/utils'
-import { PeerID } from '../../src/helpers/PeerID'
-import { keyFromPeerDescriptor, peerIdFromPeerDescriptor } from '../../src/helpers/peerIdFromPeerDescriptor'
 import { SortedContactList } from '../../src/dht/contact/SortedContactList'
-import { Contact } from '../../src/dht/contact/Contact'
-import { DhtAddress, getDhtAddressFromRaw, getRawFromDhtAddress } from '../../src/identifiers'
-import { createMockDataEntry } from '../utils/mock/mockDataEntry'
-import { LocalDataStore } from '../../src/dht/store/LocalDataStore'
-
-const logger = new Logger(module)
-
-jest.setTimeout(60000)
+import { createMockDataEntry, expectEqualData } from '../utils/mock/mockDataEntry'
+import { DhtAddress, createRandomDhtAddress, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor } from '../../src/identifiers'
+import { sample } from 'lodash'
+import { DataEntry, PeerDescriptor } from '../../src/proto/packages/dht/protos/DhtRpc'
 
+const DATA = createMockDataEntry()
 const NUM_NODES = 100
 const MAX_CONNECTIONS = 80
 const K = 8
+const ENTRY_POINT_INDEX = 0
+
+const getDataEntries = (node: DhtNode): DataEntry[] => {
+    // @ts-expect-error private field
+    const store = node.localDataStore
+    return Array.from(store.values(getDhtAddressFromRaw(DATA.key)))
+}
 
 describe('Replicate data from node to node in DHT', () => {
 
-    let entryPoint: DhtNode
     let nodes: DhtNode[]
-    let entrypointDescriptor: PeerDescriptor
+    let entryPointDescriptor: PeerDescriptor
     const simulator = new Simulator(LatencyType.FIXED, 20)
-    const nodesById: Map<DhtAddress, DhtNode> = new Map()
-
-    if (!fs.existsSync('test/data/nodeids.json')) {
-        console.log('ground truth data does not exist yet, generating..')
-        execSync('npm run prepare-kademlia-simulation')
-    }
-
-    const dhtIds: Array<{ type: string, data: Array<number> }> = JSON.parse(fs.readFileSync('test/data/nodeids.json').toString())
-    /*
-    const getRandomNode = () => {
-        return nodes[Math.floor(Math.random() * nodes.length)]
-    }
-    */
-
-    const getEntries = (key: DhtAddress, localDataStore: LocalDataStore) => {
-        return Array.from(localDataStore.values(key))
-    }
 
     beforeEach(async () => {
+        const entryPoint = await createMockConnectionDhtNode(simulator, createRandomDhtAddress(), K, MAX_CONNECTIONS)
+        entryPointDescriptor = entryPoint.getLocalPeerDescriptor()
+        await entryPoint.joinDht([entryPointDescriptor])
         nodes = []
-        entryPoint = await createMockConnectionDhtNode(simulator,
-            getDhtAddressFromRaw(Uint8Array.from(dhtIds[0].data)), K, MAX_CONNECTIONS)
         nodes.push(entryPoint)
-        nodesById.set(entryPoint.getNodeId(), entryPoint)
-
-        entrypointDescriptor = {
-            nodeId: getRawFromDhtAddress(entryPoint.getNodeId()),
-            type: NodeType.NODEJS
-        }
-
-        nodes.push(entryPoint)
-
         for (let i = 1; i < NUM_NODES; i++) {
-            const node = await createMockConnectionDhtNode(simulator,
-                getDhtAddressFromRaw(Uint8Array.from(dhtIds[i].data)), K, MAX_CONNECTIONS)
-            nodesById.set(node.getNodeId(), node)
+            const node = await createMockConnectionDhtNode(
+                simulator,
+                createRandomDhtAddress(),
+                K,
+                MAX_CONNECTIONS
+            )
             nodes.push(node)
         }
-    })
+    }, 60000)
 
     afterEach(async () => {
-        await Promise.all([
-            ...nodes.map(async (node) => await node.stop())
-        ])
-        logger.info('nodes stopped')
-    })
+        await Promise.all(nodes.map(async (node) => await node.stop()))
+    }, 60000)
 
     afterAll(async () => {
         simulator.stop()
     })
 
     it('Data replicates to the closest node no matter where it is stored', async () => {
-        const dataKey = getDhtAddressFromRaw(PeerID.fromString('3232323e12r31r3').value)
-        const data = createMockDataEntry({ key: dataKey })
-
         // calculate offline which node is closest to the data
-
-        const sortedList = new SortedContactList<Contact>({ 
-            referenceId: dataKey,
+        const sortedList = new SortedContactList<DhtNode>({ 
+            referenceId: getDhtAddressFromRaw(DATA.key),
             maxSize: 10000, 
             allowToContainReferenceId: true, 
             emitEvents: false 
         })
-
-        nodes.forEach((node) => {
-            sortedList.addContact(new Contact(node.getLocalPeerDescriptor())
-            )
-        })
+        nodes.forEach((node) => sortedList.addContact(node))
 
         const closest = sortedList.getAllContacts()
-
-        logger.info('Nodes sorted according to distance to data are: ')
-        closest.forEach((contact) => {
-            logger.info(contact.getNodeId())
-        })
-
-        logger.info('node 0 joining to the DHT')
-
-        await nodes[0].joinDht([entrypointDescriptor])
-
-        logger.info('storing data to node 0')
-        const successfulStorers = await nodes[0].storeDataToDht(dataKey, data.data!)
+        const successfulStorers = await nodes[0].storeDataToDht(getDhtAddressFromRaw(DATA.key), DATA.data!)
         expect(successfulStorers.length).toBe(1)
-        logger.info('data successfully stored to node 0')
-
-        logger.info('Nodes sorted according to distance to data with storing nodes marked are: ')
 
-        closest.forEach((contact) => {
-            const node = nodesById.get(contact.getNodeId())!
-            let hasDataMarker = ''
-            
-            // @ts-expect-error private field
-            const store = node.localDataStore
-            if (getEntries(dataKey, store).length > 0) {
-                hasDataMarker = '<-'
-            }
-
-            // eslint-disable-next-line max-len
-            logger.info(peerIdFromPeerDescriptor(contact.getPeerDescriptor()) + ' ' + node.getNodeId() + hasDataMarker)
-        })
-
-        logger.info(NUM_NODES + ' nodes joining layer0 DHT')
         await Promise.all(
-            nodes.map((node) => {
-                if (keyFromPeerDescriptor(node.getLocalPeerDescriptor()) != '0') {
-                    node.joinDht([entrypointDescriptor])
+            nodes.map(async (node, i) => {
+                if (i !== ENTRY_POINT_INDEX) {
+                    await node.joinDht([entryPointDescriptor])
                 }
             })
         )
-
-        logger.info('completed ' + NUM_NODES + ' nodes joining layer0 DHT')
-
         await waitNodesReadyForTesting(nodes)
 
-        logger.info('After join of 99 nodes: nodes sorted according to distance to data with storing nodes marked are: ')
-
-        closest.forEach((contact) => {
-            const node = nodesById.get(contact.getNodeId())!
-            let hasDataMarker = ''
-
-            // @ts-expect-error private field
-            const store = node.localDataStore
-            if (getEntries(dataKey, store).length > 0) {
-                hasDataMarker = '<-'
-            }
-
-            logger.info(node.getNodeId() + hasDataMarker)
-        })
-
-        const closestNode = nodesById.get(closest[0].getNodeId())!
-
-        // @ts-expect-error private field
-        const store = closestNode.localDataStore
-        expect(getEntries(dataKey, store).length).toBeGreaterThanOrEqual(1)
+        const data = getDataEntries(closest[0])
+        expect(data).toHaveLength(1)
+        expectEqualData(data[0], DATA)
     }, 180000)
 
-    it('Data replicates to the last remaining node if all other nodes leave gracefully', async () => {
-        const dataKey = getDhtAddressFromRaw(PeerID.fromString('3232323e12r31r3').value)
-        const data = createMockDataEntry({ key: dataKey })
-
-        logger.info(NUM_NODES + ' nodes joining layer0 DHT')
+    it('Data replicates to the other nodes when storers are stopped', async () => {
         await Promise.all(
-            nodes.map((node) => {
-                node.joinDht([entrypointDescriptor])
+            nodes.map(async (node, i) => {
+                if (i !== ENTRY_POINT_INDEX) {
+                    await node.joinDht([entryPointDescriptor])
+                }
             })
         )
-
-        logger.info('completed ' + NUM_NODES + ' nodes joining layer0 DHT')
-
         await waitNodesReadyForTesting(nodes)
 
         const randomIndex = Math.floor(Math.random() * nodes.length)
-        logger.info('storing data to a random node: ' + randomIndex)
-
-        const successfulStorers = await nodes[randomIndex].storeDataToDht(dataKey, data.data!)
-
-        logger.info('data successfully stored to ' + successfulStorers + ' nodes')
-
-        const randomIndices = []
-        for (let i = 0; i < nodes.length; i++) {
-            randomIndices.push(i)
-        }
-
-        console.error(randomIndices)
-        while (randomIndices.length > 1) {
-            const index = Math.floor(Math.random() * randomIndices.length)
-            const nodeIndex = randomIndices[index]
-            randomIndices.splice(index, 1)
-            // @ts-expect-error private field
-            const store = nodes[nodeIndex].localDataStore
-            logger.info('Stopping node ' + nodeIndex + ' ' +
-                ((getEntries(dataKey, store).length > 0) ? ', has data' : ' does not have data'))
-
-            await nodes[nodeIndex].stop()
-        }
-
-        logger.info('after random graceful leaving, node ' + randomIndices[0] + ' is left')
-
-        // @ts-expect-error private field
-        const firstStore = nodes[randomIndices[0]].localDataStore
-        logger.info('data of ' + randomIndices[0] + ' was ' + getEntries(dataKey, firstStore))
-        expect(getEntries(dataKey, firstStore).length).toBeGreaterThanOrEqual(1)
-
+        const storerDescriptors = await nodes[randomIndex].storeDataToDht(getDhtAddressFromRaw(DATA.key), DATA.data!)
+        const stoppedNodeIds: DhtAddress[] = []
+        await Promise.all(storerDescriptors.map(async (storerDescriptor) => {
+            const storer = nodes.find((n) => n.getNodeId() === getNodeIdFromPeerDescriptor(storerDescriptor))!
+            await storer.stop()
+            stoppedNodeIds.push(storer.getNodeId())
+        }))
+
+        const randomNonStoppedNode = sample(nodes.filter((n) => !stoppedNodeIds.includes(n.getNodeId())))!
+        const data = await randomNonStoppedNode.getDataFromDht(getDhtAddressFromRaw(DATA.key))
+        expect(data).toHaveLength(1)
+        expectEqualData(data[0], DATA)
     }, 180000)
 })

package/test/integration/RouteMessage.test.ts

@@ -3,20 +3,14 @@ import { Message, MessageType, NodeType, PeerDescriptor, RouteMessageWrapper } f
 import { RpcMessage } from '../../src/proto/packages/proto-rpc/protos/ProtoRpc'
 import { Logger, runAndWaitForEvents3, waitForCondition } from '@streamr/utils'
 import { createMockConnectionDhtNode, createWrappedClosestPeersRequest } from '../utils/utils'
-import { PeerID, PeerIDKey } from '../../src/helpers/PeerID'
 import { Simulator } from '../../src/connection/simulator/Simulator'
 import { v4 } from 'uuid'
 import { Any } from '../../src/proto/google/protobuf/any'
 import { RoutingMode } from '../../src/dht/routing/RoutingSession'
-import { createRandomDhtAddress, getDhtAddressFromRaw, getRawFromDhtAddress } from '../../src/identifiers'
+import { DhtAddress, createRandomDhtAddress, getRawFromDhtAddress } from '../../src/identifiers'
 
 const logger = new Logger(module)
 
-// TODO refactor the test to not to use PeerID
-const getPeerId = (node: DhtNode) => {
-    return PeerID.fromValue(getRawFromDhtAddress(node.getNodeId()))
-}
-
 const NUM_NODES = 30
 
 describe('Route Message With Mock Connections', () => {
@@ -27,7 +21,6 @@ describe('Route Message With Mock Connections', () => {
     let routerNodes: DhtNode[]
     let simulator: Simulator
     let entryPointDescriptor: PeerDescriptor
-    const receiveMatrix: Array<Array<number>> = []
 
     beforeEach(async () => {
         routerNodes = []
@@ -43,7 +36,7 @@ describe('Route Message With Mock Connections', () => {
         destinationNode = await createMockConnectionDhtNode(simulator, createRandomDhtAddress())
 
         for (let i = 1; i < NUM_NODES; i++) {
-            const node = await createMockConnectionDhtNode(simulator, getDhtAddressFromRaw(PeerID.fromString(`${i}`).value))
+            const node = await createMockConnectionDhtNode(simulator, createRandomDhtAddress())
             routerNodes.push(node)
         }
 
@@ -87,21 +80,21 @@ describe('Route Message With Mock Connections', () => {
                 requestId: v4(),
                 sourcePeer: sourceNode.getLocalPeerDescriptor(),
                 reachableThrough: [],
-                routingPath: []
-
+                routingPath: [],
+                parallelRootNodeIds: []
             })
         }], [[destinationNode, 'message']], 20000)
     }, 30000)
 
     it('Receives multiple messages', async () => {
-        const numOfMessages = 20
+        const messageCount = 20
         let receivedMessages = 0
         destinationNode.on('message', () => {
             receivedMessages += 1
         })
         const rpcWrapper = createWrappedClosestPeersRequest(sourceNode.getLocalPeerDescriptor())
 
-        for (let i = 0; i < numOfMessages; i++) {
+        for (let i = 0; i < messageCount; i++) {
             const message: Message = {
                 serviceId: 'unknown',
                 messageId: v4(),
@@ -119,40 +112,22 @@ describe('Route Message With Mock Connections', () => {
                 requestId: v4(),
                 sourcePeer: sourceNode.getLocalPeerDescriptor(),
                 reachableThrough: [],
-                routingPath: []
+                routingPath: [],
+                parallelRootNodeIds: []
             })
         }
-        await waitForCondition(() => receivedMessages === numOfMessages)
+        await waitForCondition(() => receivedMessages === messageCount)
     })
 
     it('From all to all', async () => {
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        for (const i in routerNodes) {
-            const arr: Array<number> = []
-            // eslint-disable-next-line @typescript-eslint/no-unused-vars
-            for (const j in routerNodes) {
-                arr.push(0)
-            }
-            receiveMatrix.push(arr)
-        }
-
-        const numsOfReceivedMessages: Record<PeerIDKey, number> = {}
+        const receivedMessageCounts: Record<DhtAddress, number> = {}
         routerNodes.forEach((node) => {
-            numsOfReceivedMessages[getPeerId(node).toKey()] = 0
-            node.on('message', (msg: Message) => {
-                numsOfReceivedMessages[getPeerId(node).toKey()] = numsOfReceivedMessages[getPeerId(node).toKey()] + 1
-                try {
-                    const target = receiveMatrix[parseInt(getPeerId(node).toString()) - 1]
-                    target[parseInt(PeerID.fromValue(msg.sourceDescriptor!.nodeId).toString()) - 1]++
-                } catch (e) {
-                    console.error(e)
-                }
-                if (parseInt(getPeerId(node).toString()) > routerNodes.length || parseInt(getPeerId(node).toString()) === 0) {
-                    console.error(getPeerId(node).toString())
-                }
+            const key = node.getNodeId()
+            receivedMessageCounts[key] = 0
+            node.on('message', () => {
+                receivedMessageCounts[key] = receivedMessageCounts[key] + 1
             })
-        }
-        )
+        })
         await Promise.all(
             routerNodes.map(async (node) =>
                 Promise.all(routerNodes.map(async (receiver) => {
@@ -175,17 +150,17 @@ describe('Route Message With Mock Connections', () => {
                             sourcePeer: node.getLocalPeerDescriptor(),
                             requestId: v4(),
                             reachableThrough: [],
-                            routingPath: []
+                            routingPath: [],
+                            parallelRootNodeIds: []
                         })
                     }
                 }))
             )
         )
-        await waitForCondition(() => numsOfReceivedMessages[PeerID.fromString('1').toKey()] >= routerNodes.length - 1
-            , 30000)
+        await waitForCondition(() => receivedMessageCounts[routerNodes[0].getNodeId()] >= routerNodes.length - 1, 30000)
         await Promise.all(
-            Object.keys(numsOfReceivedMessages).map(async (key) =>
-                waitForCondition(() => numsOfReceivedMessages[key as PeerIDKey] >= routerNodes.length - 1, 30000)
+            Object.keys(receivedMessageCounts).map(async (key) =>
+                waitForCondition(() => receivedMessageCounts[key as DhtAddress] >= routerNodes.length - 1, 30000)
             )
         )
 
@@ -211,7 +186,8 @@ describe('Route Message With Mock Connections', () => {
             requestId: v4(),
             sourcePeer: sourceNode.getLocalPeerDescriptor(),
             reachableThrough: [entryPointDescriptor],
-            routingPath: []
+            routingPath: [],
+            parallelRootNodeIds: []
         }
 
         const rpcMessage: RpcMessage = {
@@ -241,7 +217,8 @@ describe('Route Message With Mock Connections', () => {
             sourcePeer: sourceNode.getLocalPeerDescriptor(),
             target: entryPoint.getLocalPeerDescriptor()!.nodeId,
             reachableThrough: [],
-            routingPath: []
+            routingPath: [],
+            parallelRootNodeIds: []
         }
 
         await runAndWaitForEvents3<DhtNodeEvents>([() => {

package/test/integration/RouterRpcRemote.test.ts

@@ -4,14 +4,15 @@ import { Message, MessageType, RouteMessageAck, RouteMessageWrapper } from '../.
 import { RouterRpcClient } from '../../src/proto/packages/dht/protos/DhtRpc.client'
 import { RpcMessage } from '../../src/proto/packages/proto-rpc/protos/ProtoRpc'
 import { createMockPeerDescriptor, createWrappedClosestPeersRequest, mockRouterRpc } from '../utils/utils'
+import { DhtCallContext } from '../../src/rpc-protocol/DhtCallContext'
 
 const SERVICE_ID = 'test'
 
 describe('RemoteRouter', () => {
 
     let remoteRouter: RouterRpcRemote
-    let clientRpcCommunicator: RpcCommunicator
-    let serverRpcCommunicator: RpcCommunicator
+    let clientRpcCommunicator: RpcCommunicator<DhtCallContext>
+    let serverRpcCommunicator: RpcCommunicator<DhtCallContext>
     const clientPeerDescriptor = createMockPeerDescriptor()
     const serverPeerDescriptor = createMockPeerDescriptor()
 
@@ -45,7 +46,8 @@ describe('RemoteRouter', () => {
             sourcePeer: clientPeerDescriptor,
             target: serverPeerDescriptor.nodeId,
             reachableThrough: [],
-            routingPath: []
+            routingPath: [],
+            parallelRootNodeIds: []
         })
         expect(routable).toEqual(true)
     })
@@ -68,7 +70,8 @@ describe('RemoteRouter', () => {
             sourcePeer: clientPeerDescriptor,
             target: serverPeerDescriptor.nodeId,
             reachableThrough: [],
-            routingPath: []
+            routingPath: [],
+            parallelRootNodeIds: []
         })
         expect(routable).toEqual(false)
     })

package/test/integration/ScaleDownDht.test.ts

@@ -2,10 +2,9 @@ import { LatencyType, Simulator } from '../../src/connection/simulator/Simulator
 import { DhtNode } from '../../src/dht/DhtNode'
 import { NodeType, PeerDescriptor } from '../../src/proto/packages/dht/protos/DhtRpc'
 import { createMockConnectionDhtNode } from '../utils/utils'
-import { areEqualPeerDescriptors, getNodeIdFromPeerDescriptor } from '../../src/helpers/peerIdFromPeerDescriptor'
 import { Logger } from '@streamr/utils'
 import { getRandomRegion } from '../../src/connection/simulator/pings'
-import { createRandomDhtAddress, getRawFromDhtAddress } from '../../src/identifiers'
+import { areEqualPeerDescriptors, createRandomDhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../src/identifiers'
 
 const logger = new Logger(module)
 
@@ -55,7 +54,7 @@ describe('Scaling down a Dht network', () => {
             const stoppingPeerDescriptor = nodes[nodeIndex].getLocalPeerDescriptor()
             await nodes[nodeIndex].stop()
             const nodeIsCleaned = nodes.every((node) =>
-                node.getAllConnectionPeerDescriptors().every((peer) => {
+                node.getConnections().every((peer) => {
                     if (areEqualPeerDescriptors(peer, stoppingPeerDescriptor)) {
                         logger.error(getNodeIdFromPeerDescriptor(node.getLocalPeerDescriptor()) + ', ' 
                             + getNodeIdFromPeerDescriptor(stoppingPeerDescriptor) + ' cleaning up failed')

package/test/integration/SimultaneousConnections.test.ts

@@ -8,6 +8,7 @@ import { RpcMessage } from '../../src/proto/packages/proto-rpc/protos/ProtoRpc'
 import { createMockPeerDescriptor } from '../utils/utils'
 import { getRandomRegion } from '../../src/connection/simulator/pings'
 import { MockTransport } from '../utils/mock/Transport'
+import { getNodeIdFromPeerDescriptor } from '../../src/identifiers'
 
 const BASE_MESSAGE: Message = {
     serviceId: 'serviceId',
@@ -78,8 +79,8 @@ describe('SimultaneousConnections', () => {
             simTransport1.send(msg1),
             simTransport2.send(msg2)
         ])
-        await waitForCondition(() => simTransport2.hasConnection(peerDescriptor1))
-        await waitForCondition(() => simTransport1.hasConnection(peerDescriptor2))
+        await waitForCondition(() => simTransport2.hasConnection(getNodeIdFromPeerDescriptor(peerDescriptor1)))
+        await waitForCondition(() => simTransport1.hasConnection(getNodeIdFromPeerDescriptor(peerDescriptor2)))
     })
 
     describe('Websocket 2 servers', () => {
@@ -159,8 +160,8 @@ describe('SimultaneousConnections', () => {
                 connectionManager2.send(msg2)
             ])
 
-            await waitForCondition(() => connectionManager1.hasConnection(wsPeerDescriptor2))
-            await waitForCondition(() => connectionManager2.hasConnection(wsPeerDescriptor1))
+            await waitForCondition(() => connectionManager1.hasConnection(getNodeIdFromPeerDescriptor(wsPeerDescriptor2)))
+            await waitForCondition(() => connectionManager2.hasConnection(getNodeIdFromPeerDescriptor(wsPeerDescriptor1)))
         })
     })
 
@@ -239,8 +240,8 @@ describe('SimultaneousConnections', () => {
                 connectionManager2.send(msg2)
             ])
 
-            await waitForCondition(() => connectionManager1.hasConnection(wsPeerDescriptor2))
-            await waitForCondition(() => connectionManager2.hasConnection(wsPeerDescriptor1))
+            await waitForCondition(() => connectionManager1.hasConnection(getNodeIdFromPeerDescriptor(wsPeerDescriptor2)))
+            await waitForCondition(() => connectionManager2.hasConnection(getNodeIdFromPeerDescriptor(wsPeerDescriptor1)))
         })
     })
 
@@ -307,8 +308,8 @@ describe('SimultaneousConnections', () => {
                 connectionManager2.send(msg2)
             ])
 
-            await waitForCondition(() => connectionManager1.hasConnection(wrtcPeerDescriptor2))
-            await waitForCondition(() => connectionManager2.hasConnection(wrtcPeerDescriptor1))
+            await waitForCondition(() => connectionManager1.hasConnection(getNodeIdFromPeerDescriptor(wrtcPeerDescriptor2)))
+            await waitForCondition(() => connectionManager2.hasConnection(getNodeIdFromPeerDescriptor(wrtcPeerDescriptor1)))
         })
     })
 

package/test/integration/Store.test.ts

@@ -1,7 +1,6 @@
 import { LatencyType, Simulator } from '../../src/connection/simulator/Simulator'
 import { DhtNode } from '../../src/dht/DhtNode'
-import { getDhtAddressFromRaw } from '../../src/identifiers'
-import { getNodeIdFromPeerDescriptor } from '../../src/helpers/peerIdFromPeerDescriptor'
+import { getDhtAddressFromRaw, getNodeIdFromPeerDescriptor } from '../../src/identifiers'
 import { PeerDescriptor } from '../../src/proto/packages/dht/protos/DhtRpc'
 import { createMockDataEntry, expectEqualData } from '../utils/mock/mockDataEntry'
 import { createMockConnectionDhtNode, createMockPeerDescriptor, waitConnectionManagersReadyForTesting } from '../utils/utils'

package/test/integration/StoreAndDelete.test.ts

@@ -4,6 +4,7 @@ import { createMockConnectionDhtNode, waitConnectionManagersReadyForTesting } fr
 import { createMockDataEntry, expectEqualData } from '../utils/mock/mockDataEntry'
 import { createRandomDhtAddress } from '../../src/identifiers'
 import { getDhtAddressFromRaw } from '../../src/identifiers'
+import { wait } from '@streamr/utils'
 
 const NUM_NODES = 5
 const MAX_CONNECTIONS = 5
@@ -42,6 +43,8 @@ describe('Storing data in DHT', () => {
         const successfulStorers = await storingNode.storeDataToDht(getDhtAddressFromRaw(entry.key), entry.data!)
         expect(successfulStorers.length).toBeGreaterThan(4)
         await storingNode.deleteDataFromDht(getDhtAddressFromRaw(entry.key), true)
+        // Wait for the delete operation to propagate
+        await wait(500)
         const fetchingNode = getRandomNode()
         const results = await fetchingNode.getDataFromDht(getDhtAddressFromRaw(entry.key))
         results.forEach((result) => {
@@ -56,6 +59,8 @@ describe('Storing data in DHT', () => {
         const successfulStorers1 = await storingNode.storeDataToDht(getDhtAddressFromRaw(entry.key), entry.data!)
         expect(successfulStorers1.length).toBeGreaterThan(4)
         await storingNode.deleteDataFromDht(getDhtAddressFromRaw(entry.key), true)
+        // Wait for the delete operation to propagate
+        await wait(500)
         const fetchingNode = getRandomNode()
         const results1 = await fetchingNode.getDataFromDht(getDhtAddressFromRaw(entry.key))
         results1.forEach((result) => {

package/test/integration/StoreOnDhtWithThreeNodes.test.ts

@@ -0,0 +1,59 @@
+import { createMockConnectionDhtNode } from '../utils/utils'
+import { DhtNode } from '../../src/dht/DhtNode'
+import { Simulator } from '../../src/connection/simulator/Simulator'
+import { createMockDataEntry, expectEqualData } from '../utils/mock/mockDataEntry'
+import { getDhtAddressFromRaw } from '../../src/identifiers'
+
+describe('Storing data in DHT with two peers', () => {
+
+    let entryPoint: DhtNode
+    let node1: DhtNode
+    let node2: DhtNode
+    let simulator: Simulator | undefined
+
+    beforeEach(async () => {
+        simulator = new Simulator()
+        entryPoint = await createMockConnectionDhtNode(simulator)
+        node1 = await createMockConnectionDhtNode(simulator)
+        node2 = await createMockConnectionDhtNode(simulator)
+
+        await entryPoint.start()
+        await node1.start()
+        await node2.start()
+
+        await entryPoint.joinDht([entryPoint.getLocalPeerDescriptor()])
+        node1.joinDht([entryPoint.getLocalPeerDescriptor()]).catch(() => {})
+        node2.joinDht([entryPoint.getLocalPeerDescriptor()]).catch(() => {})
+        await Promise.all([
+            node1.waitForNetworkConnectivity(),
+            node2.waitForNetworkConnectivity()
+        ])
+    })
+
+    afterEach(async () => {
+        await entryPoint.stop()
+        await node1.stop()
+        await node2.stop()
+        simulator?.stop()
+    })
+
+    it('Node can store on three peer DHT', async () => {
+        const storedData1 = createMockDataEntry()
+        const storedData2 = createMockDataEntry()
+        // Here we effectively test that fetching "null" data doesn't take too long. A test timeout here indicates an issue. 
+        await node1.getDataFromDht(getDhtAddressFromRaw(storedData1.key))
+        await node2.getDataFromDht(getDhtAddressFromRaw(storedData1.key))
+        
+        await node1.storeDataToDht(getDhtAddressFromRaw(storedData1.key), storedData1.data!)
+        await node2.storeDataToDht(getDhtAddressFromRaw(storedData1.key), storedData1.data!)
+        await entryPoint.storeDataToDht(getDhtAddressFromRaw(storedData2.key), storedData2.data!)
+        const foundData1 = await node1.getDataFromDht(getDhtAddressFromRaw(storedData1.key))
+        const foundData2 = await node2.getDataFromDht(getDhtAddressFromRaw(storedData1.key))
+        const foundData3 = await entryPoint.getDataFromDht(getDhtAddressFromRaw(storedData2.key))
+        expectEqualData(foundData1[0], storedData1)
+        expectEqualData(foundData1[1], storedData1)
+        expectEqualData(foundData2[0], storedData1)
+        expectEqualData(foundData2[1], storedData1)
+        expectEqualData(foundData3[0], storedData2)
+    }, 30000)
+})

package/test/integration/StoreOnDhtWithTwoNodes.test.ts

@@ -42,7 +42,7 @@ describe('Storing data in DHT with two peers', () => {
 
     it('Can store on one peer DHT', async () => {
         await otherNode.stop()
-        await waitForCondition(() => entryPoint.getNumberOfNeighbors() === 0)
+        await waitForCondition(() => entryPoint.getNeighborCount() === 0)
         const storedData = createMockDataEntry()
         await entryPoint.storeDataToDht(getDhtAddressFromRaw(storedData.key), storedData.data!)
         const foundData = await entryPoint.getDataFromDht(getDhtAddressFromRaw(storedData.key))