@streamr/dht 100.0.0-pretestnet.6 → 100.0.0-rc.1

package/src/dht/find/Finder.ts

@@ -1,275 +0,0 @@
-import {
-    DataEntry,
-    Message,
-    MessageType,
-    NodeType,
-    PeerDescriptor,
-    FindRequest,
-    RouteMessageAck,
-    RouteMessageWrapper
-} from '../../proto/packages/dht/protos/DhtRpc'
-import { PeerID, PeerIDKey } from '../../helpers/PeerID'
-import { RoutingErrors, IRouter } from '../routing/Router'
-import { RoutingMode } from '../routing/RoutingSession'
-import { areEqualPeerDescriptors, peerIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
-import { Logger, runAndWaitForEvents3 } from '@streamr/utils'
-import { RoutingRpcCommunicator } from '../../transport/RoutingRpcCommunicator'
-import { FindSessionRpcRemote } from './FindSessionRpcRemote'
-import { v4 } from 'uuid'
-import { FindSession, FindSessionEvents } from './FindSession'
-import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
-import { ITransport } from '../../transport/ITransport'
-import { LocalDataStore } from '../store/LocalDataStore'
-import { ListeningRpcCommunicator } from '../../transport/ListeningRpcCommunicator'
-import { FindSessionRpcClient } from '../../proto/packages/dht/protos/DhtRpc.client'
-import { toProtoRpcClient } from '@streamr/proto-rpc'
-import { SortedContactList } from '../contact/SortedContactList'
-import { getPreviousPeer } from '../routing/getPreviousPeer'
-import { createRouteMessageAck } from '../routing/RouterRpcLocal'
-import { ServiceID } from '../../types/ServiceID'
-import { FindRpcLocal } from './FindRpcLocal'
-
-interface FinderConfig {
-    rpcCommunicator: RoutingRpcCommunicator
-    sessionTransport: ITransport
-    connections: Map<PeerIDKey, DhtNodeRpcRemote>
-    router: IRouter
-    localPeerDescriptor: PeerDescriptor
-    serviceId: ServiceID
-    localDataStore: LocalDataStore
-    addContact: (contact: PeerDescriptor, setActive?: boolean) => void
-    isPeerCloserToIdThanSelf: (peer1: PeerDescriptor, compareToId: PeerID) => boolean
-}
-
-export interface IFinder {
-    startFind(idToFind: Uint8Array, fetchData?: boolean): Promise<FindResult>
-}
-
-export interface FindResult { closestNodes: Array<PeerDescriptor>, dataEntries?: Array<DataEntry> }
-
-const logger = new Logger(module)
-
-export class Finder implements IFinder {
-
-    private readonly rpcCommunicator: RoutingRpcCommunicator
-    private readonly sessionTransport: ITransport
-    private readonly connections: Map<PeerIDKey, DhtNodeRpcRemote>
-    private readonly router: IRouter
-    private readonly localPeerDescriptor: PeerDescriptor
-    private readonly serviceId: ServiceID
-    private readonly localDataStore: LocalDataStore
-    private readonly isPeerCloserToIdThanSelf: (peer1: PeerDescriptor, compareToId: PeerID) => boolean
-    private ongoingSessions: Map<string, FindSession> = new Map()
-    private stopped = false
-
-    constructor(config: FinderConfig) {
-        this.rpcCommunicator = config.rpcCommunicator
-        this.sessionTransport = config.sessionTransport
-        this.connections = config.connections
-        this.router = config.router
-        this.localPeerDescriptor = config.localPeerDescriptor
-        this.serviceId = config.serviceId
-        this.localDataStore = config.localDataStore
-        this.isPeerCloserToIdThanSelf = config.isPeerCloserToIdThanSelf
-        this.registerLocalRpcMethods(config)
-    }
-
-    private registerLocalRpcMethods(config: FinderConfig) {
-        const rpcLocal = new FindRpcLocal({
-            doRouteFindRequest: (routedMessage: RouteMessageWrapper) => this.doRouteFindRequest(routedMessage),
-            addContact: (contact: PeerDescriptor, setActive?: boolean) => config.addContact(contact, setActive),
-            isMostLikelyDuplicate: (requestId: string) => this.router.isMostLikelyDuplicate(requestId),
-            addToDuplicateDetector: (requestId: string) => this.router.addToDuplicateDetector(requestId)
-        })
-        this.rpcCommunicator.registerRpcMethod(
-            RouteMessageWrapper,
-            RouteMessageAck,
-            'routeFindRequest',
-            async (routedMessage: RouteMessageWrapper) => {
-                if (this.stopped) {
-                    return createRouteMessageAck(routedMessage, 'routeFindRequest() service is not running')
-                } else {
-                    return rpcLocal.routeFindRequest(routedMessage)
-                }
-            }
-        )
-    }
-
-    public async startFind(
-        idToFind: Uint8Array,
-        fetchData: boolean = false,
-        excludedPeer?: PeerDescriptor
-    ): Promise<FindResult> {
-        if (this.stopped) {
-            return { closestNodes: [] }
-        }
-        const sessionId = v4()
-        const session = new FindSession({
-            serviceId: sessionId,
-            transport: this.sessionTransport,
-            kademliaIdToFind: idToFind,
-            localPeerId: peerIdFromPeerDescriptor(this.localPeerDescriptor),
-            waitedRoutingPathCompletions: this.connections.size > 1 ? 2 : 1,
-            fetchData
-        })
-        if (this.connections.size === 0) {
-            const data = this.localDataStore.getEntry(PeerID.fromValue(idToFind))
-            session.doSendFindResponse(
-                [this.localPeerDescriptor],
-                [this.localPeerDescriptor],
-                data ? Array.from(data.values()) : [],
-                true
-            )
-            return session.getResults()
-        }
-        const routeMessage = this.wrapFindRequest(idToFind, sessionId, fetchData)
-        this.ongoingSessions.set(sessionId, session)
-        try {
-            await runAndWaitForEvents3<FindSessionEvents>(
-                [() => this.doRouteFindRequest(routeMessage, excludedPeer)],
-                [[session, 'findCompleted']],
-                15000
-            )
-        } catch (err) {
-            logger.debug(`doRouteFindRequest failed with error ${err}`)
-        }
-        this.findAndReportLocalData(idToFind, fetchData, [], this.localPeerDescriptor, sessionId)
-        this.ongoingSessions.delete(sessionId)
-        session.stop()
-        return session.getResults()
-    }
-
-    private wrapFindRequest(idToFind: Uint8Array, sessionId: string, fetchData: boolean): RouteMessageWrapper {
-        const targetDescriptor: PeerDescriptor = {
-            kademliaId: idToFind,
-            type: NodeType.VIRTUAL
-        }
-        const request: FindRequest = {
-            sessionId,
-            fetchData
-        }
-        const msg: Message = {
-            messageType: MessageType.FIND_REQUEST,
-            messageId: v4(),
-            serviceId: this.serviceId,
-            body: {
-                oneofKind: 'findRequest',
-                findRequest: request
-            }
-        }
-        const routeMessage: RouteMessageWrapper = {
-            message: msg,
-            requestId: v4(),
-            destinationPeer: targetDescriptor,
-            sourcePeer: this.localPeerDescriptor,
-            reachableThrough: [],
-            routingPath: []
-        }
-        return routeMessage
-    }
-
-    private findAndReportLocalData(
-        idToFind: Uint8Array,
-        fetchData: boolean,
-        routingPath: PeerDescriptor[],
-        sourcePeer: PeerDescriptor,
-        sessionId: string
-    ): boolean {
-        if (fetchData) {
-            const data = this.localDataStore.getEntry(PeerID.fromValue(idToFind))
-            if (data.size > 0) {
-                this.sendFindResponse(routingPath, sourcePeer, sessionId, [], data, true)
-                return true
-            }
-        }
-        return false
-    }
-
-    private findLocalData(idToFind: Uint8Array, fetchData: boolean): Map<PeerIDKey, DataEntry> | undefined {
-        if (fetchData) {
-            return this.localDataStore.getEntry(PeerID.fromValue(idToFind))
-        }
-        return undefined
-    }
-
-    private sendFindResponse(
-        routingPath: PeerDescriptor[],
-        targetPeerDescriptor: PeerDescriptor,
-        serviceId: ServiceID,
-        closestNodes: PeerDescriptor[],
-        data: Map<PeerIDKey, DataEntry> | undefined,
-        noCloserNodesFound: boolean = false
-    ): void {
-        const dataEntries = data ? Array.from(data.values(), DataEntry.create.bind(DataEntry)) : []
-        const isOwnNode = areEqualPeerDescriptors(this.localPeerDescriptor, targetPeerDescriptor)
-        if (isOwnNode && this.ongoingSessions.has(serviceId)) {
-            this.ongoingSessions.get(serviceId)!
-                .doSendFindResponse(routingPath, closestNodes, dataEntries, noCloserNodesFound)
-        } else {
-            const remoteCommunicator = new ListeningRpcCommunicator(serviceId, this.sessionTransport, { rpcRequestTimeout: 15000 })
-            const rpcRemote = new FindSessionRpcRemote(
-                this.localPeerDescriptor,
-                targetPeerDescriptor,
-                serviceId,
-                toProtoRpcClient(new FindSessionRpcClient(remoteCommunicator.getRpcClientTransport())),
-                10000
-            )
-            rpcRemote.sendFindResponse(routingPath, closestNodes, dataEntries, noCloserNodesFound)
-            remoteCommunicator.destroy()
-        }
-    }
-
-    private doRouteFindRequest(routedMessage: RouteMessageWrapper, excludedPeer?: PeerDescriptor): RouteMessageAck {
-        if (this.stopped) {
-            return createRouteMessageAck(routedMessage, 'DhtNode Stopped')
-        }
-        const idToFind = peerIdFromPeerDescriptor(routedMessage.destinationPeer!)
-        const msg = routedMessage.message
-        const findRequest = msg?.body.oneofKind === 'findRequest' ? msg.body.findRequest : undefined
-        const closestPeersToDestination = this.getClosestConnections(routedMessage.destinationPeer!.kademliaId, 5)
-        const data = this.findLocalData(idToFind.value, findRequest!.fetchData)
-        if (areEqualPeerDescriptors(this.localPeerDescriptor, routedMessage.destinationPeer!)) {
-            this.sendFindResponse(routedMessage.routingPath, routedMessage.sourcePeer!, findRequest!.sessionId,
-                closestPeersToDestination, data, true)
-            return createRouteMessageAck(routedMessage)
-        }
-        const ack = this.router.doRouteMessage(routedMessage, RoutingMode.FIND, excludedPeer)
-        if (ack.error === RoutingErrors.NO_CANDIDATES_FOUND) {
-            logger.trace(`routeFindRequest Node found no candidates`)
-            this.sendFindResponse(routedMessage.routingPath, routedMessage.sourcePeer!, findRequest!.sessionId,
-                closestPeersToDestination, data, true)
-        } else if (ack.error) {
-            return ack
-        } else {
-            const noCloserContactsFound = (
-                closestPeersToDestination.length > 0
-                && getPreviousPeer(routedMessage)
-                && !this.isPeerCloserToIdThanSelf(closestPeersToDestination[0], idToFind)
-            )
-            this.sendFindResponse(routedMessage.routingPath, routedMessage.sourcePeer!, findRequest!.sessionId,
-                closestPeersToDestination, data, noCloserContactsFound)
-        }
-        return ack
-    }
-
-    private getClosestConnections(kademliaId: Uint8Array, limit: number): PeerDescriptor[] {
-        const connectedPeers = Array.from(this.connections.values())
-        const closestPeers = new SortedContactList<DhtNodeRpcRemote>(
-            PeerID.fromValue(kademliaId),
-            limit,
-            undefined,
-            true,
-            undefined
-        )
-        closestPeers.addContacts(connectedPeers)
-        return closestPeers.getClosestContacts(limit).map((peer) => peer.getPeerDescriptor())
-    }
-
-    public stop(): void {
-        this.stopped = true
-        this.ongoingSessions.forEach((session, _id) => {
-            session.stop()
-        })
-        this.ongoingSessions.clear()
-    }
-}

package/src/dht/routing/FindRpcRemote.ts

@@ -1,40 +0,0 @@
-import { RouteMessageWrapper } from '../../proto/packages/dht/protos/DhtRpc'
-import { v4 } from 'uuid'
-import { getNodeIdFromPeerDescriptor } from '../../helpers/peerIdFromPeerDescriptor'
-import { Remote } from '../contact/Remote'
-import { Logger } from '@streamr/utils'
-import { IFindRpcClient } from '../../proto/packages/dht/protos/DhtRpc.client'
-import { getPreviousPeer } from './getPreviousPeer'
-
-const logger = new Logger(module)
-
-export class FindRpcRemote extends Remote<IFindRpcClient> {
-
-    async routeFindRequest(params: RouteMessageWrapper): Promise<boolean> {
-        const message: RouteMessageWrapper = {
-            destinationPeer: params.destinationPeer,
-            sourcePeer: params.sourcePeer,
-            message: params.message,
-            requestId: params.requestId ?? v4(),
-            reachableThrough: params.reachableThrough ?? [],
-            routingPath: params.routingPath
-        }
-        const options = this.formDhtRpcOptions()
-        try {
-            const ack = await this.getClient().routeFindRequest(message, options)
-            if (ack.error.length > 0) {
-                logger.debug('Next hop responded with error ' + ack.error)
-                return false
-            }
-        } catch (err) {
-            const previousPeer = getPreviousPeer(params)
-            const fromNode = previousPeer
-                ? getNodeIdFromPeerDescriptor(previousPeer)
-                : getNodeIdFromPeerDescriptor(params.sourcePeer!)
-            // eslint-disable-next-line max-len
-            logger.debug(`Failed to send routeFindRequest message from ${fromNode} to ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())} with: ${err}`)
-            return false
-        }
-        return true
-    }
-}

package/src/helpers/PeerID.ts

@@ -1,88 +0,0 @@
-import { BrandedString } from '@streamr/utils'
-import { UUID } from './UUID'
-import { IllegalArguments } from './errors'
-import crypto from 'crypto'
-
-export type PeerIDKey = BrandedString<'PeerIDKey'>
-
-export class PeerID {
-    // avoid creating a new instance for every operation
-    private static readonly textEncoder = new TextEncoder() 
-    private static readonly textDecoder = new TextDecoder()
-    
-    private readonly data!: Uint8Array
-    private readonly key: PeerIDKey  // precompute often-used form of data
-
-    protected constructor({ ip, value, stringValue }: { ip?: string, value?: Uint8Array, stringValue?: string } = {}) {
-        if (ip) {
-            this.data = new Uint8Array(20)
-            const ipNum = this.ip2Int(ip)
-            const view = new DataView(this.data.buffer)
-            view.setInt32(0, ipNum)
-
-            this.data.set((new UUID()).value, 4)
-        } else if (value) {
-            this.data = new Uint8Array(value.slice(0))
-        } else if (stringValue) {
-            const ab = PeerID.textEncoder.encode(stringValue) //toUTF8Array(stringValue)
-            this.data = ab
-        } else {
-            throw new IllegalArguments('Constructor of PeerID must be given either ip, value or stringValue')
-        }
-
-        this.key = Buffer.from(this.data).toString('hex') as PeerIDKey
-    }
-
-    static fromIp(ip: string): PeerID {
-        return new PeerID({ ip })
-    }
-
-    static fromValue(value: Uint8Array): PeerID {
-        return new PeerID({ value })
-    }
-
-    static fromKey(value: PeerIDKey): PeerID {
-        return new PeerID({ value: Buffer.from(value, 'hex') })
-    }
-
-    static fromString(stringValue: string): PeerID {
-        return new PeerID({ stringValue })
-    }
-
-    // TODO convert to static method?
-    // eslint-disable-next-line class-methods-use-this
-    private ip2Int(ip: string): number {
-        return ip.split('.').map((octet, index, array) => {
-            return parseInt(octet) * Math.pow(256, (array.length - index - 1))
-        }).reduce((prev, curr) => {
-            return prev + curr
-        })
-    }
-
-    equals(other: PeerID): boolean {
-        return (Buffer.compare(this.data, other.value) === 0)
-    }
-
-    toString(): string {
-        return PeerID.textDecoder.decode(this.data) //utf8ArrayToString(this.data)
-    }
-
-    toKey(): PeerIDKey {
-        return this.key
-    }
-
-    get value(): Uint8Array {
-        return this.data
-    }
-
-    hasSmallerHashThan(other: PeerID): boolean {
-        const myId = this.toKey()
-        const theirId = other.toKey()
-        return PeerID.offeringHash(myId + ',' + theirId) < PeerID.offeringHash(theirId + ',' + myId)
-    }
-
-    private static offeringHash(idPair: string): number {
-        const buffer = crypto.createHash('md5').update(idPair).digest()
-        return buffer.readInt32LE(0)
-    }
-}

package/src/helpers/UUID.ts

@@ -1,35 +0,0 @@
-import { v4, parse, stringify } from 'uuid'
-import { binaryToHex } from '@streamr/utils'
-
-export class UUID {
-    private buf: Uint8Array
-
-    constructor(other?: (UUID | Uint8Array | string)) {
-        if (!other) {
-            this.buf = new Uint8Array(16)
-            v4(null, this.buf)
-        } else if (other.constructor === UUID) {
-            this.buf = other.buf
-        } else if (typeof other === 'string') {
-            this.buf = new Uint8Array(parse(other))
-        } else {
-            this.buf = other as Uint8Array
-        }
-    }
-
-    toString(): string {
-        return stringify(this.buf)
-    }
-
-    toHex(): string {
-        return binaryToHex(this.buf)
-    }
-
-    equals(other: UUID): boolean {
-        return (Buffer.compare(this.buf, other.value) === 0)
-    }
-
-    get value(): Uint8Array {
-        return this.buf
-    }
-}

package/src/helpers/kademliaId.ts

@@ -1,8 +0,0 @@
-import crypto from 'crypto'
-
-// https://www.scs.stanford.edu/~dm/home/papers/kpos.pdf
-const KADEMLIA_ID_LENGTH_IN_BYTES = 20
-
-export const createRandomKademliaId = (): Uint8Array => {
-    return crypto.randomBytes(KADEMLIA_ID_LENGTH_IN_BYTES)
-}

package/src/helpers/peerIdFromPeerDescriptor.ts

@@ -1,20 +0,0 @@
-import { binaryToHex } from '@streamr/utils'
-import { PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
-import { PeerID, PeerIDKey } from './PeerID'
-
-export const peerIdFromPeerDescriptor = (peerDescriptor: PeerDescriptor): PeerID => {
-    return PeerID.fromValue(peerDescriptor.kademliaId)
-}
-
-// TODO could move getNodeIdFromPeerDescriptor (and NodeID) from trackerless-network
-export const getNodeIdFromPeerDescriptor = (peerDescriptor: PeerDescriptor): string => {
-    return binaryToHex(peerDescriptor.kademliaId)
-}
-
-export const keyFromPeerDescriptor = (peerDescriptor: PeerDescriptor): PeerIDKey => {
-    return PeerID.fromValue(peerDescriptor.kademliaId).toKey()
-}
-
-export const areEqualPeerDescriptors = (peerDescriptor1: PeerDescriptor, peerDescriptor2: PeerDescriptor): boolean => {
-    return peerIdFromPeerDescriptor(peerDescriptor1).equals(peerIdFromPeerDescriptor(peerDescriptor2))
-}

package/test/integration/MigrateData.test.ts

@@ -1,204 +0,0 @@
-/* eslint-disable no-console */
-import { LatencyType, Simulator } from '../../src/connection/simulator/Simulator'
-import { DhtNode } from '../../src/dht/DhtNode'
-import { NodeType, PeerDescriptor } from '../../src/proto/packages/dht/protos/DhtRpc'
-import { createMockConnectionDhtNode, waitNodesReadyForTesting } from '../utils/utils'
-import { execSync } from 'child_process'
-import fs from 'fs'
-import { Logger } from '@streamr/utils'
-import { PeerID } from '../../src/helpers/PeerID'
-import { getNodeIdFromPeerDescriptor, keyFromPeerDescriptor, peerIdFromPeerDescriptor } from '../../src/helpers/peerIdFromPeerDescriptor'
-import { Any } from '../../src/proto/google/protobuf/any'
-import { SortedContactList } from '../../src/dht/contact/SortedContactList'
-import { Contact } from '../../src/dht/contact/Contact'
-
-const logger = new Logger(module)
-
-jest.setTimeout(60000)
-
-describe('Migrating data from node to node in DHT', () => {
-    let entryPoint: DhtNode
-    let nodes: DhtNode[]
-    let entrypointDescriptor: PeerDescriptor
-    const simulator = new Simulator(LatencyType.FIXED, 20)
-    const NUM_NODES = 100
-    const MAX_CONNECTIONS = 80
-    const K = 8
-
-    const nodesById: Map<string, DhtNode> = new Map()
-
-    if (!fs.existsSync('test/data/nodeids.json')) {
-        console.log('ground truth data does not exist yet, generating..')
-        execSync('npm run prepare-kademlia-simulation')
-    }
-
-    const dhtIds: Array<{ type: string, data: Array<number> }> = JSON.parse(fs.readFileSync('test/data/nodeids.json').toString())
-    /*
-    const getRandomNode = () => {
-        return nodes[Math.floor(Math.random() * nodes.length)]
-    }
-    */
-    beforeEach(async () => {
-        nodes = []
-        const entryPointId = '0'
-        entryPoint = await createMockConnectionDhtNode(entryPointId, simulator,
-            Uint8Array.from(dhtIds[0].data), K, MAX_CONNECTIONS)
-        nodes.push(entryPoint)
-        nodesById.set(entryPoint.getNodeId().toKey(), entryPoint)
-
-        entrypointDescriptor = {
-            kademliaId: entryPoint.getNodeId().value,
-            type: NodeType.NODEJS
-        }
-
-        nodes.push(entryPoint)
-
-        for (let i = 1; i < NUM_NODES; i++) {
-            const nodeId = `${i}`
-
-            const node = await createMockConnectionDhtNode(nodeId, simulator,
-                Uint8Array.from(dhtIds[i].data), K, MAX_CONNECTIONS)
-            nodesById.set(node.getNodeId().toKey(), node)
-            nodes.push(node)
-        }
-    })
-
-    afterEach(async () => {
-        await Promise.all([
-            ...nodes.map(async (node) => await node.stop())
-        ])
-        logger.info('nodes stopped')
-    })
-
-    afterAll(async () => {
-        simulator.stop()
-    })
-
-    it('Data migrates to the closest node no matter where it is stored', async () => {
-        const dataKey = PeerID.fromString('3232323e12r31r3')
-        const data = Any.pack(entrypointDescriptor, PeerDescriptor)
-
-        // calculate offline which node is closest to the data
-
-        const sortedList = new SortedContactList<Contact>(dataKey, 10000)
-
-        nodes.forEach((node) => {
-            sortedList.addContact(new Contact(node.getLocalPeerDescriptor())
-            )
-        })
-
-        const closest = sortedList.getAllContacts()
-
-        logger.info('Nodes sorted according to distance to data are: ')
-        closest.forEach((contact) => {
-            logger.info(getNodeIdFromPeerDescriptor(contact.getPeerDescriptor()))
-        })
-
-        logger.info('node 0 joining to the DHT')
-
-        await nodes[0].joinDht([entrypointDescriptor])
-
-        logger.info('storing data to node 0')
-        const successfulStorers = await nodes[0].storeDataToDht(dataKey.value, data)
-        expect(successfulStorers.length).toBe(1)
-        logger.info('data successfully stored to node 0')
-
-        logger.info('Nodes sorted according to distance to data with storing nodes marked are: ')
-
-        closest.forEach((contact) => {
-            const node = nodesById.get(PeerID.fromValue(contact.getPeerDescriptor().kademliaId).toKey())!
-            let hasDataMarker = ''
-            
-            // @ts-expect-error private field
-            if (node.localDataStore.getEntry(dataKey)) {
-                hasDataMarker = '<-'
-            }
-
-            // eslint-disable-next-line max-len
-            logger.info(peerIdFromPeerDescriptor(contact.getPeerDescriptor()) + ' ' + getNodeIdFromPeerDescriptor(node.getLocalPeerDescriptor()) + hasDataMarker)
-        })
-
-        logger.info(NUM_NODES + ' nodes joining layer0 DHT')
-        await Promise.all(
-            nodes.map((node) => {
-                if (keyFromPeerDescriptor(node.getLocalPeerDescriptor()) != '0') {
-                    node.joinDht([entrypointDescriptor])
-                }
-            })
-        )
-
-        logger.info('completed ' + NUM_NODES + ' nodes joining layer0 DHT')
-
-        await waitNodesReadyForTesting(nodes)
-
-        logger.info('After join of 99 nodes: nodes sorted according to distance to data with storing nodes marked are: ')
-
-        closest.forEach((contact) => {
-            const node = nodesById.get(PeerID.fromValue(contact.getPeerDescriptor().kademliaId).toKey())!
-            let hasDataMarker = ''
-
-            // @ts-expect-error private field
-
-            if (node.localDataStore.getEntry(dataKey)) {
-                hasDataMarker = '<-'
-            }
-
-            logger.info(getNodeIdFromPeerDescriptor(node.getLocalPeerDescriptor()) + hasDataMarker)
-        })
-
-        const closestNode = nodesById.get(PeerID.fromValue(closest[0].getPeerDescriptor().kademliaId).toKey())!
-
-        // @ts-expect-error private field
-        expect(closestNode.localDataStore.getEntry(dataKey)).toBeTruthy()
-    }, 180000)
-
-    it('Data migrates to the last remaining node if all other nodes leave gracefully', async () => {
-        const dataKey = PeerID.fromString('3232323e12r31r3')
-        const data = Any.pack(entrypointDescriptor, PeerDescriptor)
-
-        logger.info(NUM_NODES + ' nodes joining layer0 DHT')
-        await Promise.all(
-            nodes.map((node) => {
-                node.joinDht([entrypointDescriptor])
-            })
-        )
-
-        logger.info('completed ' + NUM_NODES + ' nodes joining layer0 DHT')
-
-        await waitNodesReadyForTesting(nodes)
-
-        const randomIndex = Math.floor(Math.random() * nodes.length)
-        logger.info('storing data to a random node: ' + randomIndex)
-
-        const successfulStorers = await nodes[randomIndex].storeDataToDht(dataKey.value, data)
-
-        logger.info('data successfully stored to ' + successfulStorers + ' nodes')
-
-        const randomIndices = []
-        for (let i = 0; i < nodes.length; i++) {
-            randomIndices.push(i)
-        }
-
-        console.error(randomIndices)
-        while (randomIndices.length > 1) {
-            const index = Math.floor(Math.random() * randomIndices.length)
-            const nodeIndex = randomIndices[index]
-            randomIndices.splice(index, 1)
-
-            logger.info('Stopping node ' + nodeIndex + ' ' +
-                // @ts-expect-error private field
-                (nodes[nodeIndex].localDataStore.getEntry(dataKey) ? ', has data' : ' does not have data'))
-
-            await nodes[nodeIndex].stop()
-        }
-
-        logger.info('after random graceful leaving, node ' + randomIndices[0] + ' is left')
-
-        // @ts-expect-error private field
-        logger.info('data of ' + randomIndices[0] + ' was ' + nodes[randomIndices[0]].localDataStore.getEntry(dataKey))
-
-        // @ts-expect-error private field
-        expect(nodes[randomIndices[0]].localDataStore.getEntry(dataKey)).toBeTruthy()
-
-    }, 180000)
-})