@streamr/trackerless-network 0.0.1-tatum.5 → 0.0.1-tatum.7

package/src/logic/StreamPartEntryPointDiscovery.ts

@@ -1,240 +0,0 @@
-import { createHash } from 'crypto'
-import {
-    isSamePeerDescriptor,
-    PeerDescriptor,
-    RecursiveFindResult,
-    DataEntry
-} from '@streamr/dht'
-import { Any } from '../proto/google/protobuf/any'
-import { Logger, setAbortableTimeout, wait } from '@streamr/utils'
-import { StreamPartDelivery } from './StreamrNode'
-import { StreamPartID } from '@streamr/protocol'
-import { NodeID, getNodeIdFromPeerDescriptor } from '../identifiers'
-import { ILayer1 } from './ILayer1'
-
-export const streamPartIdToDataKey = (streamPartId: StreamPartID): Uint8Array => {
-    return new Uint8Array(createHash('md5').update(streamPartId).digest())
-}
-
-const parseEntryPointData = (dataEntries: DataEntry[]): PeerDescriptor[] => {
-    return dataEntries.filter((entry) => !entry.deleted).map((entry) => Any.unpack(entry.data!, PeerDescriptor))
-}
-
-interface FindEntryPointsResult {
-    entryPointsFromDht: boolean
-    discoveredEntryPoints: PeerDescriptor[]
-}
-
-const exponentialRunOff = async (
-    task: () => Promise<void>,
-    description: string,
-    abortSignal: AbortSignal,
-    baseDelay = 500,
-    maxAttempts = 6
-): Promise<void> => {
-    for (let i = 1; i <= maxAttempts; i++) {
-        if (abortSignal.aborted) {
-            return
-        }
-        const factor = 2 ** i
-        const delay = baseDelay * factor
-        try {
-            await task()
-        } catch (e: any) {
-            logger.trace(`${description} failed, retrying in ${delay} ms`)
-        }
-        try { // Abort controller throws unexpected errors in destroy?
-            await wait(delay, abortSignal)
-        } catch (err) {
-            logger.trace(`${err}`)
-        }
-    }
-}
-
-const logger = new Logger(module)
-
-const ENTRYPOINT_STORE_LIMIT = 8
-export const NETWORK_SPLIT_AVOIDANCE_LIMIT = 4
-
-interface StreamPartEntryPointDiscoveryConfig {
-    streamParts: Map<string, StreamPartDelivery>
-    ownPeerDescriptor: PeerDescriptor
-    getEntryPointData: (key: Uint8Array) => Promise<RecursiveFindResult>
-    getEntryPointDataViaNode: (key: Uint8Array, node: PeerDescriptor) => Promise<DataEntry[]>
-    storeEntryPointData: (key: Uint8Array, data: Any) => Promise<PeerDescriptor[]>
-    deleteEntryPointData: (key: Uint8Array) => Promise<void>
-    cacheInterval?: number
-}
-
-export class StreamPartEntryPointDiscovery {
-    private readonly abortController: AbortController
-    private readonly config: StreamPartEntryPointDiscoveryConfig
-    private readonly servicedStreamParts: Map<StreamPartID, NodeJS.Timeout>
-    private readonly cacheInterval: number
-    private readonly networkSplitAvoidedNodes: Map<StreamPartID, Set<NodeID>> = new Map()
-
-    constructor(config: StreamPartEntryPointDiscoveryConfig) {
-        this.config = config
-        this.abortController = new AbortController()
-        this.cacheInterval = this.config.cacheInterval ?? 60000
-        this.servicedStreamParts = new Map()
-    }
-
-    async discoverEntryPointsFromDht(
-        streamPartId: StreamPartID,
-        knownEntryPointCount: number,
-        forwardingNode?: PeerDescriptor
-    ): Promise<FindEntryPointsResult> {
-        if (knownEntryPointCount > 0) {
-            return {
-                entryPointsFromDht: false,
-                discoveredEntryPoints: []
-            }
-        }
-        const discoveredEntryPoints = await this.discoverEntryPoints(streamPartId, forwardingNode)
-        if (discoveredEntryPoints.length === 0) {
-            discoveredEntryPoints.push(this.config.ownPeerDescriptor)
-        }
-        return {
-            discoveredEntryPoints,
-            entryPointsFromDht: true
-        }
-    }
-
-    private async discoverEntryPoints(streamPartId: StreamPartID, forwardingNode?: PeerDescriptor): Promise<PeerDescriptor[]> {
-        const dataKey = streamPartIdToDataKey(streamPartId)
-        let discoveredEntryPoints = forwardingNode ? 
-            await this.queryEntryPointsViaNode(dataKey, forwardingNode) : await this.queryEntrypoints(dataKey)
-    
-        if (this.networkSplitAvoidedNodes.has(streamPartId)) {
-            const filtered = discoveredEntryPoints.filter((node) => 
-                !this.networkSplitAvoidedNodes.get(streamPartId)!.has(getNodeIdFromPeerDescriptor(node)))
-            // If all discovered entry points have previously beed detected as offline, try again
-            if (filtered.length > 0) {
-                discoveredEntryPoints = filtered
-            }
-        }
-        return discoveredEntryPoints
-    }
-
-    private async queryEntrypoints(key: Uint8Array): Promise<PeerDescriptor[]> {
-        logger.trace(`Finding data from dht node ${this.config.ownPeerDescriptor.nodeName}`)
-        try {
-            const results = await this.config.getEntryPointData(key)
-            if (results.dataEntries) {
-                return parseEntryPointData(results.dataEntries)
-            } else {
-                return []
-            }
-        } catch (err) {
-            return []
-        }
-    }
-
-    private async queryEntryPointsViaNode(key: Uint8Array, node: PeerDescriptor): Promise<PeerDescriptor[]> {
-        logger.trace(`Finding data via node ${this.config.ownPeerDescriptor.nodeName}`)
-        try {
-            const results = await this.config.getEntryPointDataViaNode(key, node)
-            if (results) {
-                return parseEntryPointData(results)
-            } else {
-                return []
-            }
-        } catch (err) {
-            return []
-        }
-    }
-
-    async storeSelfAsEntryPointIfNecessary(
-        streamPartId: StreamPartID,
-        entryPointsFromDht: boolean,
-        currentEntrypointCount: number
-    ): Promise<void> {
-        if (!this.config.streamParts.has(streamPartId) || !entryPointsFromDht) {
-            return
-        }
-        if ((this.config.streamParts.get(streamPartId)! as { layer1: ILayer1 }).layer1!.getBucketSize() < NETWORK_SPLIT_AVOIDANCE_LIMIT) {
-            await this.storeSelfAsEntryPoint(streamPartId)
-            setImmediate(() => this.avoidNetworkSplit(streamPartId))
-        } else if (currentEntrypointCount < ENTRYPOINT_STORE_LIMIT) {
-            await this.storeSelfAsEntryPoint(streamPartId)
-        }
-    }
-
-    private async storeSelfAsEntryPoint(streamPartId: StreamPartID): Promise<void> {
-        const ownPeerDescriptor = this.config.ownPeerDescriptor
-        const dataToStore = Any.pack(ownPeerDescriptor, PeerDescriptor)
-        try {
-            await this.config.storeEntryPointData(streamPartIdToDataKey(streamPartId), dataToStore)
-            this.keepSelfAsEntryPoint(streamPartId)
-        } catch (err) {
-            logger.warn(`Failed to store self as entrypoint for ${streamPartId}`)
-        }
-    }
-
-    private keepSelfAsEntryPoint(streamPartId: StreamPartID): void {
-        if (!this.config.streamParts.has(streamPartId) || this.servicedStreamParts.has(streamPartId)) {
-            return
-        }
-        this.servicedStreamParts.set(streamPartId, setTimeout(async () => {
-            if (!this.config.streamParts.has(streamPartId)) {
-                this.servicedStreamParts.delete(streamPartId)
-                return
-            }
-            logger.trace(`Attempting to keep self as entrypoint for ${streamPartId}`)
-            try {
-                const discovered = await this.discoverEntryPoints(streamPartId)
-                if (discovered.length < ENTRYPOINT_STORE_LIMIT 
-                    || discovered.some((peerDescriptor) => isSamePeerDescriptor(peerDescriptor, this.config.ownPeerDescriptor))) {
-                    await this.storeSelfAsEntryPoint(streamPartId)
-                    this.servicedStreamParts.delete(streamPartId)
-                    this.keepSelfAsEntryPoint(streamPartId)
-                } else {
-                    this.servicedStreamParts.delete(streamPartId)
-                }
-            } catch (err) {
-                logger.debug(`Failed to keep self as entrypoint for ${streamPartId}`)
-            }
-        }, this.cacheInterval))
-    }
-
-    private async avoidNetworkSplit(streamPartId: StreamPartID): Promise<void> {
-        await exponentialRunOff(async () => {
-            if (this.config.streamParts.has(streamPartId)) {
-                const stream = this.config.streamParts.get(streamPartId)! as { layer1: ILayer1 }
-                const rediscoveredEntrypoints = await this.discoverEntryPoints(streamPartId)
-                await stream.layer1!.joinDht(rediscoveredEntrypoints, false, false)
-                if (stream.layer1!.getBucketSize() < NETWORK_SPLIT_AVOIDANCE_LIMIT) {
-                    // Filter out nodes that are not in the k-bucket, assumed to be offline
-                    const nodesToAvoid = rediscoveredEntrypoints.filter((peer) => !stream.layer1!.getKBucketPeers().includes(peer))
-                    this.addAvoidedNodes(streamPartId, nodesToAvoid)
-                    throw new Error(`Network split is still possible`)
-                }
-            }
-        }, 'avoid network split', this.abortController.signal)
-        this.networkSplitAvoidedNodes.delete(streamPartId)
-        logger.trace(`Network split avoided`)
-    }
-
-    private addAvoidedNodes(streamPartId: StreamPartID, nodesToAvoid: PeerDescriptor[]): void {
-        if (!this.networkSplitAvoidedNodes.has(streamPartId)) {
-            this.networkSplitAvoidedNodes.set(streamPartId, new Set())
-        }
-        nodesToAvoid.forEach((node) => this.networkSplitAvoidedNodes.get(streamPartId)!.add(getNodeIdFromPeerDescriptor(node)))
-    }
-
-    removeSelfAsEntryPoint(streamPartId: StreamPartID): void {
-        if (this.servicedStreamParts.has(streamPartId)) {
-            setAbortableTimeout(() => this.config.deleteEntryPointData(streamPartIdToDataKey(streamPartId)), 0, this.abortController.signal)
-            clearTimeout(this.servicedStreamParts.get(streamPartId))
-            this.servicedStreamParts.delete(streamPartId)
-        }
-    }
-
-    async destroy(): Promise<void> {
-        this.servicedStreamParts.forEach((_, streamPartId) => this.removeSelfAsEntryPoint(streamPartId))
-        this.servicedStreamParts.clear()
-        this.abortController.abort()
-    }
-
-}

package/test/unit/StreamPartEntrypointDiscovery.test.ts

@@ -1,164 +0,0 @@
-import { PeerDescriptor, RecursiveFindResult, isSamePeerDescriptor } from '@streamr/dht'
-import { StreamPartIDUtils } from '@streamr/protocol'
-import { wait } from '@streamr/utils'
-import { range } from 'lodash'
-import { getNodeIdFromPeerDescriptor } from '../../src/identifiers'
-import { StreamPartEntryPointDiscovery } from '../../src/logic/StreamPartEntryPointDiscovery'
-import { StreamPartDelivery } from '../../src/logic/StreamrNode'
-import { Any } from '../../src/proto/google/protobuf/any'
-import { DataEntry } from '../../src/proto/packages/dht/protos/DhtRpc'
-import { MockLayer1 } from '../utils/mock/MockLayer1'
-import { createMockPeerDescriptor } from '../utils/utils'
-
-describe('StreamPartEntryPointDiscovery', () => {
-
-    let entryPointDiscoveryWithData: StreamPartEntryPointDiscovery
-    let entryPointDiscoveryWithoutData: StreamPartEntryPointDiscovery
-    let storeCalled: number
-    let streamParts = new Map<string, StreamPartDelivery>()
-
-    const peerDescriptor = createMockPeerDescriptor({
-        nodeName: 'fake'
-    })
-    const deletedPeerDescriptor = createMockPeerDescriptor({
-        nodeName: 'deleted'
-    })
-
-    const fakeData: DataEntry = {
-        data: Any.pack(peerDescriptor, PeerDescriptor),
-        ttl: 1000,
-        storer: peerDescriptor,
-        kademliaId: Uint8Array.from([1, 2, 3]),
-        stale: false,
-        deleted: false
-    }
-
-    const fakeDeletedData: DataEntry = {
-        data: Any.pack(deletedPeerDescriptor, PeerDescriptor),
-        ttl: 1000,
-        storer: deletedPeerDescriptor,
-        kademliaId: Uint8Array.from([1, 2, 3]),
-        stale: false,
-        deleted: true
-    }
-
-    const streamPartId = StreamPartIDUtils.parse('stream#0')
-
-    const fakeGetEntryPointData = async (_key: Uint8Array): Promise<RecursiveFindResult> => {
-        return {
-            closestNodes: [peerDescriptor],
-            dataEntries: [fakeData, fakeDeletedData]
-        }
-    }
-
-    const fakegetEntryPointDataViaNode = async (_key: Uint8Array, _node: PeerDescriptor): Promise<DataEntry[]> => {
-        return [fakeData]
-    }
-
-    const fakeStoreEntryPointData = async (_key: Uint8Array, _data: Any): Promise<PeerDescriptor[]> => {
-        storeCalled++
-        return [peerDescriptor]
-    }
-
-    const fakeEmptyGetEntryPointData = async (_key: Uint8Array): Promise<RecursiveFindResult> => {
-        return {
-            closestNodes: [],
-            dataEntries: []
-        }
-    }
-
-    const fakeDeleteEntryPointData = async (_key: Uint8Array): Promise<void> => {}
-
-    const addNodesToStream = (layer1: MockLayer1, count: number) => {
-        range(count).forEach(() => {
-            layer1.addNewRandomPeerToKBucket()
-            layer1.addNewRandomPeerToKBucket()
-            layer1.addNewRandomPeerToKBucket()
-            layer1.addNewRandomPeerToKBucket()
-        })
-    } 
-
-    let layer1: MockLayer1
-
-    beforeEach(() => {
-        storeCalled = 0
-        streamParts = new Map()
-        layer1 = new MockLayer1(getNodeIdFromPeerDescriptor(peerDescriptor))
-        streamParts.set(streamPartId, { layer1 } as any)
-        entryPointDiscoveryWithData = new StreamPartEntryPointDiscovery({
-            ownPeerDescriptor: peerDescriptor,
-            streamParts: streamParts,
-            getEntryPointData: fakeGetEntryPointData,
-            getEntryPointDataViaNode: fakegetEntryPointDataViaNode,
-            storeEntryPointData: fakeStoreEntryPointData,
-            deleteEntryPointData: fakeDeleteEntryPointData,
-            cacheInterval: 2000
-        })
-        entryPointDiscoveryWithoutData = new StreamPartEntryPointDiscovery({
-            ownPeerDescriptor: peerDescriptor,
-            streamParts: new Map<string, StreamPartDelivery>(),
-            getEntryPointData: fakeEmptyGetEntryPointData,
-            getEntryPointDataViaNode: fakegetEntryPointDataViaNode,
-            storeEntryPointData: fakeStoreEntryPointData,
-            deleteEntryPointData: fakeDeleteEntryPointData,
-            cacheInterval: 2000
-        })
-    })
-
-    afterEach(() => {
-        entryPointDiscoveryWithData.destroy()
-    })
-
-    it('discoverEntryPointsFromDht has known entrypoints', async () => {
-        const res = await entryPointDiscoveryWithData.discoverEntryPointsFromDht(streamPartId, 1)
-        expect(res.entryPointsFromDht).toEqual(false)
-        expect(res.discoveredEntryPoints).toEqual([])
-    })
-
-    it('discoverEntryPointsFromDht does not have known entrypoints', async () => {
-        const res = await entryPointDiscoveryWithData.discoverEntryPointsFromDht(streamPartId, 0)
-        expect(res.discoveredEntryPoints.length).toBe(1)
-        expect(isSamePeerDescriptor(res.discoveredEntryPoints[0], peerDescriptor)).toBe(true)
-    })
-
-    it('discoverEntryPointsfromDht on an empty stream', async () => {
-        const res = await entryPointDiscoveryWithoutData.discoverEntryPointsFromDht(streamPartId, 0)
-        expect(res.entryPointsFromDht).toEqual(true)
-        expect(res.discoveredEntryPoints.length).toBe(1)
-        expect(isSamePeerDescriptor(res.discoveredEntryPoints[0], peerDescriptor)).toBe(true)  // ownPeerDescriptor
-    })
-
-    it('store on empty stream', async () => {
-        await entryPointDiscoveryWithData.storeSelfAsEntryPointIfNecessary(streamPartId, true, 0)
-        expect(storeCalled).toEqual(1)
-    })
-
-    it('store on non-empty stream without known entry points', async () => {
-        addNodesToStream(layer1, 4)
-        await entryPointDiscoveryWithData.storeSelfAsEntryPointIfNecessary(streamPartId, false, 0)
-        expect(storeCalled).toEqual(0)
-    })
-
-    it('store on stream without saturated entrypoint count', async () => {
-        addNodesToStream(layer1, 4)
-        await entryPointDiscoveryWithData.storeSelfAsEntryPointIfNecessary(streamPartId, true, 0)
-        expect(storeCalled).toEqual(1)
-    })
-
-    it('will keep recaching until stream stopped', async () => {
-        await entryPointDiscoveryWithData.storeSelfAsEntryPointIfNecessary(streamPartId, true, 0)
-        expect(storeCalled).toEqual(1)
-        await wait(4500)
-        entryPointDiscoveryWithData.removeSelfAsEntryPoint(streamPartId)
-        expect(storeCalled).toEqual(3)
-    })
-
-    it('will stop recaching is stream is left', async () => {
-        await entryPointDiscoveryWithData.storeSelfAsEntryPointIfNecessary(streamPartId, true, 0)
-        expect(storeCalled).toEqual(1)
-        streamParts.delete(streamPartId)
-        await wait(4500)
-        expect(storeCalled).toEqual(1)
-    })
-
-})