@streamr/trackerless-network 102.0.0-beta.1 → 102.0.0-beta.3

package/src/logic/neighbor-discovery/NeighborUpdateManager.ts

@@ -1,69 +0,0 @@
-import { DhtAddress, ListeningRpcCommunicator, PeerDescriptor, toNodeId } from '@streamr/dht'
-import { Logger, StreamPartID, scheduleAtInterval } from '@streamr/utils'
-import { NeighborUpdate } from '../../../generated/packages/trackerless-network/protos/NetworkRpc'
-import { NeighborUpdateRpcClient } from '../../../generated/packages/trackerless-network/protos/NetworkRpc.client'
-import { NodeList } from '../NodeList'
-import { NeighborFinder } from './NeighborFinder'
-import { NeighborUpdateRpcLocal } from './NeighborUpdateRpcLocal'
-import { NeighborUpdateRpcRemote } from './NeighborUpdateRpcRemote'
-
-interface NeighborUpdateManagerOptions {
-    localPeerDescriptor: PeerDescriptor
-    neighbors: NodeList
-    nearbyNodeView: NodeList
-    neighborFinder: NeighborFinder
-    streamPartId: StreamPartID
-    rpcCommunicator: ListeningRpcCommunicator
-    neighborUpdateInterval: number
-    neighborTargetCount: number
-    ongoingHandshakes: Set<DhtAddress>
-}
-
-const logger = new Logger(module)
-
-export class NeighborUpdateManager {
-
-    private readonly abortController: AbortController
-    private readonly options: NeighborUpdateManagerOptions
-    private readonly rpcLocal: NeighborUpdateRpcLocal
-
-    constructor(options: NeighborUpdateManagerOptions) {
-        this.abortController = new AbortController()
-        this.rpcLocal = new NeighborUpdateRpcLocal(options)
-        this.options = options
-        this.options.rpcCommunicator.registerRpcMethod(NeighborUpdate, NeighborUpdate, 'neighborUpdate',
-            (req: NeighborUpdate, context) => this.rpcLocal.neighborUpdate(req, context))
-    }
-
-    async start(): Promise<void> {
-        await scheduleAtInterval(() => this.updateNeighborInfo(), this.options.neighborUpdateInterval, false, this.abortController.signal)
-    }
-
-    stop(): void {
-        this.abortController.abort()
-    }
-
-    private async updateNeighborInfo(): Promise<void> {
-        logger.trace(`Updating neighbor info to nodes`)
-        const neighborDescriptors = this.options.neighbors.getAll().map((neighbor) => neighbor.getPeerDescriptor())
-        const startTime = Date.now()
-        await Promise.allSettled(this.options.neighbors.getAll().map(async (neighbor) => {
-            const res = await this.createRemote(neighbor.getPeerDescriptor()).updateNeighbors(this.options.streamPartId, neighborDescriptors)
-            const nodeId = toNodeId(neighbor.getPeerDescriptor())
-            this.options.neighbors.get(nodeId)!.setRtt(Date.now() - startTime)
-            if (res.removeMe) {
-                this.options.neighbors.remove(nodeId)
-                this.options.neighborFinder.start([nodeId])
-            }
-        }))
-    }
-
-    private createRemote(targetPeerDescriptor: PeerDescriptor): NeighborUpdateRpcRemote {
-        return new NeighborUpdateRpcRemote(
-            this.options.localPeerDescriptor,
-            targetPeerDescriptor,
-            this.options.rpcCommunicator,
-            NeighborUpdateRpcClient
-        )
-    }
-}

package/src/logic/neighbor-discovery/NeighborUpdateRpcLocal.ts

@@ -1,75 +0,0 @@
-import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
-import { DhtAddress, DhtCallContext, ListeningRpcCommunicator, PeerDescriptor, toNodeId } from '@streamr/dht'
-import { StreamPartID } from '@streamr/utils'
-import { NeighborUpdate } from '../../../generated/packages/trackerless-network/protos/NetworkRpc'
-import { ContentDeliveryRpcClient } from '../../../generated/packages/trackerless-network/protos/NetworkRpc.client'
-import { INeighborUpdateRpc } from '../../../generated/packages/trackerless-network/protos/NetworkRpc.server'
-import { ContentDeliveryRpcRemote } from '../ContentDeliveryRpcRemote'
-import { NodeList } from '../NodeList'
-import { NeighborFinder } from './NeighborFinder'
-
-interface NeighborUpdateRpcLocalOptions {
-    localPeerDescriptor: PeerDescriptor
-    streamPartId: StreamPartID
-    neighbors: NodeList
-    nearbyNodeView: NodeList
-    neighborFinder: NeighborFinder
-    rpcCommunicator: ListeningRpcCommunicator
-    neighborTargetCount: number
-    ongoingHandshakes: Set<DhtAddress>
-}
-
-export class NeighborUpdateRpcLocal implements INeighborUpdateRpc {
-
-    private readonly options: NeighborUpdateRpcLocalOptions
-
-    constructor(options: NeighborUpdateRpcLocalOptions) {
-        this.options = options
-    }
-
-    private updateContacts(neighborDescriptors: PeerDescriptor[]): void {
-        const ownNodeId = toNodeId(this.options.localPeerDescriptor)
-        const newPeerDescriptors = neighborDescriptors.filter((peerDescriptor) => {
-            const nodeId = toNodeId(peerDescriptor)
-            return nodeId !== ownNodeId && !this.options.neighbors.getIds().includes(nodeId)
-        })
-        newPeerDescriptors.forEach((peerDescriptor) => this.options.nearbyNodeView.add(
-            new ContentDeliveryRpcRemote(
-                this.options.localPeerDescriptor,
-                peerDescriptor,
-                this.options.rpcCommunicator,
-                ContentDeliveryRpcClient
-            ))
-        )
-    }
-
-    private createResponse(removeMe: boolean): NeighborUpdate {
-        return {
-            streamPartId: this.options.streamPartId,
-            neighborDescriptors: this.options.neighbors.getAll().map((neighbor) => neighbor.getPeerDescriptor()),
-            removeMe
-        }
-    }
-
-    // INeighborUpdateRpc server method
-    async neighborUpdate(message: NeighborUpdate, context: ServerCallContext): Promise<NeighborUpdate> {
-        const senderPeerDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
-        const remoteNodeId = toNodeId(senderPeerDescriptor)
-        this.updateContacts(message.neighborDescriptors)
-        if (!this.options.neighbors.has(remoteNodeId) && !this.options.ongoingHandshakes.has(remoteNodeId)) {
-            return this.createResponse(true)
-        } else {
-            const isOverNeighborCount = this.options.neighbors.size() > this.options.neighborTargetCount
-                // Motivation: We don't know the remote's neighborTargetCount setting here. We only ask to cut connections
-                // if the remote has a "sufficient" number of neighbors, where "sufficient" means our neighborTargetCount
-                // setting.
-                && message.neighborDescriptors.length > this.options.neighborTargetCount
-            if (!isOverNeighborCount) {
-                this.options.neighborFinder.start()
-            } else {
-                this.options.neighbors.remove(remoteNodeId)
-            }
-            return this.createResponse(isOverNeighborCount)
-        }
-    }
-}

package/src/logic/neighbor-discovery/NeighborUpdateRpcRemote.ts

@@ -1,35 +0,0 @@
-import { PeerDescriptor, RpcRemote, toNodeId } from '@streamr/dht'
-import { Logger, StreamPartID } from '@streamr/utils'
-import { NeighborUpdate } from '../../../generated/packages/trackerless-network/protos/NetworkRpc'
-import { NeighborUpdateRpcClient } from '../../../generated/packages/trackerless-network/protos/NetworkRpc.client'
-
-const logger = new Logger(module)
-
-interface UpdateNeighborsResponse {
-    peerDescriptors: PeerDescriptor[]
-    removeMe: boolean
-}
-
-export class NeighborUpdateRpcRemote extends RpcRemote<NeighborUpdateRpcClient> {
-
-    async updateNeighbors(streamPartId: StreamPartID, neighbors: PeerDescriptor[]): Promise<UpdateNeighborsResponse> {
-        const request: NeighborUpdate = {
-            streamPartId,
-            neighborDescriptors: neighbors,
-            removeMe: false
-        }
-        try {
-            const response = await this.getClient().neighborUpdate(request, this.formDhtRpcOptions())
-            return {
-                peerDescriptors: response.neighborDescriptors,
-                removeMe: response.removeMe
-            }
-        } catch (err: any) {
-            logger.debug(`updateNeighbors to ${toNodeId(this.getPeerDescriptor())} failed`, { err })
-            return {
-                peerDescriptors: [],
-                removeMe: true
-            }
-        }
-    }
-}

package/src/logic/node-info/NodeInfoClient.ts

@@ -1,23 +0,0 @@
-import { DhtCallContext, PeerDescriptor } from '@streamr/dht'
-import { RpcCommunicator } from '@streamr/proto-rpc'
-import { NodeInfo } from '../../types'
-import { NodeInfoRpcClient } from '../../../generated/packages/trackerless-network/protos/NetworkRpc.client'
-import { NodeInfoRpcRemote } from './NodeInfoRpcRemote'
-
-export class NodeInfoClient {
-    private readonly ownPeerDescriptor: PeerDescriptor
-    private readonly rpcCommunicator: RpcCommunicator<DhtCallContext>
-
-    constructor(ownPeerDescriptor: PeerDescriptor, rpcCommunicator: RpcCommunicator<DhtCallContext>) {
-        this.ownPeerDescriptor = ownPeerDescriptor
-        this.rpcCommunicator = rpcCommunicator
-    }
-
-    async getInfo(node: PeerDescriptor): Promise<NodeInfo> {
-        const remote = new NodeInfoRpcRemote(this.ownPeerDescriptor, node, this.rpcCommunicator, NodeInfoRpcClient)
-        // TODO remove casting when we validate NodeInfoResponse messages and therefore can annotate
-        // each of the field as required in the decorated type
-        return remote.getInfo() as unknown as NodeInfo
-    }
-
-}

package/src/logic/node-info/NodeInfoRpcLocal.ts

@@ -1,28 +0,0 @@
-import { NodeInfoRequest, NodeInfoResponse } from '../../../generated/packages/trackerless-network/protos/NetworkRpc'
-import { INodeInfoRpc } from '../../../generated/packages/trackerless-network/protos/NetworkRpc.server'
-import { NetworkStack } from '../../NetworkStack'
-import { ListeningRpcCommunicator } from '@streamr/dht'
-
-export const NODE_INFO_RPC_SERVICE_ID = 'system/node-info-rpc'
-
-export class NodeInfoRpcLocal implements INodeInfoRpc {
-    
-    private readonly stack: NetworkStack
-    private readonly rpcCommunicator: ListeningRpcCommunicator
- 
-    constructor(stack: NetworkStack, rpcCommunicator: ListeningRpcCommunicator) {
-        this.stack = stack
-        this.rpcCommunicator = rpcCommunicator
-        this.registerDefaultServerMethods()
-    }
-
-    private registerDefaultServerMethods(): void {
-        this.rpcCommunicator.registerRpcMethod(NodeInfoRequest, NodeInfoResponse, 'getInfo',
-            () => this.getInfo())
-    }
- 
-    async getInfo(): Promise<NodeInfoResponse> {
-        return this.stack.createNodeInfo()
-    }
-
-}

package/src/logic/node-info/NodeInfoRpcRemote.ts

@@ -1,11 +0,0 @@
-import { NodeInfoResponse } from '../../../generated/packages/trackerless-network/protos/NetworkRpc'
-import { NodeInfoRpcClient } from '../../../generated/packages/trackerless-network/protos/NetworkRpc.client'
-import { RpcRemote } from '@streamr/dht'
-
-export class NodeInfoRpcRemote extends RpcRemote<NodeInfoRpcClient> {
-
-    async getInfo(): Promise<NodeInfoResponse> {
-        return this.getClient().getInfo({}, this.formDhtRpcOptions())
-    }
-
-}

package/src/logic/propagation/FifoMapWithTTL.ts

@@ -1,116 +0,0 @@
-import { Node, Yallist } from 'yallist'
-
-interface Item<K, V> {
-    value: V
-    dropQueueNode: Node<K>
-    expiresAt: number
-}
-
-export interface FifoMapWithTtlOptions<K> {
-    ttlInMs: number
-    maxSize: number
-    onItemDropped?: (key: K) => void
-    timeProvider?: () => number
-    debugMode?: boolean
-}
-
-/**
- * A "Map" implementation with a maximum size and TTL expiration on entries.
- *
- * When full, room is made for new entries by dropping existing by FIFO method.
- *
- * Entries have a TTL after which they are considered stale. Stale items are
- * not returned when querying.
- *
- */
-export class FifoMapWithTTL<K, V> {
-    // class invariant: the keys present in `items` and `dropQueue` are the same set.
-    private readonly items = new Map<K, Item<K, V>>()
-    private readonly dropQueue = Yallist.create<K>() // queue is used to determine deletion order when full
-    private readonly ttlInMs: number
-    private readonly maxSize: number
-    private readonly onItemDropped: (key: K) => void
-    private readonly timeProvider: () => number
-
-    constructor({
-        ttlInMs,
-        maxSize,
-        onItemDropped = () => {},
-        timeProvider = Date.now
-    }: FifoMapWithTtlOptions<K>) {
-        if (ttlInMs < 0) {
-            throw new Error(`ttlInMs (${ttlInMs}) cannot be < 0`)
-        }
-        if (maxSize < 0) {
-            throw new Error(`maxSize (${maxSize}) cannot be < 0`)
-        }
-        this.ttlInMs = ttlInMs
-        this.maxSize = maxSize
-        this.onItemDropped = onItemDropped
-        this.timeProvider = timeProvider
-    }
-
-    set(key: K, value: V): void {
-        if (this.maxSize === 0) {
-            return
-        }
-        if (this.items.size > this.maxSize) {
-            throw new Error('assertion error: maximum size exceeded')
-        }
-
-        // delete an existing entry if exists
-        this.delete(key)
-
-        // make room for new entry
-        if (this.items.size === this.maxSize) {
-            const keyToDel = this.dropQueue.shift()
-            if (keyToDel === undefined) {
-                throw new Error('assertion error: queue empty but still have items')
-            }
-            this.items.delete(keyToDel)
-            this.onItemDropped(keyToDel)
-        }
-
-        // add entry
-        const dropQueueNode = new Node<K>(key)
-        this.dropQueue.pushNode(dropQueueNode)
-        this.items.set(key, {
-            value,
-            dropQueueNode,
-            expiresAt: this.timeProvider() + this.ttlInMs
-        })
-    }
-
-    delete(key: K): void {
-        const item = this.items.get(key)
-        if (item !== undefined) {
-            this.items.delete(key)
-            this.dropQueue.removeNode(item.dropQueueNode)
-            this.onItemDropped(key)
-        }
-    }
-
-    get(key: K): V | undefined {
-        const item = this.items.get(key)
-        if (item === undefined) {
-            return undefined
-        }
-        if (item.expiresAt <= this.timeProvider()) {
-            this.delete(key)
-            return undefined
-        }
-        return item.value
-    }
-
-    values(): V[] {
-        const keys = [...this.items.keys()]
-        const values = []
-        for (const key of keys) {
-            const value = this.get(key)
-            if (value !== undefined) {
-                values.push(value)
-            }
-        }
-        return values
-    }
-}

package/src/logic/propagation/Propagation.ts

@@ -1,84 +0,0 @@
-import { DhtAddress } from '@streamr/dht'
-import { StreamMessage } from '../../../generated/packages/trackerless-network/protos/NetworkRpc'
-import { PropagationTask, PropagationTaskStore } from './PropagationTaskStore'
-
-type SendToNeighborFn = (neighborId: DhtAddress, msg: StreamMessage) => Promise<void>
-
-interface ConstructorOptions {
-    sendToNeighbor: SendToNeighborFn
-    minPropagationTargets: number
-    ttl?: number
-    maxMessages?: number
-}
-
-const DEFAULT_MAX_MESSAGES = 150
-const DEFAULT_TTL = 10 * 1000
-
-/**
- * Message propagation logic of a node. Given a message, this class will actively attempt to propagate it to
- * `minPropagationTargets` neighbors until success or TTL expiration.
- *
- * Setting `minPropagationTargets = 0` effectively disables any propagation reattempts. A message will then
- * only be propagated exactly once, to neighbors that are present at that moment, in a fire-and-forget manner.
- */
-
-export class Propagation {
-    private readonly sendToNeighbor: SendToNeighborFn
-    private readonly minPropagationTargets: number
-    private readonly activeTaskStore: PropagationTaskStore
-
-    constructor({
-        sendToNeighbor,
-        minPropagationTargets,
-        ttl = DEFAULT_TTL,
-        maxMessages = DEFAULT_MAX_MESSAGES
-    }: ConstructorOptions) {
-        this.sendToNeighbor = sendToNeighbor
-        this.minPropagationTargets = minPropagationTargets
-        this.activeTaskStore = new PropagationTaskStore(ttl, maxMessages)
-    }
-
-    /**
-     * Node should invoke this when it learns about a new message
-     */
-    feedUnseenMessage(message: StreamMessage, targets: DhtAddress[], source: DhtAddress | null): void {
-        const task = {
-            message,
-            source,
-            handledNeighbors: new Set<DhtAddress>()
-        }
-        this.activeTaskStore.add(task)
-        for (const target of targets) {
-            this.sendAndAwaitThenMark(task, target)
-        }
-    }
-
-    /**
-     * Node should invoke this when it learns about a new node stream assignment
-     */
-    onNeighborJoined(neighborId: DhtAddress): void {
-        const tasks = this.activeTaskStore.get()
-        for (const task of tasks) {
-            this.sendAndAwaitThenMark(task, neighborId)
-        }
-    }
-
-    private sendAndAwaitThenMark({ message, source, handledNeighbors }: PropagationTask, neighborId: DhtAddress): void {
-        if (!handledNeighbors.has(neighborId) && neighborId !== source) {
-            (async () => {
-                try {
-                    await this.sendToNeighbor(neighborId, message)
-                } catch {
-                    return
-                }
-                // Side-note: due to asynchronicity, the task being modified at this point could already be stale and
-                // deleted from `activeTaskStore`. However, as modifying it or re-deleting it is pretty much
-                // inconsequential at this point, leaving the logic as is.
-                handledNeighbors.add(neighborId)
-                if (handledNeighbors.size >= this.minPropagationTargets) {
-                    this.activeTaskStore.delete(message.messageId!)
-                }
-            })()
-        }
-    }
-}

package/src/logic/propagation/PropagationTaskStore.ts

@@ -1,41 +0,0 @@
-import { DhtAddress } from '@streamr/dht'
-import { MessageRef, StreamMessage } from '../../../generated/packages/trackerless-network/protos/NetworkRpc'
-import { FifoMapWithTTL } from './FifoMapWithTTL'
-
-export interface PropagationTask {
-    message: StreamMessage
-    source: string | null
-    handledNeighbors: Set<DhtAddress>
-}
-
-/**
- * Keeps track of propagation tasks for the needs of message propagation logic.
- *
- * Properties:
- * - Allows fetching propagation tasks by StreamPartID
- * - Upper bound on number of tasks stored, replacement policy if FIFO
- * - Items have a TTL, after which they are considered stale and not returned when querying
-**/
-export class PropagationTaskStore {
-    private readonly tasks: FifoMapWithTTL<MessageRef, PropagationTask>
-
-    constructor(ttlInMs: number, maxTasks: number) {
-        this.tasks = new FifoMapWithTTL<MessageRef, PropagationTask>({
-            ttlInMs,
-            maxSize: maxTasks
-        })
-    }
-
-    get(): PropagationTask[] {
-        return this.tasks.values()
-    }
-
-    add(task: PropagationTask): void {
-        const messageId = task.message.messageId!
-        this.tasks.set(messageId, task)
-    }
-
-    delete(messageId: MessageRef): void {
-        this.tasks.delete(messageId) // causes `onKeyDropped` to be invoked
-    }
-}