@streamr/trackerless-network 0.0.1-tatum.7 → 100.0.0-pretestnet.0

package/src/logic/{ILayer0.ts → Layer0Node.ts}

@@ -1,11 +1,15 @@
 import { DataEntry, ITransport, PeerDescriptor } from '@streamr/dht'
 import { Any } from '../proto/google/protobuf/any'
 
-export interface ILayer0 extends ITransport {
+export interface Layer0Node extends ITransport {
+    joinDht(entryPointDescriptors: PeerDescriptor[]): Promise<void>
+    hasJoined(): boolean
     getPeerDescriptor(): PeerDescriptor
     getDataFromDht(key: Uint8Array): Promise<DataEntry[]>
     storeDataToDht(key: Uint8Array, data: Any): Promise<PeerDescriptor[]>
     deleteDataFromDht(key: Uint8Array): Promise<void>
     waitForNetworkConnectivity(): Promise<void>
+    getTransport(): ITransport
+    start(): Promise<void>
     stop(): Promise<void>
 }

package/src/logic/{ILayer1.ts → Layer1Node.ts}

@@ -1,16 +1,16 @@
 import { PeerDescriptor } from '@streamr/dht'
 
-export interface ILayer1Events {
+export interface Layer1NodeEvents {
     newContact: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
     contactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
     newRandomContact: (peerDescriptor: PeerDescriptor, randomPeers: PeerDescriptor[]) => void
     randomContactRemoved: (peerDescriptor: PeerDescriptor, randomPeers: PeerDescriptor[]) => void
 }
 
-export interface ILayer1 {
-    on<T extends keyof ILayer1Events>(eventName: T, listener: (peerDescriptor: PeerDescriptor, peers: PeerDescriptor[]) => void): void
-    once<T extends keyof ILayer1Events>(eventName: T, listener: (peerDescriptor: PeerDescriptor, peers: PeerDescriptor[]) => void): void
-    off<T extends keyof ILayer1Events>(eventName: T, listener: (peerDescriptor: PeerDescriptor, peers: PeerDescriptor[]) => void): void
+export interface Layer1Node {
+    on<T extends keyof Layer1NodeEvents>(eventName: T, listener: (peerDescriptor: PeerDescriptor, peers: PeerDescriptor[]) => void): void
+    once<T extends keyof Layer1NodeEvents>(eventName: T, listener: (peerDescriptor: PeerDescriptor, peers: PeerDescriptor[]) => void): void
+    off<T extends keyof Layer1NodeEvents>(eventName: T, listener: (peerDescriptor: PeerDescriptor, peers: PeerDescriptor[]) => void): void
     removeContact: (peerDescriptor: PeerDescriptor, removeFromOpenInternetPeers?: boolean) => void
     getClosestContacts: (maxCount?: number) => PeerDescriptor[]
     getKBucketPeers: () => PeerDescriptor[]

package/src/logic/RandomGraphNode.ts

@@ -30,7 +30,7 @@ import { IInspector } from './inspect/Inspector'
 import { TemporaryConnectionRpcLocal } from './temporary-connection/TemporaryConnectionRpcLocal'
 import { markAndCheckDuplicate } from './utils'
 import { NodeID, getNodeIdFromPeerDescriptor } from '../identifiers'
-import { ILayer1 } from './ILayer1'
+import { Layer1Node } from './Layer1Node'
 import { StreamPartID } from '@streamr/protocol'
 
 export interface Events {
@@ -40,8 +40,8 @@ export interface Events {
 
 export interface StrictRandomGraphNodeConfig {
     streamPartId: StreamPartID
-    layer1: ILayer1
-    P2PTransport: ITransport
+    layer1Node: Layer1Node
+    transport: ITransport
     connectionLocker: ConnectionLocker
     ownPeerDescriptor: PeerDescriptor
     nodeViewSize: number
@@ -86,7 +86,7 @@ export class RandomGraphNode extends EventEmitter<Events> {
                 || this.config.proxyConnectionRpcLocal?.getConnection(senderId )?.remote
                 // TODO: check integrity of notifier?
                 if (contact) {
-                    this.config.layer1.removeContact(contact.getPeerDescriptor(), true)
+                    this.config.layer1Node.removeContact(contact.getPeerDescriptor(), true)
                     this.config.targetNeighbors.remove(contact.getPeerDescriptor())
                     this.config.nearbyNodeView.remove(contact.getPeerDescriptor())
                     this.config.connectionLocker.unlockConnection(contact.getPeerDescriptor(), this.config.streamPartId)
@@ -102,31 +102,31 @@ export class RandomGraphNode extends EventEmitter<Events> {
         this.started = true
         this.registerDefaultServerMethods()
         addManagedEventListener<any, any>(
-            this.config.layer1 as any,
+            this.config.layer1Node as any,
             'newContact',
             (_peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => this.newContact(closestPeers),
             this.abortController.signal
         )
         addManagedEventListener<any, any>(
-            this.config.layer1 as any,
+            this.config.layer1Node as any,
             'contactRemoved',
             (_peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => this.removedContact(closestPeers),
             this.abortController.signal
         )
         addManagedEventListener<any, any>(
-            this.config.layer1 as any,
+            this.config.layer1Node as any,
             'newRandomContact',
             (_peerDescriptor: PeerDescriptor, randomPeers: PeerDescriptor[]) => this.newRandomContact(randomPeers),
             this.abortController.signal
         )   
         addManagedEventListener<any, any>(
-            this.config.layer1 as any,
+            this.config.layer1Node as any,
             'randomContactRemoved',
             (_peerDescriptor: PeerDescriptor, randomPeers: PeerDescriptor[]) => this.removedRandomContact(randomPeers),
             this.abortController.signal
         )   
         addManagedEventListener<any, any>(
-            this.config.P2PTransport as any,
+            this.config.transport as any,
             'disconnected',
             (peerDescriptor: PeerDescriptor) => this.onNodeDisconnected(peerDescriptor),
             this.abortController.signal
@@ -193,7 +193,7 @@ export class RandomGraphNode extends EventEmitter<Events> {
                 toProtoRpcClient(new DeliveryRpcClient(this.config.rpcCommunicator.getRpcClientTransport()))
             )
         ))
-        for (const descriptor of this.config.layer1.getKBucketPeers()) {
+        for (const descriptor of this.config.layer1Node.getKBucketPeers()) {
             if (this.config.nearbyNodeView.size() >= this.config.nodeViewSize) {
                 break
             }
@@ -251,10 +251,10 @@ export class RandomGraphNode extends EventEmitter<Events> {
 
     private getNeighborCandidatesFromLayer1(): PeerDescriptor[] {
         const uniqueNodes = new Set<PeerDescriptor>()
-        this.config.layer1.getClosestContacts(this.config.nodeViewSize).forEach((peer: PeerDescriptor) => {
+        this.config.layer1Node.getClosestContacts(this.config.nodeViewSize).forEach((peer: PeerDescriptor) => {
             uniqueNodes.add(peer)
         })
-        this.config.layer1.getKBucketPeers().forEach((peer: PeerDescriptor) => {
+        this.config.layer1Node.getKBucketPeers().forEach((peer: PeerDescriptor) => {
             uniqueNodes.add(peer)
         })
         return Array.from(uniqueNodes)

package/src/logic/StreamrNode.ts

@@ -17,8 +17,8 @@ import { EventEmitter } from 'eventemitter3'
 import { sampleSize } from 'lodash'
 import { NodeID, getNodeIdFromPeerDescriptor } from '../identifiers'
 import { ProxyDirection, StreamMessage } from '../proto/packages/trackerless-network/protos/NetworkRpc'
-import { ILayer0 } from './ILayer0'
-import { ILayer1 } from './ILayer1'
+import { Layer0Node } from './Layer0Node'
+import { Layer1Node } from './Layer1Node'
 import { RandomGraphNode } from './RandomGraphNode'
 import { NETWORK_SPLIT_AVOIDANCE_LIMIT, EntryPointDiscovery } from './EntryPointDiscovery'
 import { createRandomGraphNode } from './createRandomGraphNode'
@@ -29,7 +29,7 @@ export type StreamPartDelivery = {
     stop: () => void
 } & ({ 
     proxied: false
-    layer1: ILayer1
+    layer1Node: Layer1Node
     node: RandomGraphNode
     entryPointDiscovery: EntryPointDiscovery
 } | {
@@ -59,9 +59,9 @@ export interface StreamrNodeConfig {
 
 // TODO rename class?
 export class StreamrNode extends EventEmitter<Events> {
-    private P2PTransport?: ITransport
+    private transport?: ITransport
     private connectionLocker?: ConnectionLocker
-    private layer0?: ILayer0
+    private layer0Node?: Layer0Node
     private readonly metricsContext: MetricsContext
     private readonly metrics: Metrics
     private readonly config: StreamrNodeConfig
@@ -82,14 +82,14 @@ export class StreamrNode extends EventEmitter<Events> {
         this.metricsContext.addMetrics('node', this.metrics)
     }
 
-    async start(startedAndJoinedLayer0: ILayer0, transport: ITransport, connectionLocker: ConnectionLocker): Promise<void> {
+    async start(startedAndJoinedLayer0Node: Layer0Node, transport: ITransport, connectionLocker: ConnectionLocker): Promise<void> {
         if (this.started || this.destroyed) {
             return
         }
-        logger.info(`Starting new StreamrNode with id ${getNodeIdFromPeerDescriptor(startedAndJoinedLayer0.getPeerDescriptor())}`)
+        logger.info(`Starting new StreamrNode with id ${getNodeIdFromPeerDescriptor(startedAndJoinedLayer0Node.getPeerDescriptor())}`)
         this.started = true
-        this.layer0 = startedAndJoinedLayer0
-        this.P2PTransport = transport
+        this.layer0Node = startedAndJoinedLayer0Node
+        this.transport = transport
         this.connectionLocker = connectionLocker
         cleanUp = () => this.destroy()
     }
@@ -103,10 +103,11 @@ export class StreamrNode extends EventEmitter<Events> {
         this.streamParts.forEach((streamPart) => streamPart.stop())
         this.streamParts.clear()
         this.removeAllListeners()
-        await this.layer0!.stop()
-        await this.P2PTransport!.stop()
-        this.layer0 = undefined
-        this.P2PTransport = undefined
+        // TODO stopping should be in NetworkStack#stop?
+        await this.layer0Node!.stop()
+        await this.transport!.stop()
+        this.layer0Node = undefined
+        this.transport = undefined
         this.connectionLocker = undefined
     }
 
@@ -132,31 +133,31 @@ export class StreamrNode extends EventEmitter<Events> {
         if (streamPart !== undefined) {
             return
         }
-        const layer1 = this.createLayer1Node(streamPartId, this.knownStreamPartEntryPoints.get(streamPartId) ?? [])
-        const node = this.createRandomGraphNode(streamPartId, layer1)
+        const layer1Node = this.createLayer1Node(streamPartId, this.knownStreamPartEntryPoints.get(streamPartId) ?? [])
+        const node = this.createRandomGraphNode(streamPartId, layer1Node)
         const entryPointDiscovery = new EntryPointDiscovery({
             streamPartId,
             ownPeerDescriptor: this.getPeerDescriptor(),
-            layer1,
-            getEntryPointData: (key) => this.layer0!.getDataFromDht(key),
-            storeEntryPointData: (key, data) => this.layer0!.storeDataToDht(key, data),
+            layer1Node,
+            getEntryPointData: (key) => this.layer0Node!.getDataFromDht(key),
+            storeEntryPointData: (key, data) => this.layer0Node!.storeDataToDht(key, data),
             deleteEntryPointData: async (key) => {
                 if (this.destroyed) {
                     return 
                 }
-                return this.layer0!.deleteDataFromDht(key)
+                return this.layer0Node!.deleteDataFromDht(key)
             }
         })
         streamPart = {
             proxied: false,
-            layer1,
+            layer1Node,
             node,
             entryPointDiscovery,
             broadcast: (msg: StreamMessage) => node.broadcast(msg),
             stop: () => {
                 entryPointDiscovery.destroy()
                 node.stop()
-                layer1.stop()
+                layer1Node.stop()
             }
         }
         this.streamParts.set(streamPartId, streamPart)
@@ -179,24 +180,24 @@ export class StreamrNode extends EventEmitter<Events> {
             // leaveStreamPart has been called (or leaveStreamPart called, and then setProxies called)
             return
         }
-        await streamPart.layer1.start()
+        await streamPart.layer1Node.start()
         await streamPart.node.start()
         let entryPoints = this.knownStreamPartEntryPoints.get(streamPartId) ?? []
         const discoveryResult = await entryPointDiscovery.discoverEntryPointsFromDht(
             entryPoints.length
         )
         entryPoints = entryPoints.concat(discoveryResult.discoveredEntryPoints)
-        await streamPart.layer1.joinDht(sampleSize(entryPoints, NETWORK_SPLIT_AVOIDANCE_LIMIT))
+        await streamPart.layer1Node.joinDht(sampleSize(entryPoints, NETWORK_SPLIT_AVOIDANCE_LIMIT))
         if (discoveryResult.entryPointsFromDht) {
             await entryPointDiscovery.storeSelfAsEntryPointIfNecessary(entryPoints.length)
         }
     }
 
-    private createLayer1Node = (streamPartId: StreamPartID, entryPoints: PeerDescriptor[]): ILayer1 => {
+    private createLayer1Node = (streamPartId: StreamPartID, entryPoints: PeerDescriptor[]): Layer1Node => {
         return new DhtNode({
-            transport: this.layer0!,
+            transport: this.layer0Node!,
             serviceId: 'layer1::' + streamPartId,
-            peerDescriptor: this.layer0!.getPeerDescriptor(),
+            peerDescriptor: this.layer0Node!.getPeerDescriptor(),
             entryPoints,
             numberOfNodesPerKBucket: 4,
             rpcRequestTimeout: 5000,
@@ -204,13 +205,13 @@ export class StreamrNode extends EventEmitter<Events> {
         })
     }
 
-    private createRandomGraphNode = (streamPartId: StreamPartID, layer1: ILayer1) => {
+    private createRandomGraphNode = (streamPartId: StreamPartID, layer1Node: Layer1Node) => {
         return createRandomGraphNode({
             streamPartId,
-            P2PTransport: this.P2PTransport!,
-            layer1,
+            transport: this.transport!,
+            layer1Node,
             connectionLocker: this.connectionLocker!,
-            ownPeerDescriptor: this.layer0!.getPeerDescriptor(),
+            ownPeerDescriptor: this.layer0Node!.getPeerDescriptor(),
             minPropagationTargets: this.config.streamPartitionMinPropagationTargets,
             numOfTargetNeighbors: this.config.streamPartitionNumOfNeighbors,
             acceptProxyConnections: this.config.acceptProxyConnections
@@ -255,8 +256,8 @@ export class StreamrNode extends EventEmitter<Events> {
 
     private createProxyClient(streamPartId: StreamPartID): ProxyClient {
         return new ProxyClient({
-            P2PTransport: this.P2PTransport!,
-            ownPeerDescriptor: this.layer0!.getPeerDescriptor(),
+            transport: this.transport!,
+            ownPeerDescriptor: this.layer0Node!.getPeerDescriptor(),
             streamPartId,
             connectionLocker: this.connectionLocker!,
             minPropagationTargets: this.config.streamPartitionMinPropagationTargets
@@ -291,11 +292,11 @@ export class StreamrNode extends EventEmitter<Events> {
     }
 
     getPeerDescriptor(): PeerDescriptor {
-        return this.layer0!.getPeerDescriptor()
+        return this.layer0Node!.getPeerDescriptor()
     }
 
     getNodeId(): NodeID {
-        return getNodeIdFromPeerDescriptor(this.layer0!.getPeerDescriptor())
+        return getNodeIdFromPeerDescriptor(this.layer0Node!.getPeerDescriptor())
     }
 
     getNeighbors(streamPartId: StreamPartID): NodeID[] {

package/src/logic/createRandomGraphNode.ts

@@ -28,7 +28,7 @@ const createConfigWithDefaults = (config: RandomGraphNodeConfig): StrictRandomGr
     const ownNodeId = getNodeIdFromPeerDescriptor(config.ownPeerDescriptor)
     const rpcCommunicator = config.rpcCommunicator ?? new ListeningRpcCommunicator(
         formStreamPartDeliveryServiceId(config.streamPartId),
-        config.P2PTransport
+        config.transport
     )
     const numOfTargetNeighbors = config.numOfTargetNeighbors ?? 4
     const maxNumberOfContacts = config.maxNumberOfContacts ?? 20

package/src/logic/proxy/ProxyClient.ts

@@ -43,7 +43,7 @@ export const retry = async <T>(task: () => Promise<T>, description: string, abor
 }
 
 interface ProxyClientConfig {
-    P2PTransport: ITransport
+    transport: ITransport
     ownPeerDescriptor: PeerDescriptor
     streamPartId: StreamPartID
     connectionLocker: ConnectionLocker
@@ -76,7 +76,7 @@ export class ProxyClient extends EventEmitter {
     constructor(config: ProxyClientConfig) {
         super()
         this.config = config
-        this.rpcCommunicator = new ListeningRpcCommunicator(formStreamPartDeliveryServiceId(config.streamPartId), config.P2PTransport)
+        this.rpcCommunicator = new ListeningRpcCommunicator(formStreamPartDeliveryServiceId(config.streamPartId), config.transport)
         this.targetNeighbors = new NodeList(getNodeIdFromPeerDescriptor(this.config.ownPeerDescriptor), 1000)
         this.deliveryRpcLocal = new DeliveryRpcLocal({
             ownPeerDescriptor: this.config.ownPeerDescriptor,
@@ -241,7 +241,7 @@ export class ProxyClient extends EventEmitter {
     async start(): Promise<void> {
         this.registerDefaultServerMethods()
         addManagedEventListener<any, any>(
-            this.config.P2PTransport as any,
+            this.config.transport as any,
             'disconnected',
             (peerDescriptor: PeerDescriptor) => this.onNodeDisconnected(peerDescriptor),
             this.abortController.signal

package/src/proto/packages/dht/protos/DhtRpc.client.ts

@@ -2,6 +2,8 @@
 // @generated from protobuf file "packages/dht/protos/DhtRpc.proto" (package "dht", syntax proto3)
 // tslint:disable
 import { ExternalApiService } from "./DhtRpc";
+import type { ExternalStoreDataResponse } from "./DhtRpc";
+import type { ExternalStoreDataRequest } from "./DhtRpc";
 import type { FindDataResponse } from "./DhtRpc";
 import type { FindDataRequest } from "./DhtRpc";
 import { ConnectionLocker } from "./DhtRpc";
@@ -353,6 +355,10 @@ export interface IExternalApiServiceClient {
      * @generated from protobuf rpc: findData(dht.FindDataRequest) returns (dht.FindDataResponse);
      */
     findData(input: FindDataRequest, options?: RpcOptions): UnaryCall<FindDataRequest, FindDataResponse>;
+    /**
+     * @generated from protobuf rpc: externalStoreData(dht.ExternalStoreDataRequest) returns (dht.ExternalStoreDataResponse);
+     */
+    externalStoreData(input: ExternalStoreDataRequest, options?: RpcOptions): UnaryCall<ExternalStoreDataRequest, ExternalStoreDataResponse>;
 }
 /**
  * @generated from protobuf service dht.ExternalApiService
@@ -370,4 +376,11 @@ export class ExternalApiServiceClient implements IExternalApiServiceClient, Serv
         const method = this.methods[0], opt = this._transport.mergeOptions(options);
         return stackIntercept<FindDataRequest, FindDataResponse>("unary", this._transport, method, opt, input);
     }
+    /**
+     * @generated from protobuf rpc: externalStoreData(dht.ExternalStoreDataRequest) returns (dht.ExternalStoreDataResponse);
+     */
+    externalStoreData(input: ExternalStoreDataRequest, options?: RpcOptions): UnaryCall<ExternalStoreDataRequest, ExternalStoreDataResponse> {
+        const method = this.methods[1], opt = this._transport.mergeOptions(options);
+        return stackIntercept<ExternalStoreDataRequest, ExternalStoreDataResponse>("unary", this._transport, method, opt, input);
+    }
 }

package/src/proto/packages/dht/protos/DhtRpc.server.ts

@@ -1,6 +1,8 @@
 // @generated by protobuf-ts 2.9.1 with parameter server_generic,generate_dependencies,long_type_number
 // @generated from protobuf file "packages/dht/protos/DhtRpc.proto" (package "dht", syntax proto3)
 // tslint:disable
+import { ExternalStoreDataResponse } from "./DhtRpc";
+import { ExternalStoreDataRequest } from "./DhtRpc";
 import { FindDataResponse } from "./DhtRpc";
 import { FindDataRequest } from "./DhtRpc";
 import { DisconnectNoticeResponse } from "./DhtRpc";
@@ -145,4 +147,8 @@ export interface IExternalApiService<T = ServerCallContext> {
      * @generated from protobuf rpc: findData(dht.FindDataRequest) returns (dht.FindDataResponse);
      */
     findData(request: FindDataRequest, context: T): Promise<FindDataResponse>;
+    /**
+     * @generated from protobuf rpc: externalStoreData(dht.ExternalStoreDataRequest) returns (dht.ExternalStoreDataResponse);
+     */
+    externalStoreData(request: ExternalStoreDataRequest, context: T): Promise<ExternalStoreDataResponse>;
 }