@streamr/trackerless-network 0.0.1-tatum.0

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@streamr/trackerless-network",
+  "version": "0.0.1-tatum.0",
+  "description": "Minimal and extendable implementation of the Streamr Network node.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/streamr-dev/network-monorepo.git",
+    "directory": "packages/trackerless-network"
+  },
+  "main": "dist/src/exports.js",
+  "types": "dist/src/exports.d.ts",
+  "license": "STREAMR NETWORK OPEN SOURCE LICENSE",
+  "author": "Streamr Network AG <contact@streamr.network>",
+  "scripts": {
+    "build": "tsc -b tsconfig.node.json",
+    "build-browser": "webpack --mode=development --progress",
+    "generate-protoc-code": "./proto.sh",
+    "check": "tsc -p ./tsconfig.jest.json --noEmit",
+    "clean": "jest --clearCache || true; rm -rf dist *.tsbuildinfo node_modules/.cache || true",
+    "coverage": "jest --coverage",
+    "eslint": "eslint --cache --cache-location=node_modules/.cache/.eslintcache/ '*/**/*.{js,ts}'",
+    "test": "jest",
+    "test-browser": "karma start karma.config.js",
+    "test-unit": "jest test/unit",
+    "test-integration": "jest test/integration",
+    "test-e2e": "jest test/end-to-end",
+    "network": "ts-node bin/network",
+    "run-joining-benchmark": "node --max-old-space-size=24288 dist/test/benchmark/first-message.js"
+  },
+  "dependencies": {
+    "@protobuf-ts/runtime": "^2.8.2",
+    "@protobuf-ts/runtime-rpc": "^2.8.2",
+    "@streamr/dht": "0.0.1-tatum.0",
+    "@streamr/proto-rpc": "0.0.1-tatum.0",
+    "@streamr/protocol": "0.0.1-tatum.0",
+    "@streamr/test-utils": "0.0.1-tatum.0",
+    "@streamr/utils": "0.0.1-tatum.0",
+    "eventemitter3": "^5.0.0",
+    "lodash": "^4.17.21",
+    "yallist": "^4.0.0"
+  },
+  "devDependencies": {
+    "@streamr/browser-test-runner": "^0.0.1",
+    "@types/lodash": "^4.14.197",
+    "@types/yallist": "^4.0.1",
+    "commander": "^11.0.0",
+    "expect": "^29.6.2",
+    "express": "^4.17.1",
+    "ts-essentials": "^9.3.0",
+    "ts-loader": "^9.4.4",
+    "ts-node": "^10.9.1"
+  }
+}

package/proto.sh

@@ -0,0 +1,2 @@
+mkdir -p ./src/proto
+npx protoc --ts_out ./src/proto --ts_opt server_generic,generate_dependencies,long_type_number --proto_path ../.. packages/trackerless-network/protos/NetworkRpc.proto --experimental_allow_proto3_optional

package/protos/NetworkRpc.proto

@@ -0,0 +1,161 @@
+syntax = "proto3";
+
+option optimize_for = CODE_SIZE;
+
+import "google/protobuf/empty.proto";
+import "google/protobuf/any.proto";
+import "packages/dht/protos/DhtRpc.proto";
+
+service NetworkRpc {
+  rpc sendData (StreamMessage) returns (google.protobuf.Empty);
+  rpc leaveStreamNotice (LeaveStreamNotice) returns (google.protobuf.Empty);
+}
+
+service ProxyConnectionRpc {
+  rpc requestConnection (ProxyConnectionRequest) returns (ProxyConnectionResponse);
+}
+
+service HandshakeRpc {
+  rpc handshake (StreamHandshakeRequest) returns (StreamHandshakeResponse);
+  rpc interleaveNotice (InterleaveNotice) returns (google.protobuf.Empty);
+}
+
+service NeighborUpdateRpc {
+  rpc neighborUpdate (NeighborUpdate) returns (NeighborUpdate);
+}
+
+service TemporaryConnectionRpc {
+  rpc openConnection (TemporaryConnectionRequest) returns (TemporaryConnectionResponse);
+}
+
+message MessageRef {
+  int64 timestamp = 1;
+  int32 sequenceNumber = 2;
+  string messageChainId = 3;
+  string streamId = 4;
+  int32 streamPartition = 5;
+  string publisherId = 6;
+}
+
+message ContentMessage {
+  string body = 1;
+}
+
+enum StreamMessageType {
+  MESSAGE = 0;
+  GROUP_KEY_REQUEST = 1;
+  GROUP_KEY_RESPONSE = 2;
+}
+
+enum EncryptionType {
+  NONE = 0;
+  RSA = 1;
+  AES = 2;
+}
+
+message EncryptedGroupKey {
+  string groupKeyId = 1;
+  string encryptedGroupKeyHex = 2;
+  optional string serialized = 3;
+}
+
+message StreamMessage {
+  StreamMessageType messageType = 1;
+  optional EncryptionType encryptionType = 2;
+  bytes content = 3;
+  string signature = 4;
+  MessageRef messageRef = 5;
+  optional MessageRef previousMessageRef = 6;
+  optional string groupKeyId = 7;
+  optional EncryptedGroupKey newGroupKey = 8;
+}
+
+message GroupKeyRequest {
+  string requestId = 1;
+  string recipient = 2;
+  string rsaPublicKey = 3;
+  repeated string groupKeyIds = 4;
+}
+
+message GroupKeyResponse {
+  string requestId = 1;
+  string recipient = 2;
+  repeated EncryptedGroupKey encryptedGroupKeys = 3;
+}
+
+message Layer2Message {
+  Layer2Type type = 1;
+}
+
+enum Layer2Type {
+  Data = 0;
+}
+
+message StreamHandshakeRequest {
+  string randomGraphId = 1;
+  string senderId = 2;
+  string requestId = 3;
+  optional string concurrentHandshakeTargetId = 4;
+  repeated string neighbors = 5;
+  dht.PeerDescriptor senderDescriptor = 6;
+  optional string interleavingFrom = 7;
+}
+
+message StreamHandshakeResponse {
+  bool accepted = 1;
+  string requestId = 2;
+  optional dht.PeerDescriptor interleaveTarget = 3;
+}
+
+message InterleaveNotice {
+  string senderId = 1;
+  string randomGraphId = 2;
+  dht.PeerDescriptor interleaveTarget = 3;
+}
+
+message LeaveStreamNotice {
+  string randomGraphId = 1;
+  string senderId = 2;
+}
+
+message NeighborUpdate {
+  string senderId = 1;
+  string randomGraphId = 2;
+  bool removeMe = 3;
+  repeated dht.PeerDescriptor neighborDescriptors = 4;
+}
+
+message StreamEntryPoint {
+  dht.PeerDescriptor peerDescriptor = 1;
+  bool firstToJoin = 2;
+}
+
+message ProxyConnectionRequest {
+  string senderId = 1;
+  string streamId = 2;
+  uint32 streamPartition = 3; 
+  ProxyDirection direction = 4;
+  string userId = 5;
+  dht.PeerDescriptor senderDescriptor = 6;
+}
+
+message ProxyConnectionResponse {
+  string senderId = 1;
+  string streamId = 2;
+  uint32 streamPartition = 3;
+  ProxyDirection direction = 4;  
+  bool accepted = 5;
+}
+
+message TemporaryConnectionRequest {
+  string senderId = 1;
+}
+
+message TemporaryConnectionResponse {
+  bool accepted = 1;
+}
+
+enum ProxyDirection {
+  PUBLISH = 0;
+  SUBSCRIBE = 1;
+}

package/src/NameDirectory.ts

@@ -0,0 +1,44 @@
+/*
+ * Get human-readable names for some Ethereum addresses in Streamr test environment
+ * The list doesn't contain all known contract addresses or nodes, but we can add more
+ * addresses if needed.
+ *
+ * Currently contains hardcoded names for all streamr-docker-dev Broker/Tracker entities
+ * -> in the future each node receives the peer names from Tracker
+ *    and we can remove the hardcoded values
+ */
+
+import { EthereumAddress } from '@streamr/utils'
+
+const NAMES: Record<string, string> = {
+    '0xb9e7cebf7b03ae26458e32a059488386b05798e8': 'T1',
+    '0x0540a3e144cdd81f402e7772c76a5808b71d2d30': 'T2',
+    '0xf2c195be194a2c91e93eacb1d6d55a00552a85e2': 'T3',
+    '0xde1112f631486cfc759a50196853011528bc5fa0': 'S1',
+    '0xde222e8603fcf641f928e5f66a0cbf4de70d5352': 'B1',
+    '0xde3331ca6b8b636e0b82bf08e941f727b8927442': 'B2',
+    '0x6ccdd5d866ea766f6df5965aa98deccd629ff222': 'DOCKER_DEV_STREAM_REGISTRY',
+    '0xd04af489677001444280366dd0885b03daade71d': 'DOCKER_DEV_STREAM_STORAGE_REGISTRY',
+    '0x231b810d98702782963472e1d60a25496999e75d': 'DOCKER_DEV_STORAGE_NODE_REGISTRY'
+}
+
+// eslint-disable-next-line @typescript-eslint/no-extraneous-class
+export class NameDirectory {
+
+    static MAX_FALLBACK_NAME_LENGTH = 8
+
+    // if name is not known, creates a short name from the address
+    static getName(address: EthereumAddress | string | undefined): string | undefined {
+        if (address === undefined) {
+            return undefined
+        }
+        const name = NAMES[address.toLowerCase()]
+        if (name !== undefined) {
+            return name
+        } else {
+            return (address.length > NameDirectory.MAX_FALLBACK_NAME_LENGTH)
+                ? address.substring(0, this.MAX_FALLBACK_NAME_LENGTH)
+                : address
+        }
+    }
+}

package/src/NetworkNode.ts

@@ -0,0 +1,169 @@
+import { StreamMessage, StreamPartID, StreamMessageType } from '@streamr/protocol'
+import { PeerDescriptor, PeerIDKey } from '@streamr/dht'
+import { StreamMessageTranslator } from './logic/protocol-integration/stream-message/StreamMessageTranslator'
+import { NetworkOptions, NetworkStack } from './NetworkStack'
+import { MetricsContext } from '@streamr/utils'
+import { ProxyDirection } from './proto/packages/trackerless-network/protos/NetworkRpc'
+
+/*
+Convenience wrapper for building client-facing functionality. Used by client.
+*/
+
+export class NetworkNode {
+
+    readonly stack: NetworkStack
+    private readonly options: NetworkOptions
+    private stopped = false
+    constructor(opts: NetworkOptions) {
+        this.options = opts
+        this.stack = new NetworkStack(opts)
+    }
+
+    async start(doJoin?: boolean): Promise<void> {
+        await this.stack.start(doJoin)
+    }
+
+    setExtraMetadata(metadata: Record<string, unknown>): void {
+        this.stack.getStreamrNode().setExtraMetadata(metadata)
+    }
+
+    async inspect(node: PeerDescriptor, streamPartId: StreamPartID): Promise<boolean> {
+        return this.stack.getStreamrNode().inspect(node, streamPartId)
+    }
+
+    async publish(streamMessage: StreamMessage): Promise<void> {
+        const streamPartId = streamMessage.getStreamPartID()
+        if (this.stack.getStreamrNode().isProxiedStreamPart(streamPartId, ProxyDirection.SUBSCRIBE) 
+            && streamMessage.messageType === StreamMessageType.MESSAGE) {
+            throw new Error(`Cannot publish content data to ${streamPartId} as proxy subscribe connections have been set`)
+        }
+
+        await this.stack.joinLayer0IfRequired(streamPartId)
+        const msg = StreamMessageTranslator.toProtobuf(streamMessage)
+        this.stack.getStreamrNode().publishToStream(streamPartId, msg)
+    }
+
+    async subscribe(streamPartId: StreamPartID): Promise<void> {
+        if (this.stack.getStreamrNode().isProxiedStreamPart(streamPartId, ProxyDirection.PUBLISH)) {
+            throw new Error(`Cannot subscribe to ${streamPartId} as proxy publish connections have been set`)
+        }
+        await this.stack.joinLayer0IfRequired(streamPartId)
+        this.stack.getStreamrNode().subscribeToStream(streamPartId)
+    }
+
+    async setProxies(
+        streamPartId: StreamPartID,
+        contactPeerDescriptors: PeerDescriptor[],
+        direction: ProxyDirection,
+        getUserId: () => Promise<string>,
+        connectionCount?: number
+    ): Promise<void> {
+        if (this.options.networkNode.acceptProxyConnections) {
+            throw new Error('cannot set proxies when acceptProxyConnections=true')
+        }
+        await this.stack.getStreamrNode().setProxies(streamPartId, contactPeerDescriptors, direction, getUserId, connectionCount)
+    }
+
+    addMessageListener<T>(cb: (msg: StreamMessage<T>) => void): void {
+        this.stack.getStreamrNode().on('newMessage', (msg) => {
+            const translated = StreamMessageTranslator.toClientProtocol<T>(msg)
+            return cb(translated)
+        })
+    }
+
+    setStreamPartEntryPoints(streamPartId: StreamPartID, contactPeerDescriptors: PeerDescriptor[]): void {
+        this.stack.getStreamrNode()!.setStreamPartEntryPoints(streamPartId, contactPeerDescriptors)
+    }
+
+    removeMessageListener<T>(cb: (msg: StreamMessage<T>) => void): void {
+        if (this.stopped) {
+            return
+        }
+        this.stack.getStreamrNode().off('newMessage', (msg) => {
+            const translated = StreamMessageTranslator.toClientProtocol<T>(msg)
+            return cb(translated)
+        })
+    }
+
+    async subscribeAndWaitForJoin(
+        streamPartId: StreamPartID,
+        timeout?: number,
+        expectedNeighbors?: number
+    ): Promise<number> {
+        if (this.stack.getStreamrNode()!.isProxiedStreamPart(streamPartId, ProxyDirection.PUBLISH)) {
+            throw new Error(`Cannot subscribe to ${streamPartId} as proxy publish connections have been set`)
+        }
+        return this.stack.getStreamrNode().waitForJoinAndSubscribe(streamPartId, timeout, expectedNeighbors)
+    }
+
+    async waitForJoinAndPublish(streamMessage: StreamMessage, timeout?: number): Promise<number> {
+        const streamPartId = streamMessage.getStreamPartID()
+        const msg = StreamMessageTranslator.toProtobuf(streamMessage)
+
+        if (this.stack.getStreamrNode()!.isProxiedStreamPart(streamPartId, ProxyDirection.SUBSCRIBE)) {
+            throw new Error(`Cannot publish to ${streamPartId} as proxy subscribe connections have been set`)
+        }
+
+        return this.stack.getStreamrNode().waitForJoinAndPublish(streamPartId, msg, timeout)
+    }
+
+    unsubscribe(streamPartId: StreamPartID): void {
+        if (this.stopped) {
+            return
+        }
+        this.stack.getStreamrNode().unsubscribeFromStream(streamPartId)
+    }
+
+    getNeighborsForStreamPart(streamPartId: StreamPartID): ReadonlyArray<string> {
+        return this.hasStreamPart(streamPartId)
+            ? this.stack.getStreamrNode().getStream(streamPartId)!.layer2.getTargetNeighborStringIds()
+            : []
+    }
+
+    hasStreamPart(streamPartId: StreamPartID): boolean {
+        return this.stack.getStreamrNode().hasStream(streamPartId)
+    }
+
+    hasProxyConnection(streamPartId: StreamPartID, contactNodeId: string, direction: ProxyDirection): boolean {
+        return this.stack.getStreamrNode()!.hasProxyConnection(streamPartId, contactNodeId as PeerIDKey, direction)
+    }
+
+    // eslint-disable-next-line class-methods-use-this
+    getRtt(_nodeId: string): number | undefined {
+        throw new Error('Not implemented')
+    }
+
+    async stop(): Promise<void> {
+        this.stopped = true
+        await this.stack.stop()
+    }
+
+    getPeerDescriptor(): PeerDescriptor {
+        return this.stack.getLayer0DhtNode().getPeerDescriptor()
+    }
+
+    getMetricsContext(): MetricsContext {
+        return this.stack.getMetricsContext()
+    }
+
+    getNodeId(): string {
+        return this.stack.getStreamrNode().getNodeId()
+    }
+
+    getNodeStringId(): string {
+        return this.stack.getStreamrNode().getNodeStringId()
+    }
+
+    getStreamParts(): StreamPartID[] {
+        return this.stack.getStreamrNode().getStreamParts()
+    }
+
+    getNeighbors(): string[] {
+        return this.stack.getStreamrNode().getNeighbors()
+    }
+
+    // eslint-disable-next-line class-methods-use-this
+    getDiagnosticInfo(): Record<string, unknown> {
+        return {}
+    }
+}

package/src/NetworkStack.ts

@@ -0,0 +1,144 @@
+import { ConnectionManager, DhtNode, DhtNodeOptions, isSamePeerDescriptor } from '@streamr/dht'
+import { StreamrNode, StreamrNodeConfig } from './logic/StreamrNode'
+import { MetricsContext, waitForEvent3 } from '@streamr/utils'
+import { EventEmitter } from 'eventemitter3'
+import { StreamPartID } from '@streamr/protocol'
+
+interface ReadynessEvents {
+    done: () => void
+}
+
+class ReadynessListener {
+
+    private readonly emitter = new EventEmitter<ReadynessEvents>()
+    private readonly networkStack: NetworkStack
+    private readonly dhtNode: DhtNode
+
+    constructor(networkStack: NetworkStack, dhtNode: DhtNode) {
+        this.networkStack = networkStack
+        this.dhtNode = dhtNode
+        this.networkStack.on('stopped', this.onStopped)
+        this.dhtNode.on('connected', this.onConnected)
+    }
+
+    private onConnected = () => {
+        this.networkStack.off('stopped', this.onStopped)
+        this.dhtNode.off('connected', this.onConnected)
+        this.emitter.emit('done')
+    }
+
+    private onStopped = () => {
+        this.networkStack.off('stopped', this.onStopped)
+        this.dhtNode.off('connected', this.onConnected)
+        this.emitter.emit('done')
+    }
+
+    public async waitUntilReady(timeout: number): Promise<void> {
+        if (this.dhtNode.getNumberOfConnections() > 0) {
+            return
+        } else {
+            await waitForEvent3<ReadynessEvents>(this.emitter, 'done', timeout)
+        }
+    }
+}
+
+export interface NetworkOptions {
+    layer0: DhtNodeOptions
+    networkNode: StreamrNodeConfig
+    metricsContext?: MetricsContext
+}
+
+export interface NetworkStackEvents {
+    stopped: () => void
+}
+
+export class NetworkStack extends EventEmitter<NetworkStackEvents> {
+
+    private connectionManager?: ConnectionManager
+    private layer0DhtNode?: DhtNode
+    private streamrNode?: StreamrNode
+    private readonly metricsContext: MetricsContext
+    private readonly options: NetworkOptions
+    private readonly firstConnectionTimeout: number
+    private dhtJoinRequired = true
+
+    constructor(options: NetworkOptions) {
+        super()
+        this.options = options
+        this.metricsContext = options.metricsContext ?? new MetricsContext()
+        this.layer0DhtNode = new DhtNode({
+            ...options.layer0,
+            metricsContext: this.metricsContext
+        })
+        this.streamrNode = new StreamrNode({
+            ...options.networkNode,
+            nodeName: options.networkNode.nodeName ?? options.layer0.nodeName,
+            metricsContext: this.metricsContext
+        })
+        this.firstConnectionTimeout = options.networkNode.firstConnectionTimeout ?? 5000
+    }
+
+    async start(doJoin = true): Promise<void> {
+        await this.layer0DhtNode!.start()
+        this.connectionManager = this.layer0DhtNode!.getTransport() as ConnectionManager
+        if (this.options.layer0.entryPoints!.some((entryPoint) => 
+            isSamePeerDescriptor(entryPoint, this.layer0DhtNode!.getPeerDescriptor())
+        )) {
+            this.dhtJoinRequired = false
+            await this.layer0DhtNode?.joinDht(this.options.layer0.entryPoints!)
+            await this.streamrNode?.start(this.layer0DhtNode!, this.connectionManager!, this.connectionManager!)
+        } else {
+            if (doJoin) {
+                this.dhtJoinRequired = false
+                await this.joinDht()
+            }
+            await this.streamrNode?.start(this.layer0DhtNode!, this.connectionManager!, this.connectionManager!)
+        }
+    }
+
+    private async joinDht(): Promise<void> {
+        setImmediate(() => {
+            this.layer0DhtNode?.joinDht(this.options.layer0.entryPoints!)
+        })
+        await this.waitForFirstConnection()
+    }
+
+    private async waitForFirstConnection(): Promise<void> {
+        const readynessListener = new ReadynessListener(this, this.layer0DhtNode!)
+        await readynessListener.waitUntilReady(this.firstConnectionTimeout)
+    }
+
+    async joinLayer0IfRequired(streamPartId: StreamPartID): Promise<void> {
+        if (this.isJoinRequired(streamPartId)) {
+            this.dhtJoinRequired = false
+            await this.joinDht()
+        } else if (this.layer0DhtNode!.getNumberOfConnections() < 1) {
+            await this.waitForFirstConnection()
+        }
+    }
+
+    private isJoinRequired(streamPartId: StreamPartID): boolean {
+        return this.dhtJoinRequired && !this.layer0DhtNode!.hasJoined() && this.streamrNode!.isJoinRequired(streamPartId)
+    }
+
+    getStreamrNode(): StreamrNode {
+        return this.streamrNode!
+    }
+
+    getLayer0DhtNode(): DhtNode {
+        return this.layer0DhtNode!
+    }
+
+    getMetricsContext(): MetricsContext {
+        return this.metricsContext
+    }
+
+    async stop(): Promise<void> {
+        await this.streamrNode!.destroy()
+        this.streamrNode = undefined
+        this.layer0DhtNode = undefined
+        this.connectionManager = undefined
+        this.emit('stopped')
+    }
+
+}

package/src/exports.ts

@@ -0,0 +1,6 @@
+export { NetworkStack, NetworkOptions } from './NetworkStack'
+export { NetworkNode } from './NetworkNode'
+export { StreamrNodeConfig } from './logic/StreamrNode'
+export { NameDirectory } from './NameDirectory'
+export { NodeId } from './identifiers'
+export { ProxyDirection } from './proto/packages/trackerless-network/protos/NetworkRpc'

package/src/identifiers.ts

@@ -0,0 +1 @@
+export type NodeId = string