@streamr/dht 0.0.1-tatum.0

package/karma.config.js

@@ -0,0 +1,20 @@
+/* eslint-disable @typescript-eslint/no-require-imports */
+const path = require('path')
+const { createKarmaConfig, createWebpackConfig } = require('@streamr/browser-test-runner')
+
+const TEST_PATHS = [
+    'test/unit/**/*.ts',
+    './test/integration/**/!(DhtWith*|MigrateData*).ts/',
+    'test/end-to-end/**/*.ts'
+]
+
+const NodeWebRtcConnection = path.resolve(__dirname, 'src/connection/WebRTC/NodeWebRtcConnection.ts')
+const BrowserWebRtcConnection = path.resolve(__dirname, 'src/connection/WebRTC/BrowserWebRtcConnection.ts')
+
+module.exports = createKarmaConfig(TEST_PATHS, createWebpackConfig({
+    entry: './src/exports.ts',
+    libraryName: 'dht',
+    alias: {
+        [NodeWebRtcConnection]: BrowserWebRtcConnection
+    }
+}))

package/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "@streamr/dht",
+  "version": "0.0.1-tatum.0",
+  "description": "Streamr Network DHT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/streamr-dev/network.git",
+    "directory": "packages/dht"
+  },
+  "main": "dist/src/exports.js",
+  "types": "dist/src/exports.d.ts",
+  "license": "STREAMR NETWORK OPEN SOURCE LICENSE",
+  "author": "Streamr Network AG <contact@streamr.network>",
+  "scripts": {
+    "generate-protoc-code": "./proto.sh",
+    "build": "tsc -b tsconfig.node.json",
+    "build-browser": "webpack --mode=development --progress",
+    "check": "tsc -p ./tsconfig.jest.json --noEmit",
+    "clean": "jest --clearCache || true; rm -rf dist *.tsbuildinfo node_modules/.cache || true",
+    "eslint": "eslint --cache --cache-location=node_modules/.cache/.eslintcache/ '*/**/*.{js,ts}'",
+    "prepublishOnly": "npm run clean && NODE_ENV=production tsc -b tsconfig.node.json",
+    "test": "jest test/unit test/integration test/end-to-end",
+    "test-browser": "karma start karma.config.js",
+    "test-unit": "jest test/unit",
+    "test-integration": "jest test/integration",
+    "test-end-to-end": "jest test/end-to-end",
+    "benchmark": "jest test/benchmark",
+    "prepare-kademlia-simulation": "cd test/data && node --max-old-space-size=8096 -- ../../../../node_modules/.bin/ts-node -P ../../tsconfig.node.json generateGroundTruthData.ts",
+    "run-kademlia-simulation": "ts-node --project tsconfig.node.json $NODE_DEBUG_OPTION --files test/benchmark/kademlia-simulation/KademliaSimulation.ts"
+  },
+  "dependencies": {
+    "@protobuf-ts/runtime": "^2.8.2",
+    "@protobuf-ts/runtime-rpc": "^2.8.2",
+    "@streamr/proto-rpc": "0.0.1-tatum.0",
+    "@streamr/utils": "0.0.1-tatum.0",
+    "eventemitter3": "^5.0.0",
+    "heap": "^0.2.6",
+    "ipaddr.js": "^2.0.1",
+    "k-bucket": "^5.1.0",
+    "lodash": "^4.17.21",
+    "node-datachannel": "^0.4.3",
+    "sinon": "15.1.2",
+    "uuid": "^9.0.0",
+    "websocket": "^1.0.34"
+  },
+  "devDependencies": {
+    "@streamr/browser-test-runner": "^0.0.1",
+    "@streamr/test-utils": "0.0.1-tatum.0",
+    "@types/express": "^4.17.13",
+    "@types/heap": "^0.2.28",
+    "@types/k-bucket": "^5.0.1",
+    "@types/lodash": "^4.14.197",
+    "@types/sinon": "^10.0.15",
+    "@types/uuid": "^9.0.2",
+    "@types/websocket": "^1.0.3",
+    "express": "^4.17.1",
+    "patch-package": "^8.0.0",
+    "ts-node": "^10.9.1"
+  },
+  "optionalDependencies": {
+    "bufferutil": "^4.0.5",
+    "utf-8-validate": "^6.0.3"
+  }
+}

package/proto.sh

@@ -0,0 +1,3 @@
+mkdir -p ./src/proto
+# npx protoc --ts_out ./src/proto --ts_opt server_generic,generate_dependencies --proto_path ../.. packages/trackerless-network/protos/NetworkRpc.proto
+npx protoc --ts_out ./src/proto --ts_opt server_generic,generate_dependencies,long_type_number --proto_path ../.. packages/dht/protos/DhtRpc.proto

package/protos/DhtRpc.proto

@@ -0,0 +1,330 @@
+syntax = "proto3";
+option optimize_for = CODE_SIZE;
+
+package dht;
+
+import "google/protobuf/empty.proto";
+import "google/protobuf/any.proto";
+import "google/protobuf/timestamp.proto";
+import "packages/proto-rpc/protos/ProtoRpc.proto";
+
+service DhtRpcService {
+  rpc getClosestPeers (ClosestPeersRequest) returns (ClosestPeersResponse);
+  rpc ping (PingRequest) returns (PingResponse);
+  rpc leaveNotice (LeaveNotice) returns (google.protobuf.Empty);
+}
+
+service RoutingService {
+  rpc routeMessage (RouteMessageWrapper) returns (RouteMessageAck);
+  rpc forwardMessage (RouteMessageWrapper) returns (RouteMessageAck);
+  rpc findRecursively (RouteMessageWrapper) returns (RouteMessageAck);
+}
+
+service StoreService {
+  rpc storeData (StoreDataRequest) returns (StoreDataResponse);
+  rpc migrateData (MigrateDataRequest) returns (MigrateDataResponse);
+  rpc deleteData (DeleteDataRequest) returns (DeleteDataResponse);
+}
+
+service RecursiveFindSessionService {
+  rpc reportRecursiveFindResult(RecursiveFindReport) returns (google.protobuf.Empty);
+}
+
+service WebSocketConnectorService {
+  rpc requestConnection (WebSocketConnectionRequest) returns (WebSocketConnectionResponse);
+}
+
+service WebRtcConnectorService {
+  rpc requestConnection (WebRtcConnectionRequest) returns (google.protobuf.Empty);
+  rpc rtcOffer (RtcOffer) returns (google.protobuf.Empty);
+  rpc rtcAnswer (RtcAnswer) returns (google.protobuf.Empty);
+  rpc iceCandidate (IceCandidate) returns (google.protobuf.Empty);
+}
+
+service ConnectionLocker {
+  rpc lockRequest (LockRequest) returns (LockResponse);
+  rpc unlockRequest (UnlockRequest) returns (google.protobuf.Empty);
+  rpc gracefulDisconnect (DisconnectNotice) returns (DisconnectNoticeResponse);
+}
+
+service ExternalApiService {
+  rpc findData (FindDataRequest) returns (FindDataResponse);
+}
+
+// Used inside RpcMessage
+
+message StoreDataRequest {
+  bytes kademliaId = 1;
+  google.protobuf.Any data = 2;
+  uint32 ttl = 3;
+  google.protobuf.Timestamp storerTime = 4;
+}
+
+message StoreDataResponse {
+  string error = 1;
+}
+
+message MigrateDataRequest {
+  DataEntry dataEntry = 1;
+}
+  
+message MigrateDataResponse {
+  string error = 1;
+}
+
+message DeleteDataRequest {
+  bytes kademliaId = 1;
+}
+
+message DeleteDataResponse {
+  bool deleted = 1;
+}
+
+message DataEntry {
+  PeerDescriptor storer = 1;
+  bytes kademliaId = 2;
+  google.protobuf.Any data = 3;
+  google.protobuf.Timestamp storedAt = 4; 
+  uint32 ttl = 5;   // milliseconds
+  bool stale = 6;
+  bool deleted = 7;
+  google.protobuf.Timestamp storerTime = 8;
+}
+
+message ClosestPeersRequest {
+  bytes kademliaId = 1;
+  string requestId = 2;
+}
+
+message ClosestPeersResponse {
+  repeated PeerDescriptor peers = 1;
+  string requestId = 2;
+}
+
+enum FindMode {
+  NODE = 0;
+  DATA = 1;
+}
+
+message RecursiveFindRequest {
+  string recursiveFindSessionId = 1;
+  FindMode findMode = 2;
+}
+
+message RecursiveFindReport {
+  repeated PeerDescriptor nodes = 1;
+  repeated DataEntry dataEntries = 2;
+  optional bool noCloserNodesFound = 3;
+  repeated PeerDescriptor routingPath = 4;
+}
+
+message PingRequest {
+  string requestId = 1;
+}
+
+message PingResponse {
+  string requestId = 1;
+}
+
+message LeaveNotice {
+  string serviceId = 1;
+}
+
+message PeerDescriptor {
+  bytes kademliaId = 1;
+  NodeType type = 2;
+  ConnectivityMethod udp = 3;
+  ConnectivityMethod tcp = 4;
+  ConnectivityMethod websocket = 5;
+  optional bool openInternet = 6;
+  optional uint32 region = 7;
+  optional string nodeName = 8;
+}
+
+message ConnectivityMethod {
+  uint32 port = 2;
+  string ip = 3;
+}
+
+enum NodeType {
+  NODEJS = 0;
+  BROWSER = 1;
+  VIRTUAL = 3;
+}
+
+message ConnectivityReportRequest {
+  uint32 port = 1;
+  string requestId = 2;
+}
+
+message ConnectivityReportResponse {
+  string open_internet = 1;
+  string ip = 2;
+  string natType = 3;
+  string requestId = 4;
+}
+
+enum RpcResponseError {
+  SERVER_TIMOUT = 0;
+  CLIENT_TIMEOUT = 1;
+  SERVER_ERROR = 2;
+  UNKNOWN_RPC_METHOD = 3;
+}
+
+message RouteMessageWrapper {
+  PeerDescriptor sourcePeer = 1;
+  string requestId = 2;
+  PeerDescriptor destinationPeer = 3;
+  PeerDescriptor previousPeer = 4;
+  Message message = 5;
+  repeated PeerDescriptor reachableThrough = 6;
+  repeated PeerDescriptor routingPath = 7;
+}
+
+message RouteMessageAck {
+  PeerDescriptor sourcePeer = 1;
+  string requestId = 2;
+  PeerDescriptor destinationPeer = 3;
+  string error = 4;
+}
+
+// Correspond to the MessageType Enum
+
+message ConnectivityRequest {
+  uint32 port = 1;
+}
+
+message ConnectivityResponse {
+  bool open_internet = 1;
+  string ip = 2;
+  string natType = 3;
+  ConnectivityMethod websocket = 4;
+}
+
+message HandshakeRequest {
+  bytes sourceId = 1;
+  string protocolVersion = 2;
+  PeerDescriptor peerDescriptor = 3;
+}
+
+message HandshakeResponse {
+  bytes sourceId = 1;
+  string protocolVersion = 2;
+  PeerDescriptor peerDescriptor = 3;
+  optional string responseError = 4;
+}
+
+// Wraps all messages
+
+enum MessageType {
+  CONNECTIVITY_REQUEST = 0;
+  CONNECTIVITY_RESPONSE = 1;
+  HANDSHAKE_REQUEST = 2;
+  HANDSHAKE_RESPONSE = 3;
+  RPC = 4;
+  RECURSIVE_FIND_REQUEST = 5;
+}
+
+message Message {
+  string messageId = 1;
+  MessageType messageType = 2;
+  PeerDescriptor sourceDescriptor = 3;
+  PeerDescriptor targetDescriptor = 4;
+  string serviceId = 5; // id of the RPC service
+  oneof body {
+    ConnectivityRequest connectivityRequest = 6;
+    ConnectivityResponse connectivityResponse = 7;
+    HandshakeRequest handshakeRequest = 8;
+    HandshakeResponse handshakeResponse = 9;
+    protorpc.RpcMessage rpcMessage = 10;
+    RecursiveFindRequest RecursiveFindRequest = 11;
+  };
+}
+
+// Connector Messages
+
+// WebSocket
+message WebSocketConnectionRequest {
+  PeerDescriptor requester = 1;
+  PeerDescriptor target = 2;
+  string ip = 3;
+  uint32 port = 4;
+}
+
+message WebSocketConnectionResponse {
+  PeerDescriptor requester = 1;
+  PeerDescriptor target = 2;
+  bool accepted = 3;
+  optional string reason = 4;
+}
+
+// WebRTC
+message WebRtcConnectionRequest {
+  PeerDescriptor requester = 1;
+  PeerDescriptor target = 2;
+  string connectionId = 3;
+}
+
+message RtcOffer {
+  PeerDescriptor requester = 1;
+  PeerDescriptor target = 2;
+  string description = 3;
+  string connectionId = 4;
+}
+
+message RtcAnswer {
+  PeerDescriptor requester = 1;
+  PeerDescriptor target = 2;
+  string description = 3;
+  string connectionId = 4;
+}
+
+message IceCandidate {
+  string candidate = 1;
+  string mid = 2;
+  string connectionId = 3;
+  PeerDescriptor requester = 4;
+  PeerDescriptor target = 5;
+}
+
+message LockRequest {
+  string protocolVersion = 1;
+  PeerDescriptor peerDescriptor = 2;
+  string serviceId = 3;
+}
+
+message UnlockRequest {
+  string protocolVersion = 1;
+  PeerDescriptor peerDescriptor = 2;
+  string serviceId = 3;
+}
+
+message LockResponse {
+  bool accepted = 1;
+  optional bool reason = 2;
+}
+
+enum DisconnectMode {
+  NORMAL = 0;
+  LEAVING = 1;
+}
+
+message DisconnectNotice {
+  string protocolVersion = 1;
+  PeerDescriptor peerDescriptor = 2;
+  DisconnectMode disconnecMode = 3;
+}
+
+message DisconnectNoticeResponse {
+  string protocolVersion = 1;
+}
+
+message FindDataRequest {
+  bytes kademliaId = 1;
+  PeerDescriptor requestor = 2;
+}
+
+message FindDataResponse {
+  repeated DataEntry dataEntries = 1;
+  optional string error = 2;
+}

package/protos/tests.proto

@@ -0,0 +1,16 @@
+
+syntax = "proto3";
+option optimize_for = CODE_SIZE;
+
+package tests;
+
+import "google/protobuf/any.proto";
+
+message TestMessage {
+    string messageId = 1;
+    google.protobuf.Any body = 2;
+  }
+
+message SomeMessage {
+    string juttu = 1;
+}

package/src/connection/Connection.ts

@@ -0,0 +1,23 @@
+import EventEmitter from 'eventemitter3'
+import { PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
+import { ConnectionID, ConnectionType, ConnectionEvents } from './IConnection'
+
+export class Connection extends EventEmitter<ConnectionEvents> {
+    public connectionId: ConnectionID
+    public connectionType: ConnectionType
+    private peerDescriptor?: PeerDescriptor
+    
+    constructor(connectionType: ConnectionType) {
+        super()
+        this.connectionId = new ConnectionID()
+        this.connectionType = connectionType
+    }
+
+    setPeerDescriptor(peerDescriptor: PeerDescriptor): void {
+        this.peerDescriptor = peerDescriptor
+    }
+    
+    getPeerDescriptor(): PeerDescriptor | undefined {
+        return this.peerDescriptor
+    }
+}

package/src/connection/ConnectionLockHandler.ts

@@ -0,0 +1,105 @@
+// Connection locks are independent of the existence of connections
+// that is why this class is needed
+
+import { PeerIDKey } from '../helpers/PeerID'
+
+type ServiceName = string
+
+export class ConnectionLockHandler {
+
+    private localLocks: Map<PeerIDKey, Set<ServiceName>> = new Map()
+    private remoteLocks: Map<PeerIDKey, Set<ServiceName>> = new Map()
+    private weakLocks: Set<PeerIDKey> = new Set()
+
+    public getNumberOfLocalLockedConnections(): number {
+        return this.localLocks.size
+    }
+
+    public getNumberOfRemoteLockedConnections(): number {
+        return this.remoteLocks.size
+    }
+
+    public getNumberOfWeakLockedConnections(): number {
+        return this.weakLocks.size
+    }
+
+    public isLocalLocked(id: PeerIDKey, serviceName?: ServiceName): boolean {
+        if (!serviceName) {
+            return this.localLocks.has(id)
+        } else {
+            return this.localLocks.has(id) && this.localLocks.get(id)!.has(serviceName)
+        }
+    }
+
+    public isRemoteLocked(id: PeerIDKey, serviceName?: ServiceName): boolean {
+        if (!serviceName) {
+            return this.remoteLocks.has(id)
+        } else {
+            if (this.remoteLocks.has(id) && this.remoteLocks.get(id)!.has(serviceName)) {
+                return true
+            } else {
+                return false
+            }
+        }
+    }
+
+    public isWeakLocked(id: PeerIDKey): boolean {
+        return this.weakLocks.has(id)
+    }
+
+    public isLocked(id: PeerIDKey): boolean {
+        return (this.isLocalLocked(id) || this.isRemoteLocked(id) || this.isWeakLocked(id))
+    }
+
+    public addLocalLocked(id: PeerIDKey, serviceName: ServiceName): void {
+        if (!this.localLocks.has(id)) {
+            this.localLocks.set(id, new Set())
+        }
+        this.localLocks.get(id)!.add(serviceName)
+    }
+
+    public addRemoteLocked(id: PeerIDKey, serviceName: ServiceName): void {
+        if (!this.remoteLocks.has(id)) {
+            this.remoteLocks.set(id, new Set())
+        }
+        this.remoteLocks.get(id)!.add(serviceName)
+    }
+
+    public addWeakLocked(id: PeerIDKey): void {
+        this.weakLocks.add(id)
+    }
+
+    public removeLocalLocked(id: PeerIDKey, serviceName: ServiceName): void {
+        if (this.localLocks.has(id)) {
+            this.localLocks.get(id)?.delete(serviceName)
+            if (this.localLocks.get(id)?.size === 0) {
+                this.localLocks.delete(id)
+            }
+        }
+    }
+
+    public removeRemoteLocked(id: PeerIDKey, serviceName: ServiceName): void {
+        if (this.remoteLocks.has(id)) {
+            this.remoteLocks.get(id)?.delete(serviceName)
+            if (this.remoteLocks.get(id)?.size === 0) {
+                this.remoteLocks.delete(id)
+            }
+        }
+    }
+
+    public removeWeakLocked(id: PeerIDKey): void {
+        this.weakLocks.delete(id)
+    }
+
+    public clearAllLocks(id: PeerIDKey): void {
+        this.localLocks.delete(id)
+        this.remoteLocks.delete(id)
+        this.weakLocks.delete(id)
+    }
+
+    public clear(): void {
+        this.localLocks.clear()
+        this.remoteLocks.clear()
+        this.weakLocks.clear()
+    }
+}