@streamr/dht 102.2.0-rc.0 → 102.2.0-rc.2

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@streamr/dht",
-  "version": "102.2.0-rc.0",
+  "version": "102.2.0-rc.2",
   "description": "Streamr Network DHT",
   "repository": {
     "type": "git",
@@ -9,15 +9,20 @@
   },
   "license": "STREAMR NETWORK OPEN SOURCE LICENSE",
   "author": "Streamr Network AG <contact@streamr.network>",
+  "files": [
+    "dist",
+    "!*.tsbuildinfo",
+    "README.md"
+  ],
   "dependencies": {
     "@js-sdsl/ordered-map": "^4.4.2",
     "@protobuf-ts/runtime": "^2.8.2",
     "@protobuf-ts/runtime-rpc": "^2.8.2",
-    "@streamr/autocertifier-client": "102.2.0-rc.0",
-    "@streamr/cdn-location": "102.2.0-rc.0",
-    "@streamr/geoip-location": "102.2.0-rc.0",
-    "@streamr/proto-rpc": "102.2.0-rc.0",
-    "@streamr/utils": "102.2.0-rc.0",
+    "@streamr/autocertifier-client": "102.2.0-rc.2",
+    "@streamr/cdn-location": "102.2.0-rc.2",
+    "@streamr/geoip-location": "102.2.0-rc.2",
+    "@streamr/proto-rpc": "102.2.0-rc.2",
+    "@streamr/utils": "102.2.0-rc.2",
     "eventemitter3": "^5.0.0",
     "heap": "^0.2.6",
     "ipaddr.js": "^2.0.1",
@@ -31,7 +36,7 @@
   },
   "devDependencies": {
     "@streamr/browser-test-runner": "^0.0.1",
-    "@streamr/test-utils": "102.2.0-rc.0",
+    "@streamr/test-utils": "102.2.0-rc.2",
     "@types/heap": "^0.2.34",
     "@types/k-bucket": "^5.0.1",
     "@types/lodash": "^4.17.15",
@@ -40,7 +45,7 @@
     "jest-leak-detector": "^27.3.1",
     "jest-matcher-utils": "^29.7.0",
     "ts-essentials": "^10.0.4",
-    "tsx": "^4.19.3"
+    "ts-node": "^10.9.2"
   },
   "optionalDependencies": {
     "bufferutil": "^4.0.9",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@streamr/dht",
-  "version": "102.2.0-rc.0",
+  "version": "102.2.0-rc.2",
   "description": "Streamr Network DHT",
   "repository": {
     "type": "git",
@@ -16,6 +16,11 @@
   },
   "license": "STREAMR NETWORK OPEN SOURCE LICENSE",
   "author": "Streamr Network AG <contact@streamr.network>",
+  "files": [
+    "dist",
+    "!*.tsbuildinfo",
+    "README.md"
+  ],
   "scripts": {
     "prebuild": "./proto.sh",
     "postbuild": "./scripts/postbuild.sh",
@@ -35,11 +40,11 @@
     "@js-sdsl/ordered-map": "^4.4.2",
     "@protobuf-ts/runtime": "^2.8.2",
     "@protobuf-ts/runtime-rpc": "^2.8.2",
-    "@streamr/autocertifier-client": "102.2.0-rc.0",
-    "@streamr/cdn-location": "102.2.0-rc.0",
-    "@streamr/geoip-location": "102.2.0-rc.0",
-    "@streamr/proto-rpc": "102.2.0-rc.0",
-    "@streamr/utils": "102.2.0-rc.0",
+    "@streamr/autocertifier-client": "102.2.0-rc.2",
+    "@streamr/cdn-location": "102.2.0-rc.2",
+    "@streamr/geoip-location": "102.2.0-rc.2",
+    "@streamr/proto-rpc": "102.2.0-rc.2",
+    "@streamr/utils": "102.2.0-rc.2",
     "eventemitter3": "^5.0.0",
     "heap": "^0.2.6",
     "ipaddr.js": "^2.0.1",
@@ -53,7 +58,7 @@
   },
   "devDependencies": {
     "@streamr/browser-test-runner": "^0.0.1",
-    "@streamr/test-utils": "102.2.0-rc.0",
+    "@streamr/test-utils": "102.2.0-rc.2",
     "@types/heap": "^0.2.34",
     "@types/k-bucket": "^5.0.1",
     "@types/lodash": "^4.17.15",
@@ -62,7 +67,7 @@
     "jest-leak-detector": "^27.3.1",
     "jest-matcher-utils": "^29.7.0",
     "ts-essentials": "^10.0.4",
-    "tsx": "^4.19.3"
+    "ts-node": "^10.9.2"
   },
   "optionalDependencies": {
     "bufferutil": "^4.0.9",

package/dist/README.md

@@ -1,120 +0,0 @@
-# @streamr/dht
-
-A connectionful Kademlia based P2P distributed hash table (DHT). Implements an in-memory key-value data store where multiple creators of data store values behind a single key. The library is entirely browser compatible.
-
-All communication between peers utilizes [protobuf](https://protobuf.dev/), transported with the help of the custom `@streamr/proto-rpc` library.
-
-Connections in the DHT are established using WebSocket or WebRTC connections. The nodes will decide internally based on peers' connectivity information which connection types to use.
-
-The DHT also provides an interface for sending messages from peer to peer with the `DhtNode#send` function. The sent message is routed over the network to the target node.
-
-## Running a node
-
-Running a bare DhtNode in the Streamr Network is heavily discouraged as it requires advanced configuration. However, if you do wish to run a node here are some recommended configurations.
-
-(the following values are examples. For working production or test values check the @streamr/sdk default network configuration)
-
-```js
-const DhtNode = new DhtNode({
-  // Add a list of known entry points to the network
-  entryPoints: [{
-    kademliaId: new Uint8Array([1, 2, 3])
-  }], 
-  // a list of STUN and TURN servers. Critically important when opening webrtc conenctions behind NATs.
-  iceServers: [{
-    url: "stun.l.google.com",
-    port: 19302
-  }],
-  // A range of ports that the node will attempt to start a WebSocket server in. If you wish to use an exact port give equal values to min and max. If left unspecified the node will start without a server.
-  websocketPortRange: {
-    min: 30000,
-    max: 30500
-  },
-  ...
-})
-```
-
-## WebSocket Server TLS configuration
-
-Setting up TLS is important if you wish to allow nodes running in the browser to connect to your node over websocket. If left unconfigured webrtc connections will be used in such cases.
-
-There two ways to configure TLS for a DhtNode's WS server.
-
-### Auto-certification
-
-When starting a node configure it to automatically fetch TLS certificates for a randomly generated domain name as such:
-
-```js
-const DhtNode = new DhtNode({
-  ...
-  websocketServerEnableTls: true,
-  autoCertifierUrl: 'http://example-autocertifier-url:30000/',
-  autoCertifierConfigFile: '~/.streamr/certificate.json'
-  ...
-})
-```
-
-### Custom TLS configuration
-
-If you have your own domain name and/or TLS certificates you can configure the node to use them as follows:
-
-```js
-const DhtNode = new DhtNode({
-  tlsCertificate: {
-    certFileName: 'path/to/file'
-    privateKeyFileName: 'path/to/file'
-  },
-  websocketHost: 'custom-domain.com' // If using a custom domain name give it here to ensure that connectivity checking is correctly pointed to the server 
-})
-```
-
-### Setting up a custom DHT network
-
-To setup a custom DHT network outside the Streamr Networks Amoy or Polygon environments, you simply need to set up one or multiple entry point DhtNodes to the network and point all newly joining DhtNodes to them. It is also a good idea to configure the entry points to know each other. This makes it possible to restart them without causing network partitioning.
-
-## Development
-
-### local testing and develpment setup
-
-When running a node for testing or develpment a few configurations are important.
-
-```js
-const DhtNode = new DhtNode({
-  entryPoints: [], // Point this to your local dev entry point!
-  iceServers: [], // Keep empty to ensure webrtc connections are local
-  webrtcAllowPrivateAddresses: true, // Make sure that the value is true in local develpment
-  websocketServerEnableTls: false, // Keep as false to ensure that auto-certification is not attempted
-  websocketHost: '127.0.0.1', // Use 127.0.0.1 instead of localhost during develpment! 
-})
-```
-
-### Generating Protobuf code
-
-After making changes to the protobuf definitions you should regenerate the protobuf code with the following command.
-
-```bash
-  npm run generate-protoc-code
-  # Or
-  ./proto.sh
-```
-
-## Running DHT simulations
-
-Generate test data
-
-```bash
-npm run prepare-dht-simulation
-```
-
-Run simulation
-
-```bash
-npm run run-dht-simulation
-```
-
-In order to change number of nodes, or other simulation settings,
-
-* Edit the chages to the file 'test/simulation/data/generatedhtids.ts'.
-* Then generate new test data by running 'npm run prepare-dht-simulation'
-* Edit the same changes to file 'src/simulation/DhtSimulation.ts'
-* Run the simulation with new settings using 'npm run run-dht-simulation '

package/jest.config.ts

@@ -1,12 +0,0 @@
-import type { Config } from '@jest/types'
-import defaultConfig from '../../jest.config'
-
-const config: Config.InitialOptions = {
-    ...defaultConfig,
-    setupFilesAfterEnv: [
-        ...defaultConfig.setupFilesAfterEnv,
-        './test/utils/customMatchers.ts',
-    ],
-}
-
-export default config

package/karma-setup.js

@@ -1,2 +0,0 @@
-import * as customMatchers from './test/utils/customMatchers'
-window.expect.extend(customMatchers)

package/karma.config.js

@@ -1,35 +0,0 @@
-/* eslint-disable @typescript-eslint/no-require-imports */
-const path = require('path')
-const { createKarmaConfig, createWebpackConfig } = require('@streamr/browser-test-runner')
-
-const TEST_PATHS = [
-    'test/unit/**/!(connectivityRequestHandler*).ts',
-    './test/integration/**/!(DhtWith*|ReplicateData*|GeoIpConnectivityChecking*).ts/',
-    './test/end-to-end/**/!(RecoveryFromFailedAutoCertification*|memory-leak*|GeoIpLayer0*).ts'
-]
-
-const NodeWebrtcConnection = path.resolve(__dirname, 'src/connection/webrtc/NodeWebrtcConnection.ts')
-const BrowserWebrtcConnection = path.resolve(__dirname, 'src/connection/webrtc/BrowserWebrtcConnection.ts')
-const NodeWebsocketClientConnection = path.resolve(__dirname, 'src/connection/websocket/NodeWebsocketClientConnection.ts')
-const BrowserWebsocketClientConnection = path.resolve(__dirname, 'src/connection/websocket/BrowserWebsocketClientConnection.ts')
-
-module.exports = createKarmaConfig(
-    TEST_PATHS,
-    createWebpackConfig({
-        entry: './src/exports.ts',
-        libraryName: 'dht',
-        alias: {
-            [NodeWebrtcConnection]: BrowserWebrtcConnection,
-            [NodeWebsocketClientConnection]: BrowserWebsocketClientConnection
-        },
-        fallback: {
-            module: false
-        },
-        externals: {
-            http: 'HTTP',
-            ws: 'WebSocket',
-            'node-datachannel': 'commonjs node-datachannel',
-        }
-    }),
-    __dirname
-)

package/proto.sh

@@ -1,2 +0,0 @@
-mkdir -p ./generated
-npx protoc --ts_out ./generated --ts_opt server_generic,generate_dependencies,long_type_number --proto_path ../.. packages/dht/protos/DhtRpc.proto

package/protos/DhtRpc.proto

@@ -1,317 +0,0 @@
-syntax = "proto3";
-option optimize_for = CODE_SIZE;
-
-package dht;
-
-import "google/protobuf/empty.proto";
-import "google/protobuf/any.proto";
-import "google/protobuf/timestamp.proto";
-import "packages/proto-rpc/protos/ProtoRpc.proto";
-
-service DhtNodeRpc {
-  // TODO rename to getClosestNeighbors (breaking change)
-  rpc getClosestPeers (ClosestPeersRequest) returns (ClosestPeersResponse);
-  // TODO rename to getClosestRingContacts (breaking change)
-  rpc getClosestRingPeers (ClosestRingPeersRequest) returns (ClosestRingPeersResponse);
-  rpc ping (PingRequest) returns (PingResponse);
-  rpc leaveNotice (LeaveNotice) returns (google.protobuf.Empty);
-}
-
-service RouterRpc {
-  rpc routeMessage (RouteMessageWrapper) returns (RouteMessageAck);
-  rpc forwardMessage (RouteMessageWrapper) returns (RouteMessageAck);
-}
-
-service RecursiveOperationRpc {
-  rpc routeRequest (RouteMessageWrapper) returns (RouteMessageAck);
-}
-
-service StoreRpc {
-  rpc storeData (StoreDataRequest) returns (StoreDataResponse);
-  rpc replicateData (ReplicateDataRequest) returns (google.protobuf.Empty);
-}
-
-service RecursiveOperationSessionRpc {
-  rpc sendResponse(RecursiveOperationResponse) returns (google.protobuf.Empty);
-}
-
-service WebsocketClientConnectorRpc {
-  rpc requestConnection (WebsocketConnectionRequest) returns (google.protobuf.Empty);
-}
-
-service WebrtcConnectorRpc {
-  rpc requestConnection (WebrtcConnectionRequest) returns (google.protobuf.Empty);
-  rpc rtcOffer (RtcOffer) returns (google.protobuf.Empty);
-  rpc rtcAnswer (RtcAnswer) returns (google.protobuf.Empty);
-  rpc iceCandidate (IceCandidate) returns (google.protobuf.Empty);
-}
-
-service ConnectionLockRpc {
-  rpc lockRequest (LockRequest) returns (LockResponse);
-  rpc unlockRequest (UnlockRequest) returns (google.protobuf.Empty);
-  rpc gracefulDisconnect (DisconnectNotice) returns (google.protobuf.Empty);
-  rpc setPrivate (SetPrivateRequest) returns (google.protobuf.Empty);
-}
-
-service ExternalApiRpc {
-  rpc externalFetchData (ExternalFetchDataRequest) returns (ExternalFetchDataResponse);
-  rpc externalStoreData (ExternalStoreDataRequest) returns (ExternalStoreDataResponse);
-}
-
-// Used inside RpcMessage
-
-message StoreDataRequest {
-  bytes key = 1;
-  google.protobuf.Any data = 2;
-  bytes creator = 3;
-  google.protobuf.Timestamp createdAt = 4;
-  uint32 ttl = 5;
-}
-
-message StoreDataResponse {
-}
-
-message ExternalStoreDataRequest {
-  bytes key = 1;
-  google.protobuf.Any data = 2;
-}
-
-message ExternalStoreDataResponse {
-  repeated PeerDescriptor storers = 1;
-}
-
-message ReplicateDataRequest {
-  DataEntry entry = 1;
-}
-
-message DataEntry {
-  bytes key = 1;
-  google.protobuf.Any data = 2;
-  bytes creator = 3;
-  google.protobuf.Timestamp createdAt = 4;
-  google.protobuf.Timestamp storedAt = 5; 
-  uint32 ttl = 6;   // milliseconds
-  bool stale = 7;
-  bool deleted = 8;
-}
-
-// TODO rename to ClosestNeighborsRequest
-message ClosestPeersRequest {
-  bytes nodeId = 1;
-  string requestId = 2;
-}
-
-// TODO rename to ClosestNeighborsResponse
-message ClosestPeersResponse {
-  repeated PeerDescriptor peers = 1;
-  string requestId = 2;
-}
-
-// TODO rename to ClosestRingContactsRequest
-message ClosestRingPeersRequest {
-  bytes ringId = 1;
-  string requestId = 2;
-}
-
-// TODO rename to ClosestRingContactsResponse
-message ClosestRingPeersResponse {
-  repeated PeerDescriptor leftPeers = 1;
-  repeated PeerDescriptor rightPeers = 2;
-  string requestId = 3;
-}
-
-message RecursiveOperationRequest {
-  string sessionId = 1;
-  RecursiveOperation operation = 2;
-}
-
-enum RecursiveOperation {
-  FIND_CLOSEST_NODES = 0;
-  FETCH_DATA = 1;
-  DELETE_DATA = 2;
-}
-
-message RecursiveOperationResponse {
-  repeated PeerDescriptor closestConnectedNodes = 1;
-  repeated DataEntry dataEntries = 2;
-  bool noCloserNodesFound = 3;
-  repeated PeerDescriptor routingPath = 4;
-}
-
-message PingRequest {
-  string requestId = 1;
-}
-
-message PingResponse {
-  string requestId = 1;
-}
-
-message LeaveNotice {
-}
-
-message PeerDescriptor {
-  bytes nodeId = 1;
-  NodeType type = 2;
-  ConnectivityMethod udp = 3;
-  ConnectivityMethod tcp = 4;
-  ConnectivityMethod websocket = 5;
-  optional uint32 region = 6;
-  optional uint32 ipAddress = 7;
-  optional bytes publicKey = 8;
-  // signature of fields 2-8
-  optional bytes signature = 9; 
-}
-
-message ConnectivityMethod {
-  uint32 port = 1;
-  string host = 2;
-  bool tls = 3;
-}
-
-enum NodeType {
-  NODEJS = 0;
-  BROWSER = 1;
-}
-
-enum RpcResponseError {
-  SERVER_TIMOUT = 0;
-  CLIENT_TIMEOUT = 1;
-  SERVER_ERROR = 2;
-  UNKNOWN_RPC_METHOD = 3;
-}
-
-message RouteMessageWrapper {
-  string requestId = 1;
-  PeerDescriptor sourcePeer = 2;
-  bytes target = 3;
-  Message message = 4;
-  repeated PeerDescriptor reachableThrough = 5;
-  repeated PeerDescriptor routingPath = 6;
-  repeated string parallelRootNodeIds = 7;
-}
-
-message RouteMessageAck {
-  string requestId = 1;
-  optional RouteMessageError error = 2;
-}
-
-enum RouteMessageError {
-  NO_TARGETS = 0;
-  DUPLICATE = 1;
-  // TODO: can this be removed? If DhtNode is already stopped the server side requests
-  // should not be processed
-  STOPPED = 2;
-}
-
-message ConnectivityRequest {
-  uint32 port = 1;
-  bool tls = 2;
-  optional string host = 3;
-  bool allowSelfSignedCertificate = 4;
-}
-
-message ConnectivityResponse {
-  string host = 1;
-  string natType = 2;
-  ConnectivityMethod websocket = 3;
-  uint32 ipAddress = 4;
-  string protocolVersion = 5;
-  optional double latitude = 6;
-  optional double longitude = 7;
-}
-
-message HandshakeRequest {
-  PeerDescriptor sourcePeerDescriptor = 1;
-  optional PeerDescriptor targetPeerDescriptor = 2;
-  string protocolVersion = 3;
-  string applicationVersion = 4;
-}
-
-message HandshakeResponse {
-  PeerDescriptor sourcePeerDescriptor = 1;
-  optional HandshakeError error = 2;
-  string protocolVersion = 3;
-  string applicationVersion = 4;
-}
-
-enum HandshakeError {
-  DUPLICATE_CONNECTION = 0;
-  INVALID_TARGET_PEER_DESCRIPTOR = 1;
-  UNSUPPORTED_PROTOCOL_VERSION = 2;
-}
-
-// Wraps all messages
-
-message Message {
-  string messageId = 1;
-  PeerDescriptor sourceDescriptor = 2;
-  PeerDescriptor targetDescriptor = 3;
-  string serviceId = 4; // id of the RPC service
-  oneof body {
-    protorpc.RpcMessage rpcMessage = 5;
-    ConnectivityRequest connectivityRequest = 6;
-    ConnectivityResponse connectivityResponse = 7;
-    HandshakeRequest handshakeRequest = 8;
-    HandshakeResponse handshakeResponse = 9;
-    RecursiveOperationRequest recursiveOperationRequest = 10;
-  };
-}
-
-// Connector Messages
-
-// WebSocket
-message WebsocketConnectionRequest {
-}
-
-// WebRTC
-message WebrtcConnectionRequest {
-}
-
-message RtcOffer {
-  string description = 1;
-  string connectionId = 2;
-}
-
-message RtcAnswer {
-  string description = 1;
-  string connectionId = 2;
-}
-
-message IceCandidate {
-  string candidate = 1;
-  string mid = 2;
-  string connectionId = 3;
-}
-
-message LockRequest {
-  string lockId = 1;
-}
-
-message UnlockRequest {
-  string lockId = 1;
-}
-
-message LockResponse {
-  bool accepted = 1;
-}
-
-enum DisconnectMode {
-  NORMAL = 0;
-  LEAVING = 1;
-}
-
-message DisconnectNotice {
-  DisconnectMode disconnectMode = 1;
-}
-
-message SetPrivateRequest {
-  bool isPrivate = 1;
-}
-
-message ExternalFetchDataRequest {
-  bytes key = 1;
-}
-
-message ExternalFetchDataResponse {
-  repeated DataEntry entries = 1;
-}

package/protos/tests.proto

@@ -1,16 +0,0 @@
-
-syntax = "proto3";
-option optimize_for = CODE_SIZE;
-
-package tests;
-
-import "google/protobuf/any.proto";
-
-message TestMessage {
-    string messageId = 1;
-    google.protobuf.Any body = 2;
-  }
-
-message SomeMessage {
-    string juttu = 1;
-}

package/scripts/postbuild.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-mkdir -p dist
-
-SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-
-cd "${SCRIPT_DIR}/.."
-
-# Sanitize the final package.json
-npx tsx scripts/rewrite-package.ts
-
-# Copy assets
-cp -f README.md dist

package/scripts/rewrite-package.ts

@@ -1,45 +0,0 @@
-/**
- * This script moves the package.json file into the dist folder while adjusting
- * relative pathnames for exports. Since the compiled output is inside "dist",
- * but the original package.json references paths as if it's in the root, we
- * need to strip "dist" from all export fields (main, types, …).
- *
- * This ensures that consumers of the package import the correct files
- * without referencing "dist" in their paths, maintaining proper module resolution.
- */
-
-import pkg from '../package.json' assert { type: 'json' }
-import * as fs from 'node:fs'
-import path, { dirname } from 'node:path'
-import { fileURLToPath } from 'url'
-
-function fixPathname(pathname: string): string {
-    return pathname.startsWith('./dist')
-        ? `./${path.relative('./dist', pathname)}`
-        : pathname
-}
-
-const { main, types, browser, scripts: _scripts, ...rest } = pkg
-
-const newPkg = {
-    ...rest,
-    main: fixPathname(main),
-    types: fixPathname(types),
-    browser: Object.entries(browser).reduce(
-        (memo, [fromPathname, toPathname]) => ({
-            ...memo,
-            [fixPathname(fromPathname)]:
-                typeof toPathname === 'string'
-                    ? fixPathname(toPathname)
-                    : toPathname,
-        }),
-        {}
-    ),
-}
-
-const dist = path.resolve(
-    dirname(fileURLToPath(import.meta.url)),
-    '../dist/package.json'
-)
-
-fs.writeFileSync(dist, JSON.stringify(newPkg, null, 2))