@streamr/dht 100.0.0-pretestnet.6 → 100.0.0-testnet-one.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (260) hide show
  1. package/dist/src/connection/ConnectionLockHandler.js +2 -2
  2. package/dist/src/connection/ConnectionLockHandler.js.map +1 -1
  3. package/dist/src/connection/ConnectionLockRpcRemote.d.ts +2 -2
  4. package/dist/src/connection/ConnectionLockRpcRemote.js +3 -27
  5. package/dist/src/connection/ConnectionLockRpcRemote.js.map +1 -1
  6. package/dist/src/connection/ConnectionManager.d.ts +0 -1
  7. package/dist/src/connection/ConnectionManager.js +11 -7
  8. package/dist/src/connection/ConnectionManager.js.map +1 -1
  9. package/dist/src/connection/ConnectorFacade.d.ts +2 -2
  10. package/dist/src/connection/ConnectorFacade.js +1 -2
  11. package/dist/src/connection/ConnectorFacade.js.map +1 -1
  12. package/dist/src/connection/ManagedConnection.js +1 -0
  13. package/dist/src/connection/ManagedConnection.js.map +1 -1
  14. package/dist/src/connection/connectivityChecker.d.ts +9 -0
  15. package/dist/src/connection/connectivityChecker.js +122 -0
  16. package/dist/src/connection/connectivityChecker.js.map +1 -0
  17. package/dist/src/connection/connectivityRequestHandler.d.ts +2 -0
  18. package/dist/src/connection/connectivityRequestHandler.js +79 -0
  19. package/dist/src/connection/connectivityRequestHandler.js.map +1 -0
  20. package/dist/src/connection/simulator/Simulator.js +3 -2
  21. package/dist/src/connection/simulator/Simulator.js.map +1 -1
  22. package/dist/src/connection/simulator/SimulatorConnection.js +1 -1
  23. package/dist/src/connection/simulator/SimulatorConnection.js.map +1 -1
  24. package/dist/src/connection/webrtc/NodeWebrtcConnection.js +1 -1
  25. package/dist/src/connection/webrtc/NodeWebrtcConnection.js.map +1 -1
  26. package/dist/src/connection/webrtc/WebrtcConnector.js +1 -1
  27. package/dist/src/connection/webrtc/WebrtcConnector.js.map +1 -1
  28. package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.js +1 -1
  29. package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.js.map +1 -1
  30. package/dist/src/connection/webrtc/WebrtcConnectorRpcRemote.d.ts +2 -2
  31. package/dist/src/connection/webrtc/WebrtcConnectorRpcRemote.js +2 -2
  32. package/dist/src/connection/webrtc/WebrtcConnectorRpcRemote.js.map +1 -1
  33. package/dist/src/connection/webrtc/iceServerAsString.js +1 -1
  34. package/dist/src/connection/webrtc/iceServerAsString.js.map +1 -1
  35. package/dist/src/connection/websocket/ClientWebsocket.d.ts +1 -0
  36. package/dist/src/connection/websocket/ClientWebsocket.js +7 -3
  37. package/dist/src/connection/websocket/ClientWebsocket.js.map +1 -1
  38. package/dist/src/connection/websocket/ServerWebsocket.d.ts +4 -0
  39. package/dist/src/connection/websocket/ServerWebsocket.js +32 -21
  40. package/dist/src/connection/websocket/ServerWebsocket.js.map +1 -1
  41. package/dist/src/connection/websocket/WebsocketConnector.d.ts +0 -2
  42. package/dist/src/connection/websocket/WebsocketConnector.js +61 -16
  43. package/dist/src/connection/websocket/WebsocketConnector.js.map +1 -1
  44. package/dist/src/connection/websocket/WebsocketConnectorRpcLocal.d.ts +1 -1
  45. package/dist/src/connection/websocket/WebsocketConnectorRpcLocal.js +8 -11
  46. package/dist/src/connection/websocket/WebsocketConnectorRpcLocal.js.map +1 -1
  47. package/dist/src/connection/websocket/WebsocketConnectorRpcRemote.d.ts +4 -4
  48. package/dist/src/connection/websocket/WebsocketConnectorRpcRemote.js +5 -39
  49. package/dist/src/connection/websocket/WebsocketConnectorRpcRemote.js.map +1 -1
  50. package/dist/src/connection/websocket/WebsocketServer.js +21 -4
  51. package/dist/src/connection/websocket/WebsocketServer.js.map +1 -1
  52. package/dist/src/dht/DhtNode.d.ts +13 -23
  53. package/dist/src/dht/DhtNode.js +97 -226
  54. package/dist/src/dht/DhtNode.js.map +1 -1
  55. package/dist/src/dht/DhtNodeRpcLocal.d.ts +1 -4
  56. package/dist/src/dht/DhtNodeRpcLocal.js +1 -5
  57. package/dist/src/dht/DhtNodeRpcLocal.js.map +1 -1
  58. package/dist/src/dht/DhtNodeRpcRemote.d.ts +3 -3
  59. package/dist/src/dht/DhtNodeRpcRemote.js +4 -4
  60. package/dist/src/dht/DhtNodeRpcRemote.js.map +1 -1
  61. package/dist/src/dht/ExternalApiRpcLocal.d.ts +4 -4
  62. package/dist/src/dht/ExternalApiRpcLocal.js +5 -12
  63. package/dist/src/dht/ExternalApiRpcLocal.js.map +1 -1
  64. package/dist/src/dht/ExternalApiRpcRemote.d.ts +3 -3
  65. package/dist/src/dht/ExternalApiRpcRemote.js +5 -5
  66. package/dist/src/dht/ExternalApiRpcRemote.js.map +1 -1
  67. package/dist/src/dht/PeerManager.d.ts +52 -0
  68. package/dist/src/dht/PeerManager.js +273 -0
  69. package/dist/src/dht/PeerManager.js.map +1 -0
  70. package/dist/src/dht/contact/ContactList.d.ts +1 -1
  71. package/dist/src/dht/contact/ContactList.js +1 -0
  72. package/dist/src/dht/contact/ContactList.js.map +1 -1
  73. package/dist/src/dht/contact/{Remote.d.ts → RpcRemote.d.ts} +3 -3
  74. package/dist/src/dht/contact/{Remote.js → RpcRemote.js} +8 -8
  75. package/dist/src/dht/contact/RpcRemote.js.map +1 -0
  76. package/dist/src/dht/contact/SortedContactList.d.ts +20 -6
  77. package/dist/src/dht/contact/SortedContactList.js +55 -24
  78. package/dist/src/dht/contact/SortedContactList.js.map +1 -1
  79. package/dist/src/dht/discovery/DiscoverySession.d.ts +4 -14
  80. package/dist/src/dht/discovery/DiscoverySession.js +15 -26
  81. package/dist/src/dht/discovery/DiscoverySession.js.map +1 -1
  82. package/dist/src/dht/discovery/PeerDiscovery.d.ts +2 -9
  83. package/dist/src/dht/discovery/PeerDiscovery.js +11 -19
  84. package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
  85. package/dist/src/dht/find/FindRpcLocal.js +2 -1
  86. package/dist/src/dht/find/FindRpcLocal.js.map +1 -1
  87. package/dist/src/dht/find/FindSession.d.ts +6 -6
  88. package/dist/src/dht/find/FindSession.js +18 -13
  89. package/dist/src/dht/find/FindSession.js.map +1 -1
  90. package/dist/src/dht/find/FindSessionRpcLocal.d.ts +1 -1
  91. package/dist/src/dht/find/FindSessionRpcRemote.d.ts +2 -2
  92. package/dist/src/dht/find/FindSessionRpcRemote.js +2 -2
  93. package/dist/src/dht/find/FindSessionRpcRemote.js.map +1 -1
  94. package/dist/src/dht/find/Finder.d.ts +4 -4
  95. package/dist/src/dht/find/Finder.js +55 -42
  96. package/dist/src/dht/find/Finder.js.map +1 -1
  97. package/dist/src/dht/routing/FindRpcRemote.d.ts +2 -2
  98. package/dist/src/dht/routing/FindRpcRemote.js +7 -5
  99. package/dist/src/dht/routing/FindRpcRemote.js.map +1 -1
  100. package/dist/src/dht/routing/Router.d.ts +3 -7
  101. package/dist/src/dht/routing/Router.js +29 -22
  102. package/dist/src/dht/routing/Router.js.map +1 -1
  103. package/dist/src/dht/routing/RouterRpcLocal.d.ts +2 -2
  104. package/dist/src/dht/routing/RouterRpcLocal.js +4 -3
  105. package/dist/src/dht/routing/RouterRpcLocal.js.map +1 -1
  106. package/dist/src/dht/routing/RouterRpcRemote.d.ts +2 -2
  107. package/dist/src/dht/routing/RouterRpcRemote.js +13 -8
  108. package/dist/src/dht/routing/RouterRpcRemote.js.map +1 -1
  109. package/dist/src/dht/routing/RoutingSession.d.ts +1 -1
  110. package/dist/src/dht/routing/RoutingSession.js +23 -11
  111. package/dist/src/dht/routing/RoutingSession.js.map +1 -1
  112. package/dist/src/dht/routing/getPreviousPeer.js.map +1 -1
  113. package/dist/src/dht/store/LocalDataStore.d.ts +3 -3
  114. package/dist/src/dht/store/LocalDataStore.js +18 -17
  115. package/dist/src/dht/store/LocalDataStore.js.map +1 -1
  116. package/dist/src/dht/store/StoreRpcLocal.d.ts +10 -9
  117. package/dist/src/dht/store/StoreRpcLocal.js +108 -102
  118. package/dist/src/dht/store/StoreRpcLocal.js.map +1 -1
  119. package/dist/src/dht/store/StoreRpcRemote.d.ts +4 -5
  120. package/dist/src/dht/store/StoreRpcRemote.js +6 -15
  121. package/dist/src/dht/store/StoreRpcRemote.js.map +1 -1
  122. package/dist/src/exports.d.ts +1 -1
  123. package/dist/src/exports.js +4 -4
  124. package/dist/src/exports.js.map +1 -1
  125. package/dist/src/helpers/PeerID.d.ts +1 -0
  126. package/dist/src/helpers/PeerID.js +9 -4
  127. package/dist/src/helpers/PeerID.js.map +1 -1
  128. package/dist/src/helpers/UUID.js +1 -1
  129. package/dist/src/helpers/UUID.js.map +1 -1
  130. package/dist/src/helpers/nodeId.d.ts +1 -0
  131. package/dist/src/helpers/{kademliaId.js → nodeId.js} +4 -4
  132. package/dist/src/helpers/nodeId.js.map +1 -0
  133. package/dist/src/helpers/peerIdFromPeerDescriptor.js +4 -4
  134. package/dist/src/helpers/peerIdFromPeerDescriptor.js.map +1 -1
  135. package/dist/src/proto/packages/dht/protos/DhtRpc.client.d.ts +5 -16
  136. package/dist/src/proto/packages/dht/protos/DhtRpc.client.js +2 -9
  137. package/dist/src/proto/packages/dht/protos/DhtRpc.client.js.map +1 -1
  138. package/dist/src/proto/packages/dht/protos/DhtRpc.d.ts +80 -95
  139. package/dist/src/proto/packages/dht/protos/DhtRpc.js +67 -66
  140. package/dist/src/proto/packages/dht/protos/DhtRpc.js.map +1 -1
  141. package/dist/src/proto/packages/dht/protos/DhtRpc.server.d.ts +3 -10
  142. package/dist/src/transport/RoutingRpcCommunicator.js +2 -0
  143. package/dist/src/transport/RoutingRpcCommunicator.js.map +1 -1
  144. package/karma.config.js +2 -2
  145. package/package.json +5 -5
  146. package/protos/DhtRpc.proto +36 -36
  147. package/src/connection/ConnectionLockHandler.ts +2 -2
  148. package/src/connection/ConnectionLockRpcRemote.ts +3 -4
  149. package/src/connection/ConnectionManager.ts +19 -17
  150. package/src/connection/ConnectorFacade.ts +5 -7
  151. package/src/connection/ManagedConnection.ts +1 -0
  152. package/src/connection/connectivityChecker.ts +102 -0
  153. package/src/connection/connectivityRequestHandler.ts +79 -0
  154. package/src/connection/simulator/Simulator.ts +3 -2
  155. package/src/connection/simulator/SimulatorConnection.ts +1 -1
  156. package/src/connection/webrtc/BrowserWebrtcConnection.ts +10 -10
  157. package/src/connection/webrtc/NodeWebrtcConnection.ts +1 -1
  158. package/src/connection/webrtc/WebrtcConnector.ts +1 -1
  159. package/src/connection/webrtc/WebrtcConnectorRpcLocal.ts +1 -1
  160. package/src/connection/webrtc/WebrtcConnectorRpcRemote.ts +2 -2
  161. package/src/connection/webrtc/iceServerAsString.ts +1 -1
  162. package/src/connection/websocket/ClientWebsocket.ts +6 -2
  163. package/src/connection/websocket/ServerWebsocket.ts +40 -25
  164. package/src/connection/websocket/WebsocketConnector.ts +43 -22
  165. package/src/connection/websocket/WebsocketConnectorRpcLocal.ts +9 -11
  166. package/src/connection/websocket/WebsocketConnectorRpcRemote.ts +7 -16
  167. package/src/connection/websocket/WebsocketServer.ts +20 -5
  168. package/src/dht/DhtNode.ts +123 -280
  169. package/src/dht/DhtNodeRpcLocal.ts +2 -9
  170. package/src/dht/DhtNodeRpcRemote.ts +4 -4
  171. package/src/dht/ExternalApiRpcLocal.ts +8 -13
  172. package/src/dht/ExternalApiRpcRemote.ts +5 -5
  173. package/src/dht/PeerManager.ts +330 -0
  174. package/src/dht/contact/ContactList.ts +3 -2
  175. package/src/dht/contact/{Remote.ts → RpcRemote.ts} +7 -6
  176. package/src/dht/contact/SortedContactList.ts +87 -44
  177. package/src/dht/discovery/DiscoverySession.ts +19 -44
  178. package/src/dht/discovery/PeerDiscovery.ts +16 -28
  179. package/src/dht/find/FindRpcLocal.ts +2 -2
  180. package/src/dht/find/FindSession.ts +25 -20
  181. package/src/dht/find/FindSessionRpcLocal.ts +1 -1
  182. package/src/dht/find/FindSessionRpcRemote.ts +2 -2
  183. package/src/dht/find/Finder.ts +84 -64
  184. package/src/dht/routing/FindRpcRemote.ts +7 -5
  185. package/src/dht/routing/Router.ts +30 -25
  186. package/src/dht/routing/RouterRpcLocal.ts +5 -5
  187. package/src/dht/routing/RouterRpcRemote.ts +13 -10
  188. package/src/dht/routing/RoutingSession.ts +22 -17
  189. package/src/dht/routing/getPreviousPeer.ts +1 -1
  190. package/src/dht/store/LocalDataStore.ts +18 -17
  191. package/src/dht/store/StoreRpcLocal.ts +118 -113
  192. package/src/dht/store/StoreRpcRemote.ts +7 -23
  193. package/src/exports.ts +1 -1
  194. package/src/helpers/PeerID.ts +8 -4
  195. package/src/helpers/UUID.ts +1 -1
  196. package/src/helpers/{kademliaId.ts → nodeId.ts} +1 -1
  197. package/src/helpers/peerIdFromPeerDescriptor.ts +6 -6
  198. package/src/proto/packages/dht/protos/DhtRpc.client.ts +6 -20
  199. package/src/proto/packages/dht/protos/DhtRpc.server.ts +3 -10
  200. package/src/proto/packages/dht/protos/DhtRpc.ts +103 -135
  201. package/src/transport/RoutingRpcCommunicator.ts +2 -0
  202. package/test/benchmark/Find.test.ts +5 -5
  203. package/test/benchmark/KademliaCorrectness.test.ts +3 -3
  204. package/test/benchmark/SortedContactListBenchmark.test.ts +150 -0
  205. package/test/benchmark/WebsocketServerMemoryLeak.test.ts +41 -0
  206. package/test/benchmark/kademlia-simulation/Contact.ts +1 -1
  207. package/test/benchmark/kademlia-simulation/KademliaSimulation.ts +1 -1
  208. package/test/benchmark/kademlia-simulation/SimulationNode.ts +6 -1
  209. package/test/end-to-end/Layer0-Layer1.test.ts +1 -1
  210. package/test/end-to-end/Layer0.test.ts +4 -4
  211. package/test/end-to-end/Layer0MixedConnectionTypes.test.ts +11 -11
  212. package/test/end-to-end/Layer0Webrtc-Layer1.test.ts +6 -6
  213. package/test/end-to-end/Layer0Webrtc.test.ts +2 -2
  214. package/test/end-to-end/Layer1-Scale-WebSocket.test.ts +3 -3
  215. package/test/end-to-end/Layer1-Scale-Webrtc.test.ts +3 -3
  216. package/test/end-to-end/RecoveryFromFailedAutoCertification.test.ts +1 -1
  217. package/test/end-to-end/WebsocketConnectionRequest.test.ts +1 -1
  218. package/test/end-to-end/memory-leak.test.ts +9 -12
  219. package/test/integration/ConnectionLocking.test.ts +2 -2
  220. package/test/integration/ConnectionManager.test.ts +14 -14
  221. package/test/integration/DhtJoinPeerDiscovery.test.ts +3 -3
  222. package/test/integration/DhtNodeExternalAPI.test.ts +10 -7
  223. package/test/integration/DhtNodeRpcRemote.test.ts +4 -4
  224. package/test/integration/DhtRpc.test.ts +6 -6
  225. package/test/integration/Find.test.ts +3 -3
  226. package/test/integration/Layer1-scale.test.ts +3 -3
  227. package/test/integration/Mock-Layer1-Layer0.test.ts +16 -16
  228. package/test/integration/MultipleEntryPointJoining.test.ts +7 -7
  229. package/test/integration/{MigrateData.test.ts → ReplicateData.test.ts} +15 -10
  230. package/test/integration/RouteMessage.test.ts +2 -2
  231. package/test/integration/RouterRpcRemote.test.ts +2 -2
  232. package/test/integration/RpcErrors.test.ts +2 -2
  233. package/test/integration/ScaleDownDht.test.ts +4 -2
  234. package/test/integration/SimultaneousConnections.test.ts +89 -57
  235. package/test/integration/Store.test.ts +33 -13
  236. package/test/integration/StoreAndDelete.test.ts +19 -17
  237. package/test/integration/StoreOnDhtWithTwoNodes.test.ts +21 -21
  238. package/test/integration/StoreRpcRemote.test.ts +3 -3
  239. package/test/integration/WebrtcConnectionManagement.test.ts +2 -2
  240. package/test/integration/WebrtcConnectorRpc.test.ts +1 -1
  241. package/test/integration/WebsocketConnectionManagement.test.ts +41 -3
  242. package/test/integration/WebsocketConnectorRpc.test.ts +5 -7
  243. package/test/unit/ConnectivityHelpers.test.ts +4 -4
  244. package/test/unit/Finder.test.ts +69 -23
  245. package/test/unit/LocalDataStore.test.ts +60 -43
  246. package/test/unit/RandomContactList.test.ts +2 -2
  247. package/test/unit/Router.test.ts +19 -11
  248. package/test/unit/RoutingSession.test.ts +76 -0
  249. package/test/unit/SortedContactList.test.ts +17 -12
  250. package/test/unit/WebsocketConnector.test.ts +1 -1
  251. package/test/unit/connectivityRequestHandler.test.ts +71 -0
  252. package/test/utils/mock/Router.ts +1 -1
  253. package/test/utils/utils.ts +24 -22
  254. package/dist/src/connection/ConnectivityChecker.d.ts +0 -17
  255. package/dist/src/connection/ConnectivityChecker.js +0 -208
  256. package/dist/src/connection/ConnectivityChecker.js.map +0 -1
  257. package/dist/src/dht/contact/Remote.js.map +0 -1
  258. package/dist/src/helpers/kademliaId.d.ts +0 -1
  259. package/dist/src/helpers/kademliaId.js.map +0 -1
  260. package/src/connection/ConnectivityChecker.ts +0 -199
@@ -1,6 +1,5 @@
1
1
  import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
2
2
  import { Logger } from '@streamr/utils'
3
- import KBucket from 'k-bucket'
4
3
  import { getNodeIdFromPeerDescriptor } from '../helpers/peerIdFromPeerDescriptor'
5
4
  import { Empty } from '../proto/google/protobuf/empty'
6
5
  import {
@@ -13,12 +12,11 @@ import {
13
12
  } from '../proto/packages/dht/protos/DhtRpc'
14
13
  import { IDhtNodeRpc } from '../proto/packages/dht/protos/DhtRpc.server'
15
14
  import { DhtCallContext } from '../rpc-protocol/DhtCallContext'
16
- import { DhtNodeRpcRemote } from './DhtNodeRpcRemote'
17
15
 
18
16
  interface DhtNodeRpcLocalConfig {
19
- bucket: KBucket<DhtNodeRpcRemote>
20
17
  serviceId: string
21
18
  peerDiscoveryQueryBatchSize: number
19
+ getClosestPeersTo: (nodeId: Uint8Array, limit: number) => PeerDescriptor[]
22
20
  addNewContact: (contact: PeerDescriptor) => void
23
21
  removeContact: (contact: PeerDescriptor) => void
24
22
  }
@@ -36,17 +34,12 @@ export class DhtNodeRpcLocal implements IDhtNodeRpc {
36
34
  async getClosestPeers(request: ClosestPeersRequest, context: ServerCallContext): Promise<ClosestPeersResponse> {
37
35
  this.config.addNewContact((context as DhtCallContext).incomingSourceDescriptor!)
38
36
  const response = {
39
- peers: this.getClosestPeerDescriptors(request.kademliaId, this.config.peerDiscoveryQueryBatchSize),
37
+ peers: this.config.getClosestPeersTo(request.nodeId, this.config.peerDiscoveryQueryBatchSize),
40
38
  requestId: request.requestId
41
39
  }
42
40
  return response
43
41
  }
44
42
 
45
- private getClosestPeerDescriptors(kademliaId: Uint8Array, limit: number): PeerDescriptor[] {
46
- const closestPeers = this.config.bucket.closest(kademliaId, limit)
47
- return closestPeers.map((rpcRemote: DhtNodeRpcRemote) => rpcRemote.getPeerDescriptor())
48
- }
49
-
50
43
  async ping(request: PingRequest, context: ServerCallContext): Promise<PingResponse> {
51
44
  logger.trace('received ping request: ' + getNodeIdFromPeerDescriptor((context as DhtCallContext).incomingSourceDescriptor!))
52
45
  setImmediate(() => {
@@ -8,7 +8,7 @@ import {
8
8
  import { v4 } from 'uuid'
9
9
  import { Logger } from '@streamr/utils'
10
10
  import { ProtoRpcClient } from '@streamr/proto-rpc'
11
- import { Remote } from './contact/Remote'
11
+ import { RpcRemote } from './contact/RpcRemote'
12
12
  import { PeerID } from '../helpers/PeerID'
13
13
  import { getNodeIdFromPeerDescriptor, peerIdFromPeerDescriptor } from '../helpers/peerIdFromPeerDescriptor'
14
14
  import { ServiceID } from '../types/ServiceID'
@@ -21,7 +21,7 @@ export interface KBucketContact {
21
21
  vectorClock: number
22
22
  }
23
23
 
24
- export class DhtNodeRpcRemote extends Remote<IDhtNodeRpcClient> implements KBucketContact {
24
+ export class DhtNodeRpcRemote extends RpcRemote<IDhtNodeRpcClient> implements KBucketContact {
25
25
 
26
26
  private static counter = 0
27
27
  public vectorClock: number
@@ -39,10 +39,10 @@ export class DhtNodeRpcRemote extends Remote<IDhtNodeRpcClient> implements KBuck
39
39
  this.vectorClock = DhtNodeRpcRemote.counter++
40
40
  }
41
41
 
42
- async getClosestPeers(kademliaId: Uint8Array): Promise<PeerDescriptor[]> {
42
+ async getClosestPeers(nodeId: Uint8Array): Promise<PeerDescriptor[]> {
43
43
  logger.trace(`Requesting getClosestPeers on ${this.getServiceId()} from ${getNodeIdFromPeerDescriptor(this.getPeerDescriptor())}`)
44
44
  const request: ClosestPeersRequest = {
45
- kademliaId,
45
+ nodeId,
46
46
  requestId: v4()
47
47
  }
48
48
  try {
@@ -4,6 +4,7 @@ import {
4
4
  ExternalFindDataResponse,
5
5
  ExternalStoreDataRequest,
6
6
  ExternalStoreDataResponse,
7
+ FindAction,
7
8
  PeerDescriptor
8
9
  } from '../proto/packages/dht/protos/DhtRpc'
9
10
  import { ServerCallContext } from '@protobuf-ts/runtime-rpc'
@@ -12,8 +13,8 @@ import { FindResult } from './find/Finder'
12
13
  import { Any } from '../proto/google/protobuf/any'
13
14
 
14
15
  interface ExternalApiRpcLocalConfig {
15
- startFind: (idToFind: Uint8Array, fetchData: boolean, excludedPeer: PeerDescriptor) => Promise<FindResult>
16
- storeDataToDht: (key: Uint8Array, data: Any) => Promise<PeerDescriptor[]>
16
+ startFind: (idToFind: Uint8Array, action: FindAction, excludedPeer: PeerDescriptor) => Promise<FindResult>
17
+ storeDataToDht: (key: Uint8Array, data: Any, creator: PeerDescriptor) => Promise<PeerDescriptor[]>
17
18
  }
18
19
 
19
20
  export class ExternalApiRpcLocal implements IExternalApiRpc {
@@ -26,19 +27,13 @@ export class ExternalApiRpcLocal implements IExternalApiRpc {
26
27
 
27
28
  async externalFindData(findDataRequest: ExternalFindDataRequest, context: ServerCallContext): Promise<ExternalFindDataResponse> {
28
29
  const senderPeerDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
29
- const result = await this.config.startFind(findDataRequest.kademliaId, true, senderPeerDescriptor)
30
- if (result.dataEntries) {
31
- return ExternalFindDataResponse.create({ dataEntries: result.dataEntries })
32
- } else {
33
- return ExternalFindDataResponse.create({
34
- dataEntries: [],
35
- error: 'Could not find data with the given key'
36
- })
37
- }
30
+ const result = await this.config.startFind(findDataRequest.key, FindAction.FETCH_DATA, senderPeerDescriptor)
31
+ return ExternalFindDataResponse.create({ entries: result.dataEntries ?? [] })
38
32
  }
39
33
 
40
- async externalStoreData(request: ExternalStoreDataRequest): Promise<ExternalStoreDataResponse> {
41
- const result = await this.config.storeDataToDht(request.key, request.data!)
34
+ async externalStoreData(request: ExternalStoreDataRequest, context: ServerCallContext): Promise<ExternalStoreDataResponse> {
35
+ const senderPeerDescriptor = (context as DhtCallContext).incomingSourceDescriptor!
36
+ const result = await this.config.storeDataToDht(request.key, request.data!, senderPeerDescriptor)
42
37
  return ExternalStoreDataResponse.create({
43
38
  storers: result
44
39
  })
@@ -1,20 +1,20 @@
1
1
  import { Any } from '../proto/google/protobuf/any'
2
2
  import { DataEntry, ExternalStoreDataRequest, ExternalFindDataRequest, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
3
3
  import { IExternalApiRpcClient } from '../proto/packages/dht/protos/DhtRpc.client'
4
- import { Remote } from './contact/Remote'
4
+ import { RpcRemote } from './contact/RpcRemote'
5
5
 
6
- export class ExternalApiRpcRemote extends Remote<IExternalApiRpcClient> {
6
+ export class ExternalApiRpcRemote extends RpcRemote<IExternalApiRpcClient> {
7
7
 
8
- async externalFindData(idToFind: Uint8Array): Promise<DataEntry[]> {
8
+ async externalFindData(key: Uint8Array): Promise<DataEntry[]> {
9
9
  const request: ExternalFindDataRequest = {
10
- kademliaId: idToFind
10
+ key
11
11
  }
12
12
  const options = this.formDhtRpcOptions({
13
13
  timeout: 10000
14
14
  })
15
15
  try {
16
16
  const data = await this.getClient().externalFindData(request, options)
17
- return data.dataEntries
17
+ return data.entries
18
18
  } catch (err) {
19
19
  return []
20
20
  }
@@ -0,0 +1,330 @@
1
+ import {
2
+ Logger
3
+ } from '@streamr/utils'
4
+ import KBucket from 'k-bucket'
5
+ import { PeerID, PeerIDKey } from '../helpers/PeerID'
6
+ import {
7
+ getNodeIdFromPeerDescriptor,
8
+ keyFromPeerDescriptor,
9
+ peerIdFromPeerDescriptor
10
+ } from '../helpers/peerIdFromPeerDescriptor'
11
+ import {
12
+ PeerDescriptor
13
+ } from '../proto/packages/dht/protos/DhtRpc'
14
+ import { DhtNodeRpcRemote } from './DhtNodeRpcRemote'
15
+ import { RandomContactList } from './contact/RandomContactList'
16
+ import { SortedContactList } from './contact/SortedContactList'
17
+ import { ConnectionManager } from '../connection/ConnectionManager'
18
+ import EventEmitter from 'eventemitter3'
19
+
20
+ const logger = new Logger(module)
21
+
22
+ interface PeerManagerConfig {
23
+ numberOfNodesPerKBucket: number
24
+ maxContactListSize: number
25
+ peerDiscoveryQueryBatchSize: number
26
+ ownPeerId: PeerID
27
+ connectionManager: ConnectionManager
28
+ isLayer0: boolean
29
+ createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => DhtNodeRpcRemote
30
+ }
31
+
32
+ export interface PeerManagerEvents {
33
+ newContact: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
34
+ contactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
35
+ newRandomContact: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
36
+ randomContactRemoved: (peerDescriptor: PeerDescriptor, closestPeers: PeerDescriptor[]) => void
37
+ kBucketEmpty: () => void
38
+ }
39
+
40
+ export const getDistance = (peerId1: Uint8Array, peerId2: Uint8Array): number => {
41
+ return KBucket.distance(peerId1, peerId2)
42
+ }
43
+
44
+ export class PeerManager extends EventEmitter<PeerManagerEvents> {
45
+
46
+ // Glossary:
47
+ // * 'neighbors' are the nodes that are our neighbors according to
48
+ // the protocol of the layer we are in
49
+ // * 'connections' are the nodes that are connected to this node on Layer0
50
+ // * 'contacts' are all non-unresponsive nodes that we know about
51
+
52
+ // The kademlia k-bucket
53
+ private bucket?: KBucket<DhtNodeRpcRemote>
54
+ // Nodes that are connected to this node on Layer0
55
+ public readonly connections: Map<PeerIDKey, DhtNodeRpcRemote> = new Map()
56
+ // All nodes that we know about
57
+ private contacts?: SortedContactList<DhtNodeRpcRemote>
58
+ private randomPeers?: RandomContactList<DhtNodeRpcRemote>
59
+ private readonly config: PeerManagerConfig
60
+ private stopped: boolean = false
61
+
62
+ constructor(config: PeerManagerConfig) {
63
+ super()
64
+ this.config = config
65
+ this.initKBuckets()
66
+ }
67
+
68
+ private initKBuckets() {
69
+ this.bucket = new KBucket<DhtNodeRpcRemote>({
70
+ localNodeId: this.config.ownPeerId.value,
71
+ numberOfNodesPerKBucket: this.config.numberOfNodesPerKBucket,
72
+ numberOfNodesToPing: this.config.numberOfNodesPerKBucket
73
+ })
74
+
75
+ this.bucket.on('ping', (oldContacts: DhtNodeRpcRemote[], newContact: DhtNodeRpcRemote) => this.onKBucketPing(oldContacts, newContact))
76
+ this.bucket.on('removed', (contact: DhtNodeRpcRemote) => this.onKBucketRemoved(contact))
77
+ this.bucket.on('added', (contact: DhtNodeRpcRemote) => this.onKBucketAdded(contact))
78
+ this.bucket.on('updated', () => {
79
+ // TODO: Update contact info to the connection manager and reconnect
80
+ })
81
+ this.contacts = new SortedContactList({
82
+ referenceId: this.config.ownPeerId,
83
+ maxSize: this.config.maxContactListSize,
84
+ allowToContainReferenceId: false,
85
+ emitEvents: true
86
+ })
87
+ this.contacts.on('contactRemoved', (removedContact: DhtNodeRpcRemote, activeContacts: DhtNodeRpcRemote[]) => {
88
+ if (this.stopped) {
89
+ return
90
+ }
91
+ this.emit('contactRemoved', removedContact.getPeerDescriptor(), activeContacts.map((c) => c.getPeerDescriptor()))
92
+ this.randomPeers!.addContact(this.config.createDhtNodeRpcRemote(removedContact.getPeerDescriptor()))
93
+ })
94
+ this.contacts.on('newContact', (newContact: DhtNodeRpcRemote, activeContacts: DhtNodeRpcRemote[]) =>
95
+ this.emit('newContact', newContact.getPeerDescriptor(), activeContacts.map((c) => c.getPeerDescriptor()))
96
+ )
97
+ this.randomPeers = new RandomContactList(this.config.ownPeerId, this.config.maxContactListSize)
98
+ this.randomPeers.on('contactRemoved', (removedContact: DhtNodeRpcRemote, activeContacts: DhtNodeRpcRemote[]) =>
99
+ this.emit('randomContactRemoved', removedContact.getPeerDescriptor(), activeContacts.map((c) => c.getPeerDescriptor()))
100
+ )
101
+ this.randomPeers.on('newContact', (newContact: DhtNodeRpcRemote, activeContacts: DhtNodeRpcRemote[]) =>
102
+ this.emit('newRandomContact', newContact.getPeerDescriptor(), activeContacts.map((c) => c.getPeerDescriptor()))
103
+ )
104
+ }
105
+
106
+ private onKBucketPing(oldContacts: DhtNodeRpcRemote[], newContact: DhtNodeRpcRemote): void {
107
+ if (this.stopped) {
108
+ return
109
+ }
110
+ const sortingList: SortedContactList<DhtNodeRpcRemote> = new SortedContactList({
111
+ referenceId: this.config.ownPeerId,
112
+ maxSize: 100, // TODO use config option or named constant?
113
+ allowToContainReferenceId: false,
114
+ emitEvents: false
115
+ })
116
+ sortingList.addContacts(oldContacts)
117
+ const sortedContacts = sortingList.getAllContacts()
118
+ this.config.connectionManager?.weakUnlockConnection(sortedContacts[sortedContacts.length - 1].getPeerDescriptor())
119
+ this.bucket?.remove(sortedContacts[sortedContacts.length - 1].getPeerId().value)
120
+ this.bucket!.add(newContact)
121
+ }
122
+
123
+ private onKBucketRemoved(contact: DhtNodeRpcRemote): void {
124
+ if (this.stopped) {
125
+ return
126
+ }
127
+ this.config.connectionManager?.weakUnlockConnection(contact.getPeerDescriptor())
128
+ logger.trace(`Removed contact ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
129
+ if (this.bucket!.count() === 0) {
130
+ this.emit('kBucketEmpty')
131
+ }
132
+ }
133
+
134
+ private onKBucketAdded(contact: DhtNodeRpcRemote): void {
135
+ if (this.stopped) {
136
+ return
137
+ }
138
+ if (!contact.getPeerId().equals(this.config.ownPeerId)) {
139
+ // Important to lock here, before the ping result is known
140
+ this.config.connectionManager?.weakLockConnection(contact.getPeerDescriptor())
141
+ if (this.connections.has(contact.getPeerId().toKey())) {
142
+ logger.trace(`Added new contact ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
143
+ } else { // open connection by pinging
144
+ logger.trace('starting ping ' + getNodeIdFromPeerDescriptor(contact.getPeerDescriptor()))
145
+ contact.ping().then((result) => {
146
+ if (result) {
147
+ logger.trace(`Added new contact ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
148
+ } else {
149
+ logger.trace('ping failed ' + getNodeIdFromPeerDescriptor(contact.getPeerDescriptor()))
150
+ this.config.connectionManager?.weakUnlockConnection(contact.getPeerDescriptor())
151
+ this.removeContact(contact.getPeerDescriptor())
152
+ this.addClosestContactToBucket()
153
+ }
154
+ return
155
+ }).catch((_e) => {
156
+ this.config.connectionManager?.weakUnlockConnection(contact.getPeerDescriptor())
157
+ this.removeContact(contact.getPeerDescriptor())
158
+ this.addClosestContactToBucket()
159
+ })
160
+ }
161
+ }
162
+ }
163
+
164
+ private addClosestContactToBucket(): void {
165
+ if (this.stopped) {
166
+ return
167
+ }
168
+ const closest = this.getClosestActiveContactNotInBucket()
169
+ if (closest) {
170
+ this.handleNewPeers([closest.getPeerDescriptor()])
171
+ }
172
+ }
173
+
174
+ private getClosestActiveContactNotInBucket(): DhtNodeRpcRemote | undefined {
175
+ for (const contactId of this.contacts!.getContactIds()) {
176
+ if (!this.bucket!.get(contactId.value) && this.contacts!.isActive(contactId)) {
177
+ return this.contacts!.getContact(contactId)!.contact
178
+ }
179
+ }
180
+ return undefined
181
+ }
182
+
183
+ handleConnected(peerDescriptor: PeerDescriptor): void {
184
+ if (PeerID.fromValue(peerDescriptor.nodeId).equals(this.config.ownPeerId)) {
185
+ logger.error('handleConnected() to self')
186
+ }
187
+ const rpcRemote = this.config.createDhtNodeRpcRemote(peerDescriptor)
188
+ if (!this.connections.has(PeerID.fromValue(rpcRemote.id).toKey())) {
189
+ this.connections.set(PeerID.fromValue(rpcRemote.id).toKey(), rpcRemote)
190
+ logger.trace('connectionschange add ' + this.connections.size)
191
+ } else {
192
+ logger.trace('new connection not set to connections, there is already a connection with the peer ID')
193
+ }
194
+ logger.trace('connected: ' + getNodeIdFromPeerDescriptor(peerDescriptor) + ' ' + this.connections.size)
195
+ }
196
+
197
+ handleDisconnected(peerDescriptor: PeerDescriptor, gracefulLeave: boolean): void {
198
+ logger.trace('disconnected: ' + getNodeIdFromPeerDescriptor(peerDescriptor))
199
+ this.connections.delete(keyFromPeerDescriptor(peerDescriptor))
200
+ if (this.config.isLayer0) {
201
+ this.bucket!.remove(peerDescriptor.nodeId)
202
+ if (gracefulLeave === true) {
203
+ logger.trace(getNodeIdFromPeerDescriptor(peerDescriptor) + ' ' + 'onTransportDisconnected with gracefulLeave ' + gracefulLeave)
204
+ this.removeContact(peerDescriptor)
205
+ } else {
206
+ logger.trace(getNodeIdFromPeerDescriptor(peerDescriptor) + ' ' + 'onTransportDisconnected with gracefulLeave ' + gracefulLeave)
207
+ }
208
+ }
209
+ }
210
+
211
+ handlePeerLeaving(peerDescriptor: PeerDescriptor): void {
212
+ this.removeContact(peerDescriptor)
213
+ }
214
+
215
+ private removeContact(contact: PeerDescriptor): void {
216
+ if (this.stopped) {
217
+ return
218
+ }
219
+ logger.trace(`Removing contact ${getNodeIdFromPeerDescriptor(contact)}`)
220
+ const peerId = peerIdFromPeerDescriptor(contact)
221
+ this.bucket!.remove(peerId.value)
222
+ this.contacts!.removeContact(peerId)
223
+ this.randomPeers!.removeContact(peerId)
224
+ }
225
+
226
+ stop(): void {
227
+ this.stopped = true
228
+ this.bucket!.toArray().forEach((rpcRemote: DhtNodeRpcRemote) => {
229
+ rpcRemote.leaveNotice()
230
+ this.bucket!.remove(rpcRemote.id)
231
+ })
232
+ this.bucket!.removeAllListeners()
233
+ this.contacts!.stop()
234
+ this.randomPeers!.stop()
235
+ this.connections.clear()
236
+ }
237
+
238
+ getClosestNeighborsTo(kademliaId: Uint8Array, limit?: number, excludeSet?: Set<PeerIDKey>): DhtNodeRpcRemote[] {
239
+ const closest = new SortedContactList<DhtNodeRpcRemote>({
240
+ referenceId: PeerID.fromValue(kademliaId),
241
+ allowToContainReferenceId: true,
242
+ emitEvents: false
243
+ })
244
+ this.bucket!.toArray().map((contact) => closest.addContact(contact))
245
+ // TODO should set the excludeSet and limit to SortedContactList constructor and remove these line
246
+ return closest.getClosestContacts(limit).filter((contact) => {
247
+ if (!excludeSet) {
248
+ return true
249
+ } else {
250
+ return !excludeSet.has(contact.getPeerId().toKey())
251
+ }
252
+ })
253
+ }
254
+
255
+ // TODO reduce copy-paste?
256
+ getClosestContactsTo(kademliaId: Uint8Array, limit?: number, excludeSet?: Set<PeerIDKey>): DhtNodeRpcRemote[] {
257
+ const closest = new SortedContactList<DhtNodeRpcRemote>({
258
+ referenceId: PeerID.fromValue(kademliaId),
259
+ allowToContainReferenceId: true,
260
+ emitEvents: false
261
+ })
262
+ this.contacts!.getAllContacts().map((contact) => closest.addContact(contact))
263
+ // TODO should set the excludeSet and limit to SortedContactList constructor and remove these line
264
+ return closest.getClosestContacts(limit).filter((contact) => {
265
+ if (!excludeSet) {
266
+ return true
267
+ } else {
268
+ return !excludeSet.has(contact.getPeerId().toKey())
269
+ }
270
+ })
271
+ }
272
+
273
+ getNumberOfContacts(excludeSet?: Set<PeerIDKey>): number {
274
+ return this.contacts!.getAllContacts().filter((contact) => {
275
+ if (!excludeSet) {
276
+ return true
277
+ } else {
278
+ return !excludeSet.has(contact.getPeerId().toKey())
279
+ }
280
+ }).length
281
+ }
282
+
283
+ getNumberOfConnections(): number {
284
+ return this.connections.size
285
+ }
286
+
287
+ getNumberOfNeighbors(): number {
288
+ return this.bucket!.count()
289
+ }
290
+
291
+ getNeighbors(): PeerDescriptor[] {
292
+ return this.bucket!.toArray().map((rpcRemote: DhtNodeRpcRemote) => rpcRemote.getPeerDescriptor())
293
+ }
294
+
295
+ handlePeerActive(peerId: PeerID): void {
296
+ this.contacts!.setActive(peerId)
297
+ }
298
+
299
+ handlePeerUnresponsive(peerId: PeerID): void {
300
+ this.bucket!.remove(peerId.value)
301
+ this.contacts!.removeContact(peerId)
302
+ }
303
+
304
+ handleNewPeers(peerDescriptors: PeerDescriptor[], setActive?: boolean): void {
305
+ if (this.stopped) {
306
+ return
307
+ }
308
+ peerDescriptors.forEach((contact) => {
309
+ const peerId = peerIdFromPeerDescriptor(contact)
310
+ if (!peerId.equals(this.config.ownPeerId)) {
311
+ logger.trace(`Adding new contact ${getNodeIdFromPeerDescriptor(contact)}`)
312
+ const remote = this.config.createDhtNodeRpcRemote(contact)
313
+ const isInBucket = (this.bucket!.get(contact.nodeId) !== null)
314
+ const isInNeighborList = (this.contacts!.getContact(peerId) !== undefined)
315
+ if (isInBucket || isInNeighborList) {
316
+ this.randomPeers!.addContact(remote)
317
+ }
318
+ if (!isInBucket) {
319
+ this.bucket!.add(remote)
320
+ }
321
+ if (!isInNeighborList) {
322
+ this.contacts!.addContact(remote)
323
+ }
324
+ if (setActive) {
325
+ this.contacts!.setActive(peerId)
326
+ }
327
+ }
328
+ })
329
+ }
330
+ }
@@ -19,6 +19,7 @@ export interface Events<C> {
19
19
  export class ContactList<C extends { getPeerId: () => PeerID }> extends EventEmitter<Events<C>> {
20
20
 
21
21
  protected contactsById: Map<PeerIDKey, ContactState<C>> = new Map()
22
+ // TODO move this to SortedContactList
22
23
  protected contactIds: PeerID[] = []
23
24
  protected ownId: PeerID
24
25
  protected maxSize: number
@@ -35,8 +36,8 @@ export class ContactList<C extends { getPeerId: () => PeerID }> extends EventEmi
35
36
  this.defaultContactQueryLimit = defaultContactQueryLimit
36
37
  }
37
38
 
38
- public getContact(id: PeerID): ContactState<C> {
39
- return this.contactsById.get(id.toKey())!
39
+ public getContact(id: PeerID): ContactState<C> | undefined {
40
+ return this.contactsById.get(id.toKey())
40
41
  }
41
42
 
42
43
  public getSize(): number {
@@ -15,7 +15,7 @@ const WEBRTC_TIMEOUT = 15000
15
15
  // default timeout for existing connections
16
16
  export const EXISTING_CONNECTION_TIMEOUT = 5000
17
17
 
18
- const getRpcTimeout = (localPeerDescriptor: PeerDescriptor, remotePeerDescriptor: PeerDescriptor): number => {
18
+ const getTimeout = (localPeerDescriptor: PeerDescriptor, remotePeerDescriptor: PeerDescriptor): number => {
19
19
  const connectionType = expectedConnectionType(localPeerDescriptor, remotePeerDescriptor)
20
20
  if (connectionType === ConnectionType.WEBSOCKET_CLIENT) {
21
21
  return WEBSOCKET_CLIENT_TIMEOUT
@@ -27,25 +27,26 @@ const getRpcTimeout = (localPeerDescriptor: PeerDescriptor, remotePeerDescriptor
27
27
  return WEBRTC_TIMEOUT
28
28
  }
29
29
 
30
- export abstract class Remote<T> {
30
+ export abstract class RpcRemote<T> {
31
31
 
32
32
  private readonly localPeerDescriptor: PeerDescriptor
33
33
  private readonly remotePeerDescriptor: PeerDescriptor
34
34
  private readonly serviceId: ServiceID
35
35
  private readonly client: ProtoRpcClient<T>
36
- private readonly rpcTimeout?: number
36
+ private readonly timeout?: number
37
+
37
38
  constructor(
38
39
  localPeerDescriptor: PeerDescriptor,
39
40
  remotePeerDescriptor: PeerDescriptor,
40
41
  serviceId: ServiceID,
41
42
  client: ProtoRpcClient<T>,
42
- rpcTimeout?: number
43
+ timeout?: number
43
44
  ) {
44
45
  this.localPeerDescriptor = localPeerDescriptor
45
46
  this.remotePeerDescriptor = remotePeerDescriptor
46
47
  this.client = client
47
48
  this.serviceId = serviceId
48
- this.rpcTimeout = rpcTimeout
49
+ this.timeout = timeout
49
50
  }
50
51
 
51
52
  getPeerDescriptor(): PeerDescriptor {
@@ -68,7 +69,7 @@ export abstract class Remote<T> {
68
69
  return {
69
70
  sourceDescriptor: this.localPeerDescriptor,
70
71
  targetDescriptor: this.remotePeerDescriptor,
71
- timeout: this.rpcTimeout ?? getRpcTimeout(this.localPeerDescriptor, this.remotePeerDescriptor),
72
+ timeout: this.timeout ?? getTimeout(this.localPeerDescriptor, this.remotePeerDescriptor),
72
73
  ...opts
73
74
  }
74
75
  }