@streamr/dht 100.2.1 → 100.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/dist/package.json +6 -6
  2. package/dist/src/connection/ConnectionLockStates.js +4 -6
  3. package/dist/src/connection/ConnectionLockStates.js.map +1 -1
  4. package/dist/src/connection/ConnectionManager.d.ts +1 -0
  5. package/dist/src/connection/ConnectionManager.js +8 -2
  6. package/dist/src/connection/ConnectionManager.js.map +1 -1
  7. package/dist/src/connection/ManagedConnection.js +7 -9
  8. package/dist/src/connection/ManagedConnection.js.map +1 -1
  9. package/dist/src/connection/connectivityChecker.js +0 -3
  10. package/dist/src/connection/connectivityChecker.js.map +1 -1
  11. package/dist/src/connection/websocket/WebsocketConnector.js +1 -1
  12. package/dist/src/connection/websocket/WebsocketConnector.js.map +1 -1
  13. package/dist/src/dht/DhtNode.d.ts +10 -7
  14. package/dist/src/dht/DhtNode.js +51 -32
  15. package/dist/src/dht/DhtNode.js.map +1 -1
  16. package/dist/src/dht/DhtNodeRpcLocal.d.ts +2 -2
  17. package/dist/src/dht/DhtNodeRpcLocal.js +5 -4
  18. package/dist/src/dht/DhtNodeRpcLocal.js.map +1 -1
  19. package/dist/src/dht/DhtNodeRpcRemote.js +1 -0
  20. package/dist/src/dht/DhtNodeRpcRemote.js.map +1 -1
  21. package/dist/src/dht/PeerManager.d.ts +14 -14
  22. package/dist/src/dht/PeerManager.js +23 -59
  23. package/dist/src/dht/PeerManager.js.map +1 -1
  24. package/dist/src/dht/contact/ContactList.d.ts +2 -2
  25. package/dist/src/dht/contact/RandomContactList.js +2 -2
  26. package/dist/src/dht/contact/RandomContactList.js.map +1 -1
  27. package/dist/src/dht/contact/RingContactList.d.ts +2 -5
  28. package/dist/src/dht/contact/RingContactList.js +3 -11
  29. package/dist/src/dht/contact/RingContactList.js.map +1 -1
  30. package/dist/src/dht/contact/SortedContactList.d.ts +4 -1
  31. package/dist/src/dht/contact/SortedContactList.js +4 -5
  32. package/dist/src/dht/contact/SortedContactList.js.map +1 -1
  33. package/dist/src/dht/contact/getClosestContacts.d.ts +7 -0
  34. package/dist/src/dht/contact/getClosestContacts.js +18 -0
  35. package/dist/src/dht/contact/getClosestContacts.js.map +1 -0
  36. package/dist/src/dht/discovery/DiscoverySession.d.ts +8 -8
  37. package/dist/src/dht/discovery/DiscoverySession.js +30 -35
  38. package/dist/src/dht/discovery/DiscoverySession.js.map +1 -1
  39. package/dist/src/dht/discovery/PeerDiscovery.d.ts +1 -2
  40. package/dist/src/dht/discovery/PeerDiscovery.js +6 -8
  41. package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
  42. package/dist/src/dht/discovery/RingDiscoverySession.d.ts +4 -4
  43. package/dist/src/dht/discovery/RingDiscoverySession.js +13 -13
  44. package/dist/src/dht/discovery/RingDiscoverySession.js.map +1 -1
  45. package/dist/src/dht/recursive-operation/RecursiveOperationManager.d.ts +2 -2
  46. package/dist/src/dht/recursive-operation/RecursiveOperationManager.js +15 -15
  47. package/dist/src/dht/recursive-operation/RecursiveOperationManager.js.map +1 -1
  48. package/dist/src/dht/recursive-operation/RecursiveOperationSession.d.ts +1 -1
  49. package/dist/src/dht/recursive-operation/RecursiveOperationSession.js +11 -13
  50. package/dist/src/dht/recursive-operation/RecursiveOperationSession.js.map +1 -1
  51. package/dist/src/dht/recursive-operation/RecursiveOperationSessionRpcLocal.js +1 -1
  52. package/dist/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.d.ts +1 -1
  53. package/dist/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.js +2 -2
  54. package/dist/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.js.map +1 -1
  55. package/dist/src/dht/routing/Router.d.ts +1 -2
  56. package/dist/src/dht/routing/Router.js +2 -3
  57. package/dist/src/dht/routing/Router.js.map +1 -1
  58. package/dist/src/dht/routing/RoutingSession.d.ts +1 -2
  59. package/dist/src/dht/routing/RoutingSession.js +2 -2
  60. package/dist/src/dht/routing/RoutingSession.js.map +1 -1
  61. package/dist/src/dht/store/StoreRpcLocal.js +3 -3
  62. package/dist/src/dht/store/StoreRpcLocal.js.map +1 -1
  63. package/dist/src/proto/packages/dht/protos/DhtRpc.client.d.ts +8 -0
  64. package/dist/src/proto/packages/dht/protos/DhtRpc.client.js +4 -0
  65. package/dist/src/proto/packages/dht/protos/DhtRpc.client.js.map +1 -1
  66. package/dist/src/proto/packages/dht/protos/DhtRpc.d.ts +10 -2
  67. package/dist/src/proto/packages/dht/protos/DhtRpc.js +1 -1
  68. package/dist/src/proto/packages/dht/protos/DhtRpc.js.map +1 -1
  69. package/dist/src/proto/packages/dht/protos/DhtRpc.server.d.ts +4 -0
  70. package/dist/src/transport/ITransport.d.ts +3 -0
  71. package/dist/src/transport/ITransport.js.map +1 -1
  72. package/package.json +6 -6
  73. package/protos/DhtRpc.proto +7 -1
  74. package/src/connection/ConnectionLockStates.ts +3 -5
  75. package/src/connection/ConnectionManager.ts +9 -2
  76. package/src/connection/ManagedConnection.ts +11 -14
  77. package/src/connection/connectivityChecker.ts +0 -2
  78. package/src/connection/webrtc/BrowserWebrtcConnection.ts +0 -1
  79. package/src/connection/websocket/WebsocketConnector.ts +1 -1
  80. package/src/dht/DhtNode.ts +63 -44
  81. package/src/dht/DhtNodeRpcLocal.ts +7 -6
  82. package/src/dht/DhtNodeRpcRemote.ts +1 -0
  83. package/src/dht/PeerManager.ts +38 -73
  84. package/src/dht/contact/ContactList.ts +2 -2
  85. package/src/dht/contact/RandomContactList.ts +2 -3
  86. package/src/dht/contact/RingContactList.ts +6 -16
  87. package/src/dht/contact/SortedContactList.ts +9 -10
  88. package/src/dht/contact/getClosestContacts.ts +22 -0
  89. package/src/dht/discovery/DiscoverySession.ts +35 -45
  90. package/src/dht/discovery/PeerDiscovery.ts +6 -9
  91. package/src/dht/discovery/RingDiscoverySession.ts +13 -13
  92. package/src/dht/recursive-operation/RecursiveOperationManager.ts +16 -16
  93. package/src/dht/recursive-operation/RecursiveOperationSession.ts +11 -13
  94. package/src/dht/recursive-operation/RecursiveOperationSessionRpcLocal.ts +1 -1
  95. package/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.ts +2 -2
  96. package/src/dht/routing/Router.ts +3 -5
  97. package/src/dht/routing/RoutingSession.ts +3 -4
  98. package/src/dht/store/StoreRpcLocal.ts +3 -3
  99. package/src/proto/packages/dht/protos/DhtRpc.client.ts +8 -0
  100. package/src/proto/packages/dht/protos/DhtRpc.server.ts +4 -0
  101. package/src/proto/packages/dht/protos/DhtRpc.ts +11 -3
  102. package/src/transport/ITransport.ts +3 -0
  103. package/test/end-to-end/Layer0MixedConnectionTypes.test.ts +3 -5
  104. package/test/integration/DhtNode.test.ts +81 -0
  105. package/test/integration/Layer1-scale.test.ts +1 -1
  106. package/test/integration/ScaleDownDht.test.ts +7 -4
  107. package/test/unit/DiscoverySession.test.ts +85 -0
  108. package/test/unit/PeerManager.test.ts +3 -17
  109. package/test/unit/RecursiveOperationManager.test.ts +5 -3
  110. package/test/unit/Router.test.ts +2 -2
  111. package/test/unit/RoutingSession.test.ts +2 -2
  112. package/test/unit/SortedContactList.test.ts +4 -4
  113. package/test/unit/getClosestContacts.test.ts +28 -0
  114. package/test/utils/FakeTransport.ts +34 -7
  115. package/test/utils/mock/Router.ts +0 -3
  116. package/test/utils/mock/Transport.ts +10 -0
  117. package/test/utils/topology.ts +80 -0
  118. package/test/RandomGraphSimulation.ts +0 -53
@@ -3,7 +3,6 @@ import { RoutingMode, RoutingRemoteContact, RoutingSession, RoutingSessionEvents
3
3
  import { Logger, executeSafePromise, raceEvents3, withTimeout } from '@streamr/utils'
4
4
  import { RoutingRpcCommunicator } from '../../transport/RoutingRpcCommunicator'
5
5
  import { DuplicateDetector } from './DuplicateDetector'
6
- import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
7
6
  import { v4 } from 'uuid'
8
7
  import { RouterRpcLocal, createRouteMessageAck } from './RouterRpcLocal'
9
8
  import { DhtAddress, areEqualPeerDescriptors, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor } from '../../identifiers'
@@ -12,8 +11,8 @@ import { RoutingTablesCache } from './RoutingTablesCache'
12
11
  export interface RouterConfig {
13
12
  rpcCommunicator: RoutingRpcCommunicator
14
13
  localPeerDescriptor: PeerDescriptor
15
- connections: Map<DhtAddress, DhtNodeRpcRemote>
16
14
  handleMessage: (message: Message) => void
15
+ getConnections: () => PeerDescriptor[]
17
16
  }
18
17
 
19
18
  interface ForwardingTableEntry {
@@ -158,17 +157,16 @@ export class Router {
158
157
  routedMessage.parallelRootNodeIds.forEach((nodeId) => {
159
158
  excludedNodeIds.add(nodeId as DhtAddress)
160
159
  })
161
- logger.trace('routing session created with connections: ' + this.config.connections.size)
162
160
  return new RoutingSession({
163
161
  rpcCommunicator: this.config.rpcCommunicator,
164
162
  localPeerDescriptor: this.config.localPeerDescriptor,
165
163
  routedMessage,
166
- connections: this.config.connections,
167
164
  // TODO use config option or named constant?
168
165
  parallelism: areEqualPeerDescriptors(this.config.localPeerDescriptor, routedMessage.sourcePeer!) ? 2 : 1,
169
166
  mode,
170
167
  excludedNodeIds,
171
- routingTablesCache: this.routingTablesCache
168
+ routingTablesCache: this.routingTablesCache,
169
+ getConnections: this.config.getConnections
172
170
  })
173
171
  }
174
172
 
@@ -1,4 +1,3 @@
1
- import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
2
1
  import { SortedContactList } from '../contact/SortedContactList'
3
2
  import { Logger } from '@streamr/utils'
4
3
  import EventEmitter from 'eventemitter3'
@@ -69,11 +68,11 @@ interface RoutingSessionConfig {
69
68
  rpcCommunicator: RoutingRpcCommunicator
70
69
  localPeerDescriptor: PeerDescriptor
71
70
  routedMessage: RouteMessageWrapper
72
- connections: Map<DhtAddress, DhtNodeRpcRemote>
73
71
  parallelism: number
74
72
  mode: RoutingMode
75
73
  excludedNodeIds: Set<DhtAddress>
76
74
  routingTablesCache: RoutingTablesCache
75
+ getConnections: () => PeerDescriptor[]
77
76
  }
78
77
 
79
78
  export class RoutingSession extends EventEmitter<RoutingSessionEvents> {
@@ -174,9 +173,9 @@ export class RoutingSession extends EventEmitter<RoutingSessionEvents> {
174
173
  allowToContainReferenceId: true,
175
174
  nodeIdDistanceLimit: previousId
176
175
  })
177
- const contacts = Array.from(this.config.connections.values())
176
+ const contacts = this.config.getConnections()
178
177
  .map((peer) => new RoutingRemoteContact(
179
- peer.getPeerDescriptor(),
178
+ peer,
180
179
  this.config.localPeerDescriptor,
181
180
  this.config.rpcCommunicator
182
181
  ))
@@ -32,7 +32,7 @@ export class StoreRpcLocal implements IStoreRpc {
32
32
  async storeData(request: StoreDataRequest): Promise<StoreDataResponse> {
33
33
  logger.trace('storeData()')
34
34
  const key = getDhtAddressFromRaw(request.key)
35
- const selfIsOneOfClosestPeers = this.config.selfIsWithinRedundancyFactor(key)
35
+ const selfIsWithinRedundancyFactor = this.config.selfIsWithinRedundancyFactor(key)
36
36
  this.config.localDataStore.storeEntry({
37
37
  key: request.key,
38
38
  data: request.data,
@@ -40,10 +40,10 @@ export class StoreRpcLocal implements IStoreRpc {
40
40
  createdAt: request.createdAt,
41
41
  storedAt: Timestamp.now(),
42
42
  ttl: request.ttl,
43
- stale: !selfIsOneOfClosestPeers,
43
+ stale: !selfIsWithinRedundancyFactor,
44
44
  deleted: false
45
45
  })
46
- if (!selfIsOneOfClosestPeers) {
46
+ if (!selfIsWithinRedundancyFactor) {
47
47
  this.config.localDataStore.setAllEntriesAsStale(key)
48
48
  }
49
49
  return {}
@@ -47,10 +47,14 @@ import type { RpcOptions } from "@protobuf-ts/runtime-rpc";
47
47
  */
48
48
  export interface IDhtNodeRpcClient {
49
49
  /**
50
+ * TODO rename to getClosestNeighbors (breaking change)
51
+ *
50
52
  * @generated from protobuf rpc: getClosestPeers(dht.ClosestPeersRequest) returns (dht.ClosestPeersResponse);
51
53
  */
52
54
  getClosestPeers(input: ClosestPeersRequest, options?: RpcOptions): UnaryCall<ClosestPeersRequest, ClosestPeersResponse>;
53
55
  /**
56
+ * TODO rename to getClosestRingContacts (breaking change)
57
+ *
54
58
  * @generated from protobuf rpc: getClosestRingPeers(dht.ClosestRingPeersRequest) returns (dht.ClosestRingPeersResponse);
55
59
  */
56
60
  getClosestRingPeers(input: ClosestRingPeersRequest, options?: RpcOptions): UnaryCall<ClosestRingPeersRequest, ClosestRingPeersResponse>;
@@ -73,6 +77,8 @@ export class DhtNodeRpcClient implements IDhtNodeRpcClient, ServiceInfo {
73
77
  constructor(private readonly _transport: RpcTransport) {
74
78
  }
75
79
  /**
80
+ * TODO rename to getClosestNeighbors (breaking change)
81
+ *
76
82
  * @generated from protobuf rpc: getClosestPeers(dht.ClosestPeersRequest) returns (dht.ClosestPeersResponse);
77
83
  */
78
84
  getClosestPeers(input: ClosestPeersRequest, options?: RpcOptions): UnaryCall<ClosestPeersRequest, ClosestPeersResponse> {
@@ -80,6 +86,8 @@ export class DhtNodeRpcClient implements IDhtNodeRpcClient, ServiceInfo {
80
86
  return stackIntercept<ClosestPeersRequest, ClosestPeersResponse>("unary", this._transport, method, opt, input);
81
87
  }
82
88
  /**
89
+ * TODO rename to getClosestRingContacts (breaking change)
90
+ *
83
91
  * @generated from protobuf rpc: getClosestRingPeers(dht.ClosestRingPeersRequest) returns (dht.ClosestRingPeersResponse);
84
92
  */
85
93
  getClosestRingPeers(input: ClosestRingPeersRequest, options?: RpcOptions): UnaryCall<ClosestRingPeersRequest, ClosestRingPeersResponse> {
@@ -34,10 +34,14 @@ import { ServerCallContext } from "@protobuf-ts/runtime-rpc";
34
34
  */
35
35
  export interface IDhtNodeRpc<T = ServerCallContext> {
36
36
  /**
37
+ * TODO rename to getClosestNeighbors (breaking change)
38
+ *
37
39
  * @generated from protobuf rpc: getClosestPeers(dht.ClosestPeersRequest) returns (dht.ClosestPeersResponse);
38
40
  */
39
41
  getClosestPeers(request: ClosestPeersRequest, context: T): Promise<ClosestPeersResponse>;
40
42
  /**
43
+ * TODO rename to getClosestRingContacts (breaking change)
44
+ *
41
45
  * @generated from protobuf rpc: getClosestRingPeers(dht.ClosestRingPeersRequest) returns (dht.ClosestRingPeersResponse);
42
46
  */
43
47
  getClosestRingPeers(request: ClosestRingPeersRequest, context: T): Promise<ClosestRingPeersResponse>;
@@ -108,6 +108,8 @@ export interface DataEntry {
108
108
  deleted: boolean;
109
109
  }
110
110
  /**
111
+ * TODO rename to ClosestNeighborsRequest
112
+ *
111
113
  * @generated from protobuf message dht.ClosestPeersRequest
112
114
  */
113
115
  export interface ClosestPeersRequest {
@@ -121,6 +123,8 @@ export interface ClosestPeersRequest {
121
123
  requestId: string;
122
124
  }
123
125
  /**
126
+ * TODO rename to ClosestPeersResponse
127
+ *
124
128
  * @generated from protobuf message dht.ClosestPeersResponse
125
129
  */
126
130
  export interface ClosestPeersResponse {
@@ -134,6 +138,8 @@ export interface ClosestPeersResponse {
134
138
  requestId: string;
135
139
  }
136
140
  /**
141
+ * TODO rename to ClosestRingContactsRequest
142
+ *
137
143
  * @generated from protobuf message dht.ClosestRingPeersRequest
138
144
  */
139
145
  export interface ClosestRingPeersRequest {
@@ -147,6 +153,8 @@ export interface ClosestRingPeersRequest {
147
153
  requestId: string;
148
154
  }
149
155
  /**
156
+ * TODO rename to ClosestRingContactsResponse
157
+ *
150
158
  * @generated from protobuf message dht.ClosestRingPeersResponse
151
159
  */
152
160
  export interface ClosestRingPeersResponse {
@@ -181,9 +189,9 @@ export interface RecursiveOperationRequest {
181
189
  */
182
190
  export interface RecursiveOperationResponse {
183
191
  /**
184
- * @generated from protobuf field: repeated dht.PeerDescriptor closestConnectedPeers = 1;
192
+ * @generated from protobuf field: repeated dht.PeerDescriptor closestConnectedNodes = 1;
185
193
  */
186
- closestConnectedPeers: PeerDescriptor[];
194
+ closestConnectedNodes: PeerDescriptor[];
187
195
  /**
188
196
  * @generated from protobuf field: repeated dht.DataEntry dataEntries = 2;
189
197
  */
@@ -837,7 +845,7 @@ export const RecursiveOperationRequest = new RecursiveOperationRequest$Type();
837
845
  class RecursiveOperationResponse$Type extends MessageType<RecursiveOperationResponse> {
838
846
  constructor() {
839
847
  super("dht.RecursiveOperationResponse", [
840
- { no: 1, name: "closestConnectedPeers", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => PeerDescriptor },
848
+ { no: 1, name: "closestConnectedNodes", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => PeerDescriptor },
841
849
  { no: 2, name: "dataEntries", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => DataEntry },
842
850
  { no: 3, name: "noCloserNodesFound", kind: "scalar", T: 8 /*ScalarType.BOOL*/ },
843
851
  { no: 4, name: "routingPath", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => PeerDescriptor }
@@ -1,3 +1,4 @@
1
+ import { DhtAddress } from '../identifiers'
1
2
  import { Message, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
2
3
 
3
4
  export interface TransportEvents {
@@ -33,5 +34,7 @@ export interface ITransport {
33
34
  send(msg: Message, opts?: SendOptions): Promise<void>
34
35
  getLocalPeerDescriptor(): PeerDescriptor
35
36
  getConnections(): PeerDescriptor[]
37
+ getConnectionCount(): number
38
+ hasConnection(nodeId: DhtAddress): boolean
36
39
  stop(): void | Promise<void>
37
40
  }
@@ -72,15 +72,13 @@ describe('Layer0MixedConnectionTypes', () => {
72
72
 
73
73
  it('2 non-server peers join first', async () => {
74
74
 
75
- const promise = Promise.all([
75
+ await Promise.all([
76
76
  waitForEvent3<TransportEvents>((node3.getTransport() as ConnectionManager), 'connected'),
77
77
  waitForEvent3<TransportEvents>((node4.getTransport() as ConnectionManager), 'connected'),
78
+ node3.joinDht([epPeerDescriptor]),
79
+ node4.joinDht([epPeerDescriptor])
78
80
  ])
79
81
 
80
- node3.joinDht([epPeerDescriptor])
81
- node4.joinDht([epPeerDescriptor])
82
-
83
- await promise
84
82
  await Promise.all([
85
83
  node1.joinDht([epPeerDescriptor]),
86
84
  node2.joinDht([epPeerDescriptor]),
@@ -0,0 +1,81 @@
1
+ import { waitForCondition } from '@streamr/utils'
2
+ import { range, without } from 'lodash'
3
+ import { DhtNodeRpcLocal } from '../../src/dht/DhtNodeRpcLocal'
4
+ import { Contact } from '../../src/dht/contact/Contact'
5
+ import { SortedContactList } from '../../src/dht/contact/SortedContactList'
6
+ import { DhtAddress, DhtNode, ListeningRpcCommunicator, getNodeIdFromPeerDescriptor } from '../../src/exports'
7
+ import { ClosestPeersRequest, ClosestPeersResponse, PeerDescriptor, PingRequest, PingResponse } from '../../src/proto/packages/dht/protos/DhtRpc'
8
+ import { FakeEnvironment } from '../utils/FakeTransport'
9
+ import { createMockPeerDescriptor } from '../utils/utils'
10
+
11
+ const OTHER_NODE_COUNT = 3
12
+ const SERVICE_ID_LAYER0 = 'layer0'
13
+
14
+ const getClosestNodes = (
15
+ referenceId: DhtAddress,
16
+ nodes: PeerDescriptor[],
17
+ maxCount: number,
18
+ allowToContainReferenceId: boolean
19
+ ): PeerDescriptor[] => {
20
+ const list = new SortedContactList<Contact>({
21
+ referenceId,
22
+ allowToContainReferenceId,
23
+ maxSize: maxCount
24
+ })
25
+ list.addContacts(nodes.map((n) => new Contact(n)))
26
+ return list.getClosestContacts().map((c) => c.getPeerDescriptor())
27
+ }
28
+
29
+ describe('DhtNode', () => {
30
+
31
+ let localPeerDescriptor: PeerDescriptor
32
+ let entryPointPeerDescriptor: PeerDescriptor
33
+ let otherPeerDescriptors: PeerDescriptor[]
34
+
35
+ const startRemoteNode = (peerDescriptor: PeerDescriptor, environment: FakeEnvironment) => {
36
+ const epRpcCommunicator = new ListeningRpcCommunicator(SERVICE_ID_LAYER0, environment.createTransport(peerDescriptor))
37
+ const dhtNodeRpcLocal = new DhtNodeRpcLocal({
38
+ peerDiscoveryQueryBatchSize: undefined as any,
39
+ getClosestNeighborsTo: (nodeId: DhtAddress, maxCount: number) => getClosestNodes(nodeId, getAllPeerDescriptors(), maxCount, true),
40
+ getClosestRingContactsTo: undefined as any,
41
+ addContact: () => {},
42
+ removeContact: undefined as any,
43
+ })
44
+ epRpcCommunicator.registerRpcMethod(PingRequest, PingResponse, 'ping',
45
+ (req: PingRequest, context) => dhtNodeRpcLocal.ping(req, context))
46
+ epRpcCommunicator.registerRpcMethod(ClosestPeersRequest, ClosestPeersResponse, 'getClosestPeers',
47
+ (req: ClosestPeersRequest, context) => dhtNodeRpcLocal.getClosestPeers(req, context))
48
+ }
49
+
50
+ const getAllPeerDescriptors = () => {
51
+ return [localPeerDescriptor, entryPointPeerDescriptor, ...otherPeerDescriptors]
52
+ }
53
+
54
+ beforeAll(() => {
55
+ localPeerDescriptor = createMockPeerDescriptor()
56
+ entryPointPeerDescriptor = createMockPeerDescriptor()
57
+ otherPeerDescriptors = range(OTHER_NODE_COUNT).map(() => createMockPeerDescriptor())
58
+ })
59
+
60
+ it('start node and join DHT', async () => {
61
+ const environment = new FakeEnvironment()
62
+ startRemoteNode(entryPointPeerDescriptor, environment)
63
+ for (const other of otherPeerDescriptors) {
64
+ startRemoteNode(other, environment)
65
+ }
66
+
67
+ const localNode = new DhtNode({
68
+ peerDescriptor: localPeerDescriptor,
69
+ transport: environment.createTransport(localPeerDescriptor),
70
+ entryPoints: [entryPointPeerDescriptor]
71
+ })
72
+ await localNode.start()
73
+ await localNode.joinDht([entryPointPeerDescriptor])
74
+ await localNode.waitForNetworkConnectivity()
75
+
76
+ await waitForCondition(() => localNode.getNeighborCount() === otherPeerDescriptors.length + 1)
77
+ const expectedNodeIds = without(getAllPeerDescriptors(), localPeerDescriptor).map((n) => getNodeIdFromPeerDescriptor(n))
78
+ const actualNodeIds = localNode.getClosestContacts().map((n) => getNodeIdFromPeerDescriptor(n))
79
+ expect(actualNodeIds).toIncludeSameMembers(expectedNodeIds)
80
+ })
81
+ })
@@ -38,8 +38,8 @@ describe('Layer1', () => {
38
38
  }, 30000)
39
39
 
40
40
  afterEach(async () => {
41
- await Promise.all(nodes.map((node) => node.stop()))
42
41
  await Promise.all(layer1CleanUp.map((node) => node.stop()))
42
+ await Promise.all(nodes.map((node) => node.stop()))
43
43
  await layer0EntryPoint.stop()
44
44
  simulator.stop()
45
45
  })
@@ -4,7 +4,7 @@ import { NodeType, PeerDescriptor } from '../../src/proto/packages/dht/protos/Dh
4
4
  import { createMockConnectionDhtNode } from '../utils/utils'
5
5
  import { Logger } from '@streamr/utils'
6
6
  import { getRandomRegion } from '../../src/connection/simulator/pings'
7
- import { areEqualPeerDescriptors, createRandomDhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../src/identifiers'
7
+ import { DhtAddress, areEqualPeerDescriptors, createRandomDhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../src/identifiers'
8
8
 
9
9
  const logger = new Logger(module)
10
10
 
@@ -47,13 +47,16 @@ describe('Scaling down a Dht network', () => {
47
47
  for (let i = 1; i < nodes.length; i++) {
48
48
  randomIndices.push(i)
49
49
  }
50
+ const stoppedNodes: Set<DhtAddress> = new Set()
50
51
  while (randomIndices.length > 1) {
51
52
  const index = Math.floor(Math.random() * randomIndices.length)
52
53
  const nodeIndex = randomIndices[index]
53
54
  randomIndices.splice(index, 1)
54
- const stoppingPeerDescriptor = nodes[nodeIndex].getLocalPeerDescriptor()
55
- await nodes[nodeIndex].stop()
56
- const nodeIsCleaned = nodes.every((node) =>
55
+ const nodeToStop = nodes[nodeIndex]
56
+ const stoppingPeerDescriptor = nodeToStop.getLocalPeerDescriptor()
57
+ stoppedNodes.add(getNodeIdFromPeerDescriptor(stoppingPeerDescriptor))
58
+ await nodeToStop.stop()
59
+ const nodeIsCleaned = nodes.filter((node) => !stoppedNodes.has(node.getNodeId())).every((node) =>
57
60
  node.getConnections().every((peer) => {
58
61
  if (areEqualPeerDescriptors(peer, stoppingPeerDescriptor)) {
59
62
  logger.error(getNodeIdFromPeerDescriptor(node.getLocalPeerDescriptor()) + ', '
@@ -0,0 +1,85 @@
1
+ import { Multimap, wait } from '@streamr/utils'
2
+ import { sampleSize } from 'lodash'
3
+ import { DhtNodeRpcRemote } from '../../src/dht/DhtNodeRpcRemote'
4
+ import { PeerManager, getDistance } from '../../src/dht/PeerManager'
5
+ import { DiscoverySession } from '../../src/dht/discovery/DiscoverySession'
6
+ import { DhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../src/identifiers'
7
+ import { NodeType, PeerDescriptor } from '../../src/proto/packages/dht/protos/DhtRpc'
8
+ import { createTestTopology } from '../utils/topology'
9
+
10
+ const NODE_COUNT = 40
11
+ const MIN_NEIGHBOR_COUNT = 2 // nodes can get more neighbors when we merge network partitions
12
+ const PARALLELISM = 1
13
+ const NO_PROGRESS_LIMIT = 1
14
+ const QUERY_BATCH_SIZE = 5 // the default value in DhtNode's config, not relevant in this test
15
+
16
+ const createPeerDescriptor = (nodeId: DhtAddress): PeerDescriptor => {
17
+ return {
18
+ nodeId: getRawFromDhtAddress(nodeId),
19
+ type: NodeType.NODEJS
20
+ }
21
+ }
22
+
23
+ describe('DiscoverySession', () => {
24
+
25
+ let topology: Multimap<DhtAddress, DhtAddress>
26
+ const queriedNodes: DhtAddress[] = []
27
+
28
+ beforeAll(() => {
29
+ topology = createTestTopology(NODE_COUNT, MIN_NEIGHBOR_COUNT)
30
+ })
31
+
32
+ const createPeerManager = (localNodeId: DhtAddress): PeerManager => {
33
+ const peerManager = new PeerManager({
34
+ localNodeId,
35
+ localPeerDescriptor: createPeerDescriptor(localNodeId),
36
+ isLayer0: true,
37
+ createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => createMockRpcRemote(peerDescriptor) as any,
38
+ hasConnection: () => true
39
+ } as any)
40
+ for (const neighbor of topology.get(localNodeId)) {
41
+ peerManager.addContact(createPeerDescriptor(neighbor))
42
+ }
43
+ return peerManager
44
+ }
45
+
46
+ const createMockRpcRemote = (peerDescriptor: PeerDescriptor): Partial<DhtNodeRpcRemote> => {
47
+ const nodeId = getNodeIdFromPeerDescriptor(peerDescriptor)
48
+ return {
49
+ id: getRawFromDhtAddress(nodeId),
50
+ getPeerDescriptor: () => peerDescriptor,
51
+ getNodeId: () => nodeId,
52
+ getClosestPeers: async (referenceId: DhtAddress) => {
53
+ queriedNodes.push(nodeId)
54
+ await wait(10)
55
+ const peerManager = createPeerManager(nodeId)
56
+ return peerManager.getClosestNeighborsTo(referenceId, QUERY_BATCH_SIZE).map((remote) => remote.getPeerDescriptor())
57
+ },
58
+ ping: async () => true
59
+ }
60
+ }
61
+
62
+ it('happy path', async () => {
63
+ const nodeIds = [...topology.keys()]
64
+ const [localNodeId, targetId] = sampleSize(nodeIds, 2)
65
+ const contactedPeers = new Set<DhtAddress>()
66
+ const peerManager = createPeerManager(localNodeId)
67
+ const session = new DiscoverySession({
68
+ targetId,
69
+ parallelism: PARALLELISM,
70
+ noProgressLimit: NO_PROGRESS_LIMIT,
71
+ peerManager,
72
+ contactedPeers,
73
+ abortSignal: new AbortController().signal
74
+ })
75
+ await session.findClosestNodes(1000)
76
+ expect(queriedNodes.length).toBeGreaterThanOrEqual(1)
77
+ // Each queried node should closer to the target than the previous queried node, because we
78
+ // use parallelism=1 and noProgressLimit=1
79
+ const distancesToTarget = queriedNodes
80
+ .map((nodeId) => getDistance(getRawFromDhtAddress(nodeId), getRawFromDhtAddress(targetId)))
81
+ for (let i = 1; i < distancesToTarget.length ; i++) {
82
+ expect(distancesToTarget[i]).toBeLessThan(distancesToTarget[i - 1])
83
+ }
84
+ })
85
+ })
@@ -26,7 +26,8 @@ const createPeerManager = (
26
26
  }
27
27
  }(localPeerDescriptor, peerDescriptor, undefined as any, new MockRpcCommunicator())
28
28
  return remote
29
- }
29
+ },
30
+ hasConnection: () => false
30
31
  } as any)
31
32
  const contacts = nodeIds.map((n) => ({ nodeId: getRawFromDhtAddress(n), type: NodeType.NODEJS }))
32
33
  for (const contact of contacts) {
@@ -53,21 +54,6 @@ const getClosestContact = (contacts: PeerDescriptor[], referenceId: DhtAddress):
53
54
 
54
55
  describe('PeerManager', () => {
55
56
 
56
- it('getClosestContactsTo', () => {
57
- const nodeIds = range(10).map(() => createRandomDhtAddress())
58
- const manager = createPeerManager(nodeIds)
59
- const referenceId = createRandomDhtAddress()
60
- const excluded = new Set<DhtAddress>(sampleSize(nodeIds, 2))
61
-
62
- const actual = manager.getClosestContactsTo(referenceId, 5, excluded)
63
-
64
- const expected = sortBy(
65
- without(nodeIds, ...Array.from(excluded.values())),
66
- (n: DhtAddress) => getDistance(getRawFromDhtAddress(n), getRawFromDhtAddress(referenceId))
67
- ).slice(0, 5)
68
- expect(actual.map((n) => n.getNodeId())).toEqual(expected)
69
- })
70
-
71
57
  it('getClosestNeighborsTo', () => {
72
58
  const nodeIds = range(10).map(() => createRandomDhtAddress())
73
59
  const manager = createPeerManager(nodeIds)
@@ -99,7 +85,7 @@ describe('PeerManager', () => {
99
85
  for (const successContact of successContacts) {
100
86
  manager.addContact(successContact)
101
87
  manager.setContactActive(getNodeIdFromPeerDescriptor(successContact))
102
- manager.onContactDisconnected(getNodeIdFromPeerDescriptor(successContact), false)
88
+ manager.removeNeighbor(getNodeIdFromPeerDescriptor(successContact))
103
89
  }
104
90
  expect(manager.getNeighborCount()).toBe(0)
105
91
  manager.addContact(failureContact)
@@ -74,12 +74,12 @@ describe('RecursiveOperationManager', () => {
74
74
  return new RecursiveOperationManager({
75
75
  localPeerDescriptor: peerDescriptor1,
76
76
  router,
77
- connections: new Map(),
78
77
  serviceId: 'RecursiveOperationManager',
79
78
  localDataStore: new LocalDataStore(30 * 100),
80
79
  sessionTransport: transport,
81
80
  addContact: () => {},
82
- rpcCommunicator: rpcCommunicator as any
81
+ rpcCommunicator: rpcCommunicator as any,
82
+ createDhtNodeRpcRemote: () => undefined as any
83
83
  })
84
84
  }
85
85
 
@@ -126,6 +126,7 @@ describe('RecursiveOperationManager', () => {
126
126
  const send = jest.fn()
127
127
  const transport = {
128
128
  send,
129
+ getConnections: () => [],
129
130
  on: () => {},
130
131
  off: () => {}
131
132
  }
@@ -143,7 +144,8 @@ describe('RecursiveOperationManager', () => {
143
144
  const router = createMockRouter(RouteMessageError.DUPLICATE)
144
145
  const send = jest.fn()
145
146
  const transport = {
146
- send
147
+ send,
148
+ getConnections: () => []
147
149
  }
148
150
  const recursiveOperationManager = createRecursiveOperationManager(router as any, transport as any)
149
151
  const ack = await rpcCommunicator.callRpcMethod('routeRequest', routedMessage)
@@ -50,8 +50,8 @@ describe('Router', () => {
50
50
  router = new Router({
51
51
  localPeerDescriptor: peerDescriptor1,
52
52
  rpcCommunicator: rpcCommunicator as any,
53
- connections,
54
- handleMessage: () => {}
53
+ handleMessage: () => {},
54
+ getConnections: () => [...connections.values()].map((c) => c.getPeerDescriptor())
55
55
  })
56
56
  })
57
57
 
@@ -49,11 +49,11 @@ describe('RoutingSession', () => {
49
49
  rpcCommunicator: rpcCommunicator,
50
50
  localPeerDescriptor: mockPeerDescriptor1,
51
51
  routedMessage,
52
- connections,
53
52
  parallelism: 2,
54
53
  mode: RoutingMode.ROUTE,
55
54
  excludedNodeIds: new Set(),
56
- routingTablesCache
55
+ routingTablesCache,
56
+ getConnections: () => [...connections.values()].map((c) => c.getPeerDescriptor())
57
57
  })
58
58
  })
59
59
 
@@ -38,7 +38,7 @@ describe('SortedContactList', () => {
38
38
  expect(list.getSize()).toEqual(3)
39
39
  expect(list.getClosestContacts()).toEqual([item1, item2, item3])
40
40
  expect(list.getContactIds()).toEqual([item1.getNodeId(), item2.getNodeId(), item3.getNodeId()])
41
- expect(onContactRemoved).toBeCalledWith(item4, [item1, item2, item3])
41
+ expect(onContactRemoved).toBeCalledWith(item4)
42
42
  expect(list.getContact(item4.getNodeId())).toBeFalsy()
43
43
  })
44
44
 
@@ -63,9 +63,9 @@ describe('SortedContactList', () => {
63
63
  list.removeContact(item3.getNodeId())
64
64
  list.removeContact(createRandomDhtAddress())
65
65
  expect(list.getClosestContacts()).toEqual([item1, item4])
66
- expect(onContactRemoved).toHaveBeenNthCalledWith(1, item3, [])
67
- expect(onContactRemoved).toHaveBeenNthCalledWith(2, item2, [item1, item3, item4])
68
- expect(onContactRemoved).toHaveBeenNthCalledWith(3, item3, [item1, item4])
66
+ expect(onContactRemoved).toHaveBeenNthCalledWith(1, item3)
67
+ expect(onContactRemoved).toHaveBeenNthCalledWith(2, item2)
68
+ expect(onContactRemoved).toHaveBeenNthCalledWith(3, item3)
69
69
  })
70
70
 
71
71
  it('get closest contacts', () => {
@@ -0,0 +1,28 @@
1
+ import { range, sampleSize, sortBy, without } from 'lodash'
2
+ import { getDistance } from '../../src/dht/PeerManager'
3
+ import { getClosestContacts } from '../../src/dht/contact/getClosestContacts'
4
+ import { DhtAddress, createRandomDhtAddress, getRawFromDhtAddress } from '../../src/identifiers'
5
+
6
+ describe('getClosestContacts', () => {
7
+
8
+ it('happy path', () => {
9
+ const nodeIds = range(10).map(() => createRandomDhtAddress())
10
+ const referenceId = createRandomDhtAddress()
11
+ const excluded = new Set<DhtAddress>(sampleSize(nodeIds, 2))
12
+
13
+ const actual = getClosestContacts(
14
+ referenceId,
15
+ nodeIds.map((nodeId) => ({ getNodeId: () => nodeId })),
16
+ {
17
+ maxCount: 5,
18
+ excludedNodeIds: excluded
19
+ }
20
+ )
21
+
22
+ const expected = sortBy(
23
+ without(nodeIds, ...Array.from(excluded.values())),
24
+ (n: DhtAddress) => getDistance(getRawFromDhtAddress(n), getRawFromDhtAddress(referenceId))
25
+ ).slice(0, 5)
26
+ expect(actual.map((n) => n.getNodeId())).toEqual(expected)
27
+ })
28
+ })