@streamr/dht 100.2.4-beta.0 → 100.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/dist/package.json +8 -9
  2. package/dist/src/connection/ConnectionManager.d.ts +2 -1
  3. package/dist/src/connection/ConnectionManager.js +2 -10
  4. package/dist/src/connection/ConnectionManager.js.map +1 -1
  5. package/dist/src/connection/ConnectionsView.d.ts +7 -0
  6. package/dist/src/connection/ConnectionsView.js +3 -0
  7. package/dist/src/connection/ConnectionsView.js.map +1 -0
  8. package/dist/src/connection/ConnectorFacade.d.ts +2 -1
  9. package/dist/src/connection/ConnectorFacade.js +5 -4
  10. package/dist/src/connection/ConnectorFacade.js.map +1 -1
  11. package/dist/src/connection/connectivityRequestHandler.d.ts +2 -1
  12. package/dist/src/connection/connectivityRequestHandler.js +12 -5
  13. package/dist/src/connection/connectivityRequestHandler.js.map +1 -1
  14. package/dist/src/connection/{ManagedWebrtcConnection.d.ts → webrtc/ManagedWebrtcConnection.d.ts} +3 -3
  15. package/dist/src/connection/{ManagedWebrtcConnection.js → webrtc/ManagedWebrtcConnection.js} +2 -2
  16. package/dist/src/connection/webrtc/ManagedWebrtcConnection.js.map +1 -0
  17. package/dist/src/connection/webrtc/NodeWebrtcConnection.js +3 -26
  18. package/dist/src/connection/webrtc/NodeWebrtcConnection.js.map +1 -1
  19. package/dist/src/connection/webrtc/WebrtcConnector.js +1 -1
  20. package/dist/src/connection/webrtc/WebrtcConnector.js.map +1 -1
  21. package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.d.ts +1 -1
  22. package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.js +1 -1
  23. package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.js.map +1 -1
  24. package/dist/src/connection/websocket/WebsocketConnector.d.ts +2 -0
  25. package/dist/src/connection/websocket/WebsocketConnector.js +21 -13
  26. package/dist/src/connection/websocket/WebsocketConnector.js.map +1 -1
  27. package/dist/src/connection/websocket/WebsocketServerConnection.d.ts +2 -1
  28. package/dist/src/connection/websocket/WebsocketServerConnection.js +4 -0
  29. package/dist/src/connection/websocket/WebsocketServerConnection.js.map +1 -1
  30. package/dist/src/dht/DhtNode.d.ts +14 -11
  31. package/dist/src/dht/DhtNode.js +74 -61
  32. package/dist/src/dht/DhtNode.js.map +1 -1
  33. package/dist/src/dht/DhtNodeRpcLocal.d.ts +3 -3
  34. package/dist/src/dht/DhtNodeRpcLocal.js +5 -2
  35. package/dist/src/dht/DhtNodeRpcLocal.js.map +1 -1
  36. package/dist/src/dht/PeerManager.d.ts +14 -14
  37. package/dist/src/dht/PeerManager.js +51 -42
  38. package/dist/src/dht/PeerManager.js.map +1 -1
  39. package/dist/src/dht/contact/ContactList.d.ts +1 -2
  40. package/dist/src/dht/contact/ContactList.js +1 -3
  41. package/dist/src/dht/contact/ContactList.js.map +1 -1
  42. package/dist/src/dht/contact/RandomContactList.d.ts +1 -1
  43. package/dist/src/dht/contact/RandomContactList.js +7 -4
  44. package/dist/src/dht/contact/RandomContactList.js.map +1 -1
  45. package/dist/src/dht/contact/getClosestNodes.d.ts +6 -0
  46. package/dist/src/dht/contact/{getClosestContacts.js → getClosestNodes.js} +7 -6
  47. package/dist/src/dht/contact/getClosestNodes.js.map +1 -0
  48. package/dist/src/dht/discovery/DiscoverySession.d.ts +4 -0
  49. package/dist/src/dht/discovery/DiscoverySession.js +27 -21
  50. package/dist/src/dht/discovery/DiscoverySession.js.map +1 -1
  51. package/dist/src/dht/discovery/PeerDiscovery.d.ts +8 -5
  52. package/dist/src/dht/discovery/PeerDiscovery.js +34 -24
  53. package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
  54. package/dist/src/dht/discovery/RingDiscoverySession.d.ts +4 -4
  55. package/dist/src/dht/discovery/RingDiscoverySession.js +10 -19
  56. package/dist/src/dht/discovery/RingDiscoverySession.js.map +1 -1
  57. package/dist/src/dht/recursive-operation/RecursiveOperationManager.d.ts +2 -0
  58. package/dist/src/dht/recursive-operation/RecursiveOperationManager.js +3 -3
  59. package/dist/src/dht/recursive-operation/RecursiveOperationManager.js.map +1 -1
  60. package/dist/src/dht/store/StoreManager.d.ts +5 -6
  61. package/dist/src/dht/store/StoreManager.js +21 -76
  62. package/dist/src/dht/store/StoreManager.js.map +1 -1
  63. package/dist/src/dht/store/StoreRpcLocal.d.ts +5 -2
  64. package/dist/src/dht/store/StoreRpcLocal.js +22 -5
  65. package/dist/src/dht/store/StoreRpcLocal.js.map +1 -1
  66. package/dist/src/exports.d.ts +2 -1
  67. package/dist/src/exports.js.map +1 -1
  68. package/dist/src/helpers/version.d.ts +1 -1
  69. package/dist/src/helpers/version.js +1 -1
  70. package/dist/src/proto/google/protobuf/any.d.ts +8 -5
  71. package/dist/src/proto/google/protobuf/any.js.map +1 -1
  72. package/dist/src/proto/google/protobuf/empty.d.ts +0 -1
  73. package/dist/src/proto/google/protobuf/empty.js.map +1 -1
  74. package/dist/src/proto/google/protobuf/timestamp.d.ts +10 -1
  75. package/dist/src/proto/google/protobuf/timestamp.js.map +1 -1
  76. package/dist/src/proto/packages/dht/protos/DhtRpc.d.ts +8 -0
  77. package/dist/src/proto/packages/dht/protos/DhtRpc.js +3 -1
  78. package/dist/src/proto/packages/dht/protos/DhtRpc.js.map +1 -1
  79. package/dist/src/transport/ITransport.d.ts +0 -4
  80. package/dist/src/transport/ITransport.js.map +1 -1
  81. package/karma.config.js +2 -2
  82. package/package.json +8 -9
  83. package/protos/DhtRpc.proto +2 -0
  84. package/src/connection/ConnectionManager.ts +4 -10
  85. package/src/connection/ConnectionsView.ts +8 -0
  86. package/src/connection/ConnectorFacade.ts +7 -5
  87. package/src/connection/connectivityRequestHandler.ts +18 -5
  88. package/src/connection/{ManagedWebrtcConnection.ts → webrtc/ManagedWebrtcConnection.ts} +4 -4
  89. package/src/connection/webrtc/NodeWebrtcConnection.ts +3 -3
  90. package/src/connection/webrtc/WebrtcConnector.ts +1 -1
  91. package/src/connection/webrtc/WebrtcConnectorRpcLocal.ts +1 -1
  92. package/src/connection/websocket/WebsocketConnector.ts +26 -16
  93. package/src/connection/websocket/WebsocketServerConnection.ts +6 -1
  94. package/src/dht/DhtNode.ts +102 -74
  95. package/src/dht/DhtNodeRpcLocal.ts +12 -5
  96. package/src/dht/PeerManager.ts +58 -49
  97. package/src/dht/contact/ContactList.ts +1 -4
  98. package/src/dht/contact/RandomContactList.ts +7 -5
  99. package/src/dht/contact/{getClosestContacts.ts → getClosestNodes.ts} +8 -6
  100. package/src/dht/discovery/DiscoverySession.ts +34 -22
  101. package/src/dht/discovery/PeerDiscovery.ts +44 -30
  102. package/src/dht/discovery/RingDiscoverySession.ts +15 -29
  103. package/src/dht/recursive-operation/RecursiveOperationManager.ts +5 -3
  104. package/src/dht/store/StoreManager.ts +46 -84
  105. package/src/dht/store/StoreRpcLocal.ts +32 -9
  106. package/src/exports.ts +2 -1
  107. package/src/helpers/version.ts +1 -1
  108. package/src/proto/google/protobuf/any.ts +8 -5
  109. package/src/proto/google/protobuf/empty.ts +0 -1
  110. package/src/proto/google/protobuf/timestamp.ts +10 -1
  111. package/src/proto/packages/dht/protos/DhtRpc.ts +11 -1
  112. package/src/transport/ITransport.ts +0 -4
  113. package/test/benchmark/Find.test.ts +1 -1
  114. package/test/end-to-end/GeoIpLayer0.test.ts +55 -0
  115. package/test/end-to-end/Layer0-Layer1.test.ts +4 -4
  116. package/test/end-to-end/Layer0Webrtc-Layer1.test.ts +11 -5
  117. package/test/end-to-end/Layer1-Scale-WebSocket.test.ts +7 -1
  118. package/test/end-to-end/Layer1-Scale-Webrtc.test.ts +11 -2
  119. package/test/integration/ConnectionLocking.test.ts +1 -1
  120. package/test/integration/ConnectionManager.test.ts +3 -3
  121. package/test/integration/ConnectivityChecking.test.ts +2 -2
  122. package/test/integration/DhtNode.test.ts +5 -20
  123. package/test/integration/GeoIpConnectivityChecking.test.ts +71 -0
  124. package/test/integration/Layer1-scale.test.ts +6 -6
  125. package/test/integration/RouteMessage.test.ts +4 -0
  126. package/test/integration/ScaleDownDht.test.ts +1 -1
  127. package/test/integration/SimultaneousConnections.test.ts +2 -2
  128. package/test/integration/WebrtcConnectionManagement.test.ts +1 -1
  129. package/test/integration/WebsocketConnectionManagement.test.ts +1 -1
  130. package/test/integration/rpc-connections-over-webrpc.test.ts +1 -1
  131. package/test/unit/AutoCertifierClientFacade.test.ts +1 -1
  132. package/test/unit/DiscoverySession.test.ts +4 -2
  133. package/test/unit/PeerManager.test.ts +45 -51
  134. package/test/unit/RandomContactList.test.ts +10 -0
  135. package/test/unit/RecursiveOperationManager.test.ts +4 -2
  136. package/test/unit/StoreManager.test.ts +42 -34
  137. package/test/unit/StoreRpcLocal.test.ts +167 -0
  138. package/test/unit/WebsocketConnector.test.ts +1 -1
  139. package/test/unit/connectivityRequestHandler.test.ts +1 -1
  140. package/test/unit/getClosestNodes.test.ts +30 -0
  141. package/test/utils/FakeTransport.ts +4 -2
  142. package/test/utils/mock/MockConnectionsView.ts +18 -0
  143. package/test/utils/mock/{Transport.ts → MockTransport.ts} +0 -15
  144. package/test/utils/utils.ts +4 -1
  145. package/tsconfig.jest.json +2 -1
  146. package/tsconfig.node.json +2 -1
  147. package/dist/src/connection/ManagedWebrtcConnection.js.map +0 -1
  148. package/dist/src/dht/contact/getClosestContacts.d.ts +0 -7
  149. package/dist/src/dht/contact/getClosestContacts.js.map +0 -1
  150. package/test/unit/getClosestContacts.test.ts +0 -28
  151. /package/test/utils/mock/{Router.ts → MockRouter.ts} +0 -0
@@ -1,19 +1,19 @@
1
1
  import {
2
2
  Logger
3
3
  } from '@streamr/utils'
4
+ import EventEmitter from 'eventemitter3'
4
5
  import KBucket from 'k-bucket'
6
+ import { LockID } from '../connection/ConnectionLockStates'
7
+ import { ConnectionLocker } from '../connection/ConnectionManager'
8
+ import { DhtAddress, DhtAddressRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../identifiers'
5
9
  import {
6
10
  PeerDescriptor
7
11
  } from '../proto/packages/dht/protos/DhtRpc'
8
12
  import { DhtNodeRpcRemote } from './DhtNodeRpcRemote'
9
13
  import { RandomContactList } from './contact/RandomContactList'
10
- import { ReadonlySortedContactList, SortedContactList } from './contact/SortedContactList'
11
- import { ConnectionLocker } from '../connection/ConnectionManager'
12
- import EventEmitter from 'eventemitter3'
13
- import { DhtAddress, DhtAddressRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../identifiers'
14
14
  import { RingContactList } from './contact/RingContactList'
15
+ import { ReadonlySortedContactList, SortedContactList } from './contact/SortedContactList'
15
16
  import { RingIdRaw, getRingIdRawFromPeerDescriptor } from './contact/ringIdentifiers'
16
- import { LockID } from '../connection/ConnectionLockStates'
17
17
 
18
18
  const logger = new Logger(module)
19
19
 
@@ -28,9 +28,24 @@ interface PeerManagerConfig {
28
28
  hasConnection: (nodeId: DhtAddress) => boolean
29
29
  }
30
30
 
31
+ // Returns all offline nodes, sets contacts as active if they are online
32
+ const pingNodes = async (nodes: DhtNodeRpcRemote[], activeContacts: Set<DhtAddress>): Promise<PeerDescriptor[]> => {
33
+ const offlineNeighbors: PeerDescriptor[] = []
34
+ await Promise.allSettled(nodes.map(async (contact) => {
35
+ const isOnline = await contact.ping()
36
+ if (!isOnline) {
37
+ activeContacts.delete(contact.getNodeId())
38
+ offlineNeighbors.push(contact.getPeerDescriptor())
39
+ } else {
40
+ activeContacts.add(contact.getNodeId())
41
+ }
42
+ }))
43
+ return offlineNeighbors
44
+ }
45
+
31
46
  export interface PeerManagerEvents {
32
- closestContactAdded: (peerDescriptor: PeerDescriptor) => void
33
- closestContactRemoved: (peerDescriptor: PeerDescriptor) => void
47
+ nearbyContactAdded: (peerDescriptor: PeerDescriptor) => void
48
+ nearbyContactRemoved: (peerDescriptor: PeerDescriptor) => void
34
49
  randomContactAdded: (peerDescriptor: PeerDescriptor) => void
35
50
  randomContactRemoved: (peerDescriptor: PeerDescriptor) => void
36
51
  ringContactAdded: (peerDescriptor: PeerDescriptor) => void
@@ -51,7 +66,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
51
66
  // * 'contacts' are all non-unresponsive nodes that we know about
52
67
 
53
68
  private neighbors: KBucket<DhtNodeRpcRemote>
54
- private closestContacts: SortedContactList<DhtNodeRpcRemote>
69
+ private nearbyContacts: SortedContactList<DhtNodeRpcRemote>
55
70
  private activeContacts: Set<DhtAddress>
56
71
  private ringContacts: RingContactList<DhtNodeRpcRemote>
57
72
  private randomContacts: RandomContactList<DhtNodeRpcRemote>
@@ -79,20 +94,20 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
79
94
  this.neighbors.on('updated', () => {
80
95
  // TODO: Update contact info to the connection manager and reconnect
81
96
  })
82
- this.closestContacts = new SortedContactList({
97
+ this.nearbyContacts = new SortedContactList({
83
98
  referenceId: this.config.localNodeId,
84
99
  maxSize: this.config.maxContactListSize,
85
100
  allowToContainReferenceId: false
86
101
  })
87
- this.closestContacts.on('contactRemoved', (contact: DhtNodeRpcRemote) => {
102
+ this.nearbyContacts.on('contactRemoved', (contact: DhtNodeRpcRemote) => {
88
103
  if (this.stopped) {
89
104
  return
90
105
  }
91
- this.emit('closestContactRemoved', contact.getPeerDescriptor())
106
+ this.emit('nearbyContactRemoved', contact.getPeerDescriptor())
92
107
  this.randomContacts.addContact(this.config.createDhtNodeRpcRemote(contact.getPeerDescriptor()))
93
108
  })
94
- this.closestContacts.on('contactAdded', (contact: DhtNodeRpcRemote) =>
95
- this.emit('closestContactAdded', contact.getPeerDescriptor())
109
+ this.nearbyContacts.on('contactAdded', (contact: DhtNodeRpcRemote) =>
110
+ this.emit('nearbyContactAdded', contact.getPeerDescriptor())
96
111
  )
97
112
  this.activeContacts = new Set()
98
113
  this.randomContacts = new RandomContactList(this.config.localNodeId, this.config.maxContactListSize)
@@ -110,7 +125,6 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
110
125
  }
111
126
  const sortingList: SortedContactList<DhtNodeRpcRemote> = new SortedContactList({
112
127
  referenceId: this.config.localNodeId,
113
- maxSize: 100, // TODO use config option or named constant?
114
128
  allowToContainReferenceId: false
115
129
  })
116
130
  sortingList.addContacts(oldContacts)
@@ -151,31 +165,31 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
151
165
  logger.trace('ping failed ' + nodeId)
152
166
  this.config.connectionLocker?.weakUnlockConnection(nodeId, this.config.lockId)
153
167
  this.removeContact(nodeId)
154
- this.addClosestContactToBucket()
168
+ this.addNearbyContactToNeighbors()
155
169
  }
156
170
  }).catch((_e) => {
157
171
  this.config.connectionLocker?.weakUnlockConnection(nodeId, this.config.lockId)
158
172
  this.removeContact(nodeId)
159
- this.addClosestContactToBucket()
173
+ this.addNearbyContactToNeighbors()
160
174
  })
161
175
  }
162
176
  }
163
177
  }
164
178
 
165
- private addClosestContactToBucket(): void {
179
+ private addNearbyContactToNeighbors(): void {
166
180
  if (this.stopped) {
167
181
  return
168
182
  }
169
- const closest = this.getClosestActiveContactNotInBucket()
183
+ const closest = this.getNearbyActiveContactNotInNeighbors()
170
184
  if (closest) {
171
185
  this.addContact(closest.getPeerDescriptor())
172
186
  }
173
187
  }
174
188
 
175
- private getClosestActiveContactNotInBucket(): DhtNodeRpcRemote | undefined {
176
- for (const contactId of this.closestContacts.getContactIds()) {
189
+ private getNearbyActiveContactNotInNeighbors(): DhtNodeRpcRemote | undefined {
190
+ for (const contactId of this.nearbyContacts.getContactIds()) {
177
191
  if (!this.neighbors.get(getRawFromDhtAddress(contactId)) && this.activeContacts.has(contactId)) {
178
- return this.closestContacts.getContact(contactId)!
192
+ return this.nearbyContacts.getContact(contactId)!
179
193
  }
180
194
  }
181
195
  return undefined
@@ -186,9 +200,9 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
186
200
  return
187
201
  }
188
202
  logger.trace(`Removing contact ${nodeId}`)
189
- this.ringContacts.removeContact(this.closestContacts.getContact(nodeId))
203
+ this.ringContacts.removeContact(this.nearbyContacts.getContact(nodeId))
190
204
  this.neighbors.remove(getRawFromDhtAddress(nodeId))
191
- this.closestContacts.removeContact(nodeId)
205
+ this.nearbyContacts.removeContact(nodeId)
192
206
  this.activeContacts.delete(nodeId)
193
207
  this.randomContacts.removeContact(nodeId)
194
208
  }
@@ -197,6 +211,15 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
197
211
  this.neighbors.remove(getRawFromDhtAddress(nodeId))
198
212
  }
199
213
 
214
+ async pruneOfflineNodes(nodes: DhtNodeRpcRemote[]): Promise<void> {
215
+ logger.trace('Pruning offline nodes', { nodes: nodes.length })
216
+ const offlineNeighbors = await pingNodes(nodes, this.activeContacts)
217
+ offlineNeighbors.forEach((offlineNeighbor) => {
218
+ logger.trace('Removing offline node', { node: getNodeIdFromPeerDescriptor(offlineNeighbor) })
219
+ this.removeContact(getNodeIdFromPeerDescriptor(offlineNeighbor))
220
+ })
221
+ }
222
+
200
223
  stop(): void {
201
224
  this.stopped = true
202
225
  this.neighbors.toArray().forEach((rpcRemote: DhtNodeRpcRemote) => {
@@ -208,23 +231,12 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
208
231
  rpcRemote.leaveNotice()
209
232
  this.ringContacts.removeContact(rpcRemote)
210
233
  })
211
- this.closestContacts.stop()
234
+ this.nearbyContacts.stop()
212
235
  this.randomContacts.stop()
213
236
  }
214
237
 
215
- getClosestNeighborsTo(referenceId: DhtAddress, limit?: number, excludedNodeIds?: Set<DhtAddress>): DhtNodeRpcRemote[] {
216
- const closest = new SortedContactList<DhtNodeRpcRemote>({
217
- referenceId,
218
- allowToContainReferenceId: true,
219
- excludedNodeIds,
220
- maxSize: limit
221
- })
222
- this.neighbors.toArray().forEach((contact) => closest.addContact(contact))
223
- return closest.getClosestContacts()
224
- }
225
-
226
- getClosestContacts(): ReadonlySortedContactList<DhtNodeRpcRemote> {
227
- return this.closestContacts
238
+ getNearbyContacts(): ReadonlySortedContactList<DhtNodeRpcRemote> {
239
+ return this.nearbyContacts
228
240
  }
229
241
 
230
242
  getClosestRingContactsTo(
@@ -233,10 +245,8 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
233
245
  excludedIds?: Set<DhtAddress>
234
246
  ): { left: DhtNodeRpcRemote[], right: DhtNodeRpcRemote[] } {
235
247
  const closest = new RingContactList<DhtNodeRpcRemote>(ringIdRaw, excludedIds)
236
- for (const contact of this.closestContacts.getAllContactsInUndefinedOrder()) {
237
- closest.addContact(contact)
238
- }
239
248
  this.ringContacts.getAllContacts().map((contact) => closest.addContact(contact))
249
+ // TODO use config option or named constant?
240
250
  return closest.getClosestContacts(limit ?? 8)
241
251
  }
242
252
 
@@ -248,17 +258,16 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
248
258
  return this.ringContacts
249
259
  }
250
260
 
251
- // TODO rename getNearbyContactsCount
252
- getContactCount(excludedNodeIds?: Set<DhtAddress>): number {
253
- return this.closestContacts.getSize(excludedNodeIds)
261
+ getNearbyContactCount(excludedNodeIds?: Set<DhtAddress>): number {
262
+ return this.nearbyContacts.getSize(excludedNodeIds)
254
263
  }
255
264
 
256
265
  getNeighborCount(): number {
257
266
  return this.neighbors.count()
258
267
  }
259
268
 
260
- getNeighbors(): PeerDescriptor[] {
261
- return this.neighbors.toArray().map((rpcRemote: DhtNodeRpcRemote) => rpcRemote.getPeerDescriptor())
269
+ getNeighbors(): ReadonlyArray<DhtNodeRpcRemote> {
270
+ return this.neighbors.toArray()
262
271
  }
263
272
 
264
273
  setContactActive(nodeId: DhtAddress): void {
@@ -274,17 +283,17 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
274
283
  logger.trace(`Adding new contact ${nodeId}`)
275
284
  const remote = this.config.createDhtNodeRpcRemote(peerDescriptor)
276
285
  const isInNeighbors = (this.neighbors.get(peerDescriptor.nodeId) !== null)
277
- const isInClosestContacts = (this.closestContacts.getContact(nodeId) !== undefined)
286
+ const isInNearbyContacts = (this.nearbyContacts.getContact(nodeId) !== undefined)
278
287
  const isInRingContacts = this.ringContacts.getContact(peerDescriptor) !== undefined
279
288
 
280
- if (isInNeighbors || isInClosestContacts) {
289
+ if (isInNeighbors || isInNearbyContacts) {
281
290
  this.randomContacts.addContact(remote)
282
291
  }
283
292
  if (!isInNeighbors) {
284
293
  this.neighbors.add(remote)
285
294
  }
286
- if (!isInClosestContacts) {
287
- this.closestContacts.addContact(remote)
295
+ if (!isInNearbyContacts) {
296
+ this.nearbyContacts.addContact(remote)
288
297
  }
289
298
  if (!isInRingContacts) {
290
299
  this.ringContacts.addContact(remote)
@@ -13,17 +13,14 @@ export class ContactList<C extends { getNodeId: () => DhtAddress }> extends Even
13
13
  protected contactIds: DhtAddress[] = []
14
14
  protected localNodeId: DhtAddress
15
15
  protected maxSize: number
16
- protected defaultContactQueryLimit
17
16
 
18
17
  constructor(
19
18
  localNodeId: DhtAddress,
20
- maxSize: number,
21
- defaultContactQueryLimit = 20
19
+ maxSize: number
22
20
  ) {
23
21
  super()
24
22
  this.localNodeId = localNodeId
25
23
  this.maxSize = maxSize
26
- this.defaultContactQueryLimit = defaultContactQueryLimit
27
24
  }
28
25
 
29
26
  public getContact(id: DhtAddress): C | undefined {
@@ -8,10 +8,9 @@ export class RandomContactList<C extends { getNodeId: () => DhtAddress }> extend
8
8
  constructor(
9
9
  localNodeId: DhtAddress,
10
10
  maxSize: number,
11
- randomness = 0.20,
12
- defaultContactQueryLimit?: number
11
+ randomness = 0.20
13
12
  ) {
14
- super(localNodeId, maxSize, defaultContactQueryLimit)
13
+ super(localNodeId, maxSize)
15
14
  this.randomness = randomness
16
15
  }
17
16
 
@@ -48,7 +47,10 @@ export class RandomContactList<C extends { getNodeId: () => DhtAddress }> extend
48
47
  return false
49
48
  }
50
49
 
51
- public getContacts(limit = this.defaultContactQueryLimit): C[] {
52
- return this.contactIds.slice(0, limit).map((contactId) => this.contactsById.get(contactId)!)
50
+ public getContacts(limit?: number): C[] {
51
+ const items = (limit === undefined)
52
+ ? this.contactIds
53
+ : this.contactIds.slice(0, Math.max(limit, 0))
54
+ return items.map((contactId) => this.contactsById.get(contactId)!)
53
55
  }
54
56
  }
@@ -1,22 +1,24 @@
1
+ import { PeerDescriptor } from '../../exports'
1
2
  import { DhtAddress } from '../../identifiers'
3
+ import { Contact } from './Contact'
2
4
  import { SortedContactList } from './SortedContactList'
3
5
 
4
- export const getClosestContacts = <C extends { getNodeId: () => DhtAddress }>(
6
+ export const getClosestNodes = (
5
7
  referenceId: DhtAddress,
6
- contacts: Iterable<C>,
8
+ contacts: Iterable<PeerDescriptor>,
7
9
  opts?: {
8
10
  maxCount?: number
9
11
  excludedNodeIds?: Set<DhtAddress>
10
12
  }
11
- ): C[] => {
12
- const list = new SortedContactList<C>({
13
+ ): PeerDescriptor[] => {
14
+ const list = new SortedContactList<Contact>({
13
15
  referenceId,
14
16
  allowToContainReferenceId: true,
15
17
  excludedNodeIds: opts?.excludedNodeIds,
16
18
  maxSize: opts?.maxCount
17
19
  })
18
20
  for (const contact of contacts) {
19
- list.addContact(contact)
21
+ list.addContact(new Contact(contact))
20
22
  }
21
- return list.getClosestContacts()
23
+ return list.getClosestContacts().map((n) => n.getPeerDescriptor())
22
24
  }
@@ -4,7 +4,7 @@ import { DhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '.
4
4
  import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
5
5
  import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
6
6
  import { PeerManager, getDistance } from '../PeerManager'
7
- import { getClosestContacts } from '../contact/getClosestContacts'
7
+ import { getClosestNodes } from '../contact/getClosestNodes'
8
8
 
9
9
  const logger = new Logger(module)
10
10
 
@@ -16,6 +16,7 @@ interface DiscoverySessionConfig {
16
16
  // Note that contacted peers will be mutated by the DiscoverySession or other parallel sessions
17
17
  contactedPeers: Set<DhtAddress>
18
18
  abortSignal: AbortSignal
19
+ createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => DhtNodeRpcRemote
19
20
  }
20
21
 
21
22
  export class DiscoverySession {
@@ -39,14 +40,16 @@ export class DiscoverySession {
39
40
  }
40
41
  }
41
42
 
42
- private async fetchClosestNeighborsFromRemote(contact: DhtNodeRpcRemote): Promise<PeerDescriptor[]> {
43
+ private async fetchClosestNeighborsFromRemote(peerDescriptor: PeerDescriptor): Promise<PeerDescriptor[]> {
43
44
  if (this.config.abortSignal.aborted || this.doneGate.isOpen()) {
44
45
  return []
45
46
  }
46
- logger.trace(`Getting closest neighbors from remote: ${getNodeIdFromPeerDescriptor(contact.getPeerDescriptor())}`)
47
- this.config.contactedPeers.add(contact.getNodeId())
48
- const returnedContacts = await contact.getClosestPeers(this.config.targetId)
49
- this.config.peerManager.setContactActive(contact.getNodeId())
47
+ const nodeId = getNodeIdFromPeerDescriptor(peerDescriptor)
48
+ logger.trace(`Getting closest neighbors from remote: ${nodeId}`)
49
+ this.config.contactedPeers.add(nodeId)
50
+ const remote = this.config.createDhtNodeRpcRemote(peerDescriptor)
51
+ const returnedContacts = await remote.getClosestPeers(this.config.targetId)
52
+ this.config.peerManager.setContactActive(nodeId)
50
53
  return returnedContacts
51
54
  }
52
55
 
@@ -56,31 +59,39 @@ export class DiscoverySession {
56
59
  }
57
60
  this.ongoingRequests.delete(nodeId)
58
61
  const targetId = getRawFromDhtAddress(this.config.targetId)
59
- const oldClosestNeighbor = this.config.peerManager.getClosestNeighborsTo(this.config.targetId, 1)[0]
60
- const oldClosestDistance = getDistance(targetId, getRawFromDhtAddress(oldClosestNeighbor.getNodeId()))
62
+ const oldClosestNeighbor = this.getClosestNeighbor()
63
+ const oldClosestDistance = getDistance(targetId, oldClosestNeighbor.nodeId)
61
64
  this.addContacts(contacts)
62
- const newClosestNeighbor = this.config.peerManager.getClosestNeighborsTo(this.config.targetId, 1)[0]
63
- const newClosestDistance = getDistance(targetId, getRawFromDhtAddress(newClosestNeighbor.getNodeId()))
65
+ const newClosestNeighbor = this.getClosestNeighbor()
66
+ const newClosestDistance = getDistance(targetId, newClosestNeighbor.nodeId)
64
67
  if (newClosestDistance >= oldClosestDistance) {
65
68
  this.noProgressCounter++
66
69
  }
67
70
  }
68
71
 
69
- private onRequestFailed(peer: DhtNodeRpcRemote) {
70
- if (!this.ongoingRequests.has(peer.getNodeId())) {
72
+ private getClosestNeighbor(): PeerDescriptor {
73
+ return getClosestNodes(
74
+ this.config.targetId,
75
+ this.config.peerManager.getNeighbors().map((n) => n.getPeerDescriptor()),
76
+ { maxCount: 1 }
77
+ )[0]
78
+ }
79
+
80
+ private onRequestFailed(nodeId: DhtAddress) {
81
+ if (!this.ongoingRequests.has(nodeId)) {
71
82
  return
72
83
  }
73
- this.ongoingRequests.delete(peer.getNodeId())
74
- this.config.peerManager.removeContact(peer.getNodeId())
84
+ this.ongoingRequests.delete(nodeId)
85
+ this.config.peerManager.removeContact(nodeId)
75
86
  }
76
87
 
77
88
  private findMoreContacts(): void {
78
89
  if (this.config.abortSignal.aborted || this.doneGate.isOpen()) {
79
90
  return
80
91
  }
81
- const uncontacted = getClosestContacts(
92
+ const uncontacted = getClosestNodes(
82
93
  this.config.targetId,
83
- this.config.peerManager.getClosestContacts().getAllContactsInUndefinedOrder(),
94
+ Array.from(this.config.peerManager.getNearbyContacts().getAllContactsInUndefinedOrder(), (c) => c.getPeerDescriptor()),
84
95
  {
85
96
  maxCount: this.config.parallelism,
86
97
  excludedNodeIds: this.config.contactedPeers
@@ -90,15 +101,16 @@ export class DiscoverySession {
90
101
  this.doneGate.open()
91
102
  return
92
103
  }
93
- for (const nextPeer of uncontacted) {
104
+ for (const node of uncontacted) {
94
105
  if (this.ongoingRequests.size >= this.config.parallelism) {
95
106
  break
96
107
  }
97
- this.ongoingRequests.add(nextPeer.getNodeId())
108
+ const nodeId = getNodeIdFromPeerDescriptor(node)
109
+ this.ongoingRequests.add(nodeId)
98
110
  // eslint-disable-next-line promise/catch-or-return
99
- this.fetchClosestNeighborsFromRemote(nextPeer)
100
- .then((contacts) => this.onRequestSucceeded(nextPeer.getNodeId(), contacts))
101
- .catch(() => this.onRequestFailed(nextPeer))
111
+ this.fetchClosestNeighborsFromRemote(node)
112
+ .then((contacts) => this.onRequestSucceeded(nodeId, contacts))
113
+ .catch(() => this.onRequestFailed(nodeId))
102
114
  .finally(() => {
103
115
  this.findMoreContacts()
104
116
  })
@@ -106,7 +118,7 @@ export class DiscoverySession {
106
118
  }
107
119
 
108
120
  public async findClosestNodes(timeout: number): Promise<void> {
109
- if (this.config.peerManager.getContactCount(this.config.contactedPeers) === 0) {
121
+ if (this.config.peerManager.getNearbyContactCount(this.config.contactedPeers) === 0) {
110
122
  return
111
123
  }
112
124
  setImmediate(() => {
@@ -1,13 +1,21 @@
1
- import { DiscoverySession } from './DiscoverySession'
2
- import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
3
- import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
4
1
  import { Logger, scheduleAtInterval, setAbortableTimeout } from '@streamr/utils'
5
2
  import { ConnectionLocker } from '../../connection/ConnectionManager'
6
- import { PeerManager } from '../PeerManager'
7
- import { DhtAddress, areEqualPeerDescriptors, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../identifiers'
3
+ import {
4
+ DhtAddress,
5
+ areEqualPeerDescriptors,
6
+ createRandomDhtAddress,
7
+ getDhtAddressFromRaw,
8
+ getNodeIdFromPeerDescriptor,
9
+ getRawFromDhtAddress
10
+ } from '../../identifiers'
11
+ import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
8
12
  import { ServiceID } from '../../types/ServiceID'
9
- import { RingDiscoverySession } from './RingDiscoverySession'
13
+ import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
14
+ import { PeerManager } from '../PeerManager'
15
+ import { getClosestNodes } from '../contact/getClosestNodes'
10
16
  import { RingIdRaw, getRingIdRawFromPeerDescriptor } from '../contact/ringIdentifiers'
17
+ import { DiscoverySession } from './DiscoverySession'
18
+ import { RingDiscoverySession } from './RingDiscoverySession'
11
19
 
12
20
  interface PeerDiscoveryConfig {
13
21
  localPeerDescriptor: PeerDescriptor
@@ -17,6 +25,8 @@ interface PeerDiscoveryConfig {
17
25
  joinTimeout: number
18
26
  connectionLocker?: ConnectionLocker
19
27
  peerManager: PeerManager
28
+ abortSignal: AbortSignal
29
+ createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => DhtNodeRpcRemote
20
30
  }
21
31
 
22
32
  export const createDistantDhtAddress = (address: DhtAddress): DhtAddress => {
@@ -34,13 +44,11 @@ export class PeerDiscovery {
34
44
 
35
45
  private rejoinOngoing = false
36
46
  private joinCalled = false
37
- private readonly abortController: AbortController
38
47
  private recoveryIntervalStarted = false
39
48
  private readonly config: PeerDiscoveryConfig
40
49
 
41
50
  constructor(config: PeerDiscoveryConfig) {
42
51
  this.config = config
43
- this.abortController = new AbortController()
44
52
  }
45
53
 
46
54
  async joinDht(
@@ -102,7 +110,8 @@ export class PeerDiscovery {
102
110
  noProgressLimit: this.config.joinNoProgressLimit,
103
111
  peerManager: this.config.peerManager,
104
112
  contactedPeers,
105
- abortSignal: this.abortController.signal
113
+ abortSignal: this.config.abortSignal,
114
+ createDhtNodeRpcRemote: this.config.createDhtNodeRpcRemote
106
115
  }
107
116
  return new DiscoverySession(sessionOptions)
108
117
  }
@@ -113,7 +122,9 @@ export class PeerDiscovery {
113
122
  parallelism: this.config.parallelism,
114
123
  noProgressLimit: this.config.joinNoProgressLimit,
115
124
  peerManager: this.config.peerManager,
116
- contactedPeers
125
+ contactedPeers,
126
+ abortSignal: this.config.abortSignal,
127
+ createDhtNodeRpcRemote: this.config.createDhtNodeRpcRemote
117
128
  }
118
129
  return new RingDiscoverySession(sessionOptions)
119
130
  }
@@ -132,7 +143,7 @@ export class PeerDiscovery {
132
143
  if (retry) {
133
144
  // TODO should we catch possible promise rejection?
134
145
  // TODO use config option or named constant?
135
- setAbortableTimeout(() => this.rejoinDht(entryPointDescriptor), 1000, this.abortController.signal)
146
+ setAbortableTimeout(() => this.rejoinDht(entryPointDescriptor), 1000, this.config.abortSignal)
136
147
  }
137
148
  } else {
138
149
  await this.ensureRecoveryIntervalIsRunning()
@@ -173,7 +184,7 @@ export class PeerDiscovery {
173
184
  if (!this.isStopped()) {
174
185
  // TODO should we catch possible promise rejection?
175
186
  // TODO use config option or named constant?
176
- setAbortableTimeout(() => this.rejoinDht(entryPoint), 5000, this.abortController.signal)
187
+ setAbortableTimeout(() => this.rejoinDht(entryPoint), 5000, this.config.abortSignal)
177
188
  }
178
189
  } finally {
179
190
  this.rejoinOngoing = false
@@ -184,27 +195,37 @@ export class PeerDiscovery {
184
195
  if (!this.recoveryIntervalStarted) {
185
196
  this.recoveryIntervalStarted = true
186
197
  // TODO use config option or named constant?
187
- await scheduleAtInterval(() => this.fetchClosestNeighbors(), 60000, true, this.abortController.signal)
198
+ await scheduleAtInterval(() => this.fetchClosestAndRandomNeighbors(), 60000, true, this.config.abortSignal)
188
199
  }
189
200
  }
190
201
 
191
- private async fetchClosestNeighbors(): Promise<void> {
202
+ private async fetchClosestAndRandomNeighbors(): Promise<void> {
192
203
  if (this.isStopped()) {
193
204
  return
194
205
  }
195
206
  const localNodeId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
196
- const nodes = this.config.peerManager.getClosestNeighborsTo(
197
- localNodeId,
198
- this.config.parallelism
199
- )
200
- await Promise.allSettled(
201
- nodes.map(async (node: DhtNodeRpcRemote) => {
202
- const contacts = await node.getClosestPeers(localNodeId)
207
+ const nodes = this.getClosestNeighbors(localNodeId, this.config.parallelism)
208
+ const randomNodes = this.getClosestNeighbors(createRandomDhtAddress(), 1)
209
+ await Promise.allSettled([
210
+ ...nodes.map(async (node: PeerDescriptor) => {
211
+ const remote = this.config.createDhtNodeRpcRemote(node)
212
+ const contacts = await remote.getClosestPeers(localNodeId)
213
+ for (const contact of contacts) {
214
+ this.config.peerManager.addContact(contact)
215
+ }
216
+ }),
217
+ ...randomNodes.map(async (node: PeerDescriptor) => {
218
+ const remote = this.config.createDhtNodeRpcRemote(node)
219
+ const contacts = await remote.getClosestPeers(createRandomDhtAddress())
203
220
  for (const contact of contacts) {
204
221
  this.config.peerManager.addContact(contact)
205
222
  }
206
223
  })
207
- )
224
+ ])
225
+ }
226
+
227
+ private getClosestNeighbors(referenceId: DhtAddress, maxCount: number): PeerDescriptor[] {
228
+ return getClosestNodes(referenceId, this.config.peerManager.getNeighbors().map((n) => n.getPeerDescriptor()), { maxCount })
208
229
  }
209
230
 
210
231
  public isJoinOngoing(): boolean {
@@ -216,13 +237,6 @@ export class PeerDiscovery {
216
237
  }
217
238
 
218
239
  private isStopped() {
219
- return this.abortController.signal.aborted
220
- }
221
-
222
- public stop(): void {
223
- this.abortController.abort()
224
- this.ongoingRingDiscoverySessions.forEach((session) => {
225
- session.stop()
226
- })
240
+ return this.config.abortSignal.aborted
227
241
  }
228
242
  }