@streamr/dht 100.2.3 → 100.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/package.json +8 -9
- package/dist/src/connection/ConnectionManager.d.ts +2 -1
- package/dist/src/connection/ConnectionManager.js +2 -10
- package/dist/src/connection/ConnectionManager.js.map +1 -1
- package/dist/src/connection/ConnectionsView.d.ts +7 -0
- package/dist/src/connection/ConnectionsView.js +3 -0
- package/dist/src/connection/ConnectionsView.js.map +1 -0
- package/dist/src/connection/ConnectorFacade.d.ts +2 -1
- package/dist/src/connection/ConnectorFacade.js +5 -4
- package/dist/src/connection/ConnectorFacade.js.map +1 -1
- package/dist/src/connection/connectivityRequestHandler.d.ts +2 -1
- package/dist/src/connection/connectivityRequestHandler.js +12 -5
- package/dist/src/connection/connectivityRequestHandler.js.map +1 -1
- package/dist/src/connection/{ManagedWebrtcConnection.d.ts → webrtc/ManagedWebrtcConnection.d.ts} +3 -3
- package/dist/src/connection/{ManagedWebrtcConnection.js → webrtc/ManagedWebrtcConnection.js} +2 -2
- package/dist/src/connection/webrtc/ManagedWebrtcConnection.js.map +1 -0
- package/dist/src/connection/webrtc/NodeWebrtcConnection.js +3 -26
- package/dist/src/connection/webrtc/NodeWebrtcConnection.js.map +1 -1
- package/dist/src/connection/webrtc/WebrtcConnector.js +1 -1
- package/dist/src/connection/webrtc/WebrtcConnector.js.map +1 -1
- package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.d.ts +1 -1
- package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.js +1 -1
- package/dist/src/connection/webrtc/WebrtcConnectorRpcLocal.js.map +1 -1
- package/dist/src/connection/websocket/WebsocketConnector.d.ts +2 -0
- package/dist/src/connection/websocket/WebsocketConnector.js +21 -13
- package/dist/src/connection/websocket/WebsocketConnector.js.map +1 -1
- package/dist/src/connection/websocket/WebsocketServerConnection.d.ts +2 -1
- package/dist/src/connection/websocket/WebsocketServerConnection.js +4 -0
- package/dist/src/connection/websocket/WebsocketServerConnection.js.map +1 -1
- package/dist/src/dht/DhtNode.d.ts +14 -11
- package/dist/src/dht/DhtNode.js +74 -61
- package/dist/src/dht/DhtNode.js.map +1 -1
- package/dist/src/dht/DhtNodeRpcLocal.d.ts +3 -3
- package/dist/src/dht/DhtNodeRpcLocal.js +5 -2
- package/dist/src/dht/DhtNodeRpcLocal.js.map +1 -1
- package/dist/src/dht/PeerManager.d.ts +14 -14
- package/dist/src/dht/PeerManager.js +51 -42
- package/dist/src/dht/PeerManager.js.map +1 -1
- package/dist/src/dht/contact/ContactList.d.ts +1 -2
- package/dist/src/dht/contact/ContactList.js +1 -3
- package/dist/src/dht/contact/ContactList.js.map +1 -1
- package/dist/src/dht/contact/RandomContactList.d.ts +1 -1
- package/dist/src/dht/contact/RandomContactList.js +7 -4
- package/dist/src/dht/contact/RandomContactList.js.map +1 -1
- package/dist/src/dht/contact/getClosestNodes.d.ts +6 -0
- package/dist/src/dht/contact/{getClosestContacts.js → getClosestNodes.js} +7 -6
- package/dist/src/dht/contact/getClosestNodes.js.map +1 -0
- package/dist/src/dht/discovery/DiscoverySession.d.ts +4 -0
- package/dist/src/dht/discovery/DiscoverySession.js +27 -21
- package/dist/src/dht/discovery/DiscoverySession.js.map +1 -1
- package/dist/src/dht/discovery/PeerDiscovery.d.ts +8 -5
- package/dist/src/dht/discovery/PeerDiscovery.js +34 -24
- package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
- package/dist/src/dht/discovery/RingDiscoverySession.d.ts +4 -4
- package/dist/src/dht/discovery/RingDiscoverySession.js +10 -19
- package/dist/src/dht/discovery/RingDiscoverySession.js.map +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationManager.d.ts +2 -0
- package/dist/src/dht/recursive-operation/RecursiveOperationManager.js +3 -3
- package/dist/src/dht/recursive-operation/RecursiveOperationManager.js.map +1 -1
- package/dist/src/dht/store/StoreManager.d.ts +5 -6
- package/dist/src/dht/store/StoreManager.js +21 -76
- package/dist/src/dht/store/StoreManager.js.map +1 -1
- package/dist/src/dht/store/StoreRpcLocal.d.ts +5 -2
- package/dist/src/dht/store/StoreRpcLocal.js +22 -5
- package/dist/src/dht/store/StoreRpcLocal.js.map +1 -1
- package/dist/src/exports.d.ts +2 -1
- package/dist/src/exports.js.map +1 -1
- package/dist/src/helpers/version.d.ts +1 -1
- package/dist/src/helpers/version.js +1 -1
- package/dist/src/proto/google/protobuf/any.d.ts +8 -5
- package/dist/src/proto/google/protobuf/any.js.map +1 -1
- package/dist/src/proto/google/protobuf/empty.d.ts +0 -1
- package/dist/src/proto/google/protobuf/empty.js.map +1 -1
- package/dist/src/proto/google/protobuf/timestamp.d.ts +10 -1
- package/dist/src/proto/google/protobuf/timestamp.js.map +1 -1
- package/dist/src/proto/packages/dht/protos/DhtRpc.d.ts +8 -0
- package/dist/src/proto/packages/dht/protos/DhtRpc.js +3 -1
- package/dist/src/proto/packages/dht/protos/DhtRpc.js.map +1 -1
- package/dist/src/transport/ITransport.d.ts +0 -4
- package/dist/src/transport/ITransport.js.map +1 -1
- package/karma.config.js +2 -2
- package/package.json +8 -9
- package/protos/DhtRpc.proto +2 -0
- package/src/connection/ConnectionManager.ts +4 -10
- package/src/connection/ConnectionsView.ts +8 -0
- package/src/connection/ConnectorFacade.ts +7 -5
- package/src/connection/connectivityRequestHandler.ts +18 -5
- package/src/connection/{ManagedWebrtcConnection.ts → webrtc/ManagedWebrtcConnection.ts} +4 -4
- package/src/connection/webrtc/NodeWebrtcConnection.ts +3 -3
- package/src/connection/webrtc/WebrtcConnector.ts +1 -1
- package/src/connection/webrtc/WebrtcConnectorRpcLocal.ts +1 -1
- package/src/connection/websocket/WebsocketConnector.ts +26 -16
- package/src/connection/websocket/WebsocketServerConnection.ts +6 -1
- package/src/dht/DhtNode.ts +102 -74
- package/src/dht/DhtNodeRpcLocal.ts +12 -5
- package/src/dht/PeerManager.ts +58 -49
- package/src/dht/contact/ContactList.ts +1 -4
- package/src/dht/contact/RandomContactList.ts +7 -5
- package/src/dht/contact/{getClosestContacts.ts → getClosestNodes.ts} +8 -6
- package/src/dht/discovery/DiscoverySession.ts +34 -22
- package/src/dht/discovery/PeerDiscovery.ts +44 -30
- package/src/dht/discovery/RingDiscoverySession.ts +15 -29
- package/src/dht/recursive-operation/RecursiveOperationManager.ts +5 -3
- package/src/dht/store/StoreManager.ts +46 -84
- package/src/dht/store/StoreRpcLocal.ts +32 -9
- package/src/exports.ts +2 -1
- package/src/helpers/version.ts +1 -1
- package/src/proto/google/protobuf/any.ts +8 -5
- package/src/proto/google/protobuf/empty.ts +0 -1
- package/src/proto/google/protobuf/timestamp.ts +10 -1
- package/src/proto/packages/dht/protos/DhtRpc.ts +11 -1
- package/src/transport/ITransport.ts +0 -4
- package/test/benchmark/Find.test.ts +1 -1
- package/test/end-to-end/GeoIpLayer0.test.ts +55 -0
- package/test/end-to-end/Layer0-Layer1.test.ts +4 -4
- package/test/end-to-end/Layer0Webrtc-Layer1.test.ts +11 -5
- package/test/end-to-end/Layer1-Scale-WebSocket.test.ts +7 -1
- package/test/end-to-end/Layer1-Scale-Webrtc.test.ts +11 -2
- package/test/integration/ConnectionLocking.test.ts +1 -1
- package/test/integration/ConnectionManager.test.ts +3 -3
- package/test/integration/ConnectivityChecking.test.ts +2 -2
- package/test/integration/DhtNode.test.ts +5 -20
- package/test/integration/GeoIpConnectivityChecking.test.ts +71 -0
- package/test/integration/Layer1-scale.test.ts +6 -6
- package/test/integration/RouteMessage.test.ts +4 -0
- package/test/integration/ScaleDownDht.test.ts +1 -1
- package/test/integration/SimultaneousConnections.test.ts +2 -2
- package/test/integration/WebrtcConnectionManagement.test.ts +1 -1
- package/test/integration/WebsocketConnectionManagement.test.ts +1 -1
- package/test/integration/rpc-connections-over-webrpc.test.ts +1 -1
- package/test/unit/AutoCertifierClientFacade.test.ts +1 -1
- package/test/unit/DiscoverySession.test.ts +4 -2
- package/test/unit/PeerManager.test.ts +45 -51
- package/test/unit/RandomContactList.test.ts +10 -0
- package/test/unit/RecursiveOperationManager.test.ts +4 -2
- package/test/unit/StoreManager.test.ts +42 -34
- package/test/unit/StoreRpcLocal.test.ts +167 -0
- package/test/unit/WebsocketConnector.test.ts +1 -1
- package/test/unit/connectivityRequestHandler.test.ts +1 -1
- package/test/unit/getClosestNodes.test.ts +30 -0
- package/test/utils/FakeTransport.ts +4 -2
- package/test/utils/mock/MockConnectionsView.ts +18 -0
- package/test/utils/mock/{Transport.ts → MockTransport.ts} +0 -15
- package/test/utils/utils.ts +4 -1
- package/tsconfig.jest.json +2 -1
- package/tsconfig.node.json +2 -1
- package/dist/src/connection/ManagedWebrtcConnection.js.map +0 -1
- package/dist/src/dht/contact/getClosestContacts.d.ts +0 -7
- package/dist/src/dht/contact/getClosestContacts.js.map +0 -1
- package/test/unit/getClosestContacts.test.ts +0 -28
- /package/test/utils/mock/{Router.ts → MockRouter.ts} +0 -0
package/src/dht/PeerManager.ts
CHANGED
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
import {
|
|
2
2
|
Logger
|
|
3
3
|
} from '@streamr/utils'
|
|
4
|
+
import EventEmitter from 'eventemitter3'
|
|
4
5
|
import KBucket from 'k-bucket'
|
|
6
|
+
import { LockID } from '../connection/ConnectionLockStates'
|
|
7
|
+
import { ConnectionLocker } from '../connection/ConnectionManager'
|
|
8
|
+
import { DhtAddress, DhtAddressRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../identifiers'
|
|
5
9
|
import {
|
|
6
10
|
PeerDescriptor
|
|
7
11
|
} from '../proto/packages/dht/protos/DhtRpc'
|
|
8
12
|
import { DhtNodeRpcRemote } from './DhtNodeRpcRemote'
|
|
9
13
|
import { RandomContactList } from './contact/RandomContactList'
|
|
10
|
-
import { ReadonlySortedContactList, SortedContactList } from './contact/SortedContactList'
|
|
11
|
-
import { ConnectionLocker } from '../connection/ConnectionManager'
|
|
12
|
-
import EventEmitter from 'eventemitter3'
|
|
13
|
-
import { DhtAddress, DhtAddressRaw, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../identifiers'
|
|
14
14
|
import { RingContactList } from './contact/RingContactList'
|
|
15
|
+
import { ReadonlySortedContactList, SortedContactList } from './contact/SortedContactList'
|
|
15
16
|
import { RingIdRaw, getRingIdRawFromPeerDescriptor } from './contact/ringIdentifiers'
|
|
16
|
-
import { LockID } from '../connection/ConnectionLockStates'
|
|
17
17
|
|
|
18
18
|
const logger = new Logger(module)
|
|
19
19
|
|
|
@@ -28,9 +28,24 @@ interface PeerManagerConfig {
|
|
|
28
28
|
hasConnection: (nodeId: DhtAddress) => boolean
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
// Returns all offline nodes, sets contacts as active if they are online
|
|
32
|
+
const pingNodes = async (nodes: DhtNodeRpcRemote[], activeContacts: Set<DhtAddress>): Promise<PeerDescriptor[]> => {
|
|
33
|
+
const offlineNeighbors: PeerDescriptor[] = []
|
|
34
|
+
await Promise.allSettled(nodes.map(async (contact) => {
|
|
35
|
+
const isOnline = await contact.ping()
|
|
36
|
+
if (!isOnline) {
|
|
37
|
+
activeContacts.delete(contact.getNodeId())
|
|
38
|
+
offlineNeighbors.push(contact.getPeerDescriptor())
|
|
39
|
+
} else {
|
|
40
|
+
activeContacts.add(contact.getNodeId())
|
|
41
|
+
}
|
|
42
|
+
}))
|
|
43
|
+
return offlineNeighbors
|
|
44
|
+
}
|
|
45
|
+
|
|
31
46
|
export interface PeerManagerEvents {
|
|
32
|
-
|
|
33
|
-
|
|
47
|
+
nearbyContactAdded: (peerDescriptor: PeerDescriptor) => void
|
|
48
|
+
nearbyContactRemoved: (peerDescriptor: PeerDescriptor) => void
|
|
34
49
|
randomContactAdded: (peerDescriptor: PeerDescriptor) => void
|
|
35
50
|
randomContactRemoved: (peerDescriptor: PeerDescriptor) => void
|
|
36
51
|
ringContactAdded: (peerDescriptor: PeerDescriptor) => void
|
|
@@ -51,7 +66,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
51
66
|
// * 'contacts' are all non-unresponsive nodes that we know about
|
|
52
67
|
|
|
53
68
|
private neighbors: KBucket<DhtNodeRpcRemote>
|
|
54
|
-
private
|
|
69
|
+
private nearbyContacts: SortedContactList<DhtNodeRpcRemote>
|
|
55
70
|
private activeContacts: Set<DhtAddress>
|
|
56
71
|
private ringContacts: RingContactList<DhtNodeRpcRemote>
|
|
57
72
|
private randomContacts: RandomContactList<DhtNodeRpcRemote>
|
|
@@ -79,20 +94,20 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
79
94
|
this.neighbors.on('updated', () => {
|
|
80
95
|
// TODO: Update contact info to the connection manager and reconnect
|
|
81
96
|
})
|
|
82
|
-
this.
|
|
97
|
+
this.nearbyContacts = new SortedContactList({
|
|
83
98
|
referenceId: this.config.localNodeId,
|
|
84
99
|
maxSize: this.config.maxContactListSize,
|
|
85
100
|
allowToContainReferenceId: false
|
|
86
101
|
})
|
|
87
|
-
this.
|
|
102
|
+
this.nearbyContacts.on('contactRemoved', (contact: DhtNodeRpcRemote) => {
|
|
88
103
|
if (this.stopped) {
|
|
89
104
|
return
|
|
90
105
|
}
|
|
91
|
-
this.emit('
|
|
106
|
+
this.emit('nearbyContactRemoved', contact.getPeerDescriptor())
|
|
92
107
|
this.randomContacts.addContact(this.config.createDhtNodeRpcRemote(contact.getPeerDescriptor()))
|
|
93
108
|
})
|
|
94
|
-
this.
|
|
95
|
-
this.emit('
|
|
109
|
+
this.nearbyContacts.on('contactAdded', (contact: DhtNodeRpcRemote) =>
|
|
110
|
+
this.emit('nearbyContactAdded', contact.getPeerDescriptor())
|
|
96
111
|
)
|
|
97
112
|
this.activeContacts = new Set()
|
|
98
113
|
this.randomContacts = new RandomContactList(this.config.localNodeId, this.config.maxContactListSize)
|
|
@@ -110,7 +125,6 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
110
125
|
}
|
|
111
126
|
const sortingList: SortedContactList<DhtNodeRpcRemote> = new SortedContactList({
|
|
112
127
|
referenceId: this.config.localNodeId,
|
|
113
|
-
maxSize: 100, // TODO use config option or named constant?
|
|
114
128
|
allowToContainReferenceId: false
|
|
115
129
|
})
|
|
116
130
|
sortingList.addContacts(oldContacts)
|
|
@@ -151,31 +165,31 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
151
165
|
logger.trace('ping failed ' + nodeId)
|
|
152
166
|
this.config.connectionLocker?.weakUnlockConnection(nodeId, this.config.lockId)
|
|
153
167
|
this.removeContact(nodeId)
|
|
154
|
-
this.
|
|
168
|
+
this.addNearbyContactToNeighbors()
|
|
155
169
|
}
|
|
156
170
|
}).catch((_e) => {
|
|
157
171
|
this.config.connectionLocker?.weakUnlockConnection(nodeId, this.config.lockId)
|
|
158
172
|
this.removeContact(nodeId)
|
|
159
|
-
this.
|
|
173
|
+
this.addNearbyContactToNeighbors()
|
|
160
174
|
})
|
|
161
175
|
}
|
|
162
176
|
}
|
|
163
177
|
}
|
|
164
178
|
|
|
165
|
-
private
|
|
179
|
+
private addNearbyContactToNeighbors(): void {
|
|
166
180
|
if (this.stopped) {
|
|
167
181
|
return
|
|
168
182
|
}
|
|
169
|
-
const closest = this.
|
|
183
|
+
const closest = this.getNearbyActiveContactNotInNeighbors()
|
|
170
184
|
if (closest) {
|
|
171
185
|
this.addContact(closest.getPeerDescriptor())
|
|
172
186
|
}
|
|
173
187
|
}
|
|
174
188
|
|
|
175
|
-
private
|
|
176
|
-
for (const contactId of this.
|
|
189
|
+
private getNearbyActiveContactNotInNeighbors(): DhtNodeRpcRemote | undefined {
|
|
190
|
+
for (const contactId of this.nearbyContacts.getContactIds()) {
|
|
177
191
|
if (!this.neighbors.get(getRawFromDhtAddress(contactId)) && this.activeContacts.has(contactId)) {
|
|
178
|
-
return this.
|
|
192
|
+
return this.nearbyContacts.getContact(contactId)!
|
|
179
193
|
}
|
|
180
194
|
}
|
|
181
195
|
return undefined
|
|
@@ -186,9 +200,9 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
186
200
|
return
|
|
187
201
|
}
|
|
188
202
|
logger.trace(`Removing contact ${nodeId}`)
|
|
189
|
-
this.ringContacts.removeContact(this.
|
|
203
|
+
this.ringContacts.removeContact(this.nearbyContacts.getContact(nodeId))
|
|
190
204
|
this.neighbors.remove(getRawFromDhtAddress(nodeId))
|
|
191
|
-
this.
|
|
205
|
+
this.nearbyContacts.removeContact(nodeId)
|
|
192
206
|
this.activeContacts.delete(nodeId)
|
|
193
207
|
this.randomContacts.removeContact(nodeId)
|
|
194
208
|
}
|
|
@@ -197,6 +211,15 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
197
211
|
this.neighbors.remove(getRawFromDhtAddress(nodeId))
|
|
198
212
|
}
|
|
199
213
|
|
|
214
|
+
async pruneOfflineNodes(nodes: DhtNodeRpcRemote[]): Promise<void> {
|
|
215
|
+
logger.trace('Pruning offline nodes', { nodes: nodes.length })
|
|
216
|
+
const offlineNeighbors = await pingNodes(nodes, this.activeContacts)
|
|
217
|
+
offlineNeighbors.forEach((offlineNeighbor) => {
|
|
218
|
+
logger.trace('Removing offline node', { node: getNodeIdFromPeerDescriptor(offlineNeighbor) })
|
|
219
|
+
this.removeContact(getNodeIdFromPeerDescriptor(offlineNeighbor))
|
|
220
|
+
})
|
|
221
|
+
}
|
|
222
|
+
|
|
200
223
|
stop(): void {
|
|
201
224
|
this.stopped = true
|
|
202
225
|
this.neighbors.toArray().forEach((rpcRemote: DhtNodeRpcRemote) => {
|
|
@@ -208,23 +231,12 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
208
231
|
rpcRemote.leaveNotice()
|
|
209
232
|
this.ringContacts.removeContact(rpcRemote)
|
|
210
233
|
})
|
|
211
|
-
this.
|
|
234
|
+
this.nearbyContacts.stop()
|
|
212
235
|
this.randomContacts.stop()
|
|
213
236
|
}
|
|
214
237
|
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
referenceId,
|
|
218
|
-
allowToContainReferenceId: true,
|
|
219
|
-
excludedNodeIds,
|
|
220
|
-
maxSize: limit
|
|
221
|
-
})
|
|
222
|
-
this.neighbors.toArray().forEach((contact) => closest.addContact(contact))
|
|
223
|
-
return closest.getClosestContacts()
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
getClosestContacts(): ReadonlySortedContactList<DhtNodeRpcRemote> {
|
|
227
|
-
return this.closestContacts
|
|
238
|
+
getNearbyContacts(): ReadonlySortedContactList<DhtNodeRpcRemote> {
|
|
239
|
+
return this.nearbyContacts
|
|
228
240
|
}
|
|
229
241
|
|
|
230
242
|
getClosestRingContactsTo(
|
|
@@ -233,10 +245,8 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
233
245
|
excludedIds?: Set<DhtAddress>
|
|
234
246
|
): { left: DhtNodeRpcRemote[], right: DhtNodeRpcRemote[] } {
|
|
235
247
|
const closest = new RingContactList<DhtNodeRpcRemote>(ringIdRaw, excludedIds)
|
|
236
|
-
for (const contact of this.closestContacts.getAllContactsInUndefinedOrder()) {
|
|
237
|
-
closest.addContact(contact)
|
|
238
|
-
}
|
|
239
248
|
this.ringContacts.getAllContacts().map((contact) => closest.addContact(contact))
|
|
249
|
+
// TODO use config option or named constant?
|
|
240
250
|
return closest.getClosestContacts(limit ?? 8)
|
|
241
251
|
}
|
|
242
252
|
|
|
@@ -248,17 +258,16 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
248
258
|
return this.ringContacts
|
|
249
259
|
}
|
|
250
260
|
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
return this.closestContacts.getSize(excludedNodeIds)
|
|
261
|
+
getNearbyContactCount(excludedNodeIds?: Set<DhtAddress>): number {
|
|
262
|
+
return this.nearbyContacts.getSize(excludedNodeIds)
|
|
254
263
|
}
|
|
255
264
|
|
|
256
265
|
getNeighborCount(): number {
|
|
257
266
|
return this.neighbors.count()
|
|
258
267
|
}
|
|
259
268
|
|
|
260
|
-
getNeighbors():
|
|
261
|
-
return this.neighbors.toArray()
|
|
269
|
+
getNeighbors(): ReadonlyArray<DhtNodeRpcRemote> {
|
|
270
|
+
return this.neighbors.toArray()
|
|
262
271
|
}
|
|
263
272
|
|
|
264
273
|
setContactActive(nodeId: DhtAddress): void {
|
|
@@ -274,17 +283,17 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
|
|
|
274
283
|
logger.trace(`Adding new contact ${nodeId}`)
|
|
275
284
|
const remote = this.config.createDhtNodeRpcRemote(peerDescriptor)
|
|
276
285
|
const isInNeighbors = (this.neighbors.get(peerDescriptor.nodeId) !== null)
|
|
277
|
-
const
|
|
286
|
+
const isInNearbyContacts = (this.nearbyContacts.getContact(nodeId) !== undefined)
|
|
278
287
|
const isInRingContacts = this.ringContacts.getContact(peerDescriptor) !== undefined
|
|
279
288
|
|
|
280
|
-
if (isInNeighbors ||
|
|
289
|
+
if (isInNeighbors || isInNearbyContacts) {
|
|
281
290
|
this.randomContacts.addContact(remote)
|
|
282
291
|
}
|
|
283
292
|
if (!isInNeighbors) {
|
|
284
293
|
this.neighbors.add(remote)
|
|
285
294
|
}
|
|
286
|
-
if (!
|
|
287
|
-
this.
|
|
295
|
+
if (!isInNearbyContacts) {
|
|
296
|
+
this.nearbyContacts.addContact(remote)
|
|
288
297
|
}
|
|
289
298
|
if (!isInRingContacts) {
|
|
290
299
|
this.ringContacts.addContact(remote)
|
|
@@ -13,17 +13,14 @@ export class ContactList<C extends { getNodeId: () => DhtAddress }> extends Even
|
|
|
13
13
|
protected contactIds: DhtAddress[] = []
|
|
14
14
|
protected localNodeId: DhtAddress
|
|
15
15
|
protected maxSize: number
|
|
16
|
-
protected defaultContactQueryLimit
|
|
17
16
|
|
|
18
17
|
constructor(
|
|
19
18
|
localNodeId: DhtAddress,
|
|
20
|
-
maxSize: number
|
|
21
|
-
defaultContactQueryLimit = 20
|
|
19
|
+
maxSize: number
|
|
22
20
|
) {
|
|
23
21
|
super()
|
|
24
22
|
this.localNodeId = localNodeId
|
|
25
23
|
this.maxSize = maxSize
|
|
26
|
-
this.defaultContactQueryLimit = defaultContactQueryLimit
|
|
27
24
|
}
|
|
28
25
|
|
|
29
26
|
public getContact(id: DhtAddress): C | undefined {
|
|
@@ -8,10 +8,9 @@ export class RandomContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
8
8
|
constructor(
|
|
9
9
|
localNodeId: DhtAddress,
|
|
10
10
|
maxSize: number,
|
|
11
|
-
randomness = 0.20
|
|
12
|
-
defaultContactQueryLimit?: number
|
|
11
|
+
randomness = 0.20
|
|
13
12
|
) {
|
|
14
|
-
super(localNodeId, maxSize
|
|
13
|
+
super(localNodeId, maxSize)
|
|
15
14
|
this.randomness = randomness
|
|
16
15
|
}
|
|
17
16
|
|
|
@@ -48,7 +47,10 @@ export class RandomContactList<C extends { getNodeId: () => DhtAddress }> extend
|
|
|
48
47
|
return false
|
|
49
48
|
}
|
|
50
49
|
|
|
51
|
-
public getContacts(limit
|
|
52
|
-
|
|
50
|
+
public getContacts(limit?: number): C[] {
|
|
51
|
+
const items = (limit === undefined)
|
|
52
|
+
? this.contactIds
|
|
53
|
+
: this.contactIds.slice(0, Math.max(limit, 0))
|
|
54
|
+
return items.map((contactId) => this.contactsById.get(contactId)!)
|
|
53
55
|
}
|
|
54
56
|
}
|
|
@@ -1,22 +1,24 @@
|
|
|
1
|
+
import { PeerDescriptor } from '../../exports'
|
|
1
2
|
import { DhtAddress } from '../../identifiers'
|
|
3
|
+
import { Contact } from './Contact'
|
|
2
4
|
import { SortedContactList } from './SortedContactList'
|
|
3
5
|
|
|
4
|
-
export const
|
|
6
|
+
export const getClosestNodes = (
|
|
5
7
|
referenceId: DhtAddress,
|
|
6
|
-
contacts: Iterable<
|
|
8
|
+
contacts: Iterable<PeerDescriptor>,
|
|
7
9
|
opts?: {
|
|
8
10
|
maxCount?: number
|
|
9
11
|
excludedNodeIds?: Set<DhtAddress>
|
|
10
12
|
}
|
|
11
|
-
):
|
|
12
|
-
const list = new SortedContactList<
|
|
13
|
+
): PeerDescriptor[] => {
|
|
14
|
+
const list = new SortedContactList<Contact>({
|
|
13
15
|
referenceId,
|
|
14
16
|
allowToContainReferenceId: true,
|
|
15
17
|
excludedNodeIds: opts?.excludedNodeIds,
|
|
16
18
|
maxSize: opts?.maxCount
|
|
17
19
|
})
|
|
18
20
|
for (const contact of contacts) {
|
|
19
|
-
list.addContact(contact)
|
|
21
|
+
list.addContact(new Contact(contact))
|
|
20
22
|
}
|
|
21
|
-
return list.getClosestContacts()
|
|
23
|
+
return list.getClosestContacts().map((n) => n.getPeerDescriptor())
|
|
22
24
|
}
|
|
@@ -4,7 +4,7 @@ import { DhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '.
|
|
|
4
4
|
import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
|
|
5
5
|
import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
|
|
6
6
|
import { PeerManager, getDistance } from '../PeerManager'
|
|
7
|
-
import {
|
|
7
|
+
import { getClosestNodes } from '../contact/getClosestNodes'
|
|
8
8
|
|
|
9
9
|
const logger = new Logger(module)
|
|
10
10
|
|
|
@@ -16,6 +16,7 @@ interface DiscoverySessionConfig {
|
|
|
16
16
|
// Note that contacted peers will be mutated by the DiscoverySession or other parallel sessions
|
|
17
17
|
contactedPeers: Set<DhtAddress>
|
|
18
18
|
abortSignal: AbortSignal
|
|
19
|
+
createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => DhtNodeRpcRemote
|
|
19
20
|
}
|
|
20
21
|
|
|
21
22
|
export class DiscoverySession {
|
|
@@ -39,14 +40,16 @@ export class DiscoverySession {
|
|
|
39
40
|
}
|
|
40
41
|
}
|
|
41
42
|
|
|
42
|
-
private async fetchClosestNeighborsFromRemote(
|
|
43
|
+
private async fetchClosestNeighborsFromRemote(peerDescriptor: PeerDescriptor): Promise<PeerDescriptor[]> {
|
|
43
44
|
if (this.config.abortSignal.aborted || this.doneGate.isOpen()) {
|
|
44
45
|
return []
|
|
45
46
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
this.config.
|
|
47
|
+
const nodeId = getNodeIdFromPeerDescriptor(peerDescriptor)
|
|
48
|
+
logger.trace(`Getting closest neighbors from remote: ${nodeId}`)
|
|
49
|
+
this.config.contactedPeers.add(nodeId)
|
|
50
|
+
const remote = this.config.createDhtNodeRpcRemote(peerDescriptor)
|
|
51
|
+
const returnedContacts = await remote.getClosestPeers(this.config.targetId)
|
|
52
|
+
this.config.peerManager.setContactActive(nodeId)
|
|
50
53
|
return returnedContacts
|
|
51
54
|
}
|
|
52
55
|
|
|
@@ -56,31 +59,39 @@ export class DiscoverySession {
|
|
|
56
59
|
}
|
|
57
60
|
this.ongoingRequests.delete(nodeId)
|
|
58
61
|
const targetId = getRawFromDhtAddress(this.config.targetId)
|
|
59
|
-
const oldClosestNeighbor = this.
|
|
60
|
-
const oldClosestDistance = getDistance(targetId,
|
|
62
|
+
const oldClosestNeighbor = this.getClosestNeighbor()
|
|
63
|
+
const oldClosestDistance = getDistance(targetId, oldClosestNeighbor.nodeId)
|
|
61
64
|
this.addContacts(contacts)
|
|
62
|
-
const newClosestNeighbor = this.
|
|
63
|
-
const newClosestDistance = getDistance(targetId,
|
|
65
|
+
const newClosestNeighbor = this.getClosestNeighbor()
|
|
66
|
+
const newClosestDistance = getDistance(targetId, newClosestNeighbor.nodeId)
|
|
64
67
|
if (newClosestDistance >= oldClosestDistance) {
|
|
65
68
|
this.noProgressCounter++
|
|
66
69
|
}
|
|
67
70
|
}
|
|
68
71
|
|
|
69
|
-
private
|
|
70
|
-
|
|
72
|
+
private getClosestNeighbor(): PeerDescriptor {
|
|
73
|
+
return getClosestNodes(
|
|
74
|
+
this.config.targetId,
|
|
75
|
+
this.config.peerManager.getNeighbors().map((n) => n.getPeerDescriptor()),
|
|
76
|
+
{ maxCount: 1 }
|
|
77
|
+
)[0]
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
private onRequestFailed(nodeId: DhtAddress) {
|
|
81
|
+
if (!this.ongoingRequests.has(nodeId)) {
|
|
71
82
|
return
|
|
72
83
|
}
|
|
73
|
-
this.ongoingRequests.delete(
|
|
74
|
-
this.config.peerManager.removeContact(
|
|
84
|
+
this.ongoingRequests.delete(nodeId)
|
|
85
|
+
this.config.peerManager.removeContact(nodeId)
|
|
75
86
|
}
|
|
76
87
|
|
|
77
88
|
private findMoreContacts(): void {
|
|
78
89
|
if (this.config.abortSignal.aborted || this.doneGate.isOpen()) {
|
|
79
90
|
return
|
|
80
91
|
}
|
|
81
|
-
const uncontacted =
|
|
92
|
+
const uncontacted = getClosestNodes(
|
|
82
93
|
this.config.targetId,
|
|
83
|
-
this.config.peerManager.
|
|
94
|
+
Array.from(this.config.peerManager.getNearbyContacts().getAllContactsInUndefinedOrder(), (c) => c.getPeerDescriptor()),
|
|
84
95
|
{
|
|
85
96
|
maxCount: this.config.parallelism,
|
|
86
97
|
excludedNodeIds: this.config.contactedPeers
|
|
@@ -90,15 +101,16 @@ export class DiscoverySession {
|
|
|
90
101
|
this.doneGate.open()
|
|
91
102
|
return
|
|
92
103
|
}
|
|
93
|
-
for (const
|
|
104
|
+
for (const node of uncontacted) {
|
|
94
105
|
if (this.ongoingRequests.size >= this.config.parallelism) {
|
|
95
106
|
break
|
|
96
107
|
}
|
|
97
|
-
|
|
108
|
+
const nodeId = getNodeIdFromPeerDescriptor(node)
|
|
109
|
+
this.ongoingRequests.add(nodeId)
|
|
98
110
|
// eslint-disable-next-line promise/catch-or-return
|
|
99
|
-
this.fetchClosestNeighborsFromRemote(
|
|
100
|
-
.then((contacts) => this.onRequestSucceeded(
|
|
101
|
-
.catch(() => this.onRequestFailed(
|
|
111
|
+
this.fetchClosestNeighborsFromRemote(node)
|
|
112
|
+
.then((contacts) => this.onRequestSucceeded(nodeId, contacts))
|
|
113
|
+
.catch(() => this.onRequestFailed(nodeId))
|
|
102
114
|
.finally(() => {
|
|
103
115
|
this.findMoreContacts()
|
|
104
116
|
})
|
|
@@ -106,7 +118,7 @@ export class DiscoverySession {
|
|
|
106
118
|
}
|
|
107
119
|
|
|
108
120
|
public async findClosestNodes(timeout: number): Promise<void> {
|
|
109
|
-
if (this.config.peerManager.
|
|
121
|
+
if (this.config.peerManager.getNearbyContactCount(this.config.contactedPeers) === 0) {
|
|
110
122
|
return
|
|
111
123
|
}
|
|
112
124
|
setImmediate(() => {
|
|
@@ -1,13 +1,21 @@
|
|
|
1
|
-
import { DiscoverySession } from './DiscoverySession'
|
|
2
|
-
import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
|
|
3
|
-
import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
|
|
4
1
|
import { Logger, scheduleAtInterval, setAbortableTimeout } from '@streamr/utils'
|
|
5
2
|
import { ConnectionLocker } from '../../connection/ConnectionManager'
|
|
6
|
-
import {
|
|
7
|
-
|
|
3
|
+
import {
|
|
4
|
+
DhtAddress,
|
|
5
|
+
areEqualPeerDescriptors,
|
|
6
|
+
createRandomDhtAddress,
|
|
7
|
+
getDhtAddressFromRaw,
|
|
8
|
+
getNodeIdFromPeerDescriptor,
|
|
9
|
+
getRawFromDhtAddress
|
|
10
|
+
} from '../../identifiers'
|
|
11
|
+
import { PeerDescriptor } from '../../proto/packages/dht/protos/DhtRpc'
|
|
8
12
|
import { ServiceID } from '../../types/ServiceID'
|
|
9
|
-
import {
|
|
13
|
+
import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
|
|
14
|
+
import { PeerManager } from '../PeerManager'
|
|
15
|
+
import { getClosestNodes } from '../contact/getClosestNodes'
|
|
10
16
|
import { RingIdRaw, getRingIdRawFromPeerDescriptor } from '../contact/ringIdentifiers'
|
|
17
|
+
import { DiscoverySession } from './DiscoverySession'
|
|
18
|
+
import { RingDiscoverySession } from './RingDiscoverySession'
|
|
11
19
|
|
|
12
20
|
interface PeerDiscoveryConfig {
|
|
13
21
|
localPeerDescriptor: PeerDescriptor
|
|
@@ -17,6 +25,8 @@ interface PeerDiscoveryConfig {
|
|
|
17
25
|
joinTimeout: number
|
|
18
26
|
connectionLocker?: ConnectionLocker
|
|
19
27
|
peerManager: PeerManager
|
|
28
|
+
abortSignal: AbortSignal
|
|
29
|
+
createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => DhtNodeRpcRemote
|
|
20
30
|
}
|
|
21
31
|
|
|
22
32
|
export const createDistantDhtAddress = (address: DhtAddress): DhtAddress => {
|
|
@@ -34,13 +44,11 @@ export class PeerDiscovery {
|
|
|
34
44
|
|
|
35
45
|
private rejoinOngoing = false
|
|
36
46
|
private joinCalled = false
|
|
37
|
-
private readonly abortController: AbortController
|
|
38
47
|
private recoveryIntervalStarted = false
|
|
39
48
|
private readonly config: PeerDiscoveryConfig
|
|
40
49
|
|
|
41
50
|
constructor(config: PeerDiscoveryConfig) {
|
|
42
51
|
this.config = config
|
|
43
|
-
this.abortController = new AbortController()
|
|
44
52
|
}
|
|
45
53
|
|
|
46
54
|
async joinDht(
|
|
@@ -102,7 +110,8 @@ export class PeerDiscovery {
|
|
|
102
110
|
noProgressLimit: this.config.joinNoProgressLimit,
|
|
103
111
|
peerManager: this.config.peerManager,
|
|
104
112
|
contactedPeers,
|
|
105
|
-
abortSignal: this.
|
|
113
|
+
abortSignal: this.config.abortSignal,
|
|
114
|
+
createDhtNodeRpcRemote: this.config.createDhtNodeRpcRemote
|
|
106
115
|
}
|
|
107
116
|
return new DiscoverySession(sessionOptions)
|
|
108
117
|
}
|
|
@@ -113,7 +122,9 @@ export class PeerDiscovery {
|
|
|
113
122
|
parallelism: this.config.parallelism,
|
|
114
123
|
noProgressLimit: this.config.joinNoProgressLimit,
|
|
115
124
|
peerManager: this.config.peerManager,
|
|
116
|
-
contactedPeers
|
|
125
|
+
contactedPeers,
|
|
126
|
+
abortSignal: this.config.abortSignal,
|
|
127
|
+
createDhtNodeRpcRemote: this.config.createDhtNodeRpcRemote
|
|
117
128
|
}
|
|
118
129
|
return new RingDiscoverySession(sessionOptions)
|
|
119
130
|
}
|
|
@@ -132,7 +143,7 @@ export class PeerDiscovery {
|
|
|
132
143
|
if (retry) {
|
|
133
144
|
// TODO should we catch possible promise rejection?
|
|
134
145
|
// TODO use config option or named constant?
|
|
135
|
-
setAbortableTimeout(() => this.rejoinDht(entryPointDescriptor), 1000, this.
|
|
146
|
+
setAbortableTimeout(() => this.rejoinDht(entryPointDescriptor), 1000, this.config.abortSignal)
|
|
136
147
|
}
|
|
137
148
|
} else {
|
|
138
149
|
await this.ensureRecoveryIntervalIsRunning()
|
|
@@ -173,7 +184,7 @@ export class PeerDiscovery {
|
|
|
173
184
|
if (!this.isStopped()) {
|
|
174
185
|
// TODO should we catch possible promise rejection?
|
|
175
186
|
// TODO use config option or named constant?
|
|
176
|
-
setAbortableTimeout(() => this.rejoinDht(entryPoint), 5000, this.
|
|
187
|
+
setAbortableTimeout(() => this.rejoinDht(entryPoint), 5000, this.config.abortSignal)
|
|
177
188
|
}
|
|
178
189
|
} finally {
|
|
179
190
|
this.rejoinOngoing = false
|
|
@@ -184,27 +195,37 @@ export class PeerDiscovery {
|
|
|
184
195
|
if (!this.recoveryIntervalStarted) {
|
|
185
196
|
this.recoveryIntervalStarted = true
|
|
186
197
|
// TODO use config option or named constant?
|
|
187
|
-
await scheduleAtInterval(() => this.
|
|
198
|
+
await scheduleAtInterval(() => this.fetchClosestAndRandomNeighbors(), 60000, true, this.config.abortSignal)
|
|
188
199
|
}
|
|
189
200
|
}
|
|
190
201
|
|
|
191
|
-
private async
|
|
202
|
+
private async fetchClosestAndRandomNeighbors(): Promise<void> {
|
|
192
203
|
if (this.isStopped()) {
|
|
193
204
|
return
|
|
194
205
|
}
|
|
195
206
|
const localNodeId = getNodeIdFromPeerDescriptor(this.config.localPeerDescriptor)
|
|
196
|
-
const nodes = this.config.
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
const
|
|
207
|
+
const nodes = this.getClosestNeighbors(localNodeId, this.config.parallelism)
|
|
208
|
+
const randomNodes = this.getClosestNeighbors(createRandomDhtAddress(), 1)
|
|
209
|
+
await Promise.allSettled([
|
|
210
|
+
...nodes.map(async (node: PeerDescriptor) => {
|
|
211
|
+
const remote = this.config.createDhtNodeRpcRemote(node)
|
|
212
|
+
const contacts = await remote.getClosestPeers(localNodeId)
|
|
213
|
+
for (const contact of contacts) {
|
|
214
|
+
this.config.peerManager.addContact(contact)
|
|
215
|
+
}
|
|
216
|
+
}),
|
|
217
|
+
...randomNodes.map(async (node: PeerDescriptor) => {
|
|
218
|
+
const remote = this.config.createDhtNodeRpcRemote(node)
|
|
219
|
+
const contacts = await remote.getClosestPeers(createRandomDhtAddress())
|
|
203
220
|
for (const contact of contacts) {
|
|
204
221
|
this.config.peerManager.addContact(contact)
|
|
205
222
|
}
|
|
206
223
|
})
|
|
207
|
-
)
|
|
224
|
+
])
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
private getClosestNeighbors(referenceId: DhtAddress, maxCount: number): PeerDescriptor[] {
|
|
228
|
+
return getClosestNodes(referenceId, this.config.peerManager.getNeighbors().map((n) => n.getPeerDescriptor()), { maxCount })
|
|
208
229
|
}
|
|
209
230
|
|
|
210
231
|
public isJoinOngoing(): boolean {
|
|
@@ -216,13 +237,6 @@ export class PeerDiscovery {
|
|
|
216
237
|
}
|
|
217
238
|
|
|
218
239
|
private isStopped() {
|
|
219
|
-
return this.
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
public stop(): void {
|
|
223
|
-
this.abortController.abort()
|
|
224
|
-
this.ongoingRingDiscoverySessions.forEach((session) => {
|
|
225
|
-
session.stop()
|
|
226
|
-
})
|
|
240
|
+
return this.config.abortSignal.aborted
|
|
227
241
|
}
|
|
228
242
|
}
|