@streamr/dht 100.2.1 → 100.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +6 -6
- package/dist/src/connection/ConnectionLockStates.js +4 -6
- package/dist/src/connection/ConnectionLockStates.js.map +1 -1
- package/dist/src/connection/ConnectionManager.d.ts +1 -0
- package/dist/src/connection/ConnectionManager.js +8 -2
- package/dist/src/connection/ConnectionManager.js.map +1 -1
- package/dist/src/connection/ManagedConnection.js +7 -9
- package/dist/src/connection/ManagedConnection.js.map +1 -1
- package/dist/src/connection/connectivityChecker.js +0 -3
- package/dist/src/connection/connectivityChecker.js.map +1 -1
- package/dist/src/connection/websocket/WebsocketConnector.js +1 -1
- package/dist/src/connection/websocket/WebsocketConnector.js.map +1 -1
- package/dist/src/dht/DhtNode.d.ts +10 -7
- package/dist/src/dht/DhtNode.js +51 -32
- package/dist/src/dht/DhtNode.js.map +1 -1
- package/dist/src/dht/DhtNodeRpcLocal.d.ts +2 -2
- package/dist/src/dht/DhtNodeRpcLocal.js +5 -4
- package/dist/src/dht/DhtNodeRpcLocal.js.map +1 -1
- package/dist/src/dht/DhtNodeRpcRemote.js +1 -0
- package/dist/src/dht/DhtNodeRpcRemote.js.map +1 -1
- package/dist/src/dht/PeerManager.d.ts +14 -14
- package/dist/src/dht/PeerManager.js +23 -59
- package/dist/src/dht/PeerManager.js.map +1 -1
- package/dist/src/dht/contact/ContactList.d.ts +2 -2
- package/dist/src/dht/contact/RandomContactList.js +2 -2
- package/dist/src/dht/contact/RandomContactList.js.map +1 -1
- package/dist/src/dht/contact/RingContactList.d.ts +2 -5
- package/dist/src/dht/contact/RingContactList.js +3 -11
- package/dist/src/dht/contact/RingContactList.js.map +1 -1
- package/dist/src/dht/contact/SortedContactList.d.ts +4 -1
- package/dist/src/dht/contact/SortedContactList.js +4 -5
- package/dist/src/dht/contact/SortedContactList.js.map +1 -1
- package/dist/src/dht/contact/getClosestContacts.d.ts +7 -0
- package/dist/src/dht/contact/getClosestContacts.js +18 -0
- package/dist/src/dht/contact/getClosestContacts.js.map +1 -0
- package/dist/src/dht/discovery/DiscoverySession.d.ts +8 -8
- package/dist/src/dht/discovery/DiscoverySession.js +30 -35
- package/dist/src/dht/discovery/DiscoverySession.js.map +1 -1
- package/dist/src/dht/discovery/PeerDiscovery.d.ts +1 -2
- package/dist/src/dht/discovery/PeerDiscovery.js +6 -8
- package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
- package/dist/src/dht/discovery/RingDiscoverySession.d.ts +4 -4
- package/dist/src/dht/discovery/RingDiscoverySession.js +13 -13
- package/dist/src/dht/discovery/RingDiscoverySession.js.map +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationManager.d.ts +2 -2
- package/dist/src/dht/recursive-operation/RecursiveOperationManager.js +15 -15
- package/dist/src/dht/recursive-operation/RecursiveOperationManager.js.map +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationSession.d.ts +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationSession.js +11 -13
- package/dist/src/dht/recursive-operation/RecursiveOperationSession.js.map +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationSessionRpcLocal.js +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.d.ts +1 -1
- package/dist/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.js +2 -2
- package/dist/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.js.map +1 -1
- package/dist/src/dht/routing/Router.d.ts +1 -2
- package/dist/src/dht/routing/Router.js +2 -3
- package/dist/src/dht/routing/Router.js.map +1 -1
- package/dist/src/dht/routing/RoutingSession.d.ts +1 -2
- package/dist/src/dht/routing/RoutingSession.js +2 -2
- package/dist/src/dht/routing/RoutingSession.js.map +1 -1
- package/dist/src/dht/store/StoreRpcLocal.js +3 -3
- package/dist/src/dht/store/StoreRpcLocal.js.map +1 -1
- package/dist/src/proto/packages/dht/protos/DhtRpc.client.d.ts +8 -0
- package/dist/src/proto/packages/dht/protos/DhtRpc.client.js +4 -0
- package/dist/src/proto/packages/dht/protos/DhtRpc.client.js.map +1 -1
- package/dist/src/proto/packages/dht/protos/DhtRpc.d.ts +10 -2
- package/dist/src/proto/packages/dht/protos/DhtRpc.js +1 -1
- package/dist/src/proto/packages/dht/protos/DhtRpc.js.map +1 -1
- package/dist/src/proto/packages/dht/protos/DhtRpc.server.d.ts +4 -0
- package/dist/src/transport/ITransport.d.ts +3 -0
- package/dist/src/transport/ITransport.js.map +1 -1
- package/package.json +6 -6
- package/protos/DhtRpc.proto +7 -1
- package/src/connection/ConnectionLockStates.ts +3 -5
- package/src/connection/ConnectionManager.ts +9 -2
- package/src/connection/ManagedConnection.ts +11 -14
- package/src/connection/connectivityChecker.ts +0 -2
- package/src/connection/webrtc/BrowserWebrtcConnection.ts +0 -1
- package/src/connection/websocket/WebsocketConnector.ts +1 -1
- package/src/dht/DhtNode.ts +63 -44
- package/src/dht/DhtNodeRpcLocal.ts +7 -6
- package/src/dht/DhtNodeRpcRemote.ts +1 -0
- package/src/dht/PeerManager.ts +38 -73
- package/src/dht/contact/ContactList.ts +2 -2
- package/src/dht/contact/RandomContactList.ts +2 -3
- package/src/dht/contact/RingContactList.ts +6 -16
- package/src/dht/contact/SortedContactList.ts +9 -10
- package/src/dht/contact/getClosestContacts.ts +22 -0
- package/src/dht/discovery/DiscoverySession.ts +35 -45
- package/src/dht/discovery/PeerDiscovery.ts +6 -9
- package/src/dht/discovery/RingDiscoverySession.ts +13 -13
- package/src/dht/recursive-operation/RecursiveOperationManager.ts +16 -16
- package/src/dht/recursive-operation/RecursiveOperationSession.ts +11 -13
- package/src/dht/recursive-operation/RecursiveOperationSessionRpcLocal.ts +1 -1
- package/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.ts +2 -2
- package/src/dht/routing/Router.ts +3 -5
- package/src/dht/routing/RoutingSession.ts +3 -4
- package/src/dht/store/StoreRpcLocal.ts +3 -3
- package/src/proto/packages/dht/protos/DhtRpc.client.ts +8 -0
- package/src/proto/packages/dht/protos/DhtRpc.server.ts +4 -0
- package/src/proto/packages/dht/protos/DhtRpc.ts +11 -3
- package/src/transport/ITransport.ts +3 -0
- package/test/end-to-end/Layer0MixedConnectionTypes.test.ts +3 -5
- package/test/integration/DhtNode.test.ts +81 -0
- package/test/integration/Layer1-scale.test.ts +1 -1
- package/test/integration/ScaleDownDht.test.ts +7 -4
- package/test/unit/DiscoverySession.test.ts +85 -0
- package/test/unit/PeerManager.test.ts +3 -17
- package/test/unit/RecursiveOperationManager.test.ts +5 -3
- package/test/unit/Router.test.ts +2 -2
- package/test/unit/RoutingSession.test.ts +2 -2
- package/test/unit/SortedContactList.test.ts +4 -4
- package/test/unit/getClosestContacts.test.ts +28 -0
- package/test/utils/FakeTransport.ts +34 -7
- package/test/utils/mock/Router.ts +0 -3
- package/test/utils/mock/Transport.ts +10 -0
- package/test/utils/topology.ts +80 -0
- package/test/RandomGraphSimulation.ts +0 -53
|
@@ -3,7 +3,6 @@ import { RoutingMode, RoutingRemoteContact, RoutingSession, RoutingSessionEvents
|
|
|
3
3
|
import { Logger, executeSafePromise, raceEvents3, withTimeout } from '@streamr/utils'
|
|
4
4
|
import { RoutingRpcCommunicator } from '../../transport/RoutingRpcCommunicator'
|
|
5
5
|
import { DuplicateDetector } from './DuplicateDetector'
|
|
6
|
-
import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
|
|
7
6
|
import { v4 } from 'uuid'
|
|
8
7
|
import { RouterRpcLocal, createRouteMessageAck } from './RouterRpcLocal'
|
|
9
8
|
import { DhtAddress, areEqualPeerDescriptors, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor } from '../../identifiers'
|
|
@@ -12,8 +11,8 @@ import { RoutingTablesCache } from './RoutingTablesCache'
|
|
|
12
11
|
export interface RouterConfig {
|
|
13
12
|
rpcCommunicator: RoutingRpcCommunicator
|
|
14
13
|
localPeerDescriptor: PeerDescriptor
|
|
15
|
-
connections: Map<DhtAddress, DhtNodeRpcRemote>
|
|
16
14
|
handleMessage: (message: Message) => void
|
|
15
|
+
getConnections: () => PeerDescriptor[]
|
|
17
16
|
}
|
|
18
17
|
|
|
19
18
|
interface ForwardingTableEntry {
|
|
@@ -158,17 +157,16 @@ export class Router {
|
|
|
158
157
|
routedMessage.parallelRootNodeIds.forEach((nodeId) => {
|
|
159
158
|
excludedNodeIds.add(nodeId as DhtAddress)
|
|
160
159
|
})
|
|
161
|
-
logger.trace('routing session created with connections: ' + this.config.connections.size)
|
|
162
160
|
return new RoutingSession({
|
|
163
161
|
rpcCommunicator: this.config.rpcCommunicator,
|
|
164
162
|
localPeerDescriptor: this.config.localPeerDescriptor,
|
|
165
163
|
routedMessage,
|
|
166
|
-
connections: this.config.connections,
|
|
167
164
|
// TODO use config option or named constant?
|
|
168
165
|
parallelism: areEqualPeerDescriptors(this.config.localPeerDescriptor, routedMessage.sourcePeer!) ? 2 : 1,
|
|
169
166
|
mode,
|
|
170
167
|
excludedNodeIds,
|
|
171
|
-
routingTablesCache: this.routingTablesCache
|
|
168
|
+
routingTablesCache: this.routingTablesCache,
|
|
169
|
+
getConnections: this.config.getConnections
|
|
172
170
|
})
|
|
173
171
|
}
|
|
174
172
|
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
|
|
2
1
|
import { SortedContactList } from '../contact/SortedContactList'
|
|
3
2
|
import { Logger } from '@streamr/utils'
|
|
4
3
|
import EventEmitter from 'eventemitter3'
|
|
@@ -69,11 +68,11 @@ interface RoutingSessionConfig {
|
|
|
69
68
|
rpcCommunicator: RoutingRpcCommunicator
|
|
70
69
|
localPeerDescriptor: PeerDescriptor
|
|
71
70
|
routedMessage: RouteMessageWrapper
|
|
72
|
-
connections: Map<DhtAddress, DhtNodeRpcRemote>
|
|
73
71
|
parallelism: number
|
|
74
72
|
mode: RoutingMode
|
|
75
73
|
excludedNodeIds: Set<DhtAddress>
|
|
76
74
|
routingTablesCache: RoutingTablesCache
|
|
75
|
+
getConnections: () => PeerDescriptor[]
|
|
77
76
|
}
|
|
78
77
|
|
|
79
78
|
export class RoutingSession extends EventEmitter<RoutingSessionEvents> {
|
|
@@ -174,9 +173,9 @@ export class RoutingSession extends EventEmitter<RoutingSessionEvents> {
|
|
|
174
173
|
allowToContainReferenceId: true,
|
|
175
174
|
nodeIdDistanceLimit: previousId
|
|
176
175
|
})
|
|
177
|
-
const contacts =
|
|
176
|
+
const contacts = this.config.getConnections()
|
|
178
177
|
.map((peer) => new RoutingRemoteContact(
|
|
179
|
-
peer
|
|
178
|
+
peer,
|
|
180
179
|
this.config.localPeerDescriptor,
|
|
181
180
|
this.config.rpcCommunicator
|
|
182
181
|
))
|
|
@@ -32,7 +32,7 @@ export class StoreRpcLocal implements IStoreRpc {
|
|
|
32
32
|
async storeData(request: StoreDataRequest): Promise<StoreDataResponse> {
|
|
33
33
|
logger.trace('storeData()')
|
|
34
34
|
const key = getDhtAddressFromRaw(request.key)
|
|
35
|
-
const
|
|
35
|
+
const selfIsWithinRedundancyFactor = this.config.selfIsWithinRedundancyFactor(key)
|
|
36
36
|
this.config.localDataStore.storeEntry({
|
|
37
37
|
key: request.key,
|
|
38
38
|
data: request.data,
|
|
@@ -40,10 +40,10 @@ export class StoreRpcLocal implements IStoreRpc {
|
|
|
40
40
|
createdAt: request.createdAt,
|
|
41
41
|
storedAt: Timestamp.now(),
|
|
42
42
|
ttl: request.ttl,
|
|
43
|
-
stale: !
|
|
43
|
+
stale: !selfIsWithinRedundancyFactor,
|
|
44
44
|
deleted: false
|
|
45
45
|
})
|
|
46
|
-
if (!
|
|
46
|
+
if (!selfIsWithinRedundancyFactor) {
|
|
47
47
|
this.config.localDataStore.setAllEntriesAsStale(key)
|
|
48
48
|
}
|
|
49
49
|
return {}
|
|
@@ -47,10 +47,14 @@ import type { RpcOptions } from "@protobuf-ts/runtime-rpc";
|
|
|
47
47
|
*/
|
|
48
48
|
export interface IDhtNodeRpcClient {
|
|
49
49
|
/**
|
|
50
|
+
* TODO rename to getClosestNeighbors (breaking change)
|
|
51
|
+
*
|
|
50
52
|
* @generated from protobuf rpc: getClosestPeers(dht.ClosestPeersRequest) returns (dht.ClosestPeersResponse);
|
|
51
53
|
*/
|
|
52
54
|
getClosestPeers(input: ClosestPeersRequest, options?: RpcOptions): UnaryCall<ClosestPeersRequest, ClosestPeersResponse>;
|
|
53
55
|
/**
|
|
56
|
+
* TODO rename to getClosestRingContacts (breaking change)
|
|
57
|
+
*
|
|
54
58
|
* @generated from protobuf rpc: getClosestRingPeers(dht.ClosestRingPeersRequest) returns (dht.ClosestRingPeersResponse);
|
|
55
59
|
*/
|
|
56
60
|
getClosestRingPeers(input: ClosestRingPeersRequest, options?: RpcOptions): UnaryCall<ClosestRingPeersRequest, ClosestRingPeersResponse>;
|
|
@@ -73,6 +77,8 @@ export class DhtNodeRpcClient implements IDhtNodeRpcClient, ServiceInfo {
|
|
|
73
77
|
constructor(private readonly _transport: RpcTransport) {
|
|
74
78
|
}
|
|
75
79
|
/**
|
|
80
|
+
* TODO rename to getClosestNeighbors (breaking change)
|
|
81
|
+
*
|
|
76
82
|
* @generated from protobuf rpc: getClosestPeers(dht.ClosestPeersRequest) returns (dht.ClosestPeersResponse);
|
|
77
83
|
*/
|
|
78
84
|
getClosestPeers(input: ClosestPeersRequest, options?: RpcOptions): UnaryCall<ClosestPeersRequest, ClosestPeersResponse> {
|
|
@@ -80,6 +86,8 @@ export class DhtNodeRpcClient implements IDhtNodeRpcClient, ServiceInfo {
|
|
|
80
86
|
return stackIntercept<ClosestPeersRequest, ClosestPeersResponse>("unary", this._transport, method, opt, input);
|
|
81
87
|
}
|
|
82
88
|
/**
|
|
89
|
+
* TODO rename to getClosestRingContacts (breaking change)
|
|
90
|
+
*
|
|
83
91
|
* @generated from protobuf rpc: getClosestRingPeers(dht.ClosestRingPeersRequest) returns (dht.ClosestRingPeersResponse);
|
|
84
92
|
*/
|
|
85
93
|
getClosestRingPeers(input: ClosestRingPeersRequest, options?: RpcOptions): UnaryCall<ClosestRingPeersRequest, ClosestRingPeersResponse> {
|
|
@@ -34,10 +34,14 @@ import { ServerCallContext } from "@protobuf-ts/runtime-rpc";
|
|
|
34
34
|
*/
|
|
35
35
|
export interface IDhtNodeRpc<T = ServerCallContext> {
|
|
36
36
|
/**
|
|
37
|
+
* TODO rename to getClosestNeighbors (breaking change)
|
|
38
|
+
*
|
|
37
39
|
* @generated from protobuf rpc: getClosestPeers(dht.ClosestPeersRequest) returns (dht.ClosestPeersResponse);
|
|
38
40
|
*/
|
|
39
41
|
getClosestPeers(request: ClosestPeersRequest, context: T): Promise<ClosestPeersResponse>;
|
|
40
42
|
/**
|
|
43
|
+
* TODO rename to getClosestRingContacts (breaking change)
|
|
44
|
+
*
|
|
41
45
|
* @generated from protobuf rpc: getClosestRingPeers(dht.ClosestRingPeersRequest) returns (dht.ClosestRingPeersResponse);
|
|
42
46
|
*/
|
|
43
47
|
getClosestRingPeers(request: ClosestRingPeersRequest, context: T): Promise<ClosestRingPeersResponse>;
|
|
@@ -108,6 +108,8 @@ export interface DataEntry {
|
|
|
108
108
|
deleted: boolean;
|
|
109
109
|
}
|
|
110
110
|
/**
|
|
111
|
+
* TODO rename to ClosestNeighborsRequest
|
|
112
|
+
*
|
|
111
113
|
* @generated from protobuf message dht.ClosestPeersRequest
|
|
112
114
|
*/
|
|
113
115
|
export interface ClosestPeersRequest {
|
|
@@ -121,6 +123,8 @@ export interface ClosestPeersRequest {
|
|
|
121
123
|
requestId: string;
|
|
122
124
|
}
|
|
123
125
|
/**
|
|
126
|
+
* TODO rename to ClosestPeersResponse
|
|
127
|
+
*
|
|
124
128
|
* @generated from protobuf message dht.ClosestPeersResponse
|
|
125
129
|
*/
|
|
126
130
|
export interface ClosestPeersResponse {
|
|
@@ -134,6 +138,8 @@ export interface ClosestPeersResponse {
|
|
|
134
138
|
requestId: string;
|
|
135
139
|
}
|
|
136
140
|
/**
|
|
141
|
+
* TODO rename to ClosestRingContactsRequest
|
|
142
|
+
*
|
|
137
143
|
* @generated from protobuf message dht.ClosestRingPeersRequest
|
|
138
144
|
*/
|
|
139
145
|
export interface ClosestRingPeersRequest {
|
|
@@ -147,6 +153,8 @@ export interface ClosestRingPeersRequest {
|
|
|
147
153
|
requestId: string;
|
|
148
154
|
}
|
|
149
155
|
/**
|
|
156
|
+
* TODO rename to ClosestRingContactsResponse
|
|
157
|
+
*
|
|
150
158
|
* @generated from protobuf message dht.ClosestRingPeersResponse
|
|
151
159
|
*/
|
|
152
160
|
export interface ClosestRingPeersResponse {
|
|
@@ -181,9 +189,9 @@ export interface RecursiveOperationRequest {
|
|
|
181
189
|
*/
|
|
182
190
|
export interface RecursiveOperationResponse {
|
|
183
191
|
/**
|
|
184
|
-
* @generated from protobuf field: repeated dht.PeerDescriptor
|
|
192
|
+
* @generated from protobuf field: repeated dht.PeerDescriptor closestConnectedNodes = 1;
|
|
185
193
|
*/
|
|
186
|
-
|
|
194
|
+
closestConnectedNodes: PeerDescriptor[];
|
|
187
195
|
/**
|
|
188
196
|
* @generated from protobuf field: repeated dht.DataEntry dataEntries = 2;
|
|
189
197
|
*/
|
|
@@ -837,7 +845,7 @@ export const RecursiveOperationRequest = new RecursiveOperationRequest$Type();
|
|
|
837
845
|
class RecursiveOperationResponse$Type extends MessageType<RecursiveOperationResponse> {
|
|
838
846
|
constructor() {
|
|
839
847
|
super("dht.RecursiveOperationResponse", [
|
|
840
|
-
{ no: 1, name: "
|
|
848
|
+
{ no: 1, name: "closestConnectedNodes", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => PeerDescriptor },
|
|
841
849
|
{ no: 2, name: "dataEntries", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => DataEntry },
|
|
842
850
|
{ no: 3, name: "noCloserNodesFound", kind: "scalar", T: 8 /*ScalarType.BOOL*/ },
|
|
843
851
|
{ no: 4, name: "routingPath", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => PeerDescriptor }
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { DhtAddress } from '../identifiers'
|
|
1
2
|
import { Message, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
|
|
2
3
|
|
|
3
4
|
export interface TransportEvents {
|
|
@@ -33,5 +34,7 @@ export interface ITransport {
|
|
|
33
34
|
send(msg: Message, opts?: SendOptions): Promise<void>
|
|
34
35
|
getLocalPeerDescriptor(): PeerDescriptor
|
|
35
36
|
getConnections(): PeerDescriptor[]
|
|
37
|
+
getConnectionCount(): number
|
|
38
|
+
hasConnection(nodeId: DhtAddress): boolean
|
|
36
39
|
stop(): void | Promise<void>
|
|
37
40
|
}
|
|
@@ -72,15 +72,13 @@ describe('Layer0MixedConnectionTypes', () => {
|
|
|
72
72
|
|
|
73
73
|
it('2 non-server peers join first', async () => {
|
|
74
74
|
|
|
75
|
-
|
|
75
|
+
await Promise.all([
|
|
76
76
|
waitForEvent3<TransportEvents>((node3.getTransport() as ConnectionManager), 'connected'),
|
|
77
77
|
waitForEvent3<TransportEvents>((node4.getTransport() as ConnectionManager), 'connected'),
|
|
78
|
+
node3.joinDht([epPeerDescriptor]),
|
|
79
|
+
node4.joinDht([epPeerDescriptor])
|
|
78
80
|
])
|
|
79
81
|
|
|
80
|
-
node3.joinDht([epPeerDescriptor])
|
|
81
|
-
node4.joinDht([epPeerDescriptor])
|
|
82
|
-
|
|
83
|
-
await promise
|
|
84
82
|
await Promise.all([
|
|
85
83
|
node1.joinDht([epPeerDescriptor]),
|
|
86
84
|
node2.joinDht([epPeerDescriptor]),
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { waitForCondition } from '@streamr/utils'
|
|
2
|
+
import { range, without } from 'lodash'
|
|
3
|
+
import { DhtNodeRpcLocal } from '../../src/dht/DhtNodeRpcLocal'
|
|
4
|
+
import { Contact } from '../../src/dht/contact/Contact'
|
|
5
|
+
import { SortedContactList } from '../../src/dht/contact/SortedContactList'
|
|
6
|
+
import { DhtAddress, DhtNode, ListeningRpcCommunicator, getNodeIdFromPeerDescriptor } from '../../src/exports'
|
|
7
|
+
import { ClosestPeersRequest, ClosestPeersResponse, PeerDescriptor, PingRequest, PingResponse } from '../../src/proto/packages/dht/protos/DhtRpc'
|
|
8
|
+
import { FakeEnvironment } from '../utils/FakeTransport'
|
|
9
|
+
import { createMockPeerDescriptor } from '../utils/utils'
|
|
10
|
+
|
|
11
|
+
const OTHER_NODE_COUNT = 3
|
|
12
|
+
const SERVICE_ID_LAYER0 = 'layer0'
|
|
13
|
+
|
|
14
|
+
const getClosestNodes = (
|
|
15
|
+
referenceId: DhtAddress,
|
|
16
|
+
nodes: PeerDescriptor[],
|
|
17
|
+
maxCount: number,
|
|
18
|
+
allowToContainReferenceId: boolean
|
|
19
|
+
): PeerDescriptor[] => {
|
|
20
|
+
const list = new SortedContactList<Contact>({
|
|
21
|
+
referenceId,
|
|
22
|
+
allowToContainReferenceId,
|
|
23
|
+
maxSize: maxCount
|
|
24
|
+
})
|
|
25
|
+
list.addContacts(nodes.map((n) => new Contact(n)))
|
|
26
|
+
return list.getClosestContacts().map((c) => c.getPeerDescriptor())
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
describe('DhtNode', () => {
|
|
30
|
+
|
|
31
|
+
let localPeerDescriptor: PeerDescriptor
|
|
32
|
+
let entryPointPeerDescriptor: PeerDescriptor
|
|
33
|
+
let otherPeerDescriptors: PeerDescriptor[]
|
|
34
|
+
|
|
35
|
+
const startRemoteNode = (peerDescriptor: PeerDescriptor, environment: FakeEnvironment) => {
|
|
36
|
+
const epRpcCommunicator = new ListeningRpcCommunicator(SERVICE_ID_LAYER0, environment.createTransport(peerDescriptor))
|
|
37
|
+
const dhtNodeRpcLocal = new DhtNodeRpcLocal({
|
|
38
|
+
peerDiscoveryQueryBatchSize: undefined as any,
|
|
39
|
+
getClosestNeighborsTo: (nodeId: DhtAddress, maxCount: number) => getClosestNodes(nodeId, getAllPeerDescriptors(), maxCount, true),
|
|
40
|
+
getClosestRingContactsTo: undefined as any,
|
|
41
|
+
addContact: () => {},
|
|
42
|
+
removeContact: undefined as any,
|
|
43
|
+
})
|
|
44
|
+
epRpcCommunicator.registerRpcMethod(PingRequest, PingResponse, 'ping',
|
|
45
|
+
(req: PingRequest, context) => dhtNodeRpcLocal.ping(req, context))
|
|
46
|
+
epRpcCommunicator.registerRpcMethod(ClosestPeersRequest, ClosestPeersResponse, 'getClosestPeers',
|
|
47
|
+
(req: ClosestPeersRequest, context) => dhtNodeRpcLocal.getClosestPeers(req, context))
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const getAllPeerDescriptors = () => {
|
|
51
|
+
return [localPeerDescriptor, entryPointPeerDescriptor, ...otherPeerDescriptors]
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
beforeAll(() => {
|
|
55
|
+
localPeerDescriptor = createMockPeerDescriptor()
|
|
56
|
+
entryPointPeerDescriptor = createMockPeerDescriptor()
|
|
57
|
+
otherPeerDescriptors = range(OTHER_NODE_COUNT).map(() => createMockPeerDescriptor())
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
it('start node and join DHT', async () => {
|
|
61
|
+
const environment = new FakeEnvironment()
|
|
62
|
+
startRemoteNode(entryPointPeerDescriptor, environment)
|
|
63
|
+
for (const other of otherPeerDescriptors) {
|
|
64
|
+
startRemoteNode(other, environment)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const localNode = new DhtNode({
|
|
68
|
+
peerDescriptor: localPeerDescriptor,
|
|
69
|
+
transport: environment.createTransport(localPeerDescriptor),
|
|
70
|
+
entryPoints: [entryPointPeerDescriptor]
|
|
71
|
+
})
|
|
72
|
+
await localNode.start()
|
|
73
|
+
await localNode.joinDht([entryPointPeerDescriptor])
|
|
74
|
+
await localNode.waitForNetworkConnectivity()
|
|
75
|
+
|
|
76
|
+
await waitForCondition(() => localNode.getNeighborCount() === otherPeerDescriptors.length + 1)
|
|
77
|
+
const expectedNodeIds = without(getAllPeerDescriptors(), localPeerDescriptor).map((n) => getNodeIdFromPeerDescriptor(n))
|
|
78
|
+
const actualNodeIds = localNode.getClosestContacts().map((n) => getNodeIdFromPeerDescriptor(n))
|
|
79
|
+
expect(actualNodeIds).toIncludeSameMembers(expectedNodeIds)
|
|
80
|
+
})
|
|
81
|
+
})
|
|
@@ -38,8 +38,8 @@ describe('Layer1', () => {
|
|
|
38
38
|
}, 30000)
|
|
39
39
|
|
|
40
40
|
afterEach(async () => {
|
|
41
|
-
await Promise.all(nodes.map((node) => node.stop()))
|
|
42
41
|
await Promise.all(layer1CleanUp.map((node) => node.stop()))
|
|
42
|
+
await Promise.all(nodes.map((node) => node.stop()))
|
|
43
43
|
await layer0EntryPoint.stop()
|
|
44
44
|
simulator.stop()
|
|
45
45
|
})
|
|
@@ -4,7 +4,7 @@ import { NodeType, PeerDescriptor } from '../../src/proto/packages/dht/protos/Dh
|
|
|
4
4
|
import { createMockConnectionDhtNode } from '../utils/utils'
|
|
5
5
|
import { Logger } from '@streamr/utils'
|
|
6
6
|
import { getRandomRegion } from '../../src/connection/simulator/pings'
|
|
7
|
-
import { areEqualPeerDescriptors, createRandomDhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../src/identifiers'
|
|
7
|
+
import { DhtAddress, areEqualPeerDescriptors, createRandomDhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../src/identifiers'
|
|
8
8
|
|
|
9
9
|
const logger = new Logger(module)
|
|
10
10
|
|
|
@@ -47,13 +47,16 @@ describe('Scaling down a Dht network', () => {
|
|
|
47
47
|
for (let i = 1; i < nodes.length; i++) {
|
|
48
48
|
randomIndices.push(i)
|
|
49
49
|
}
|
|
50
|
+
const stoppedNodes: Set<DhtAddress> = new Set()
|
|
50
51
|
while (randomIndices.length > 1) {
|
|
51
52
|
const index = Math.floor(Math.random() * randomIndices.length)
|
|
52
53
|
const nodeIndex = randomIndices[index]
|
|
53
54
|
randomIndices.splice(index, 1)
|
|
54
|
-
const
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
const nodeToStop = nodes[nodeIndex]
|
|
56
|
+
const stoppingPeerDescriptor = nodeToStop.getLocalPeerDescriptor()
|
|
57
|
+
stoppedNodes.add(getNodeIdFromPeerDescriptor(stoppingPeerDescriptor))
|
|
58
|
+
await nodeToStop.stop()
|
|
59
|
+
const nodeIsCleaned = nodes.filter((node) => !stoppedNodes.has(node.getNodeId())).every((node) =>
|
|
57
60
|
node.getConnections().every((peer) => {
|
|
58
61
|
if (areEqualPeerDescriptors(peer, stoppingPeerDescriptor)) {
|
|
59
62
|
logger.error(getNodeIdFromPeerDescriptor(node.getLocalPeerDescriptor()) + ', '
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { Multimap, wait } from '@streamr/utils'
|
|
2
|
+
import { sampleSize } from 'lodash'
|
|
3
|
+
import { DhtNodeRpcRemote } from '../../src/dht/DhtNodeRpcRemote'
|
|
4
|
+
import { PeerManager, getDistance } from '../../src/dht/PeerManager'
|
|
5
|
+
import { DiscoverySession } from '../../src/dht/discovery/DiscoverySession'
|
|
6
|
+
import { DhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../src/identifiers'
|
|
7
|
+
import { NodeType, PeerDescriptor } from '../../src/proto/packages/dht/protos/DhtRpc'
|
|
8
|
+
import { createTestTopology } from '../utils/topology'
|
|
9
|
+
|
|
10
|
+
const NODE_COUNT = 40
|
|
11
|
+
const MIN_NEIGHBOR_COUNT = 2 // nodes can get more neighbors when we merge network partitions
|
|
12
|
+
const PARALLELISM = 1
|
|
13
|
+
const NO_PROGRESS_LIMIT = 1
|
|
14
|
+
const QUERY_BATCH_SIZE = 5 // the default value in DhtNode's config, not relevant in this test
|
|
15
|
+
|
|
16
|
+
const createPeerDescriptor = (nodeId: DhtAddress): PeerDescriptor => {
|
|
17
|
+
return {
|
|
18
|
+
nodeId: getRawFromDhtAddress(nodeId),
|
|
19
|
+
type: NodeType.NODEJS
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
describe('DiscoverySession', () => {
|
|
24
|
+
|
|
25
|
+
let topology: Multimap<DhtAddress, DhtAddress>
|
|
26
|
+
const queriedNodes: DhtAddress[] = []
|
|
27
|
+
|
|
28
|
+
beforeAll(() => {
|
|
29
|
+
topology = createTestTopology(NODE_COUNT, MIN_NEIGHBOR_COUNT)
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
const createPeerManager = (localNodeId: DhtAddress): PeerManager => {
|
|
33
|
+
const peerManager = new PeerManager({
|
|
34
|
+
localNodeId,
|
|
35
|
+
localPeerDescriptor: createPeerDescriptor(localNodeId),
|
|
36
|
+
isLayer0: true,
|
|
37
|
+
createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => createMockRpcRemote(peerDescriptor) as any,
|
|
38
|
+
hasConnection: () => true
|
|
39
|
+
} as any)
|
|
40
|
+
for (const neighbor of topology.get(localNodeId)) {
|
|
41
|
+
peerManager.addContact(createPeerDescriptor(neighbor))
|
|
42
|
+
}
|
|
43
|
+
return peerManager
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const createMockRpcRemote = (peerDescriptor: PeerDescriptor): Partial<DhtNodeRpcRemote> => {
|
|
47
|
+
const nodeId = getNodeIdFromPeerDescriptor(peerDescriptor)
|
|
48
|
+
return {
|
|
49
|
+
id: getRawFromDhtAddress(nodeId),
|
|
50
|
+
getPeerDescriptor: () => peerDescriptor,
|
|
51
|
+
getNodeId: () => nodeId,
|
|
52
|
+
getClosestPeers: async (referenceId: DhtAddress) => {
|
|
53
|
+
queriedNodes.push(nodeId)
|
|
54
|
+
await wait(10)
|
|
55
|
+
const peerManager = createPeerManager(nodeId)
|
|
56
|
+
return peerManager.getClosestNeighborsTo(referenceId, QUERY_BATCH_SIZE).map((remote) => remote.getPeerDescriptor())
|
|
57
|
+
},
|
|
58
|
+
ping: async () => true
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
it('happy path', async () => {
|
|
63
|
+
const nodeIds = [...topology.keys()]
|
|
64
|
+
const [localNodeId, targetId] = sampleSize(nodeIds, 2)
|
|
65
|
+
const contactedPeers = new Set<DhtAddress>()
|
|
66
|
+
const peerManager = createPeerManager(localNodeId)
|
|
67
|
+
const session = new DiscoverySession({
|
|
68
|
+
targetId,
|
|
69
|
+
parallelism: PARALLELISM,
|
|
70
|
+
noProgressLimit: NO_PROGRESS_LIMIT,
|
|
71
|
+
peerManager,
|
|
72
|
+
contactedPeers,
|
|
73
|
+
abortSignal: new AbortController().signal
|
|
74
|
+
})
|
|
75
|
+
await session.findClosestNodes(1000)
|
|
76
|
+
expect(queriedNodes.length).toBeGreaterThanOrEqual(1)
|
|
77
|
+
// Each queried node should closer to the target than the previous queried node, because we
|
|
78
|
+
// use parallelism=1 and noProgressLimit=1
|
|
79
|
+
const distancesToTarget = queriedNodes
|
|
80
|
+
.map((nodeId) => getDistance(getRawFromDhtAddress(nodeId), getRawFromDhtAddress(targetId)))
|
|
81
|
+
for (let i = 1; i < distancesToTarget.length ; i++) {
|
|
82
|
+
expect(distancesToTarget[i]).toBeLessThan(distancesToTarget[i - 1])
|
|
83
|
+
}
|
|
84
|
+
})
|
|
85
|
+
})
|
|
@@ -26,7 +26,8 @@ const createPeerManager = (
|
|
|
26
26
|
}
|
|
27
27
|
}(localPeerDescriptor, peerDescriptor, undefined as any, new MockRpcCommunicator())
|
|
28
28
|
return remote
|
|
29
|
-
}
|
|
29
|
+
},
|
|
30
|
+
hasConnection: () => false
|
|
30
31
|
} as any)
|
|
31
32
|
const contacts = nodeIds.map((n) => ({ nodeId: getRawFromDhtAddress(n), type: NodeType.NODEJS }))
|
|
32
33
|
for (const contact of contacts) {
|
|
@@ -53,21 +54,6 @@ const getClosestContact = (contacts: PeerDescriptor[], referenceId: DhtAddress):
|
|
|
53
54
|
|
|
54
55
|
describe('PeerManager', () => {
|
|
55
56
|
|
|
56
|
-
it('getClosestContactsTo', () => {
|
|
57
|
-
const nodeIds = range(10).map(() => createRandomDhtAddress())
|
|
58
|
-
const manager = createPeerManager(nodeIds)
|
|
59
|
-
const referenceId = createRandomDhtAddress()
|
|
60
|
-
const excluded = new Set<DhtAddress>(sampleSize(nodeIds, 2))
|
|
61
|
-
|
|
62
|
-
const actual = manager.getClosestContactsTo(referenceId, 5, excluded)
|
|
63
|
-
|
|
64
|
-
const expected = sortBy(
|
|
65
|
-
without(nodeIds, ...Array.from(excluded.values())),
|
|
66
|
-
(n: DhtAddress) => getDistance(getRawFromDhtAddress(n), getRawFromDhtAddress(referenceId))
|
|
67
|
-
).slice(0, 5)
|
|
68
|
-
expect(actual.map((n) => n.getNodeId())).toEqual(expected)
|
|
69
|
-
})
|
|
70
|
-
|
|
71
57
|
it('getClosestNeighborsTo', () => {
|
|
72
58
|
const nodeIds = range(10).map(() => createRandomDhtAddress())
|
|
73
59
|
const manager = createPeerManager(nodeIds)
|
|
@@ -99,7 +85,7 @@ describe('PeerManager', () => {
|
|
|
99
85
|
for (const successContact of successContacts) {
|
|
100
86
|
manager.addContact(successContact)
|
|
101
87
|
manager.setContactActive(getNodeIdFromPeerDescriptor(successContact))
|
|
102
|
-
manager.
|
|
88
|
+
manager.removeNeighbor(getNodeIdFromPeerDescriptor(successContact))
|
|
103
89
|
}
|
|
104
90
|
expect(manager.getNeighborCount()).toBe(0)
|
|
105
91
|
manager.addContact(failureContact)
|
|
@@ -74,12 +74,12 @@ describe('RecursiveOperationManager', () => {
|
|
|
74
74
|
return new RecursiveOperationManager({
|
|
75
75
|
localPeerDescriptor: peerDescriptor1,
|
|
76
76
|
router,
|
|
77
|
-
connections: new Map(),
|
|
78
77
|
serviceId: 'RecursiveOperationManager',
|
|
79
78
|
localDataStore: new LocalDataStore(30 * 100),
|
|
80
79
|
sessionTransport: transport,
|
|
81
80
|
addContact: () => {},
|
|
82
|
-
rpcCommunicator: rpcCommunicator as any
|
|
81
|
+
rpcCommunicator: rpcCommunicator as any,
|
|
82
|
+
createDhtNodeRpcRemote: () => undefined as any
|
|
83
83
|
})
|
|
84
84
|
}
|
|
85
85
|
|
|
@@ -126,6 +126,7 @@ describe('RecursiveOperationManager', () => {
|
|
|
126
126
|
const send = jest.fn()
|
|
127
127
|
const transport = {
|
|
128
128
|
send,
|
|
129
|
+
getConnections: () => [],
|
|
129
130
|
on: () => {},
|
|
130
131
|
off: () => {}
|
|
131
132
|
}
|
|
@@ -143,7 +144,8 @@ describe('RecursiveOperationManager', () => {
|
|
|
143
144
|
const router = createMockRouter(RouteMessageError.DUPLICATE)
|
|
144
145
|
const send = jest.fn()
|
|
145
146
|
const transport = {
|
|
146
|
-
send
|
|
147
|
+
send,
|
|
148
|
+
getConnections: () => []
|
|
147
149
|
}
|
|
148
150
|
const recursiveOperationManager = createRecursiveOperationManager(router as any, transport as any)
|
|
149
151
|
const ack = await rpcCommunicator.callRpcMethod('routeRequest', routedMessage)
|
package/test/unit/Router.test.ts
CHANGED
|
@@ -50,8 +50,8 @@ describe('Router', () => {
|
|
|
50
50
|
router = new Router({
|
|
51
51
|
localPeerDescriptor: peerDescriptor1,
|
|
52
52
|
rpcCommunicator: rpcCommunicator as any,
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
handleMessage: () => {},
|
|
54
|
+
getConnections: () => [...connections.values()].map((c) => c.getPeerDescriptor())
|
|
55
55
|
})
|
|
56
56
|
})
|
|
57
57
|
|
|
@@ -49,11 +49,11 @@ describe('RoutingSession', () => {
|
|
|
49
49
|
rpcCommunicator: rpcCommunicator,
|
|
50
50
|
localPeerDescriptor: mockPeerDescriptor1,
|
|
51
51
|
routedMessage,
|
|
52
|
-
connections,
|
|
53
52
|
parallelism: 2,
|
|
54
53
|
mode: RoutingMode.ROUTE,
|
|
55
54
|
excludedNodeIds: new Set(),
|
|
56
|
-
routingTablesCache
|
|
55
|
+
routingTablesCache,
|
|
56
|
+
getConnections: () => [...connections.values()].map((c) => c.getPeerDescriptor())
|
|
57
57
|
})
|
|
58
58
|
})
|
|
59
59
|
|
|
@@ -38,7 +38,7 @@ describe('SortedContactList', () => {
|
|
|
38
38
|
expect(list.getSize()).toEqual(3)
|
|
39
39
|
expect(list.getClosestContacts()).toEqual([item1, item2, item3])
|
|
40
40
|
expect(list.getContactIds()).toEqual([item1.getNodeId(), item2.getNodeId(), item3.getNodeId()])
|
|
41
|
-
expect(onContactRemoved).toBeCalledWith(item4
|
|
41
|
+
expect(onContactRemoved).toBeCalledWith(item4)
|
|
42
42
|
expect(list.getContact(item4.getNodeId())).toBeFalsy()
|
|
43
43
|
})
|
|
44
44
|
|
|
@@ -63,9 +63,9 @@ describe('SortedContactList', () => {
|
|
|
63
63
|
list.removeContact(item3.getNodeId())
|
|
64
64
|
list.removeContact(createRandomDhtAddress())
|
|
65
65
|
expect(list.getClosestContacts()).toEqual([item1, item4])
|
|
66
|
-
expect(onContactRemoved).toHaveBeenNthCalledWith(1, item3
|
|
67
|
-
expect(onContactRemoved).toHaveBeenNthCalledWith(2, item2
|
|
68
|
-
expect(onContactRemoved).toHaveBeenNthCalledWith(3, item3
|
|
66
|
+
expect(onContactRemoved).toHaveBeenNthCalledWith(1, item3)
|
|
67
|
+
expect(onContactRemoved).toHaveBeenNthCalledWith(2, item2)
|
|
68
|
+
expect(onContactRemoved).toHaveBeenNthCalledWith(3, item3)
|
|
69
69
|
})
|
|
70
70
|
|
|
71
71
|
it('get closest contacts', () => {
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { range, sampleSize, sortBy, without } from 'lodash'
|
|
2
|
+
import { getDistance } from '../../src/dht/PeerManager'
|
|
3
|
+
import { getClosestContacts } from '../../src/dht/contact/getClosestContacts'
|
|
4
|
+
import { DhtAddress, createRandomDhtAddress, getRawFromDhtAddress } from '../../src/identifiers'
|
|
5
|
+
|
|
6
|
+
describe('getClosestContacts', () => {
|
|
7
|
+
|
|
8
|
+
it('happy path', () => {
|
|
9
|
+
const nodeIds = range(10).map(() => createRandomDhtAddress())
|
|
10
|
+
const referenceId = createRandomDhtAddress()
|
|
11
|
+
const excluded = new Set<DhtAddress>(sampleSize(nodeIds, 2))
|
|
12
|
+
|
|
13
|
+
const actual = getClosestContacts(
|
|
14
|
+
referenceId,
|
|
15
|
+
nodeIds.map((nodeId) => ({ getNodeId: () => nodeId })),
|
|
16
|
+
{
|
|
17
|
+
maxCount: 5,
|
|
18
|
+
excludedNodeIds: excluded
|
|
19
|
+
}
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
const expected = sortBy(
|
|
23
|
+
without(nodeIds, ...Array.from(excluded.values())),
|
|
24
|
+
(n: DhtAddress) => getDistance(getRawFromDhtAddress(n), getRawFromDhtAddress(referenceId))
|
|
25
|
+
).slice(0, 5)
|
|
26
|
+
expect(actual.map((n) => n.getNodeId())).toEqual(expected)
|
|
27
|
+
})
|
|
28
|
+
})
|