@streamr/dht 100.2.2 → 100.2.4-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +1 -1
  2. package/dist/package.json +6 -6
  3. package/dist/src/connection/ConnectionLockStates.js +4 -6
  4. package/dist/src/connection/ConnectionLockStates.js.map +1 -1
  5. package/dist/src/connection/ConnectionManager.d.ts +1 -0
  6. package/dist/src/connection/ConnectionManager.js +8 -1
  7. package/dist/src/connection/ConnectionManager.js.map +1 -1
  8. package/dist/src/connection/ManagedConnection.js +7 -9
  9. package/dist/src/connection/ManagedConnection.js.map +1 -1
  10. package/dist/src/connection/websocket/WebsocketConnector.js +1 -0
  11. package/dist/src/connection/websocket/WebsocketConnector.js.map +1 -1
  12. package/dist/src/dht/DhtNode.d.ts +10 -7
  13. package/dist/src/dht/DhtNode.js +51 -32
  14. package/dist/src/dht/DhtNode.js.map +1 -1
  15. package/dist/src/dht/DhtNodeRpcLocal.d.ts +2 -2
  16. package/dist/src/dht/DhtNodeRpcLocal.js +5 -4
  17. package/dist/src/dht/DhtNodeRpcLocal.js.map +1 -1
  18. package/dist/src/dht/DhtNodeRpcRemote.js +1 -0
  19. package/dist/src/dht/DhtNodeRpcRemote.js.map +1 -1
  20. package/dist/src/dht/PeerManager.d.ts +14 -14
  21. package/dist/src/dht/PeerManager.js +23 -58
  22. package/dist/src/dht/PeerManager.js.map +1 -1
  23. package/dist/src/dht/contact/ContactList.d.ts +2 -2
  24. package/dist/src/dht/contact/RandomContactList.js +2 -2
  25. package/dist/src/dht/contact/RandomContactList.js.map +1 -1
  26. package/dist/src/dht/contact/RingContactList.d.ts +2 -5
  27. package/dist/src/dht/contact/RingContactList.js +3 -11
  28. package/dist/src/dht/contact/RingContactList.js.map +1 -1
  29. package/dist/src/dht/contact/SortedContactList.d.ts +4 -1
  30. package/dist/src/dht/contact/SortedContactList.js +4 -5
  31. package/dist/src/dht/contact/SortedContactList.js.map +1 -1
  32. package/dist/src/dht/contact/getClosestContacts.d.ts +7 -0
  33. package/dist/src/dht/contact/getClosestContacts.js +18 -0
  34. package/dist/src/dht/contact/getClosestContacts.js.map +1 -0
  35. package/dist/src/dht/discovery/DiscoverySession.d.ts +8 -8
  36. package/dist/src/dht/discovery/DiscoverySession.js +30 -35
  37. package/dist/src/dht/discovery/DiscoverySession.js.map +1 -1
  38. package/dist/src/dht/discovery/PeerDiscovery.d.ts +1 -2
  39. package/dist/src/dht/discovery/PeerDiscovery.js +6 -8
  40. package/dist/src/dht/discovery/PeerDiscovery.js.map +1 -1
  41. package/dist/src/dht/discovery/RingDiscoverySession.d.ts +4 -4
  42. package/dist/src/dht/discovery/RingDiscoverySession.js +13 -13
  43. package/dist/src/dht/discovery/RingDiscoverySession.js.map +1 -1
  44. package/dist/src/dht/recursive-operation/RecursiveOperationManager.d.ts +2 -2
  45. package/dist/src/dht/recursive-operation/RecursiveOperationManager.js +15 -15
  46. package/dist/src/dht/recursive-operation/RecursiveOperationManager.js.map +1 -1
  47. package/dist/src/dht/recursive-operation/RecursiveOperationSession.d.ts +1 -1
  48. package/dist/src/dht/recursive-operation/RecursiveOperationSession.js +11 -13
  49. package/dist/src/dht/recursive-operation/RecursiveOperationSession.js.map +1 -1
  50. package/dist/src/dht/recursive-operation/RecursiveOperationSessionRpcLocal.js +1 -1
  51. package/dist/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.d.ts +1 -1
  52. package/dist/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.js +2 -2
  53. package/dist/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.js.map +1 -1
  54. package/dist/src/dht/routing/Router.d.ts +1 -2
  55. package/dist/src/dht/routing/Router.js +2 -3
  56. package/dist/src/dht/routing/Router.js.map +1 -1
  57. package/dist/src/dht/routing/RoutingSession.d.ts +1 -2
  58. package/dist/src/dht/routing/RoutingSession.js +2 -2
  59. package/dist/src/dht/routing/RoutingSession.js.map +1 -1
  60. package/dist/src/dht/store/StoreRpcLocal.js +3 -3
  61. package/dist/src/dht/store/StoreRpcLocal.js.map +1 -1
  62. package/dist/src/proto/packages/dht/protos/DhtRpc.client.d.ts +8 -0
  63. package/dist/src/proto/packages/dht/protos/DhtRpc.client.js +4 -0
  64. package/dist/src/proto/packages/dht/protos/DhtRpc.client.js.map +1 -1
  65. package/dist/src/proto/packages/dht/protos/DhtRpc.d.ts +10 -2
  66. package/dist/src/proto/packages/dht/protos/DhtRpc.js +1 -1
  67. package/dist/src/proto/packages/dht/protos/DhtRpc.js.map +1 -1
  68. package/dist/src/proto/packages/dht/protos/DhtRpc.server.d.ts +4 -0
  69. package/dist/src/transport/ITransport.d.ts +3 -0
  70. package/dist/src/transport/ITransport.js.map +1 -1
  71. package/package.json +6 -6
  72. package/protos/DhtRpc.proto +7 -1
  73. package/src/connection/ConnectionLockStates.ts +3 -5
  74. package/src/connection/ConnectionManager.ts +9 -1
  75. package/src/connection/ManagedConnection.ts +11 -14
  76. package/src/connection/websocket/WebsocketConnector.ts +1 -0
  77. package/src/dht/DhtNode.ts +63 -44
  78. package/src/dht/DhtNodeRpcLocal.ts +7 -6
  79. package/src/dht/DhtNodeRpcRemote.ts +1 -0
  80. package/src/dht/PeerManager.ts +38 -72
  81. package/src/dht/contact/ContactList.ts +2 -2
  82. package/src/dht/contact/RandomContactList.ts +2 -3
  83. package/src/dht/contact/RingContactList.ts +6 -16
  84. package/src/dht/contact/SortedContactList.ts +9 -10
  85. package/src/dht/contact/getClosestContacts.ts +22 -0
  86. package/src/dht/discovery/DiscoverySession.ts +35 -45
  87. package/src/dht/discovery/PeerDiscovery.ts +6 -9
  88. package/src/dht/discovery/RingDiscoverySession.ts +13 -13
  89. package/src/dht/recursive-operation/RecursiveOperationManager.ts +16 -16
  90. package/src/dht/recursive-operation/RecursiveOperationSession.ts +11 -13
  91. package/src/dht/recursive-operation/RecursiveOperationSessionRpcLocal.ts +1 -1
  92. package/src/dht/recursive-operation/RecursiveOperationSessionRpcRemote.ts +2 -2
  93. package/src/dht/routing/Router.ts +3 -5
  94. package/src/dht/routing/RoutingSession.ts +3 -4
  95. package/src/dht/store/StoreRpcLocal.ts +3 -3
  96. package/src/proto/packages/dht/protos/DhtRpc.client.ts +8 -0
  97. package/src/proto/packages/dht/protos/DhtRpc.server.ts +4 -0
  98. package/src/proto/packages/dht/protos/DhtRpc.ts +11 -3
  99. package/src/transport/ITransport.ts +3 -0
  100. package/test/end-to-end/Layer0MixedConnectionTypes.test.ts +3 -5
  101. package/test/integration/DhtNode.test.ts +81 -0
  102. package/test/integration/Layer1-scale.test.ts +1 -1
  103. package/test/integration/ScaleDownDht.test.ts +7 -4
  104. package/test/unit/DiscoverySession.test.ts +85 -0
  105. package/test/unit/PeerManager.test.ts +3 -17
  106. package/test/unit/RecursiveOperationManager.test.ts +5 -3
  107. package/test/unit/Router.test.ts +2 -2
  108. package/test/unit/RoutingSession.test.ts +2 -2
  109. package/test/unit/SortedContactList.test.ts +4 -4
  110. package/test/unit/getClosestContacts.test.ts +28 -0
  111. package/test/utils/FakeTransport.ts +34 -7
  112. package/test/utils/mock/Transport.ts +10 -0
  113. package/test/utils/topology.ts +80 -0
  114. package/test/RandomGraphSimulation.ts +0 -53
@@ -1,4 +1,3 @@
1
- import { DhtNodeRpcRemote } from '../DhtNodeRpcRemote'
2
1
  import { SortedContactList } from '../contact/SortedContactList'
3
2
  import { Logger } from '@streamr/utils'
4
3
  import EventEmitter from 'eventemitter3'
@@ -69,11 +68,11 @@ interface RoutingSessionConfig {
69
68
  rpcCommunicator: RoutingRpcCommunicator
70
69
  localPeerDescriptor: PeerDescriptor
71
70
  routedMessage: RouteMessageWrapper
72
- connections: Map<DhtAddress, DhtNodeRpcRemote>
73
71
  parallelism: number
74
72
  mode: RoutingMode
75
73
  excludedNodeIds: Set<DhtAddress>
76
74
  routingTablesCache: RoutingTablesCache
75
+ getConnections: () => PeerDescriptor[]
77
76
  }
78
77
 
79
78
  export class RoutingSession extends EventEmitter<RoutingSessionEvents> {
@@ -174,9 +173,9 @@ export class RoutingSession extends EventEmitter<RoutingSessionEvents> {
174
173
  allowToContainReferenceId: true,
175
174
  nodeIdDistanceLimit: previousId
176
175
  })
177
- const contacts = Array.from(this.config.connections.values())
176
+ const contacts = this.config.getConnections()
178
177
  .map((peer) => new RoutingRemoteContact(
179
- peer.getPeerDescriptor(),
178
+ peer,
180
179
  this.config.localPeerDescriptor,
181
180
  this.config.rpcCommunicator
182
181
  ))
@@ -32,7 +32,7 @@ export class StoreRpcLocal implements IStoreRpc {
32
32
  async storeData(request: StoreDataRequest): Promise<StoreDataResponse> {
33
33
  logger.trace('storeData()')
34
34
  const key = getDhtAddressFromRaw(request.key)
35
- const selfIsOneOfClosestPeers = this.config.selfIsWithinRedundancyFactor(key)
35
+ const selfIsWithinRedundancyFactor = this.config.selfIsWithinRedundancyFactor(key)
36
36
  this.config.localDataStore.storeEntry({
37
37
  key: request.key,
38
38
  data: request.data,
@@ -40,10 +40,10 @@ export class StoreRpcLocal implements IStoreRpc {
40
40
  createdAt: request.createdAt,
41
41
  storedAt: Timestamp.now(),
42
42
  ttl: request.ttl,
43
- stale: !selfIsOneOfClosestPeers,
43
+ stale: !selfIsWithinRedundancyFactor,
44
44
  deleted: false
45
45
  })
46
- if (!selfIsOneOfClosestPeers) {
46
+ if (!selfIsWithinRedundancyFactor) {
47
47
  this.config.localDataStore.setAllEntriesAsStale(key)
48
48
  }
49
49
  return {}
@@ -47,10 +47,14 @@ import type { RpcOptions } from "@protobuf-ts/runtime-rpc";
47
47
  */
48
48
  export interface IDhtNodeRpcClient {
49
49
  /**
50
+ * TODO rename to getClosestNeighbors (breaking change)
51
+ *
50
52
  * @generated from protobuf rpc: getClosestPeers(dht.ClosestPeersRequest) returns (dht.ClosestPeersResponse);
51
53
  */
52
54
  getClosestPeers(input: ClosestPeersRequest, options?: RpcOptions): UnaryCall<ClosestPeersRequest, ClosestPeersResponse>;
53
55
  /**
56
+ * TODO rename to getClosestRingContacts (breaking change)
57
+ *
54
58
  * @generated from protobuf rpc: getClosestRingPeers(dht.ClosestRingPeersRequest) returns (dht.ClosestRingPeersResponse);
55
59
  */
56
60
  getClosestRingPeers(input: ClosestRingPeersRequest, options?: RpcOptions): UnaryCall<ClosestRingPeersRequest, ClosestRingPeersResponse>;
@@ -73,6 +77,8 @@ export class DhtNodeRpcClient implements IDhtNodeRpcClient, ServiceInfo {
73
77
  constructor(private readonly _transport: RpcTransport) {
74
78
  }
75
79
  /**
80
+ * TODO rename to getClosestNeighbors (breaking change)
81
+ *
76
82
  * @generated from protobuf rpc: getClosestPeers(dht.ClosestPeersRequest) returns (dht.ClosestPeersResponse);
77
83
  */
78
84
  getClosestPeers(input: ClosestPeersRequest, options?: RpcOptions): UnaryCall<ClosestPeersRequest, ClosestPeersResponse> {
@@ -80,6 +86,8 @@ export class DhtNodeRpcClient implements IDhtNodeRpcClient, ServiceInfo {
80
86
  return stackIntercept<ClosestPeersRequest, ClosestPeersResponse>("unary", this._transport, method, opt, input);
81
87
  }
82
88
  /**
89
+ * TODO rename to getClosestRingContacts (breaking change)
90
+ *
83
91
  * @generated from protobuf rpc: getClosestRingPeers(dht.ClosestRingPeersRequest) returns (dht.ClosestRingPeersResponse);
84
92
  */
85
93
  getClosestRingPeers(input: ClosestRingPeersRequest, options?: RpcOptions): UnaryCall<ClosestRingPeersRequest, ClosestRingPeersResponse> {
@@ -34,10 +34,14 @@ import { ServerCallContext } from "@protobuf-ts/runtime-rpc";
34
34
  */
35
35
  export interface IDhtNodeRpc<T = ServerCallContext> {
36
36
  /**
37
+ * TODO rename to getClosestNeighbors (breaking change)
38
+ *
37
39
  * @generated from protobuf rpc: getClosestPeers(dht.ClosestPeersRequest) returns (dht.ClosestPeersResponse);
38
40
  */
39
41
  getClosestPeers(request: ClosestPeersRequest, context: T): Promise<ClosestPeersResponse>;
40
42
  /**
43
+ * TODO rename to getClosestRingContacts (breaking change)
44
+ *
41
45
  * @generated from protobuf rpc: getClosestRingPeers(dht.ClosestRingPeersRequest) returns (dht.ClosestRingPeersResponse);
42
46
  */
43
47
  getClosestRingPeers(request: ClosestRingPeersRequest, context: T): Promise<ClosestRingPeersResponse>;
@@ -108,6 +108,8 @@ export interface DataEntry {
108
108
  deleted: boolean;
109
109
  }
110
110
  /**
111
+ * TODO rename to ClosestNeighborsRequest
112
+ *
111
113
  * @generated from protobuf message dht.ClosestPeersRequest
112
114
  */
113
115
  export interface ClosestPeersRequest {
@@ -121,6 +123,8 @@ export interface ClosestPeersRequest {
121
123
  requestId: string;
122
124
  }
123
125
  /**
126
+ * TODO rename to ClosestPeersResponse
127
+ *
124
128
  * @generated from protobuf message dht.ClosestPeersResponse
125
129
  */
126
130
  export interface ClosestPeersResponse {
@@ -134,6 +138,8 @@ export interface ClosestPeersResponse {
134
138
  requestId: string;
135
139
  }
136
140
  /**
141
+ * TODO rename to ClosestRingContactsRequest
142
+ *
137
143
  * @generated from protobuf message dht.ClosestRingPeersRequest
138
144
  */
139
145
  export interface ClosestRingPeersRequest {
@@ -147,6 +153,8 @@ export interface ClosestRingPeersRequest {
147
153
  requestId: string;
148
154
  }
149
155
  /**
156
+ * TODO rename to ClosestRingContactsResponse
157
+ *
150
158
  * @generated from protobuf message dht.ClosestRingPeersResponse
151
159
  */
152
160
  export interface ClosestRingPeersResponse {
@@ -181,9 +189,9 @@ export interface RecursiveOperationRequest {
181
189
  */
182
190
  export interface RecursiveOperationResponse {
183
191
  /**
184
- * @generated from protobuf field: repeated dht.PeerDescriptor closestConnectedPeers = 1;
192
+ * @generated from protobuf field: repeated dht.PeerDescriptor closestConnectedNodes = 1;
185
193
  */
186
- closestConnectedPeers: PeerDescriptor[];
194
+ closestConnectedNodes: PeerDescriptor[];
187
195
  /**
188
196
  * @generated from protobuf field: repeated dht.DataEntry dataEntries = 2;
189
197
  */
@@ -837,7 +845,7 @@ export const RecursiveOperationRequest = new RecursiveOperationRequest$Type();
837
845
  class RecursiveOperationResponse$Type extends MessageType<RecursiveOperationResponse> {
838
846
  constructor() {
839
847
  super("dht.RecursiveOperationResponse", [
840
- { no: 1, name: "closestConnectedPeers", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => PeerDescriptor },
848
+ { no: 1, name: "closestConnectedNodes", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => PeerDescriptor },
841
849
  { no: 2, name: "dataEntries", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => DataEntry },
842
850
  { no: 3, name: "noCloserNodesFound", kind: "scalar", T: 8 /*ScalarType.BOOL*/ },
843
851
  { no: 4, name: "routingPath", kind: "message", repeat: 1 /*RepeatType.PACKED*/, T: () => PeerDescriptor }
@@ -1,3 +1,4 @@
1
+ import { DhtAddress } from '../identifiers'
1
2
  import { Message, PeerDescriptor } from '../proto/packages/dht/protos/DhtRpc'
2
3
 
3
4
  export interface TransportEvents {
@@ -33,5 +34,7 @@ export interface ITransport {
33
34
  send(msg: Message, opts?: SendOptions): Promise<void>
34
35
  getLocalPeerDescriptor(): PeerDescriptor
35
36
  getConnections(): PeerDescriptor[]
37
+ getConnectionCount(): number
38
+ hasConnection(nodeId: DhtAddress): boolean
36
39
  stop(): void | Promise<void>
37
40
  }
@@ -72,15 +72,13 @@ describe('Layer0MixedConnectionTypes', () => {
72
72
 
73
73
  it('2 non-server peers join first', async () => {
74
74
 
75
- const promise = Promise.all([
75
+ await Promise.all([
76
76
  waitForEvent3<TransportEvents>((node3.getTransport() as ConnectionManager), 'connected'),
77
77
  waitForEvent3<TransportEvents>((node4.getTransport() as ConnectionManager), 'connected'),
78
+ node3.joinDht([epPeerDescriptor]),
79
+ node4.joinDht([epPeerDescriptor])
78
80
  ])
79
81
 
80
- node3.joinDht([epPeerDescriptor])
81
- node4.joinDht([epPeerDescriptor])
82
-
83
- await promise
84
82
  await Promise.all([
85
83
  node1.joinDht([epPeerDescriptor]),
86
84
  node2.joinDht([epPeerDescriptor]),
@@ -0,0 +1,81 @@
1
+ import { waitForCondition } from '@streamr/utils'
2
+ import { range, without } from 'lodash'
3
+ import { DhtNodeRpcLocal } from '../../src/dht/DhtNodeRpcLocal'
4
+ import { Contact } from '../../src/dht/contact/Contact'
5
+ import { SortedContactList } from '../../src/dht/contact/SortedContactList'
6
+ import { DhtAddress, DhtNode, ListeningRpcCommunicator, getNodeIdFromPeerDescriptor } from '../../src/exports'
7
+ import { ClosestPeersRequest, ClosestPeersResponse, PeerDescriptor, PingRequest, PingResponse } from '../../src/proto/packages/dht/protos/DhtRpc'
8
+ import { FakeEnvironment } from '../utils/FakeTransport'
9
+ import { createMockPeerDescriptor } from '../utils/utils'
10
+
11
+ const OTHER_NODE_COUNT = 3
12
+ const SERVICE_ID_LAYER0 = 'layer0'
13
+
14
+ const getClosestNodes = (
15
+ referenceId: DhtAddress,
16
+ nodes: PeerDescriptor[],
17
+ maxCount: number,
18
+ allowToContainReferenceId: boolean
19
+ ): PeerDescriptor[] => {
20
+ const list = new SortedContactList<Contact>({
21
+ referenceId,
22
+ allowToContainReferenceId,
23
+ maxSize: maxCount
24
+ })
25
+ list.addContacts(nodes.map((n) => new Contact(n)))
26
+ return list.getClosestContacts().map((c) => c.getPeerDescriptor())
27
+ }
28
+
29
+ describe('DhtNode', () => {
30
+
31
+ let localPeerDescriptor: PeerDescriptor
32
+ let entryPointPeerDescriptor: PeerDescriptor
33
+ let otherPeerDescriptors: PeerDescriptor[]
34
+
35
+ const startRemoteNode = (peerDescriptor: PeerDescriptor, environment: FakeEnvironment) => {
36
+ const epRpcCommunicator = new ListeningRpcCommunicator(SERVICE_ID_LAYER0, environment.createTransport(peerDescriptor))
37
+ const dhtNodeRpcLocal = new DhtNodeRpcLocal({
38
+ peerDiscoveryQueryBatchSize: undefined as any,
39
+ getClosestNeighborsTo: (nodeId: DhtAddress, maxCount: number) => getClosestNodes(nodeId, getAllPeerDescriptors(), maxCount, true),
40
+ getClosestRingContactsTo: undefined as any,
41
+ addContact: () => {},
42
+ removeContact: undefined as any,
43
+ })
44
+ epRpcCommunicator.registerRpcMethod(PingRequest, PingResponse, 'ping',
45
+ (req: PingRequest, context) => dhtNodeRpcLocal.ping(req, context))
46
+ epRpcCommunicator.registerRpcMethod(ClosestPeersRequest, ClosestPeersResponse, 'getClosestPeers',
47
+ (req: ClosestPeersRequest, context) => dhtNodeRpcLocal.getClosestPeers(req, context))
48
+ }
49
+
50
+ const getAllPeerDescriptors = () => {
51
+ return [localPeerDescriptor, entryPointPeerDescriptor, ...otherPeerDescriptors]
52
+ }
53
+
54
+ beforeAll(() => {
55
+ localPeerDescriptor = createMockPeerDescriptor()
56
+ entryPointPeerDescriptor = createMockPeerDescriptor()
57
+ otherPeerDescriptors = range(OTHER_NODE_COUNT).map(() => createMockPeerDescriptor())
58
+ })
59
+
60
+ it('start node and join DHT', async () => {
61
+ const environment = new FakeEnvironment()
62
+ startRemoteNode(entryPointPeerDescriptor, environment)
63
+ for (const other of otherPeerDescriptors) {
64
+ startRemoteNode(other, environment)
65
+ }
66
+
67
+ const localNode = new DhtNode({
68
+ peerDescriptor: localPeerDescriptor,
69
+ transport: environment.createTransport(localPeerDescriptor),
70
+ entryPoints: [entryPointPeerDescriptor]
71
+ })
72
+ await localNode.start()
73
+ await localNode.joinDht([entryPointPeerDescriptor])
74
+ await localNode.waitForNetworkConnectivity()
75
+
76
+ await waitForCondition(() => localNode.getNeighborCount() === otherPeerDescriptors.length + 1)
77
+ const expectedNodeIds = without(getAllPeerDescriptors(), localPeerDescriptor).map((n) => getNodeIdFromPeerDescriptor(n))
78
+ const actualNodeIds = localNode.getClosestContacts().map((n) => getNodeIdFromPeerDescriptor(n))
79
+ expect(actualNodeIds).toIncludeSameMembers(expectedNodeIds)
80
+ })
81
+ })
@@ -38,8 +38,8 @@ describe('Layer1', () => {
38
38
  }, 30000)
39
39
 
40
40
  afterEach(async () => {
41
- await Promise.all(nodes.map((node) => node.stop()))
42
41
  await Promise.all(layer1CleanUp.map((node) => node.stop()))
42
+ await Promise.all(nodes.map((node) => node.stop()))
43
43
  await layer0EntryPoint.stop()
44
44
  simulator.stop()
45
45
  })
@@ -4,7 +4,7 @@ import { NodeType, PeerDescriptor } from '../../src/proto/packages/dht/protos/Dh
4
4
  import { createMockConnectionDhtNode } from '../utils/utils'
5
5
  import { Logger } from '@streamr/utils'
6
6
  import { getRandomRegion } from '../../src/connection/simulator/pings'
7
- import { areEqualPeerDescriptors, createRandomDhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../src/identifiers'
7
+ import { DhtAddress, areEqualPeerDescriptors, createRandomDhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../src/identifiers'
8
8
 
9
9
  const logger = new Logger(module)
10
10
 
@@ -47,13 +47,16 @@ describe('Scaling down a Dht network', () => {
47
47
  for (let i = 1; i < nodes.length; i++) {
48
48
  randomIndices.push(i)
49
49
  }
50
+ const stoppedNodes: Set<DhtAddress> = new Set()
50
51
  while (randomIndices.length > 1) {
51
52
  const index = Math.floor(Math.random() * randomIndices.length)
52
53
  const nodeIndex = randomIndices[index]
53
54
  randomIndices.splice(index, 1)
54
- const stoppingPeerDescriptor = nodes[nodeIndex].getLocalPeerDescriptor()
55
- await nodes[nodeIndex].stop()
56
- const nodeIsCleaned = nodes.every((node) =>
55
+ const nodeToStop = nodes[nodeIndex]
56
+ const stoppingPeerDescriptor = nodeToStop.getLocalPeerDescriptor()
57
+ stoppedNodes.add(getNodeIdFromPeerDescriptor(stoppingPeerDescriptor))
58
+ await nodeToStop.stop()
59
+ const nodeIsCleaned = nodes.filter((node) => !stoppedNodes.has(node.getNodeId())).every((node) =>
57
60
  node.getConnections().every((peer) => {
58
61
  if (areEqualPeerDescriptors(peer, stoppingPeerDescriptor)) {
59
62
  logger.error(getNodeIdFromPeerDescriptor(node.getLocalPeerDescriptor()) + ', '
@@ -0,0 +1,85 @@
1
+ import { Multimap, wait } from '@streamr/utils'
2
+ import { sampleSize } from 'lodash'
3
+ import { DhtNodeRpcRemote } from '../../src/dht/DhtNodeRpcRemote'
4
+ import { PeerManager, getDistance } from '../../src/dht/PeerManager'
5
+ import { DiscoverySession } from '../../src/dht/discovery/DiscoverySession'
6
+ import { DhtAddress, getNodeIdFromPeerDescriptor, getRawFromDhtAddress } from '../../src/identifiers'
7
+ import { NodeType, PeerDescriptor } from '../../src/proto/packages/dht/protos/DhtRpc'
8
+ import { createTestTopology } from '../utils/topology'
9
+
10
+ const NODE_COUNT = 40
11
+ const MIN_NEIGHBOR_COUNT = 2 // nodes can get more neighbors when we merge network partitions
12
+ const PARALLELISM = 1
13
+ const NO_PROGRESS_LIMIT = 1
14
+ const QUERY_BATCH_SIZE = 5 // the default value in DhtNode's config, not relevant in this test
15
+
16
+ const createPeerDescriptor = (nodeId: DhtAddress): PeerDescriptor => {
17
+ return {
18
+ nodeId: getRawFromDhtAddress(nodeId),
19
+ type: NodeType.NODEJS
20
+ }
21
+ }
22
+
23
+ describe('DiscoverySession', () => {
24
+
25
+ let topology: Multimap<DhtAddress, DhtAddress>
26
+ const queriedNodes: DhtAddress[] = []
27
+
28
+ beforeAll(() => {
29
+ topology = createTestTopology(NODE_COUNT, MIN_NEIGHBOR_COUNT)
30
+ })
31
+
32
+ const createPeerManager = (localNodeId: DhtAddress): PeerManager => {
33
+ const peerManager = new PeerManager({
34
+ localNodeId,
35
+ localPeerDescriptor: createPeerDescriptor(localNodeId),
36
+ isLayer0: true,
37
+ createDhtNodeRpcRemote: (peerDescriptor: PeerDescriptor) => createMockRpcRemote(peerDescriptor) as any,
38
+ hasConnection: () => true
39
+ } as any)
40
+ for (const neighbor of topology.get(localNodeId)) {
41
+ peerManager.addContact(createPeerDescriptor(neighbor))
42
+ }
43
+ return peerManager
44
+ }
45
+
46
+ const createMockRpcRemote = (peerDescriptor: PeerDescriptor): Partial<DhtNodeRpcRemote> => {
47
+ const nodeId = getNodeIdFromPeerDescriptor(peerDescriptor)
48
+ return {
49
+ id: getRawFromDhtAddress(nodeId),
50
+ getPeerDescriptor: () => peerDescriptor,
51
+ getNodeId: () => nodeId,
52
+ getClosestPeers: async (referenceId: DhtAddress) => {
53
+ queriedNodes.push(nodeId)
54
+ await wait(10)
55
+ const peerManager = createPeerManager(nodeId)
56
+ return peerManager.getClosestNeighborsTo(referenceId, QUERY_BATCH_SIZE).map((remote) => remote.getPeerDescriptor())
57
+ },
58
+ ping: async () => true
59
+ }
60
+ }
61
+
62
+ it('happy path', async () => {
63
+ const nodeIds = [...topology.keys()]
64
+ const [localNodeId, targetId] = sampleSize(nodeIds, 2)
65
+ const contactedPeers = new Set<DhtAddress>()
66
+ const peerManager = createPeerManager(localNodeId)
67
+ const session = new DiscoverySession({
68
+ targetId,
69
+ parallelism: PARALLELISM,
70
+ noProgressLimit: NO_PROGRESS_LIMIT,
71
+ peerManager,
72
+ contactedPeers,
73
+ abortSignal: new AbortController().signal
74
+ })
75
+ await session.findClosestNodes(1000)
76
+ expect(queriedNodes.length).toBeGreaterThanOrEqual(1)
77
+ // Each queried node should closer to the target than the previous queried node, because we
78
+ // use parallelism=1 and noProgressLimit=1
79
+ const distancesToTarget = queriedNodes
80
+ .map((nodeId) => getDistance(getRawFromDhtAddress(nodeId), getRawFromDhtAddress(targetId)))
81
+ for (let i = 1; i < distancesToTarget.length ; i++) {
82
+ expect(distancesToTarget[i]).toBeLessThan(distancesToTarget[i - 1])
83
+ }
84
+ })
85
+ })
@@ -26,7 +26,8 @@ const createPeerManager = (
26
26
  }
27
27
  }(localPeerDescriptor, peerDescriptor, undefined as any, new MockRpcCommunicator())
28
28
  return remote
29
- }
29
+ },
30
+ hasConnection: () => false
30
31
  } as any)
31
32
  const contacts = nodeIds.map((n) => ({ nodeId: getRawFromDhtAddress(n), type: NodeType.NODEJS }))
32
33
  for (const contact of contacts) {
@@ -53,21 +54,6 @@ const getClosestContact = (contacts: PeerDescriptor[], referenceId: DhtAddress):
53
54
 
54
55
  describe('PeerManager', () => {
55
56
 
56
- it('getClosestContactsTo', () => {
57
- const nodeIds = range(10).map(() => createRandomDhtAddress())
58
- const manager = createPeerManager(nodeIds)
59
- const referenceId = createRandomDhtAddress()
60
- const excluded = new Set<DhtAddress>(sampleSize(nodeIds, 2))
61
-
62
- const actual = manager.getClosestContactsTo(referenceId, 5, excluded)
63
-
64
- const expected = sortBy(
65
- without(nodeIds, ...Array.from(excluded.values())),
66
- (n: DhtAddress) => getDistance(getRawFromDhtAddress(n), getRawFromDhtAddress(referenceId))
67
- ).slice(0, 5)
68
- expect(actual.map((n) => n.getNodeId())).toEqual(expected)
69
- })
70
-
71
57
  it('getClosestNeighborsTo', () => {
72
58
  const nodeIds = range(10).map(() => createRandomDhtAddress())
73
59
  const manager = createPeerManager(nodeIds)
@@ -99,7 +85,7 @@ describe('PeerManager', () => {
99
85
  for (const successContact of successContacts) {
100
86
  manager.addContact(successContact)
101
87
  manager.setContactActive(getNodeIdFromPeerDescriptor(successContact))
102
- manager.onContactDisconnected(getNodeIdFromPeerDescriptor(successContact), false)
88
+ manager.removeNeighbor(getNodeIdFromPeerDescriptor(successContact))
103
89
  }
104
90
  expect(manager.getNeighborCount()).toBe(0)
105
91
  manager.addContact(failureContact)
@@ -74,12 +74,12 @@ describe('RecursiveOperationManager', () => {
74
74
  return new RecursiveOperationManager({
75
75
  localPeerDescriptor: peerDescriptor1,
76
76
  router,
77
- connections: new Map(),
78
77
  serviceId: 'RecursiveOperationManager',
79
78
  localDataStore: new LocalDataStore(30 * 100),
80
79
  sessionTransport: transport,
81
80
  addContact: () => {},
82
- rpcCommunicator: rpcCommunicator as any
81
+ rpcCommunicator: rpcCommunicator as any,
82
+ createDhtNodeRpcRemote: () => undefined as any
83
83
  })
84
84
  }
85
85
 
@@ -126,6 +126,7 @@ describe('RecursiveOperationManager', () => {
126
126
  const send = jest.fn()
127
127
  const transport = {
128
128
  send,
129
+ getConnections: () => [],
129
130
  on: () => {},
130
131
  off: () => {}
131
132
  }
@@ -143,7 +144,8 @@ describe('RecursiveOperationManager', () => {
143
144
  const router = createMockRouter(RouteMessageError.DUPLICATE)
144
145
  const send = jest.fn()
145
146
  const transport = {
146
- send
147
+ send,
148
+ getConnections: () => []
147
149
  }
148
150
  const recursiveOperationManager = createRecursiveOperationManager(router as any, transport as any)
149
151
  const ack = await rpcCommunicator.callRpcMethod('routeRequest', routedMessage)
@@ -50,8 +50,8 @@ describe('Router', () => {
50
50
  router = new Router({
51
51
  localPeerDescriptor: peerDescriptor1,
52
52
  rpcCommunicator: rpcCommunicator as any,
53
- connections,
54
- handleMessage: () => {}
53
+ handleMessage: () => {},
54
+ getConnections: () => [...connections.values()].map((c) => c.getPeerDescriptor())
55
55
  })
56
56
  })
57
57
 
@@ -49,11 +49,11 @@ describe('RoutingSession', () => {
49
49
  rpcCommunicator: rpcCommunicator,
50
50
  localPeerDescriptor: mockPeerDescriptor1,
51
51
  routedMessage,
52
- connections,
53
52
  parallelism: 2,
54
53
  mode: RoutingMode.ROUTE,
55
54
  excludedNodeIds: new Set(),
56
- routingTablesCache
55
+ routingTablesCache,
56
+ getConnections: () => [...connections.values()].map((c) => c.getPeerDescriptor())
57
57
  })
58
58
  })
59
59
 
@@ -38,7 +38,7 @@ describe('SortedContactList', () => {
38
38
  expect(list.getSize()).toEqual(3)
39
39
  expect(list.getClosestContacts()).toEqual([item1, item2, item3])
40
40
  expect(list.getContactIds()).toEqual([item1.getNodeId(), item2.getNodeId(), item3.getNodeId()])
41
- expect(onContactRemoved).toBeCalledWith(item4, [item1, item2, item3])
41
+ expect(onContactRemoved).toBeCalledWith(item4)
42
42
  expect(list.getContact(item4.getNodeId())).toBeFalsy()
43
43
  })
44
44
 
@@ -63,9 +63,9 @@ describe('SortedContactList', () => {
63
63
  list.removeContact(item3.getNodeId())
64
64
  list.removeContact(createRandomDhtAddress())
65
65
  expect(list.getClosestContacts()).toEqual([item1, item4])
66
- expect(onContactRemoved).toHaveBeenNthCalledWith(1, item3, [])
67
- expect(onContactRemoved).toHaveBeenNthCalledWith(2, item2, [item1, item3, item4])
68
- expect(onContactRemoved).toHaveBeenNthCalledWith(3, item3, [item1, item4])
66
+ expect(onContactRemoved).toHaveBeenNthCalledWith(1, item3)
67
+ expect(onContactRemoved).toHaveBeenNthCalledWith(2, item2)
68
+ expect(onContactRemoved).toHaveBeenNthCalledWith(3, item3)
69
69
  })
70
70
 
71
71
  it('get closest contacts', () => {
@@ -0,0 +1,28 @@
1
+ import { range, sampleSize, sortBy, without } from 'lodash'
2
+ import { getDistance } from '../../src/dht/PeerManager'
3
+ import { getClosestContacts } from '../../src/dht/contact/getClosestContacts'
4
+ import { DhtAddress, createRandomDhtAddress, getRawFromDhtAddress } from '../../src/identifiers'
5
+
6
+ describe('getClosestContacts', () => {
7
+
8
+ it('happy path', () => {
9
+ const nodeIds = range(10).map(() => createRandomDhtAddress())
10
+ const referenceId = createRandomDhtAddress()
11
+ const excluded = new Set<DhtAddress>(sampleSize(nodeIds, 2))
12
+
13
+ const actual = getClosestContacts(
14
+ referenceId,
15
+ nodeIds.map((nodeId) => ({ getNodeId: () => nodeId })),
16
+ {
17
+ maxCount: 5,
18
+ excludedNodeIds: excluded
19
+ }
20
+ )
21
+
22
+ const expected = sortBy(
23
+ without(nodeIds, ...Array.from(excluded.values())),
24
+ (n: DhtAddress) => getDistance(getRawFromDhtAddress(n), getRawFromDhtAddress(referenceId))
25
+ ).slice(0, 5)
26
+ expect(actual.map((n) => n.getNodeId())).toEqual(expected)
27
+ })
28
+ })
@@ -1,11 +1,16 @@
1
1
  import { EventEmitter } from 'eventemitter3'
2
- import { ITransport, TransportEvents } from '../../src/transport/ITransport'
2
+ import { DhtAddress, getDhtAddressFromRaw, getNodeIdFromPeerDescriptor } from '../../src/identifiers'
3
3
  import { Message, PeerDescriptor } from '../../src/proto/packages/dht/protos/DhtRpc'
4
+ import { DEFAULT_SEND_OPTIONS, ITransport, SendOptions, TransportEvents } from '../../src/transport/ITransport'
4
5
 
5
6
  class FakeTransport extends EventEmitter<TransportEvents> implements ITransport {
6
7
 
7
8
  private onSend: (msg: Message) => void
8
9
  private readonly localPeerDescriptor: PeerDescriptor
10
+ // currently adds a peerDescription to the connections array when a "connect" option is seen in
11
+ // in send() call and never disconnects (TODO could add some disconnection logic? and maybe
12
+ // the connection should be seen by another FakeTransport instance, too?)
13
+ private connections: PeerDescriptor[] = []
9
14
 
10
15
  constructor(peerDescriptor: PeerDescriptor, onSend: (msg: Message) => void) {
11
16
  super()
@@ -13,19 +18,37 @@ class FakeTransport extends EventEmitter<TransportEvents> implements ITransport
13
18
  this.localPeerDescriptor = peerDescriptor
14
19
  }
15
20
 
16
- async send(msg: Message): Promise<void> {
21
+ async send(msg: Message, opts?: SendOptions): Promise<void> {
22
+ const connect = opts?.connect ?? DEFAULT_SEND_OPTIONS.connect
23
+ const targetNodeId = getNodeIdFromPeerDescriptor(msg.targetDescriptor!)
24
+ if (connect && !this.connections.some((c) => getNodeIdFromPeerDescriptor(c) === targetNodeId)) {
25
+ this.connect(msg.targetDescriptor!)
26
+ }
17
27
  msg.sourceDescriptor = this.localPeerDescriptor
18
28
  this.onSend(msg)
19
29
  }
20
30
 
21
- // eslint-disable-next-line class-methods-use-this
22
31
  getLocalPeerDescriptor(): PeerDescriptor {
23
- throw new Error('not implemented')
32
+ return this.localPeerDescriptor
33
+ }
34
+
35
+ private connect(peerDescriptor: PeerDescriptor) {
36
+ this.connections.push(peerDescriptor)
37
+ this.emit('connected', peerDescriptor)
24
38
  }
25
39
 
26
- // eslint-disable-next-line class-methods-use-this
27
40
  getConnections(): PeerDescriptor[] {
28
- throw new Error('not implemented')
41
+ return this.connections
42
+ }
43
+
44
+ // eslint-disable-next-line class-methods-use-this
45
+ getConnectionCount(): number {
46
+ return this.connections.length
47
+ }
48
+
49
+ // eslint-disable-next-line class-methods-use-this
50
+ hasConnection(nodeId: DhtAddress): boolean {
51
+ return this.connections.some((c) => getNodeIdFromPeerDescriptor(c) === nodeId)
29
52
  }
30
53
 
31
54
  // eslint-disable-next-line class-methods-use-this
@@ -39,7 +62,11 @@ export class FakeEnvironment {
39
62
 
40
63
  createTransport(peerDescriptor: PeerDescriptor): ITransport {
41
64
  const transport = new FakeTransport(peerDescriptor, (msg) => {
42
- this.transports.forEach((t) => t.emit('message', msg))
65
+ const targetNode = getDhtAddressFromRaw(msg.targetDescriptor!.nodeId)
66
+ const targetTransport = this.transports.find((t) => getNodeIdFromPeerDescriptor(t.getLocalPeerDescriptor()) === targetNode)
67
+ if (targetTransport !== undefined) {
68
+ targetTransport.emit('message', msg)
69
+ }
43
70
  })
44
71
  this.transports.push(transport)
45
72
  return transport
@@ -19,6 +19,16 @@ export class MockTransport extends EventEmitter<TransportEvents> implements ITra
19
19
  return []
20
20
  }
21
21
 
22
+ // eslint-disable-next-line class-methods-use-this
23
+ getConnectionCount(): number {
24
+ return 0
25
+ }
26
+
27
+ // eslint-disable-next-line class-methods-use-this
28
+ hasConnection(): boolean {
29
+ return false
30
+ }
31
+
22
32
  // eslint-disable-next-line class-methods-use-this
23
33
  stop(): void {
24
34